1 /*- 2 * Copyright © 2008 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eric Anholt <eric@anholt.net> 25 * 26 * Copyright (c) 2011 The FreeBSD Foundation 27 * All rights reserved. 28 * 29 * This software was developed by Konstantin Belousov under sponsorship from 30 * the FreeBSD Foundation. 31 * 32 * Redistribution and use in source and binary forms, with or without 33 * modification, are permitted provided that the following conditions 34 * are met: 35 * 1. Redistributions of source code must retain the above copyright 36 * notice, this list of conditions and the following disclaimer. 37 * 2. Redistributions in binary form must reproduce the above copyright 38 * notice, this list of conditions and the following disclaimer in the 39 * documentation and/or other materials provided with the distribution. 40 * 41 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 42 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 43 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 44 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 45 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 46 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 47 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 48 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 49 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 50 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 51 * SUCH DAMAGE. 52 * 53 * $FreeBSD: head/sys/dev/drm2/i915/i915_gem.c 253497 2013-07-20 13:52:40Z kib $ 54 */ 55 56 #include <sys/resourcevar.h> 57 #include <sys/sfbuf.h> 58 59 #include <drm/drmP.h> 60 #include <drm/i915_drm.h> 61 #include "i915_drv.h" 62 #include "intel_drv.h" 63 #include "intel_ringbuffer.h" 64 65 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj); 66 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj); 67 static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, 68 unsigned alignment, bool map_and_fenceable); 69 70 static int i915_gem_phys_pwrite(struct drm_device *dev, 71 struct drm_i915_gem_object *obj, uint64_t data_ptr, uint64_t offset, 72 uint64_t size, struct drm_file *file_priv); 73 74 static uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, 75 int tiling_mode); 76 static uint32_t i915_gem_get_gtt_alignment(struct drm_device *dev, 77 uint32_t size, int tiling_mode); 78 static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, 79 int flags); 80 static int i915_gem_object_set_cpu_read_domain_range( 81 struct drm_i915_gem_object *obj, uint64_t offset, uint64_t size); 82 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj); 83 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj); 84 static int i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj); 85 static bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj); 86 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj); 87 static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex); 88 static void i915_gem_process_flushing_list(struct intel_ring_buffer *ring, 89 uint32_t flush_domains); 90 static void i915_gem_clear_fence_reg(struct drm_device *dev, 91 struct drm_i915_fence_reg *reg); 92 static void i915_gem_reset_fences(struct drm_device *dev); 93 static void i915_gem_lowmem(void *arg); 94 95 static int i915_gem_obj_io(struct drm_device *dev, uint32_t handle, uint64_t data_ptr, 96 uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file); 97 98 MALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem"); 99 long i915_gem_wired_pages_cnt; 100 101 /* some bookkeeping */ 102 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, 103 size_t size) 104 { 105 106 dev_priv->mm.object_count++; 107 dev_priv->mm.object_memory += size; 108 } 109 110 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, 111 size_t size) 112 { 113 114 dev_priv->mm.object_count--; 115 dev_priv->mm.object_memory -= size; 116 } 117 118 static int 119 i915_gem_wait_for_error(struct drm_device *dev) 120 { 121 struct drm_i915_private *dev_priv = dev->dev_private; 122 int ret; 123 124 if (!atomic_read(&dev_priv->mm.wedged)) 125 return 0; 126 127 lockmgr(&dev_priv->error_completion_lock, LK_EXCLUSIVE); 128 while (dev_priv->error_completion == 0) { 129 ret = -lksleep(&dev_priv->error_completion, 130 &dev_priv->error_completion_lock, PCATCH, "915wco", 0); 131 if (ret != 0) { 132 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE); 133 return (ret); 134 } 135 } 136 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE); 137 138 if (atomic_read(&dev_priv->mm.wedged)) { 139 /* GPU is hung, bump the completion count to account for 140 * the token we just consumed so that we never hit zero and 141 * end up waiting upon a subsequent completion event that 142 * will never happen. 143 */ 144 lockmgr(&dev_priv->error_completion_lock, LK_EXCLUSIVE); 145 dev_priv->error_completion++; 146 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE); 147 } 148 return 0; 149 } 150 151 int i915_mutex_lock_interruptible(struct drm_device *dev) 152 { 153 int ret; 154 155 ret = i915_gem_wait_for_error(dev); 156 if (ret != 0) 157 return (ret); 158 159 ret = lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_SLEEPFAIL); 160 if (ret) 161 return -EINTR; 162 163 #if 0 164 WARN_ON(i915_verify_lists(dev)); 165 #endif 166 return 0; 167 } 168 169 static inline bool 170 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj) 171 { 172 return (obj->gtt_space && !obj->active && obj->pin_count == 0); 173 } 174 175 int 176 i915_gem_init_ioctl(struct drm_device *dev, void *data, 177 struct drm_file *file) 178 { 179 struct drm_i915_gem_init *args; 180 drm_i915_private_t *dev_priv; 181 182 dev_priv = dev->dev_private; 183 args = data; 184 185 if (args->gtt_start >= args->gtt_end || 186 (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1)) 187 return (-EINVAL); 188 189 /* 190 * XXXKIB. The second-time initialization should be guarded 191 * against. 192 */ 193 lockmgr(&dev->dev_lock, LK_EXCLUSIVE|LK_RETRY|LK_CANRECURSE); 194 i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end); 195 lockmgr(&dev->dev_lock, LK_RELEASE); 196 197 return 0; 198 } 199 200 int 201 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, 202 struct drm_file *file) 203 { 204 struct drm_i915_private *dev_priv; 205 struct drm_i915_gem_get_aperture *args; 206 struct drm_i915_gem_object *obj; 207 size_t pinned; 208 209 dev_priv = dev->dev_private; 210 args = data; 211 212 if (!(dev->driver->driver_features & DRIVER_GEM)) 213 return (-ENODEV); 214 215 pinned = 0; 216 DRM_LOCK(dev); 217 list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list) 218 pinned += obj->gtt_space->size; 219 DRM_UNLOCK(dev); 220 221 args->aper_size = dev_priv->mm.gtt_total; 222 args->aper_available_size = args->aper_size - pinned; 223 224 return (0); 225 } 226 227 int 228 i915_gem_create(struct drm_file *file, struct drm_device *dev, uint64_t size, 229 uint32_t *handle_p) 230 { 231 struct drm_i915_gem_object *obj; 232 uint32_t handle; 233 int ret; 234 235 size = roundup(size, PAGE_SIZE); 236 if (size == 0) 237 return (-EINVAL); 238 239 obj = i915_gem_alloc_object(dev, size); 240 if (obj == NULL) 241 return (-ENOMEM); 242 243 handle = 0; 244 ret = drm_gem_handle_create(file, &obj->base, &handle); 245 if (ret != 0) { 246 drm_gem_object_release(&obj->base); 247 i915_gem_info_remove_obj(dev->dev_private, obj->base.size); 248 drm_free(obj, DRM_I915_GEM); 249 return (-ret); 250 } 251 252 /* drop reference from allocate - handle holds it now */ 253 drm_gem_object_unreference(&obj->base); 254 *handle_p = handle; 255 return (0); 256 } 257 258 int 259 i915_gem_dumb_create(struct drm_file *file, 260 struct drm_device *dev, 261 struct drm_mode_create_dumb *args) 262 { 263 264 /* have to work out size/pitch and return them */ 265 args->pitch = roundup2(args->width * ((args->bpp + 7) / 8), 64); 266 args->size = args->pitch * args->height; 267 return (i915_gem_create(file, dev, args->size, &args->handle)); 268 } 269 270 int i915_gem_dumb_destroy(struct drm_file *file, 271 struct drm_device *dev, 272 uint32_t handle) 273 { 274 275 return (drm_gem_handle_delete(file, handle)); 276 } 277 278 /** 279 * Creates a new mm object and returns a handle to it. 280 */ 281 int 282 i915_gem_create_ioctl(struct drm_device *dev, void *data, 283 struct drm_file *file) 284 { 285 struct drm_i915_gem_create *args = data; 286 287 return (i915_gem_create(file, dev, args->size, &args->handle)); 288 } 289 290 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) 291 { 292 drm_i915_private_t *dev_priv; 293 294 dev_priv = obj->base.dev->dev_private; 295 return (dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && 296 obj->tiling_mode != I915_TILING_NONE); 297 } 298 299 /** 300 * Reads data from the object referenced by handle. 301 * 302 * On error, the contents of *data are undefined. 303 */ 304 int 305 i915_gem_pread_ioctl(struct drm_device *dev, void *data, 306 struct drm_file *file) 307 { 308 struct drm_i915_gem_pread *args; 309 310 args = data; 311 return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size, 312 args->offset, UIO_READ, file)); 313 } 314 315 /** 316 * Writes data to the object referenced by handle. 317 * 318 * On error, the contents of the buffer that were to be modified are undefined. 319 */ 320 int 321 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, 322 struct drm_file *file) 323 { 324 struct drm_i915_gem_pwrite *args; 325 326 args = data; 327 return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size, 328 args->offset, UIO_WRITE, file)); 329 } 330 331 /** 332 * Waits for a sequence number to be signaled, and cleans up the 333 * request and object lists appropriately for that event. 334 */ 335 int 336 i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno) 337 { 338 drm_i915_private_t *dev_priv; 339 struct drm_i915_gem_request *request; 340 uint32_t ier; 341 int flags, ret; 342 bool recovery_complete; 343 bool do_retire = true; 344 345 KASSERT(seqno != 0, ("Zero seqno")); 346 347 dev_priv = ring->dev->dev_private; 348 ret = 0; 349 350 if (atomic_read(&dev_priv->mm.wedged) != 0) { 351 /* Give the error handler a chance to run. */ 352 lockmgr(&dev_priv->error_completion_lock, LK_EXCLUSIVE); 353 recovery_complete = (&dev_priv->error_completion) > 0; 354 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE); 355 return (recovery_complete ? -EIO : -EAGAIN); 356 } 357 358 if (seqno == ring->outstanding_lazy_request) { 359 request = kmalloc(sizeof(*request), DRM_I915_GEM, 360 M_WAITOK | M_ZERO); 361 if (request == NULL) 362 return (-ENOMEM); 363 364 ret = i915_add_request(ring, NULL, request); 365 if (ret != 0) { 366 drm_free(request, DRM_I915_GEM); 367 return (ret); 368 } 369 370 seqno = request->seqno; 371 } 372 373 if (!i915_seqno_passed(ring->get_seqno(ring,false), seqno)) { 374 if (HAS_PCH_SPLIT(ring->dev)) 375 ier = I915_READ(DEIER) | I915_READ(GTIER); 376 else 377 ier = I915_READ(IER); 378 if (!ier) { 379 DRM_ERROR("something (likely vbetool) disabled " 380 "interrupts, re-enabling\n"); 381 ring->dev->driver->irq_preinstall(ring->dev); 382 ring->dev->driver->irq_postinstall(ring->dev); 383 } 384 385 lockmgr(&ring->irq_lock, LK_EXCLUSIVE); 386 if (ring->irq_get(ring)) { 387 flags = dev_priv->mm.interruptible ? PCATCH : 0; 388 while (!i915_seqno_passed(ring->get_seqno(ring,false), seqno) 389 && !atomic_read(&dev_priv->mm.wedged) && 390 ret == 0) { 391 ret = -lksleep(ring, &ring->irq_lock, flags, 392 "915gwr", 1*hz); 393 } 394 ring->irq_put(ring); 395 lockmgr(&ring->irq_lock, LK_RELEASE); 396 } else { 397 lockmgr(&ring->irq_lock, LK_RELEASE); 398 if (_intel_wait_for(ring->dev, 399 i915_seqno_passed(ring->get_seqno(ring,false), seqno) || 400 atomic_read(&dev_priv->mm.wedged), 3000, 401 0, "i915wrq") != 0) 402 ret = -EBUSY; 403 } 404 405 } 406 if (atomic_read(&dev_priv->mm.wedged)) 407 ret = -EAGAIN; 408 409 /* Directly dispatch request retiring. While we have the work queue 410 * to handle this, the waiter on a request often wants an associated 411 * buffer to have made it to the inactive list, and we would need 412 * a separate wait queue to handle that. 413 */ 414 if (ret == 0 && do_retire) 415 i915_gem_retire_requests_ring(ring); 416 417 return (ret); 418 } 419 420 /** 421 * Ensures that all rendering to the object has completed and the object is 422 * safe to unbind from the GTT or access from the CPU. 423 */ 424 int 425 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj) 426 { 427 u32 seqno; 428 int ret; 429 430 431 seqno = obj->last_rendering_seqno; 432 if (seqno == 0) 433 return 0; 434 435 if (obj->active) { 436 ret = i915_wait_seqno(obj->ring, obj->last_rendering_seqno); 437 if (ret != 0) 438 return (ret); 439 } 440 441 /* Manually manage the write flush as we may have not yet 442 * retired the buffer. 443 */ 444 if (obj->last_rendering_seqno && 445 i915_seqno_passed(seqno, obj->last_rendering_seqno)) { 446 obj->last_rendering_seqno = 0; 447 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS; 448 } 449 450 return 0; 451 } 452 453 /** 454 * Called when user space prepares to use an object with the CPU, either 455 * through the mmap ioctl's mapping or a GTT mapping. 456 */ 457 int 458 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 459 struct drm_file *file) 460 { 461 struct drm_i915_gem_set_domain *args; 462 struct drm_i915_gem_object *obj; 463 uint32_t read_domains; 464 uint32_t write_domain; 465 int ret; 466 467 if ((dev->driver->driver_features & DRIVER_GEM) == 0) 468 return (-ENODEV); 469 470 args = data; 471 read_domains = args->read_domains; 472 write_domain = args->write_domain; 473 474 if ((write_domain & I915_GEM_GPU_DOMAINS) != 0 || 475 (read_domains & I915_GEM_GPU_DOMAINS) != 0 || 476 (write_domain != 0 && read_domains != write_domain)) 477 return (-EINVAL); 478 479 ret = i915_mutex_lock_interruptible(dev); 480 if (ret != 0) 481 return (ret); 482 483 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 484 if (&obj->base == NULL) { 485 ret = -ENOENT; 486 goto unlock; 487 } 488 489 if ((read_domains & I915_GEM_DOMAIN_GTT) != 0) { 490 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); 491 if (ret == -EINVAL) 492 ret = 0; 493 } else 494 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); 495 496 drm_gem_object_unreference(&obj->base); 497 unlock: 498 DRM_UNLOCK(dev); 499 return (ret); 500 } 501 502 /** 503 * Called when user space has done writes to this buffer 504 */ 505 int 506 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, 507 struct drm_file *file) 508 { 509 struct drm_i915_gem_sw_finish *args; 510 struct drm_i915_gem_object *obj; 511 int ret; 512 513 args = data; 514 ret = 0; 515 if ((dev->driver->driver_features & DRIVER_GEM) == 0) 516 return (ENODEV); 517 ret = i915_mutex_lock_interruptible(dev); 518 if (ret != 0) 519 return (ret); 520 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 521 if (&obj->base == NULL) { 522 ret = -ENOENT; 523 goto unlock; 524 } 525 if (obj->pin_count != 0) 526 i915_gem_object_flush_cpu_write_domain(obj); 527 drm_gem_object_unreference(&obj->base); 528 unlock: 529 DRM_UNLOCK(dev); 530 return (ret); 531 } 532 533 /** 534 * Maps the contents of an object, returning the address it is mapped 535 * into. 536 * 537 * While the mapping holds a reference on the contents of the object, it doesn't 538 * imply a ref on the object itself. 539 */ 540 int 541 i915_gem_mmap_ioctl(struct drm_device *dev, void *data, 542 struct drm_file *file) 543 { 544 struct drm_i915_gem_mmap *args; 545 struct drm_gem_object *obj; 546 struct proc *p; 547 vm_map_t map; 548 vm_offset_t addr; 549 vm_size_t size; 550 int error, rv; 551 552 args = data; 553 554 if ((dev->driver->driver_features & DRIVER_GEM) == 0) 555 return (-ENODEV); 556 557 obj = drm_gem_object_lookup(dev, file, args->handle); 558 if (obj == NULL) 559 return (-ENOENT); 560 error = 0; 561 if (args->size == 0) 562 goto out; 563 p = curproc; 564 map = &p->p_vmspace->vm_map; 565 size = round_page(args->size); 566 PROC_LOCK(p); 567 if (map->size + size > p->p_rlimit[RLIMIT_VMEM].rlim_cur) { 568 PROC_UNLOCK(p); 569 error = ENOMEM; 570 goto out; 571 } 572 PROC_UNLOCK(p); 573 574 addr = 0; 575 vm_object_hold(obj->vm_obj); 576 vm_object_reference_locked(obj->vm_obj); 577 vm_object_drop(obj->vm_obj); 578 DRM_UNLOCK(dev); 579 rv = vm_map_find(map, obj->vm_obj, args->offset, &addr, args->size, 580 PAGE_SIZE, /* align */ 581 TRUE, /* fitit */ 582 VM_MAPTYPE_NORMAL, /* maptype */ 583 VM_PROT_READ | VM_PROT_WRITE, /* prot */ 584 VM_PROT_READ | VM_PROT_WRITE, /* max */ 585 MAP_SHARED /* cow */); 586 if (rv != KERN_SUCCESS) { 587 vm_object_deallocate(obj->vm_obj); 588 error = -vm_mmap_to_errno(rv); 589 } else { 590 args->addr_ptr = (uint64_t)addr; 591 } 592 DRM_LOCK(dev); 593 out: 594 drm_gem_object_unreference(obj); 595 return (error); 596 } 597 598 /** 599 * i915_gem_release_mmap - remove physical page mappings 600 * @obj: obj in question 601 * 602 * Preserve the reservation of the mmapping with the DRM core code, but 603 * relinquish ownership of the pages back to the system. 604 * 605 * It is vital that we remove the page mapping if we have mapped a tiled 606 * object through the GTT and then lose the fence register due to 607 * resource pressure. Similarly if the object has been moved out of the 608 * aperture, than pages mapped into userspace must be revoked. Removing the 609 * mapping will then trigger a page fault on the next user access, allowing 610 * fixup by i915_gem_fault(). 611 */ 612 void 613 i915_gem_release_mmap(struct drm_i915_gem_object *obj) 614 { 615 vm_object_t devobj; 616 vm_page_t m; 617 int i, page_count; 618 619 if (!obj->fault_mappable) 620 return; 621 622 devobj = cdev_pager_lookup(obj); 623 if (devobj != NULL) { 624 page_count = OFF_TO_IDX(obj->base.size); 625 626 VM_OBJECT_LOCK(devobj); 627 for (i = 0; i < page_count; i++) { 628 m = vm_page_lookup_busy_wait(devobj, i, TRUE, "915unm"); 629 if (m == NULL) 630 continue; 631 cdev_pager_free_page(devobj, m); 632 } 633 VM_OBJECT_UNLOCK(devobj); 634 vm_object_deallocate(devobj); 635 } 636 637 obj->fault_mappable = false; 638 } 639 640 static uint32_t 641 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode) 642 { 643 uint32_t gtt_size; 644 645 if (INTEL_INFO(dev)->gen >= 4 || 646 tiling_mode == I915_TILING_NONE) 647 return (size); 648 649 /* Previous chips need a power-of-two fence region when tiling */ 650 if (INTEL_INFO(dev)->gen == 3) 651 gtt_size = 1024*1024; 652 else 653 gtt_size = 512*1024; 654 655 while (gtt_size < size) 656 gtt_size <<= 1; 657 658 return (gtt_size); 659 } 660 661 /** 662 * i915_gem_get_gtt_alignment - return required GTT alignment for an object 663 * @obj: object to check 664 * 665 * Return the required GTT alignment for an object, taking into account 666 * potential fence register mapping. 667 */ 668 static uint32_t 669 i915_gem_get_gtt_alignment(struct drm_device *dev, 670 uint32_t size, 671 int tiling_mode) 672 { 673 674 /* 675 * Minimum alignment is 4k (GTT page size), but might be greater 676 * if a fence register is needed for the object. 677 */ 678 if (INTEL_INFO(dev)->gen >= 4 || 679 tiling_mode == I915_TILING_NONE) 680 return (4096); 681 682 /* 683 * Previous chips need to be aligned to the size of the smallest 684 * fence register that can contain the object. 685 */ 686 return (i915_gem_get_gtt_size(dev, size, tiling_mode)); 687 } 688 689 /** 690 * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an 691 * unfenced object 692 * @dev: the device 693 * @size: size of the object 694 * @tiling_mode: tiling mode of the object 695 * 696 * Return the required GTT alignment for an object, only taking into account 697 * unfenced tiled surface requirements. 698 */ 699 uint32_t 700 i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev, 701 uint32_t size, 702 int tiling_mode) 703 { 704 705 if (tiling_mode == I915_TILING_NONE) 706 return (4096); 707 708 /* 709 * Minimum alignment is 4k (GTT page size) for sane hw. 710 */ 711 if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev)) 712 return (4096); 713 714 /* 715 * Previous hardware however needs to be aligned to a power-of-two 716 * tile height. The simplest method for determining this is to reuse 717 * the power-of-tile object size. 718 */ 719 return (i915_gem_get_gtt_size(dev, size, tiling_mode)); 720 } 721 722 int 723 i915_gem_mmap_gtt(struct drm_file *file, 724 struct drm_device *dev, 725 uint32_t handle, 726 uint64_t *offset) 727 { 728 struct drm_i915_private *dev_priv; 729 struct drm_i915_gem_object *obj; 730 int ret; 731 732 if (!(dev->driver->driver_features & DRIVER_GEM)) 733 return (-ENODEV); 734 735 dev_priv = dev->dev_private; 736 737 ret = i915_mutex_lock_interruptible(dev); 738 if (ret != 0) 739 return (ret); 740 741 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); 742 if (&obj->base == NULL) { 743 ret = -ENOENT; 744 goto unlock; 745 } 746 747 if (obj->base.size > dev_priv->mm.gtt_mappable_end) { 748 ret = -E2BIG; 749 goto out; 750 } 751 752 if (obj->madv != I915_MADV_WILLNEED) { 753 DRM_ERROR("Attempting to mmap a purgeable buffer\n"); 754 ret = -EINVAL; 755 goto out; 756 } 757 758 ret = drm_gem_create_mmap_offset(&obj->base); 759 if (ret != 0) 760 goto out; 761 762 *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) | 763 DRM_GEM_MAPPING_KEY; 764 out: 765 drm_gem_object_unreference(&obj->base); 766 unlock: 767 DRM_UNLOCK(dev); 768 return (ret); 769 } 770 771 /** 772 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing 773 * @dev: DRM device 774 * @data: GTT mapping ioctl data 775 * @file: GEM object info 776 * 777 * Simply returns the fake offset to userspace so it can mmap it. 778 * The mmap call will end up in drm_gem_mmap(), which will set things 779 * up so we can get faults in the handler above. 780 * 781 * The fault handler will take care of binding the object into the GTT 782 * (since it may have been evicted to make room for something), allocating 783 * a fence register, and mapping the appropriate aperture address into 784 * userspace. 785 */ 786 int 787 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, 788 struct drm_file *file) 789 { 790 struct drm_i915_private *dev_priv; 791 struct drm_i915_gem_mmap_gtt *args; 792 793 dev_priv = dev->dev_private; 794 args = data; 795 796 return (i915_gem_mmap_gtt(file, dev, args->handle, &args->offset)); 797 } 798 799 /* Immediately discard the backing storage */ 800 static void 801 i915_gem_object_truncate(struct drm_i915_gem_object *obj) 802 { 803 vm_object_t vm_obj; 804 805 vm_obj = obj->base.vm_obj; 806 VM_OBJECT_LOCK(vm_obj); 807 vm_object_page_remove(vm_obj, 0, 0, false); 808 VM_OBJECT_UNLOCK(vm_obj); 809 obj->madv = __I915_MADV_PURGED; 810 } 811 812 static inline int 813 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj) 814 { 815 return obj->madv == I915_MADV_DONTNEED; 816 } 817 818 static inline void vm_page_reference(vm_page_t m) 819 { 820 vm_page_flag_set(m, PG_REFERENCED); 821 } 822 823 static void 824 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) 825 { 826 vm_page_t m; 827 int page_count, i; 828 829 BUG_ON(obj->madv == __I915_MADV_PURGED); 830 831 if (obj->tiling_mode != I915_TILING_NONE) 832 i915_gem_object_save_bit_17_swizzle(obj); 833 if (obj->madv == I915_MADV_DONTNEED) 834 obj->dirty = 0; 835 page_count = obj->base.size / PAGE_SIZE; 836 VM_OBJECT_LOCK(obj->base.vm_obj); 837 #if GEM_PARANOID_CHECK_GTT 838 i915_gem_assert_pages_not_mapped(obj->base.dev, obj->pages, page_count); 839 #endif 840 for (i = 0; i < page_count; i++) { 841 m = obj->pages[i]; 842 if (obj->dirty) 843 vm_page_dirty(m); 844 if (obj->madv == I915_MADV_WILLNEED) 845 vm_page_reference(m); 846 vm_page_busy_wait(obj->pages[i], FALSE, "i915gem"); 847 vm_page_unwire(obj->pages[i], 1); 848 vm_page_wakeup(obj->pages[i]); 849 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 850 } 851 VM_OBJECT_UNLOCK(obj->base.vm_obj); 852 obj->dirty = 0; 853 drm_free(obj->pages, DRM_I915_GEM); 854 obj->pages = NULL; 855 } 856 857 static int 858 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, 859 int flags) 860 { 861 struct drm_device *dev; 862 vm_object_t vm_obj; 863 vm_page_t m; 864 int page_count, i, j; 865 866 dev = obj->base.dev; 867 KASSERT(obj->pages == NULL, ("Obj already has pages")); 868 page_count = obj->base.size / PAGE_SIZE; 869 obj->pages = kmalloc(page_count * sizeof(vm_page_t), DRM_I915_GEM, 870 M_WAITOK); 871 vm_obj = obj->base.vm_obj; 872 VM_OBJECT_LOCK(vm_obj); 873 for (i = 0; i < page_count; i++) { 874 if ((obj->pages[i] = i915_gem_wire_page(vm_obj, i)) == NULL) 875 goto failed; 876 } 877 VM_OBJECT_UNLOCK(vm_obj); 878 if (i915_gem_object_needs_bit17_swizzle(obj)) 879 i915_gem_object_do_bit_17_swizzle(obj); 880 return (0); 881 882 failed: 883 for (j = 0; j < i; j++) { 884 m = obj->pages[j]; 885 vm_page_busy_wait(m, FALSE, "i915gem"); 886 vm_page_unwire(m, 0); 887 vm_page_wakeup(m); 888 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 889 } 890 VM_OBJECT_UNLOCK(vm_obj); 891 drm_free(obj->pages, DRM_I915_GEM); 892 obj->pages = NULL; 893 return (-EIO); 894 } 895 896 void 897 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj, 898 struct intel_ring_buffer *ring, uint32_t seqno) 899 { 900 struct drm_device *dev = obj->base.dev; 901 struct drm_i915_private *dev_priv = dev->dev_private; 902 struct drm_i915_fence_reg *reg; 903 904 obj->ring = ring; 905 KASSERT(ring != NULL, ("NULL ring")); 906 907 /* Add a reference if we're newly entering the active list. */ 908 if (!obj->active) { 909 drm_gem_object_reference(&obj->base); 910 obj->active = 1; 911 } 912 913 /* Move from whatever list we were on to the tail of execution. */ 914 list_move_tail(&obj->mm_list, &dev_priv->mm.active_list); 915 list_move_tail(&obj->ring_list, &ring->active_list); 916 917 obj->last_rendering_seqno = seqno; 918 if (obj->fenced_gpu_access) { 919 obj->last_fenced_seqno = seqno; 920 obj->last_fenced_ring = ring; 921 922 /* Bump MRU to take account of the delayed flush */ 923 if (obj->fence_reg != I915_FENCE_REG_NONE) { 924 reg = &dev_priv->fence_regs[obj->fence_reg]; 925 list_move_tail(®->lru_list, 926 &dev_priv->mm.fence_list); 927 } 928 } 929 } 930 931 static void 932 i915_gem_object_move_off_active(struct drm_i915_gem_object *obj) 933 { 934 list_del_init(&obj->ring_list); 935 obj->last_rendering_seqno = 0; 936 obj->last_fenced_seqno = 0; 937 } 938 939 static void 940 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) 941 { 942 struct drm_device *dev = obj->base.dev; 943 struct drm_i915_private *dev_priv = dev->dev_private; 944 945 if (obj->pin_count != 0) 946 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list); 947 else 948 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 949 950 KASSERT(list_empty(&obj->gpu_write_list), ("On gpu_write_list")); 951 KASSERT(obj->active, ("Object not active")); 952 obj->ring = NULL; 953 obj->last_fenced_ring = NULL; 954 955 i915_gem_object_move_off_active(obj); 956 obj->fenced_gpu_access = false; 957 958 obj->active = 0; 959 obj->pending_gpu_write = false; 960 drm_gem_object_unreference(&obj->base); 961 962 #if 1 963 KIB_NOTYET(); 964 #else 965 WARN_ON(i915_verify_lists(dev)); 966 #endif 967 } 968 969 static u32 970 i915_gem_get_seqno(struct drm_device *dev) 971 { 972 drm_i915_private_t *dev_priv = dev->dev_private; 973 u32 seqno = dev_priv->next_seqno; 974 975 /* reserve 0 for non-seqno */ 976 if (++dev_priv->next_seqno == 0) 977 dev_priv->next_seqno = 1; 978 979 return seqno; 980 } 981 982 int 983 i915_add_request(struct intel_ring_buffer *ring, struct drm_file *file, 984 struct drm_i915_gem_request *request) 985 { 986 drm_i915_private_t *dev_priv; 987 struct drm_i915_file_private *file_priv; 988 uint32_t seqno; 989 u32 request_ring_position; 990 int was_empty; 991 int ret; 992 993 KASSERT(request != NULL, ("NULL request in add")); 994 DRM_LOCK_ASSERT(ring->dev); 995 dev_priv = ring->dev->dev_private; 996 997 seqno = i915_gem_next_request_seqno(ring); 998 request_ring_position = intel_ring_get_tail(ring); 999 1000 ret = ring->add_request(ring, &seqno); 1001 if (ret != 0) 1002 return ret; 1003 1004 request->seqno = seqno; 1005 request->ring = ring; 1006 request->tail = request_ring_position; 1007 request->emitted_jiffies = ticks; 1008 was_empty = list_empty(&ring->request_list); 1009 list_add_tail(&request->list, &ring->request_list); 1010 1011 if (file != NULL) { 1012 file_priv = file->driver_priv; 1013 1014 spin_lock(&file_priv->mm.lock); 1015 request->file_priv = file_priv; 1016 list_add_tail(&request->client_list, 1017 &file_priv->mm.request_list); 1018 spin_unlock(&file_priv->mm.lock); 1019 } 1020 1021 ring->outstanding_lazy_request = 0; 1022 1023 if (!dev_priv->mm.suspended) { 1024 if (i915_enable_hangcheck) { 1025 mod_timer(&dev_priv->hangcheck_timer, 1026 round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES)); 1027 } 1028 if (was_empty) { 1029 queue_delayed_work(dev_priv->wq, 1030 &dev_priv->mm.retire_work, 1031 round_jiffies_up_relative(hz)); 1032 intel_mark_busy(dev_priv->dev); 1033 } 1034 } 1035 return (0); 1036 } 1037 1038 static inline void 1039 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request) 1040 { 1041 struct drm_i915_file_private *file_priv = request->file_priv; 1042 1043 if (!file_priv) 1044 return; 1045 1046 DRM_LOCK_ASSERT(request->ring->dev); 1047 1048 spin_lock(&file_priv->mm.lock); 1049 if (request->file_priv != NULL) { 1050 list_del(&request->client_list); 1051 request->file_priv = NULL; 1052 } 1053 spin_unlock(&file_priv->mm.lock); 1054 } 1055 1056 static void 1057 i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv, 1058 struct intel_ring_buffer *ring) 1059 { 1060 1061 if (ring->dev != NULL) 1062 DRM_LOCK_ASSERT(ring->dev); 1063 1064 while (!list_empty(&ring->request_list)) { 1065 struct drm_i915_gem_request *request; 1066 1067 request = list_first_entry(&ring->request_list, 1068 struct drm_i915_gem_request, list); 1069 1070 list_del(&request->list); 1071 i915_gem_request_remove_from_client(request); 1072 drm_free(request, DRM_I915_GEM); 1073 } 1074 1075 while (!list_empty(&ring->active_list)) { 1076 struct drm_i915_gem_object *obj; 1077 1078 obj = list_first_entry(&ring->active_list, 1079 struct drm_i915_gem_object, ring_list); 1080 1081 obj->base.write_domain = 0; 1082 list_del_init(&obj->gpu_write_list); 1083 i915_gem_object_move_to_inactive(obj); 1084 } 1085 } 1086 1087 static void 1088 i915_gem_reset_fences(struct drm_device *dev) 1089 { 1090 struct drm_i915_private *dev_priv = dev->dev_private; 1091 int i; 1092 1093 for (i = 0; i < dev_priv->num_fence_regs; i++) { 1094 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; 1095 struct drm_i915_gem_object *obj = reg->obj; 1096 1097 if (!obj) 1098 continue; 1099 1100 if (obj->tiling_mode) 1101 i915_gem_release_mmap(obj); 1102 1103 reg->obj->fence_reg = I915_FENCE_REG_NONE; 1104 reg->obj->fenced_gpu_access = false; 1105 reg->obj->last_fenced_seqno = 0; 1106 reg->obj->last_fenced_ring = NULL; 1107 i915_gem_clear_fence_reg(dev, reg); 1108 } 1109 } 1110 1111 void i915_gem_reset(struct drm_device *dev) 1112 { 1113 struct drm_i915_private *dev_priv = dev->dev_private; 1114 struct drm_i915_gem_object *obj; 1115 int i; 1116 1117 for (i = 0; i < I915_NUM_RINGS; i++) 1118 i915_gem_reset_ring_lists(dev_priv, &dev_priv->ring[i]); 1119 1120 /* Remove anything from the flushing lists. The GPU cache is likely 1121 * to be lost on reset along with the data, so simply move the 1122 * lost bo to the inactive list. 1123 */ 1124 while (!list_empty(&dev_priv->mm.flushing_list)) { 1125 obj = list_first_entry(&dev_priv->mm.flushing_list, 1126 struct drm_i915_gem_object, 1127 mm_list); 1128 1129 obj->base.write_domain = 0; 1130 list_del_init(&obj->gpu_write_list); 1131 i915_gem_object_move_to_inactive(obj); 1132 } 1133 1134 /* Move everything out of the GPU domains to ensure we do any 1135 * necessary invalidation upon reuse. 1136 */ 1137 list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) { 1138 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 1139 } 1140 1141 /* The fence registers are invalidated so clear them out */ 1142 i915_gem_reset_fences(dev); 1143 } 1144 1145 static void 1146 i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj) 1147 { 1148 struct drm_device *dev = obj->base.dev; 1149 drm_i915_private_t *dev_priv = dev->dev_private; 1150 1151 KASSERT(obj->active, ("Object not active")); 1152 list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list); 1153 1154 i915_gem_object_move_off_active(obj); 1155 } 1156 1157 /** 1158 * This function clears the request list as sequence numbers are passed. 1159 */ 1160 void 1161 i915_gem_retire_requests_ring(struct intel_ring_buffer *ring) 1162 { 1163 uint32_t seqno; 1164 1165 if (list_empty(&ring->request_list)) 1166 return; 1167 1168 seqno = ring->get_seqno(ring, true); 1169 1170 while (!list_empty(&ring->request_list)) { 1171 struct drm_i915_gem_request *request; 1172 1173 request = list_first_entry(&ring->request_list, 1174 struct drm_i915_gem_request, 1175 list); 1176 1177 if (!i915_seqno_passed(seqno, request->seqno)) 1178 break; 1179 1180 /* We know the GPU must have read the request to have 1181 * sent us the seqno + interrupt, so use the position 1182 * of tail of the request to update the last known position 1183 * of the GPU head. 1184 */ 1185 ring->last_retired_head = request->tail; 1186 1187 list_del(&request->list); 1188 i915_gem_request_remove_from_client(request); 1189 drm_free(request, DRM_I915_GEM); 1190 } 1191 1192 /* Move any buffers on the active list that are no longer referenced 1193 * by the ringbuffer to the flushing/inactive lists as appropriate. 1194 */ 1195 while (!list_empty(&ring->active_list)) { 1196 struct drm_i915_gem_object *obj; 1197 1198 obj = list_first_entry(&ring->active_list, 1199 struct drm_i915_gem_object, 1200 ring_list); 1201 1202 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno)) 1203 break; 1204 1205 if (obj->base.write_domain != 0) 1206 i915_gem_object_move_to_flushing(obj); 1207 else 1208 i915_gem_object_move_to_inactive(obj); 1209 } 1210 1211 if (unlikely(ring->trace_irq_seqno && 1212 i915_seqno_passed(seqno, ring->trace_irq_seqno))) { 1213 ring->irq_put(ring); 1214 ring->trace_irq_seqno = 0; 1215 } 1216 1217 } 1218 1219 static void 1220 i915_gem_free_object_tail(struct drm_i915_gem_object *obj); 1221 1222 void 1223 i915_gem_retire_requests(struct drm_device *dev) 1224 { 1225 drm_i915_private_t *dev_priv = dev->dev_private; 1226 struct drm_i915_gem_object *obj, *next; 1227 int i; 1228 1229 if (!list_empty(&dev_priv->mm.deferred_free_list)) { 1230 list_for_each_entry_safe(obj, next, 1231 &dev_priv->mm.deferred_free_list, mm_list) 1232 i915_gem_free_object_tail(obj); 1233 } 1234 1235 for (i = 0; i < I915_NUM_RINGS; i++) 1236 i915_gem_retire_requests_ring(&dev_priv->ring[i]); 1237 } 1238 1239 static void 1240 i915_gem_retire_work_handler(struct work_struct *work) 1241 { 1242 drm_i915_private_t *dev_priv; 1243 struct drm_device *dev; 1244 struct intel_ring_buffer *ring; 1245 bool idle; 1246 int i; 1247 1248 dev_priv = container_of(work, drm_i915_private_t, 1249 mm.retire_work.work); 1250 dev = dev_priv->dev; 1251 1252 /* Come back later if the device is busy... */ 1253 if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT)) { 1254 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 1255 round_jiffies_up_relative(hz)); 1256 return; 1257 } 1258 1259 i915_gem_retire_requests(dev); 1260 1261 /* Send a periodic flush down the ring so we don't hold onto GEM 1262 * objects indefinitely. 1263 */ 1264 idle = true; 1265 for_each_ring(ring, dev_priv, i) { 1266 if (ring->gpu_caches_dirty) 1267 i915_add_request(ring, NULL, NULL); 1268 1269 idle &= list_empty(&ring->request_list); 1270 } 1271 1272 if (!dev_priv->mm.suspended && !idle) 1273 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 1274 round_jiffies_up_relative(hz)); 1275 if (idle) 1276 intel_mark_idle(dev); 1277 1278 DRM_UNLOCK(dev); 1279 } 1280 1281 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj) 1282 { 1283 u32 old_write_domain, old_read_domains; 1284 1285 /* Act a barrier for all accesses through the GTT */ 1286 cpu_mfence(); 1287 1288 /* Force a pagefault for domain tracking on next user access */ 1289 i915_gem_release_mmap(obj); 1290 1291 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) 1292 return; 1293 1294 old_read_domains = obj->base.read_domains; 1295 old_write_domain = obj->base.write_domain; 1296 1297 obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT; 1298 obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT; 1299 1300 } 1301 1302 int 1303 i915_gem_object_unbind(struct drm_i915_gem_object *obj) 1304 { 1305 drm_i915_private_t *dev_priv; 1306 int ret; 1307 1308 dev_priv = obj->base.dev->dev_private; 1309 ret = 0; 1310 if (obj->gtt_space == NULL) 1311 return (0); 1312 if (obj->pin_count != 0) { 1313 DRM_ERROR("Attempting to unbind pinned buffer\n"); 1314 return (-EINVAL); 1315 } 1316 1317 ret = i915_gem_object_finish_gpu(obj); 1318 if (ret == -ERESTART || ret == -EINTR) 1319 return (ret); 1320 1321 i915_gem_object_finish_gtt(obj); 1322 1323 if (ret == 0) 1324 ret = i915_gem_object_set_to_cpu_domain(obj, 1); 1325 if (ret == -ERESTART || ret == -EINTR) 1326 return (ret); 1327 if (ret != 0) { 1328 i915_gem_clflush_object(obj); 1329 obj->base.read_domains = obj->base.write_domain = 1330 I915_GEM_DOMAIN_CPU; 1331 } 1332 1333 ret = i915_gem_object_put_fence(obj); 1334 if (ret == -ERESTART) 1335 return (ret); 1336 1337 i915_gem_gtt_unbind_object(obj); 1338 if (obj->has_aliasing_ppgtt_mapping) { 1339 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj); 1340 obj->has_aliasing_ppgtt_mapping = 0; 1341 } 1342 i915_gem_object_put_pages_gtt(obj); 1343 1344 list_del_init(&obj->gtt_list); 1345 list_del_init(&obj->mm_list); 1346 obj->map_and_fenceable = true; 1347 1348 drm_mm_put_block(obj->gtt_space); 1349 obj->gtt_space = NULL; 1350 obj->gtt_offset = 0; 1351 1352 if (i915_gem_object_is_purgeable(obj)) 1353 i915_gem_object_truncate(obj); 1354 1355 return (ret); 1356 } 1357 1358 int i915_gpu_idle(struct drm_device *dev) 1359 { 1360 drm_i915_private_t *dev_priv = dev->dev_private; 1361 struct intel_ring_buffer *ring; 1362 int ret, i; 1363 1364 /* Flush everything onto the inactive list. */ 1365 for_each_ring(ring, dev_priv, i) { 1366 ret = intel_ring_idle(ring); 1367 if (ret) 1368 return ret; 1369 } 1370 1371 return 0; 1372 } 1373 1374 static int 1375 sandybridge_write_fence_reg(struct drm_i915_gem_object *obj, 1376 struct intel_ring_buffer *pipelined) 1377 { 1378 struct drm_device *dev = obj->base.dev; 1379 drm_i915_private_t *dev_priv = dev->dev_private; 1380 u32 size = obj->gtt_space->size; 1381 int regnum = obj->fence_reg; 1382 uint64_t val; 1383 1384 val = (uint64_t)((obj->gtt_offset + size - 4096) & 1385 0xfffff000) << 32; 1386 val |= obj->gtt_offset & 0xfffff000; 1387 val |= (uint64_t)((obj->stride / 128) - 1) << 1388 SANDYBRIDGE_FENCE_PITCH_SHIFT; 1389 1390 if (obj->tiling_mode == I915_TILING_Y) 1391 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 1392 val |= I965_FENCE_REG_VALID; 1393 1394 if (pipelined) { 1395 int ret = intel_ring_begin(pipelined, 6); 1396 if (ret) 1397 return ret; 1398 1399 intel_ring_emit(pipelined, MI_NOOP); 1400 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2)); 1401 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8); 1402 intel_ring_emit(pipelined, (u32)val); 1403 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4); 1404 intel_ring_emit(pipelined, (u32)(val >> 32)); 1405 intel_ring_advance(pipelined); 1406 } else 1407 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val); 1408 1409 return 0; 1410 } 1411 1412 static int 1413 i965_write_fence_reg(struct drm_i915_gem_object *obj, 1414 struct intel_ring_buffer *pipelined) 1415 { 1416 struct drm_device *dev = obj->base.dev; 1417 drm_i915_private_t *dev_priv = dev->dev_private; 1418 u32 size = obj->gtt_space->size; 1419 int regnum = obj->fence_reg; 1420 uint64_t val; 1421 1422 val = (uint64_t)((obj->gtt_offset + size - 4096) & 1423 0xfffff000) << 32; 1424 val |= obj->gtt_offset & 0xfffff000; 1425 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; 1426 if (obj->tiling_mode == I915_TILING_Y) 1427 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 1428 val |= I965_FENCE_REG_VALID; 1429 1430 if (pipelined) { 1431 int ret = intel_ring_begin(pipelined, 6); 1432 if (ret) 1433 return ret; 1434 1435 intel_ring_emit(pipelined, MI_NOOP); 1436 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2)); 1437 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8); 1438 intel_ring_emit(pipelined, (u32)val); 1439 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4); 1440 intel_ring_emit(pipelined, (u32)(val >> 32)); 1441 intel_ring_advance(pipelined); 1442 } else 1443 I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val); 1444 1445 return 0; 1446 } 1447 1448 static int 1449 i915_write_fence_reg(struct drm_i915_gem_object *obj, 1450 struct intel_ring_buffer *pipelined) 1451 { 1452 struct drm_device *dev = obj->base.dev; 1453 drm_i915_private_t *dev_priv = dev->dev_private; 1454 u32 size = obj->gtt_space->size; 1455 u32 fence_reg, val, pitch_val; 1456 int tile_width; 1457 1458 if ((obj->gtt_offset & ~I915_FENCE_START_MASK) || 1459 (size & -size) != size || (obj->gtt_offset & (size - 1))) { 1460 kprintf( 1461 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n", 1462 obj->gtt_offset, obj->map_and_fenceable, size); 1463 return -EINVAL; 1464 } 1465 1466 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) 1467 tile_width = 128; 1468 else 1469 tile_width = 512; 1470 1471 /* Note: pitch better be a power of two tile widths */ 1472 pitch_val = obj->stride / tile_width; 1473 pitch_val = ffs(pitch_val) - 1; 1474 1475 val = obj->gtt_offset; 1476 if (obj->tiling_mode == I915_TILING_Y) 1477 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 1478 val |= I915_FENCE_SIZE_BITS(size); 1479 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 1480 val |= I830_FENCE_REG_VALID; 1481 1482 fence_reg = obj->fence_reg; 1483 if (fence_reg < 8) 1484 fence_reg = FENCE_REG_830_0 + fence_reg * 4; 1485 else 1486 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4; 1487 1488 if (pipelined) { 1489 int ret = intel_ring_begin(pipelined, 4); 1490 if (ret) 1491 return ret; 1492 1493 intel_ring_emit(pipelined, MI_NOOP); 1494 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1)); 1495 intel_ring_emit(pipelined, fence_reg); 1496 intel_ring_emit(pipelined, val); 1497 intel_ring_advance(pipelined); 1498 } else 1499 I915_WRITE(fence_reg, val); 1500 1501 return 0; 1502 } 1503 1504 static int 1505 i830_write_fence_reg(struct drm_i915_gem_object *obj, 1506 struct intel_ring_buffer *pipelined) 1507 { 1508 struct drm_device *dev = obj->base.dev; 1509 drm_i915_private_t *dev_priv = dev->dev_private; 1510 u32 size = obj->gtt_space->size; 1511 int regnum = obj->fence_reg; 1512 uint32_t val; 1513 uint32_t pitch_val; 1514 1515 if ((obj->gtt_offset & ~I830_FENCE_START_MASK) || 1516 (size & -size) != size || (obj->gtt_offset & (size - 1))) { 1517 kprintf( 1518 "object 0x%08x not 512K or pot-size 0x%08x aligned\n", 1519 obj->gtt_offset, size); 1520 return -EINVAL; 1521 } 1522 1523 pitch_val = obj->stride / 128; 1524 pitch_val = ffs(pitch_val) - 1; 1525 1526 val = obj->gtt_offset; 1527 if (obj->tiling_mode == I915_TILING_Y) 1528 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 1529 val |= I830_FENCE_SIZE_BITS(size); 1530 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 1531 val |= I830_FENCE_REG_VALID; 1532 1533 if (pipelined) { 1534 int ret = intel_ring_begin(pipelined, 4); 1535 if (ret) 1536 return ret; 1537 1538 intel_ring_emit(pipelined, MI_NOOP); 1539 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1)); 1540 intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4); 1541 intel_ring_emit(pipelined, val); 1542 intel_ring_advance(pipelined); 1543 } else 1544 I915_WRITE(FENCE_REG_830_0 + regnum * 4, val); 1545 1546 return 0; 1547 } 1548 1549 static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno) 1550 { 1551 return i915_seqno_passed(ring->get_seqno(ring,false), seqno); 1552 } 1553 1554 static int 1555 i915_gem_object_flush_fence(struct drm_i915_gem_object *obj, 1556 struct intel_ring_buffer *pipelined) 1557 { 1558 int ret; 1559 1560 if (obj->fenced_gpu_access) { 1561 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 1562 ret = i915_gem_flush_ring(obj->last_fenced_ring, 0, 1563 obj->base.write_domain); 1564 if (ret) 1565 return ret; 1566 } 1567 1568 obj->fenced_gpu_access = false; 1569 } 1570 1571 if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) { 1572 if (!ring_passed_seqno(obj->last_fenced_ring, 1573 obj->last_fenced_seqno)) { 1574 ret = i915_wait_seqno(obj->last_fenced_ring, 1575 obj->last_fenced_seqno); 1576 if (ret) 1577 return ret; 1578 } 1579 1580 obj->last_fenced_seqno = 0; 1581 obj->last_fenced_ring = NULL; 1582 } 1583 1584 /* Ensure that all CPU reads are completed before installing a fence 1585 * and all writes before removing the fence. 1586 */ 1587 if (obj->base.read_domains & I915_GEM_DOMAIN_GTT) 1588 cpu_mfence(); 1589 1590 return 0; 1591 } 1592 1593 int 1594 i915_gem_object_put_fence(struct drm_i915_gem_object *obj) 1595 { 1596 int ret; 1597 1598 if (obj->tiling_mode) 1599 i915_gem_release_mmap(obj); 1600 1601 ret = i915_gem_object_flush_fence(obj, NULL); 1602 if (ret) 1603 return ret; 1604 1605 if (obj->fence_reg != I915_FENCE_REG_NONE) { 1606 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 1607 1608 if (dev_priv->fence_regs[obj->fence_reg].pin_count != 0) 1609 kprintf("%s: pin_count %d\n", __func__, 1610 dev_priv->fence_regs[obj->fence_reg].pin_count); 1611 i915_gem_clear_fence_reg(obj->base.dev, 1612 &dev_priv->fence_regs[obj->fence_reg]); 1613 1614 obj->fence_reg = I915_FENCE_REG_NONE; 1615 } 1616 1617 return 0; 1618 } 1619 1620 static struct drm_i915_fence_reg * 1621 i915_find_fence_reg(struct drm_device *dev, struct intel_ring_buffer *pipelined) 1622 { 1623 struct drm_i915_private *dev_priv = dev->dev_private; 1624 struct drm_i915_fence_reg *reg, *first, *avail; 1625 int i; 1626 1627 /* First try to find a free reg */ 1628 avail = NULL; 1629 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { 1630 reg = &dev_priv->fence_regs[i]; 1631 if (!reg->obj) 1632 return reg; 1633 1634 if (!reg->pin_count) 1635 avail = reg; 1636 } 1637 1638 if (avail == NULL) 1639 return NULL; 1640 1641 /* None available, try to steal one or wait for a user to finish */ 1642 avail = first = NULL; 1643 list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) { 1644 if (reg->pin_count) 1645 continue; 1646 1647 if (first == NULL) 1648 first = reg; 1649 1650 if (!pipelined || 1651 !reg->obj->last_fenced_ring || 1652 reg->obj->last_fenced_ring == pipelined) { 1653 avail = reg; 1654 break; 1655 } 1656 } 1657 1658 if (avail == NULL) 1659 avail = first; 1660 1661 return avail; 1662 } 1663 1664 int 1665 i915_gem_object_get_fence(struct drm_i915_gem_object *obj, 1666 struct intel_ring_buffer *pipelined) 1667 { 1668 struct drm_device *dev = obj->base.dev; 1669 struct drm_i915_private *dev_priv = dev->dev_private; 1670 struct drm_i915_fence_reg *reg; 1671 int ret; 1672 1673 pipelined = NULL; 1674 ret = 0; 1675 1676 if (obj->fence_reg != I915_FENCE_REG_NONE) { 1677 reg = &dev_priv->fence_regs[obj->fence_reg]; 1678 list_move_tail(®->lru_list, &dev_priv->mm.fence_list); 1679 1680 if (obj->tiling_changed) { 1681 ret = i915_gem_object_flush_fence(obj, pipelined); 1682 if (ret) 1683 return ret; 1684 1685 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno) 1686 pipelined = NULL; 1687 1688 if (pipelined) { 1689 reg->setup_seqno = 1690 i915_gem_next_request_seqno(pipelined); 1691 obj->last_fenced_seqno = reg->setup_seqno; 1692 obj->last_fenced_ring = pipelined; 1693 } 1694 1695 goto update; 1696 } 1697 1698 if (!pipelined) { 1699 if (reg->setup_seqno) { 1700 if (!ring_passed_seqno(obj->last_fenced_ring, 1701 reg->setup_seqno)) { 1702 ret = i915_wait_seqno( 1703 obj->last_fenced_ring, 1704 reg->setup_seqno); 1705 if (ret) 1706 return ret; 1707 } 1708 1709 reg->setup_seqno = 0; 1710 } 1711 } else if (obj->last_fenced_ring && 1712 obj->last_fenced_ring != pipelined) { 1713 ret = i915_gem_object_flush_fence(obj, pipelined); 1714 if (ret) 1715 return ret; 1716 } 1717 1718 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno) 1719 pipelined = NULL; 1720 KASSERT(pipelined || reg->setup_seqno == 0, ("!pipelined")); 1721 1722 if (obj->tiling_changed) { 1723 if (pipelined) { 1724 reg->setup_seqno = 1725 i915_gem_next_request_seqno(pipelined); 1726 obj->last_fenced_seqno = reg->setup_seqno; 1727 obj->last_fenced_ring = pipelined; 1728 } 1729 goto update; 1730 } 1731 1732 return 0; 1733 } 1734 1735 reg = i915_find_fence_reg(dev, pipelined); 1736 if (reg == NULL) 1737 return -EDEADLK; 1738 1739 ret = i915_gem_object_flush_fence(obj, pipelined); 1740 if (ret) 1741 return ret; 1742 1743 if (reg->obj) { 1744 struct drm_i915_gem_object *old = reg->obj; 1745 1746 drm_gem_object_reference(&old->base); 1747 1748 if (old->tiling_mode) 1749 i915_gem_release_mmap(old); 1750 1751 ret = i915_gem_object_flush_fence(old, pipelined); 1752 if (ret) { 1753 drm_gem_object_unreference(&old->base); 1754 return ret; 1755 } 1756 1757 if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0) 1758 pipelined = NULL; 1759 1760 old->fence_reg = I915_FENCE_REG_NONE; 1761 old->last_fenced_ring = pipelined; 1762 old->last_fenced_seqno = 1763 pipelined ? i915_gem_next_request_seqno(pipelined) : 0; 1764 1765 drm_gem_object_unreference(&old->base); 1766 } else if (obj->last_fenced_seqno == 0) 1767 pipelined = NULL; 1768 1769 reg->obj = obj; 1770 list_move_tail(®->lru_list, &dev_priv->mm.fence_list); 1771 obj->fence_reg = reg - dev_priv->fence_regs; 1772 obj->last_fenced_ring = pipelined; 1773 1774 reg->setup_seqno = 1775 pipelined ? i915_gem_next_request_seqno(pipelined) : 0; 1776 obj->last_fenced_seqno = reg->setup_seqno; 1777 1778 update: 1779 obj->tiling_changed = false; 1780 switch (INTEL_INFO(dev)->gen) { 1781 case 7: 1782 case 6: 1783 ret = sandybridge_write_fence_reg(obj, pipelined); 1784 break; 1785 case 5: 1786 case 4: 1787 ret = i965_write_fence_reg(obj, pipelined); 1788 break; 1789 case 3: 1790 ret = i915_write_fence_reg(obj, pipelined); 1791 break; 1792 case 2: 1793 ret = i830_write_fence_reg(obj, pipelined); 1794 break; 1795 } 1796 1797 return ret; 1798 } 1799 1800 static int 1801 i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, 1802 unsigned alignment, bool map_and_fenceable) 1803 { 1804 struct drm_device *dev; 1805 struct drm_i915_private *dev_priv; 1806 struct drm_mm_node *free_space; 1807 uint32_t size, fence_size, fence_alignment, unfenced_alignment; 1808 bool mappable, fenceable; 1809 int ret; 1810 1811 dev = obj->base.dev; 1812 dev_priv = dev->dev_private; 1813 1814 if (obj->madv != I915_MADV_WILLNEED) { 1815 DRM_ERROR("Attempting to bind a purgeable object\n"); 1816 return (-EINVAL); 1817 } 1818 1819 fence_size = i915_gem_get_gtt_size(dev, obj->base.size, 1820 obj->tiling_mode); 1821 fence_alignment = i915_gem_get_gtt_alignment(dev, obj->base.size, 1822 obj->tiling_mode); 1823 unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(dev, 1824 obj->base.size, obj->tiling_mode); 1825 if (alignment == 0) 1826 alignment = map_and_fenceable ? fence_alignment : 1827 unfenced_alignment; 1828 if (map_and_fenceable && (alignment & (fence_alignment - 1)) != 0) { 1829 DRM_ERROR("Invalid object alignment requested %u\n", alignment); 1830 return (-EINVAL); 1831 } 1832 1833 size = map_and_fenceable ? fence_size : obj->base.size; 1834 1835 /* If the object is bigger than the entire aperture, reject it early 1836 * before evicting everything in a vain attempt to find space. 1837 */ 1838 if (obj->base.size > (map_and_fenceable ? 1839 dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) { 1840 DRM_ERROR( 1841 "Attempting to bind an object larger than the aperture\n"); 1842 return (-E2BIG); 1843 } 1844 1845 search_free: 1846 if (map_and_fenceable) 1847 free_space = drm_mm_search_free_in_range( 1848 &dev_priv->mm.gtt_space, size, alignment, 0, 1849 dev_priv->mm.gtt_mappable_end, 0); 1850 else 1851 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space, 1852 size, alignment, 0); 1853 if (free_space != NULL) { 1854 int color = 0; 1855 if (map_and_fenceable) 1856 obj->gtt_space = drm_mm_get_block_range_generic( 1857 free_space, size, alignment, color, 0, 1858 dev_priv->mm.gtt_mappable_end, 1); 1859 else 1860 obj->gtt_space = drm_mm_get_block_generic(free_space, 1861 size, alignment, color, 1); 1862 } 1863 if (obj->gtt_space == NULL) { 1864 ret = i915_gem_evict_something(dev, size, alignment, 1865 map_and_fenceable); 1866 if (ret != 0) 1867 return (ret); 1868 goto search_free; 1869 } 1870 ret = i915_gem_object_get_pages_gtt(obj, 0); 1871 if (ret != 0) { 1872 drm_mm_put_block(obj->gtt_space); 1873 obj->gtt_space = NULL; 1874 /* 1875 * i915_gem_object_get_pages_gtt() cannot return 1876 * ENOMEM, since we use vm_page_grab(VM_ALLOC_RETRY) 1877 * (which does not support operation without a flag 1878 * anyway). 1879 */ 1880 return (ret); 1881 } 1882 1883 i915_gem_gtt_bind_object(obj, obj->cache_level); 1884 if (ret != 0) { 1885 i915_gem_object_put_pages_gtt(obj); 1886 drm_mm_put_block(obj->gtt_space); 1887 obj->gtt_space = NULL; 1888 if (i915_gem_evict_everything(dev, false)) 1889 return (ret); 1890 goto search_free; 1891 } 1892 1893 list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list); 1894 list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 1895 1896 obj->gtt_offset = obj->gtt_space->start; 1897 1898 fenceable = 1899 obj->gtt_space->size == fence_size && 1900 (obj->gtt_space->start & (fence_alignment - 1)) == 0; 1901 1902 mappable = 1903 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end; 1904 obj->map_and_fenceable = mappable && fenceable; 1905 1906 return (0); 1907 } 1908 1909 void 1910 i915_gem_clflush_object(struct drm_i915_gem_object *obj) 1911 { 1912 1913 /* If we don't have a page list set up, then we're not pinned 1914 * to GPU, and we can ignore the cache flush because it'll happen 1915 * again at bind time. 1916 */ 1917 if (obj->pages == NULL) 1918 return; 1919 1920 /* If the GPU is snooping the contents of the CPU cache, 1921 * we do not need to manually clear the CPU cache lines. However, 1922 * the caches are only snooped when the render cache is 1923 * flushed/invalidated. As we always have to emit invalidations 1924 * and flushes when moving into and out of the RENDER domain, correct 1925 * snooping behaviour occurs naturally as the result of our domain 1926 * tracking. 1927 */ 1928 if (obj->cache_level != I915_CACHE_NONE) 1929 return; 1930 1931 drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE); 1932 } 1933 1934 /** Flushes the GTT write domain for the object if it's dirty. */ 1935 static void 1936 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj) 1937 { 1938 uint32_t old_write_domain; 1939 1940 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT) 1941 return; 1942 1943 /* No actual flushing is required for the GTT write domain. Writes 1944 * to it immediately go to main memory as far as we know, so there's 1945 * no chipset flush. It also doesn't land in render cache. 1946 * 1947 * However, we do have to enforce the order so that all writes through 1948 * the GTT land before any writes to the device, such as updates to 1949 * the GATT itself. 1950 */ 1951 cpu_sfence(); 1952 1953 old_write_domain = obj->base.write_domain; 1954 obj->base.write_domain = 0; 1955 } 1956 1957 /** Flushes the CPU write domain for the object if it's dirty. */ 1958 static void 1959 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) 1960 { 1961 uint32_t old_write_domain; 1962 1963 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) 1964 return; 1965 1966 i915_gem_clflush_object(obj); 1967 intel_gtt_chipset_flush(); 1968 old_write_domain = obj->base.write_domain; 1969 obj->base.write_domain = 0; 1970 } 1971 1972 static int 1973 i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj) 1974 { 1975 1976 if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0) 1977 return (0); 1978 return (i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain)); 1979 } 1980 1981 /** 1982 * Moves a single object to the GTT read, and possibly write domain. 1983 * 1984 * This function returns when the move is complete, including waiting on 1985 * flushes to occur. 1986 */ 1987 int 1988 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) 1989 { 1990 uint32_t old_write_domain, old_read_domains; 1991 int ret; 1992 1993 if (obj->gtt_space == NULL) 1994 return (-EINVAL); 1995 1996 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) 1997 return 0; 1998 1999 ret = i915_gem_object_flush_gpu_write_domain(obj); 2000 if (ret != 0) 2001 return (ret); 2002 2003 if (obj->pending_gpu_write || write) { 2004 ret = i915_gem_object_wait_rendering(obj); 2005 if (ret != 0) 2006 return (ret); 2007 } 2008 2009 i915_gem_object_flush_cpu_write_domain(obj); 2010 2011 old_write_domain = obj->base.write_domain; 2012 old_read_domains = obj->base.read_domains; 2013 2014 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0, 2015 ("In GTT write domain")); 2016 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 2017 if (write) { 2018 obj->base.read_domains = I915_GEM_DOMAIN_GTT; 2019 obj->base.write_domain = I915_GEM_DOMAIN_GTT; 2020 obj->dirty = 1; 2021 } 2022 2023 return (0); 2024 } 2025 2026 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, 2027 enum i915_cache_level cache_level) 2028 { 2029 struct drm_device *dev = obj->base.dev; 2030 drm_i915_private_t *dev_priv = dev->dev_private; 2031 int ret; 2032 2033 if (obj->cache_level == cache_level) 2034 return 0; 2035 2036 if (obj->pin_count) { 2037 DRM_DEBUG("can not change the cache level of pinned objects\n"); 2038 return -EBUSY; 2039 } 2040 2041 if (obj->gtt_space) { 2042 ret = i915_gem_object_finish_gpu(obj); 2043 if (ret != 0) 2044 return (ret); 2045 2046 i915_gem_object_finish_gtt(obj); 2047 2048 /* Before SandyBridge, you could not use tiling or fence 2049 * registers with snooped memory, so relinquish any fences 2050 * currently pointing to our region in the aperture. 2051 */ 2052 if (INTEL_INFO(obj->base.dev)->gen < 6) { 2053 ret = i915_gem_object_put_fence(obj); 2054 if (ret) 2055 return ret; 2056 } 2057 2058 if (obj->has_global_gtt_mapping) 2059 i915_gem_gtt_bind_object(obj, cache_level); 2060 if (obj->has_aliasing_ppgtt_mapping) 2061 i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt, 2062 obj, cache_level); 2063 } 2064 2065 if (cache_level == I915_CACHE_NONE) { 2066 u32 old_read_domains, old_write_domain; 2067 2068 /* If we're coming from LLC cached, then we haven't 2069 * actually been tracking whether the data is in the 2070 * CPU cache or not, since we only allow one bit set 2071 * in obj->write_domain and have been skipping the clflushes. 2072 * Just set it to the CPU cache for now. 2073 */ 2074 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 2075 ("obj %p in CPU write domain", obj)); 2076 KASSERT((obj->base.read_domains & ~I915_GEM_DOMAIN_CPU) == 0, 2077 ("obj %p in CPU read domain", obj)); 2078 2079 old_read_domains = obj->base.read_domains; 2080 old_write_domain = obj->base.write_domain; 2081 2082 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 2083 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 2084 2085 } 2086 2087 obj->cache_level = cache_level; 2088 return 0; 2089 } 2090 2091 /* 2092 * Prepare buffer for display plane (scanout, cursors, etc). 2093 * Can be called from an uninterruptible phase (modesetting) and allows 2094 * any flushes to be pipelined (for pageflips). 2095 */ 2096 int 2097 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, 2098 u32 alignment, 2099 struct intel_ring_buffer *pipelined) 2100 { 2101 u32 old_read_domains, old_write_domain; 2102 int ret; 2103 2104 ret = i915_gem_object_flush_gpu_write_domain(obj); 2105 if (ret != 0) 2106 return (ret); 2107 2108 if (pipelined != obj->ring) { 2109 ret = i915_gem_object_wait_rendering(obj); 2110 if (ret == -ERESTART || ret == -EINTR) 2111 return (ret); 2112 } 2113 2114 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE); 2115 if (ret != 0) 2116 return (ret); 2117 2118 ret = i915_gem_object_pin(obj, alignment, true); 2119 if (ret != 0) 2120 return (ret); 2121 2122 i915_gem_object_flush_cpu_write_domain(obj); 2123 2124 old_write_domain = obj->base.write_domain; 2125 old_read_domains = obj->base.read_domains; 2126 2127 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0, 2128 ("obj %p in GTT write domain", obj)); 2129 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 2130 2131 return (0); 2132 } 2133 2134 int 2135 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj) 2136 { 2137 int ret; 2138 2139 if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0) 2140 return (0); 2141 2142 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 2143 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain); 2144 if (ret != 0) 2145 return (ret); 2146 } 2147 2148 ret = i915_gem_object_wait_rendering(obj); 2149 if (ret != 0) 2150 return (ret); 2151 2152 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 2153 2154 return (0); 2155 } 2156 2157 /** 2158 * Moves a single object to the CPU read, and possibly write domain. 2159 * 2160 * This function returns when the move is complete, including waiting on 2161 * flushes to occur. 2162 */ 2163 int 2164 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) 2165 { 2166 uint32_t old_write_domain, old_read_domains; 2167 int ret; 2168 2169 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) 2170 return 0; 2171 2172 ret = i915_gem_object_flush_gpu_write_domain(obj); 2173 if (ret != 0) 2174 return (ret); 2175 2176 ret = i915_gem_object_wait_rendering(obj); 2177 if (ret) 2178 return ret; 2179 2180 i915_gem_object_flush_gtt_write_domain(obj); 2181 2182 old_write_domain = obj->base.write_domain; 2183 old_read_domains = obj->base.read_domains; 2184 2185 /* Flush the CPU cache if it's still invalid. */ 2186 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) { 2187 i915_gem_clflush_object(obj); 2188 2189 obj->base.read_domains |= I915_GEM_DOMAIN_CPU; 2190 } 2191 2192 /* It should now be out of any other write domains, and we can update 2193 * the domain values for our changes. 2194 */ 2195 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0); 2196 2197 /* If we're writing through the CPU, then the GPU read domains will 2198 * need to be invalidated at next use. 2199 */ 2200 if (write) { 2201 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 2202 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 2203 } 2204 2205 return 0; 2206 } 2207 2208 /* Throttle our rendering by waiting until the ring has completed our requests 2209 * emitted over 20 msec ago. 2210 * 2211 * Note that if we were to use the current jiffies each time around the loop, 2212 * we wouldn't escape the function with any frames outstanding if the time to 2213 * render a frame was over 20ms. 2214 * 2215 * This should get us reasonable parallelism between CPU and GPU but also 2216 * relatively low latency when blocking on a particular request to finish. 2217 */ 2218 static int 2219 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) 2220 { 2221 struct drm_i915_private *dev_priv = dev->dev_private; 2222 struct drm_i915_file_private *file_priv = file->driver_priv; 2223 unsigned long recent_enough = ticks - (20 * hz / 1000); 2224 struct drm_i915_gem_request *request; 2225 struct intel_ring_buffer *ring = NULL; 2226 u32 seqno = 0; 2227 int ret; 2228 2229 dev_priv = dev->dev_private; 2230 if (atomic_read(&dev_priv->mm.wedged)) 2231 return -EIO; 2232 2233 recent_enough = ticks - (20 * hz / 1000); 2234 ring = NULL; 2235 seqno = 0; 2236 2237 spin_lock(&file_priv->mm.lock); 2238 list_for_each_entry(request, &file_priv->mm.request_list, client_list) { 2239 if (time_after_eq(request->emitted_jiffies, recent_enough)) 2240 break; 2241 2242 ring = request->ring; 2243 seqno = request->seqno; 2244 } 2245 spin_unlock(&file_priv->mm.lock); 2246 2247 if (seqno == 0) 2248 return 0; 2249 2250 ret = 0; 2251 lockmgr(&ring->irq_lock, LK_EXCLUSIVE); 2252 if (!i915_seqno_passed(ring->get_seqno(ring,false), seqno)) { 2253 if (ring->irq_get(ring)) { 2254 while (ret == 0 && 2255 !(i915_seqno_passed(ring->get_seqno(ring,false), seqno) || 2256 atomic_read(&dev_priv->mm.wedged))) 2257 ret = -lksleep(ring, &ring->irq_lock, PCATCH, 2258 "915thr", 1*hz); 2259 ring->irq_put(ring); 2260 if (ret == 0 && atomic_read(&dev_priv->mm.wedged)) 2261 ret = -EIO; 2262 } else if (_intel_wait_for(dev, 2263 i915_seqno_passed(ring->get_seqno(ring,false), seqno) || 2264 atomic_read(&dev_priv->mm.wedged), 3000, 0, "915rtr")) { 2265 ret = -EBUSY; 2266 } 2267 } 2268 lockmgr(&ring->irq_lock, LK_RELEASE); 2269 2270 if (ret == 0) 2271 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0); 2272 2273 return ret; 2274 } 2275 2276 int 2277 i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment, 2278 bool map_and_fenceable) 2279 { 2280 struct drm_device *dev; 2281 struct drm_i915_private *dev_priv; 2282 int ret; 2283 2284 dev = obj->base.dev; 2285 dev_priv = dev->dev_private; 2286 2287 KASSERT(obj->pin_count != DRM_I915_GEM_OBJECT_MAX_PIN_COUNT, 2288 ("Max pin count")); 2289 2290 if (obj->gtt_space != NULL) { 2291 if ((alignment && obj->gtt_offset & (alignment - 1)) || 2292 (map_and_fenceable && !obj->map_and_fenceable)) { 2293 DRM_DEBUG("bo is already pinned with incorrect alignment:" 2294 " offset=%x, req.alignment=%x, req.map_and_fenceable=%d," 2295 " obj->map_and_fenceable=%d\n", 2296 obj->gtt_offset, alignment, 2297 map_and_fenceable, 2298 obj->map_and_fenceable); 2299 ret = i915_gem_object_unbind(obj); 2300 if (ret != 0) 2301 return (ret); 2302 } 2303 } 2304 2305 if (obj->gtt_space == NULL) { 2306 ret = i915_gem_object_bind_to_gtt(obj, alignment, 2307 map_and_fenceable); 2308 if (ret) 2309 return (ret); 2310 } 2311 2312 if (obj->pin_count++ == 0 && !obj->active) 2313 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list); 2314 obj->pin_mappable |= map_and_fenceable; 2315 2316 #if 1 2317 KIB_NOTYET(); 2318 #else 2319 WARN_ON(i915_verify_lists(dev)); 2320 #endif 2321 return (0); 2322 } 2323 2324 void 2325 i915_gem_object_unpin(struct drm_i915_gem_object *obj) 2326 { 2327 struct drm_device *dev; 2328 drm_i915_private_t *dev_priv; 2329 2330 dev = obj->base.dev; 2331 dev_priv = dev->dev_private; 2332 2333 #if 1 2334 KIB_NOTYET(); 2335 #else 2336 WARN_ON(i915_verify_lists(dev)); 2337 #endif 2338 2339 KASSERT(obj->pin_count != 0, ("zero pin count")); 2340 KASSERT(obj->gtt_space != NULL, ("No gtt mapping")); 2341 2342 if (--obj->pin_count == 0) { 2343 if (!obj->active) 2344 list_move_tail(&obj->mm_list, 2345 &dev_priv->mm.inactive_list); 2346 obj->pin_mappable = false; 2347 } 2348 #if 1 2349 KIB_NOTYET(); 2350 #else 2351 WARN_ON(i915_verify_lists(dev)); 2352 #endif 2353 } 2354 2355 int 2356 i915_gem_pin_ioctl(struct drm_device *dev, void *data, 2357 struct drm_file *file) 2358 { 2359 struct drm_i915_gem_pin *args; 2360 struct drm_i915_gem_object *obj; 2361 struct drm_gem_object *gobj; 2362 int ret; 2363 2364 args = data; 2365 2366 ret = i915_mutex_lock_interruptible(dev); 2367 if (ret != 0) 2368 return ret; 2369 2370 gobj = drm_gem_object_lookup(dev, file, args->handle); 2371 if (gobj == NULL) { 2372 ret = -ENOENT; 2373 goto unlock; 2374 } 2375 obj = to_intel_bo(gobj); 2376 2377 if (obj->madv != I915_MADV_WILLNEED) { 2378 DRM_ERROR("Attempting to pin a purgeable buffer\n"); 2379 ret = -EINVAL; 2380 goto out; 2381 } 2382 2383 if (obj->pin_filp != NULL && obj->pin_filp != file) { 2384 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", 2385 args->handle); 2386 ret = -EINVAL; 2387 goto out; 2388 } 2389 2390 obj->user_pin_count++; 2391 obj->pin_filp = file; 2392 if (obj->user_pin_count == 1) { 2393 ret = i915_gem_object_pin(obj, args->alignment, true); 2394 if (ret != 0) 2395 goto out; 2396 } 2397 2398 /* XXX - flush the CPU caches for pinned objects 2399 * as the X server doesn't manage domains yet 2400 */ 2401 i915_gem_object_flush_cpu_write_domain(obj); 2402 args->offset = obj->gtt_offset; 2403 out: 2404 drm_gem_object_unreference(&obj->base); 2405 unlock: 2406 DRM_UNLOCK(dev); 2407 return (ret); 2408 } 2409 2410 int 2411 i915_gem_unpin_ioctl(struct drm_device *dev, void *data, 2412 struct drm_file *file) 2413 { 2414 struct drm_i915_gem_pin *args; 2415 struct drm_i915_gem_object *obj; 2416 int ret; 2417 2418 args = data; 2419 ret = i915_mutex_lock_interruptible(dev); 2420 if (ret != 0) 2421 return (ret); 2422 2423 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 2424 if (&obj->base == NULL) { 2425 ret = -ENOENT; 2426 goto unlock; 2427 } 2428 2429 if (obj->pin_filp != file) { 2430 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", 2431 args->handle); 2432 ret = -EINVAL; 2433 goto out; 2434 } 2435 obj->user_pin_count--; 2436 if (obj->user_pin_count == 0) { 2437 obj->pin_filp = NULL; 2438 i915_gem_object_unpin(obj); 2439 } 2440 2441 out: 2442 drm_gem_object_unreference(&obj->base); 2443 unlock: 2444 DRM_UNLOCK(dev); 2445 return (ret); 2446 } 2447 2448 int 2449 i915_gem_busy_ioctl(struct drm_device *dev, void *data, 2450 struct drm_file *file) 2451 { 2452 struct drm_i915_gem_busy *args; 2453 struct drm_i915_gem_object *obj; 2454 struct drm_i915_gem_request *request; 2455 int ret; 2456 2457 args = data; 2458 2459 ret = i915_mutex_lock_interruptible(dev); 2460 if (ret != 0) 2461 return ret; 2462 2463 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 2464 if (&obj->base == NULL) { 2465 ret = -ENOENT; 2466 goto unlock; 2467 } 2468 2469 args->busy = obj->active; 2470 if (args->busy) { 2471 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 2472 ret = i915_gem_flush_ring(obj->ring, 2473 0, obj->base.write_domain); 2474 } else if (obj->ring->outstanding_lazy_request == 2475 obj->last_rendering_seqno) { 2476 request = kmalloc(sizeof(*request), DRM_I915_GEM, 2477 M_WAITOK | M_ZERO); 2478 ret = i915_add_request(obj->ring, NULL, request); 2479 if (ret != 0) 2480 drm_free(request, DRM_I915_GEM); 2481 } 2482 2483 i915_gem_retire_requests_ring(obj->ring); 2484 args->busy = obj->active; 2485 } 2486 2487 drm_gem_object_unreference(&obj->base); 2488 unlock: 2489 DRM_UNLOCK(dev); 2490 return (ret); 2491 } 2492 2493 int 2494 i915_gem_throttle_ioctl(struct drm_device *dev, void *data, 2495 struct drm_file *file_priv) 2496 { 2497 2498 return (i915_gem_ring_throttle(dev, file_priv)); 2499 } 2500 2501 int 2502 i915_gem_madvise_ioctl(struct drm_device *dev, void *data, 2503 struct drm_file *file_priv) 2504 { 2505 struct drm_i915_gem_madvise *args = data; 2506 struct drm_i915_gem_object *obj; 2507 int ret; 2508 2509 switch (args->madv) { 2510 case I915_MADV_DONTNEED: 2511 case I915_MADV_WILLNEED: 2512 break; 2513 default: 2514 return -EINVAL; 2515 } 2516 2517 ret = i915_mutex_lock_interruptible(dev); 2518 if (ret) 2519 return ret; 2520 2521 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle)); 2522 if (&obj->base == NULL) { 2523 ret = -ENOENT; 2524 goto unlock; 2525 } 2526 2527 if (obj->pin_count) { 2528 ret = -EINVAL; 2529 goto out; 2530 } 2531 2532 if (obj->madv != __I915_MADV_PURGED) 2533 obj->madv = args->madv; 2534 2535 /* if the object is no longer attached, discard its backing storage */ 2536 if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL) 2537 i915_gem_object_truncate(obj); 2538 2539 args->retained = obj->madv != __I915_MADV_PURGED; 2540 2541 out: 2542 drm_gem_object_unreference(&obj->base); 2543 unlock: 2544 DRM_UNLOCK(dev); 2545 return ret; 2546 } 2547 2548 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev, 2549 size_t size) 2550 { 2551 struct drm_i915_private *dev_priv; 2552 struct drm_i915_gem_object *obj; 2553 2554 dev_priv = dev->dev_private; 2555 2556 obj = kmalloc(sizeof(*obj), DRM_I915_GEM, M_WAITOK | M_ZERO); 2557 2558 if (drm_gem_object_init(dev, &obj->base, size) != 0) { 2559 drm_free(obj, DRM_I915_GEM); 2560 return (NULL); 2561 } 2562 2563 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 2564 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 2565 2566 if (HAS_LLC(dev)) 2567 obj->cache_level = I915_CACHE_LLC; 2568 else 2569 obj->cache_level = I915_CACHE_NONE; 2570 obj->base.driver_private = NULL; 2571 obj->fence_reg = I915_FENCE_REG_NONE; 2572 INIT_LIST_HEAD(&obj->mm_list); 2573 INIT_LIST_HEAD(&obj->gtt_list); 2574 INIT_LIST_HEAD(&obj->ring_list); 2575 INIT_LIST_HEAD(&obj->exec_list); 2576 INIT_LIST_HEAD(&obj->gpu_write_list); 2577 obj->madv = I915_MADV_WILLNEED; 2578 /* Avoid an unnecessary call to unbind on the first bind. */ 2579 obj->map_and_fenceable = true; 2580 2581 i915_gem_info_add_obj(dev_priv, size); 2582 2583 return (obj); 2584 } 2585 2586 int i915_gem_init_object(struct drm_gem_object *obj) 2587 { 2588 2589 kprintf("i915_gem_init_object called\n"); 2590 return (0); 2591 } 2592 2593 static void 2594 i915_gem_free_object_tail(struct drm_i915_gem_object *obj) 2595 { 2596 struct drm_device *dev; 2597 drm_i915_private_t *dev_priv; 2598 int ret; 2599 2600 dev = obj->base.dev; 2601 dev_priv = dev->dev_private; 2602 2603 ret = i915_gem_object_unbind(obj); 2604 if (ret == -ERESTART) { 2605 list_move(&obj->mm_list, &dev_priv->mm.deferred_free_list); 2606 return; 2607 } 2608 2609 drm_gem_free_mmap_offset(&obj->base); 2610 drm_gem_object_release(&obj->base); 2611 i915_gem_info_remove_obj(dev_priv, obj->base.size); 2612 2613 drm_free(obj->page_cpu_valid, DRM_I915_GEM); 2614 drm_free(obj->bit_17, DRM_I915_GEM); 2615 drm_free(obj, DRM_I915_GEM); 2616 } 2617 2618 void 2619 i915_gem_free_object(struct drm_gem_object *gem_obj) 2620 { 2621 struct drm_i915_gem_object *obj; 2622 struct drm_device *dev; 2623 2624 obj = to_intel_bo(gem_obj); 2625 dev = obj->base.dev; 2626 2627 while (obj->pin_count > 0) 2628 i915_gem_object_unpin(obj); 2629 2630 if (obj->phys_obj != NULL) 2631 i915_gem_detach_phys_object(dev, obj); 2632 2633 i915_gem_free_object_tail(obj); 2634 } 2635 2636 int 2637 i915_gem_do_init(struct drm_device *dev, unsigned long start, 2638 unsigned long mappable_end, unsigned long end) 2639 { 2640 drm_i915_private_t *dev_priv; 2641 unsigned long mappable; 2642 int error; 2643 2644 dev_priv = dev->dev_private; 2645 mappable = min(end, mappable_end) - start; 2646 2647 drm_mm_init(&dev_priv->mm.gtt_space, start, end - start); 2648 2649 dev_priv->mm.gtt_start = start; 2650 dev_priv->mm.gtt_mappable_end = mappable_end; 2651 dev_priv->mm.gtt_end = end; 2652 dev_priv->mm.gtt_total = end - start; 2653 dev_priv->mm.mappable_gtt_total = mappable; 2654 2655 /* Take over this portion of the GTT */ 2656 intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE); 2657 device_printf(dev->dev, 2658 "taking over the fictitious range 0x%lx-0x%lx\n", 2659 dev->agp->base + start, dev->agp->base + start + mappable); 2660 error = -vm_phys_fictitious_reg_range(dev->agp->base + start, 2661 dev->agp->base + start + mappable, VM_MEMATTR_WRITE_COMBINING); 2662 return (error); 2663 } 2664 2665 int 2666 i915_gem_idle(struct drm_device *dev) 2667 { 2668 drm_i915_private_t *dev_priv; 2669 int ret; 2670 2671 dev_priv = dev->dev_private; 2672 if (dev_priv->mm.suspended) 2673 return (0); 2674 2675 ret = i915_gpu_idle(dev); 2676 if (ret != 0) 2677 return (ret); 2678 2679 /* Under UMS, be paranoid and evict. */ 2680 if (!drm_core_check_feature(dev, DRIVER_MODESET)) { 2681 ret = i915_gem_evict_inactive(dev, false); 2682 if (ret != 0) 2683 return ret; 2684 } 2685 2686 i915_gem_reset_fences(dev); 2687 2688 /* Hack! Don't let anybody do execbuf while we don't control the chip. 2689 * We need to replace this with a semaphore, or something. 2690 * And not confound mm.suspended! 2691 */ 2692 dev_priv->mm.suspended = 1; 2693 del_timer_sync(&dev_priv->hangcheck_timer); 2694 2695 i915_kernel_lost_context(dev); 2696 i915_gem_cleanup_ringbuffer(dev); 2697 2698 /* Cancel the retire work handler, which should be idle now. */ 2699 cancel_delayed_work_sync(&dev_priv->mm.retire_work); 2700 2701 return (ret); 2702 } 2703 2704 void 2705 i915_gem_init_swizzling(struct drm_device *dev) 2706 { 2707 drm_i915_private_t *dev_priv; 2708 2709 dev_priv = dev->dev_private; 2710 2711 if (INTEL_INFO(dev)->gen < 5 || 2712 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) 2713 return; 2714 2715 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | 2716 DISP_TILE_SURFACE_SWIZZLING); 2717 2718 if (IS_GEN5(dev)) 2719 return; 2720 2721 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL); 2722 if (IS_GEN6(dev)) 2723 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB)); 2724 else 2725 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB)); 2726 } 2727 2728 int 2729 i915_gem_init_hw(struct drm_device *dev) 2730 { 2731 drm_i915_private_t *dev_priv; 2732 int ret; 2733 2734 dev_priv = dev->dev_private; 2735 2736 i915_gem_init_swizzling(dev); 2737 2738 ret = intel_init_render_ring_buffer(dev); 2739 if (ret != 0) 2740 return (ret); 2741 2742 if (HAS_BSD(dev)) { 2743 ret = intel_init_bsd_ring_buffer(dev); 2744 if (ret != 0) 2745 goto cleanup_render_ring; 2746 } 2747 2748 if (HAS_BLT(dev)) { 2749 ret = intel_init_blt_ring_buffer(dev); 2750 if (ret != 0) 2751 goto cleanup_bsd_ring; 2752 } 2753 2754 dev_priv->next_seqno = 1; 2755 i915_gem_init_ppgtt(dev); 2756 return (0); 2757 2758 cleanup_bsd_ring: 2759 intel_cleanup_ring_buffer(&dev_priv->ring[VCS]); 2760 cleanup_render_ring: 2761 intel_cleanup_ring_buffer(&dev_priv->ring[RCS]); 2762 return (ret); 2763 } 2764 2765 void 2766 i915_gem_cleanup_ringbuffer(struct drm_device *dev) 2767 { 2768 drm_i915_private_t *dev_priv; 2769 int i; 2770 2771 dev_priv = dev->dev_private; 2772 for (i = 0; i < I915_NUM_RINGS; i++) 2773 intel_cleanup_ring_buffer(&dev_priv->ring[i]); 2774 } 2775 2776 int 2777 i915_gem_entervt_ioctl(struct drm_device *dev, void *data, 2778 struct drm_file *file_priv) 2779 { 2780 drm_i915_private_t *dev_priv = dev->dev_private; 2781 int ret; 2782 2783 if (drm_core_check_feature(dev, DRIVER_MODESET)) 2784 return 0; 2785 2786 if (atomic_read(&dev_priv->mm.wedged)) { 2787 DRM_ERROR("Reenabling wedged hardware, good luck\n"); 2788 atomic_set(&dev_priv->mm.wedged, 0); 2789 } 2790 2791 DRM_LOCK(dev); 2792 dev_priv->mm.suspended = 0; 2793 2794 ret = i915_gem_init_hw(dev); 2795 if (ret != 0) { 2796 DRM_UNLOCK(dev); 2797 return ret; 2798 } 2799 2800 KASSERT(list_empty(&dev_priv->mm.active_list), ("active list")); 2801 DRM_UNLOCK(dev); 2802 2803 ret = drm_irq_install(dev); 2804 if (ret) 2805 goto cleanup_ringbuffer; 2806 2807 return 0; 2808 2809 cleanup_ringbuffer: 2810 DRM_LOCK(dev); 2811 i915_gem_cleanup_ringbuffer(dev); 2812 dev_priv->mm.suspended = 1; 2813 DRM_UNLOCK(dev); 2814 2815 return ret; 2816 } 2817 2818 int 2819 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data, 2820 struct drm_file *file_priv) 2821 { 2822 2823 if (drm_core_check_feature(dev, DRIVER_MODESET)) 2824 return 0; 2825 2826 drm_irq_uninstall(dev); 2827 return (i915_gem_idle(dev)); 2828 } 2829 2830 void 2831 i915_gem_lastclose(struct drm_device *dev) 2832 { 2833 int ret; 2834 2835 if (drm_core_check_feature(dev, DRIVER_MODESET)) 2836 return; 2837 2838 ret = i915_gem_idle(dev); 2839 if (ret != 0) 2840 DRM_ERROR("failed to idle hardware: %d\n", ret); 2841 } 2842 2843 static void 2844 init_ring_lists(struct intel_ring_buffer *ring) 2845 { 2846 2847 INIT_LIST_HEAD(&ring->active_list); 2848 INIT_LIST_HEAD(&ring->request_list); 2849 INIT_LIST_HEAD(&ring->gpu_write_list); 2850 } 2851 2852 void 2853 i915_gem_load(struct drm_device *dev) 2854 { 2855 int i; 2856 drm_i915_private_t *dev_priv = dev->dev_private; 2857 2858 INIT_LIST_HEAD(&dev_priv->mm.active_list); 2859 INIT_LIST_HEAD(&dev_priv->mm.flushing_list); 2860 INIT_LIST_HEAD(&dev_priv->mm.inactive_list); 2861 INIT_LIST_HEAD(&dev_priv->mm.pinned_list); 2862 INIT_LIST_HEAD(&dev_priv->mm.fence_list); 2863 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list); 2864 INIT_LIST_HEAD(&dev_priv->mm.gtt_list); 2865 for (i = 0; i < I915_NUM_RINGS; i++) 2866 init_ring_lists(&dev_priv->ring[i]); 2867 for (i = 0; i < I915_MAX_NUM_FENCES; i++) 2868 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); 2869 INIT_DELAYED_WORK(&dev_priv->mm.retire_work, 2870 i915_gem_retire_work_handler); 2871 dev_priv->error_completion = 0; 2872 2873 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ 2874 if (IS_GEN3(dev)) { 2875 I915_WRITE(MI_ARB_STATE, 2876 _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE)); 2877 } 2878 2879 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; 2880 2881 /* Old X drivers will take 0-2 for front, back, depth buffers */ 2882 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 2883 dev_priv->fence_reg_start = 3; 2884 2885 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) 2886 dev_priv->num_fence_regs = 16; 2887 else 2888 dev_priv->num_fence_regs = 8; 2889 2890 /* Initialize fence registers to zero */ 2891 i915_gem_reset_fences(dev); 2892 2893 i915_gem_detect_bit_6_swizzle(dev); 2894 2895 dev_priv->mm.interruptible = true; 2896 2897 dev_priv->mm.i915_lowmem = EVENTHANDLER_REGISTER(vm_lowmem, 2898 i915_gem_lowmem, dev, EVENTHANDLER_PRI_ANY); 2899 } 2900 2901 static int 2902 i915_gem_init_phys_object(struct drm_device *dev, int id, int size, int align) 2903 { 2904 drm_i915_private_t *dev_priv; 2905 struct drm_i915_gem_phys_object *phys_obj; 2906 int ret; 2907 2908 dev_priv = dev->dev_private; 2909 if (dev_priv->mm.phys_objs[id - 1] != NULL || size == 0) 2910 return (0); 2911 2912 phys_obj = kmalloc(sizeof(struct drm_i915_gem_phys_object), DRM_I915_GEM, 2913 M_WAITOK | M_ZERO); 2914 2915 phys_obj->id = id; 2916 2917 phys_obj->handle = drm_pci_alloc(dev, size, align, ~0); 2918 if (phys_obj->handle == NULL) { 2919 ret = -ENOMEM; 2920 goto free_obj; 2921 } 2922 pmap_change_attr((vm_offset_t)phys_obj->handle->vaddr, 2923 size / PAGE_SIZE, PAT_WRITE_COMBINING); 2924 2925 dev_priv->mm.phys_objs[id - 1] = phys_obj; 2926 2927 return (0); 2928 2929 free_obj: 2930 drm_free(phys_obj, DRM_I915_GEM); 2931 return (ret); 2932 } 2933 2934 static void 2935 i915_gem_free_phys_object(struct drm_device *dev, int id) 2936 { 2937 drm_i915_private_t *dev_priv; 2938 struct drm_i915_gem_phys_object *phys_obj; 2939 2940 dev_priv = dev->dev_private; 2941 if (dev_priv->mm.phys_objs[id - 1] == NULL) 2942 return; 2943 2944 phys_obj = dev_priv->mm.phys_objs[id - 1]; 2945 if (phys_obj->cur_obj != NULL) 2946 i915_gem_detach_phys_object(dev, phys_obj->cur_obj); 2947 2948 drm_pci_free(dev, phys_obj->handle); 2949 drm_free(phys_obj, DRM_I915_GEM); 2950 dev_priv->mm.phys_objs[id - 1] = NULL; 2951 } 2952 2953 void 2954 i915_gem_free_all_phys_object(struct drm_device *dev) 2955 { 2956 int i; 2957 2958 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++) 2959 i915_gem_free_phys_object(dev, i); 2960 } 2961 2962 void 2963 i915_gem_detach_phys_object(struct drm_device *dev, 2964 struct drm_i915_gem_object *obj) 2965 { 2966 vm_page_t m; 2967 struct sf_buf *sf; 2968 char *vaddr, *dst; 2969 int i, page_count; 2970 2971 if (obj->phys_obj == NULL) 2972 return; 2973 vaddr = obj->phys_obj->handle->vaddr; 2974 2975 page_count = obj->base.size / PAGE_SIZE; 2976 VM_OBJECT_LOCK(obj->base.vm_obj); 2977 for (i = 0; i < page_count; i++) { 2978 m = i915_gem_wire_page(obj->base.vm_obj, i); 2979 if (m == NULL) 2980 continue; /* XXX */ 2981 2982 VM_OBJECT_UNLOCK(obj->base.vm_obj); 2983 sf = sf_buf_alloc(m); 2984 if (sf != NULL) { 2985 dst = (char *)sf_buf_kva(sf); 2986 memcpy(dst, vaddr + IDX_TO_OFF(i), PAGE_SIZE); 2987 sf_buf_free(sf); 2988 } 2989 drm_clflush_pages(&m, 1); 2990 2991 VM_OBJECT_LOCK(obj->base.vm_obj); 2992 vm_page_reference(m); 2993 vm_page_dirty(m); 2994 vm_page_busy_wait(m, FALSE, "i915gem"); 2995 vm_page_unwire(m, 0); 2996 vm_page_wakeup(m); 2997 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 2998 } 2999 VM_OBJECT_UNLOCK(obj->base.vm_obj); 3000 intel_gtt_chipset_flush(); 3001 3002 obj->phys_obj->cur_obj = NULL; 3003 obj->phys_obj = NULL; 3004 } 3005 3006 int 3007 i915_gem_attach_phys_object(struct drm_device *dev, 3008 struct drm_i915_gem_object *obj, 3009 int id, 3010 int align) 3011 { 3012 drm_i915_private_t *dev_priv; 3013 vm_page_t m; 3014 struct sf_buf *sf; 3015 char *dst, *src; 3016 int i, page_count, ret; 3017 3018 if (id > I915_MAX_PHYS_OBJECT) 3019 return (-EINVAL); 3020 3021 if (obj->phys_obj != NULL) { 3022 if (obj->phys_obj->id == id) 3023 return (0); 3024 i915_gem_detach_phys_object(dev, obj); 3025 } 3026 3027 dev_priv = dev->dev_private; 3028 if (dev_priv->mm.phys_objs[id - 1] == NULL) { 3029 ret = i915_gem_init_phys_object(dev, id, obj->base.size, align); 3030 if (ret != 0) { 3031 DRM_ERROR("failed to init phys object %d size: %zu\n", 3032 id, obj->base.size); 3033 return (ret); 3034 } 3035 } 3036 3037 /* bind to the object */ 3038 obj->phys_obj = dev_priv->mm.phys_objs[id - 1]; 3039 obj->phys_obj->cur_obj = obj; 3040 3041 page_count = obj->base.size / PAGE_SIZE; 3042 3043 VM_OBJECT_LOCK(obj->base.vm_obj); 3044 ret = 0; 3045 for (i = 0; i < page_count; i++) { 3046 m = i915_gem_wire_page(obj->base.vm_obj, i); 3047 if (m == NULL) { 3048 ret = -EIO; 3049 break; 3050 } 3051 VM_OBJECT_UNLOCK(obj->base.vm_obj); 3052 sf = sf_buf_alloc(m); 3053 src = (char *)sf_buf_kva(sf); 3054 dst = (char *)obj->phys_obj->handle->vaddr + IDX_TO_OFF(i); 3055 memcpy(dst, src, PAGE_SIZE); 3056 sf_buf_free(sf); 3057 3058 VM_OBJECT_LOCK(obj->base.vm_obj); 3059 3060 vm_page_reference(m); 3061 vm_page_busy_wait(m, FALSE, "i915gem"); 3062 vm_page_unwire(m, 0); 3063 vm_page_wakeup(m); 3064 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 3065 } 3066 VM_OBJECT_UNLOCK(obj->base.vm_obj); 3067 3068 return (0); 3069 } 3070 3071 static int 3072 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_i915_gem_object *obj, 3073 uint64_t data_ptr, uint64_t offset, uint64_t size, 3074 struct drm_file *file_priv) 3075 { 3076 char *user_data, *vaddr; 3077 int ret; 3078 3079 vaddr = (char *)obj->phys_obj->handle->vaddr + offset; 3080 user_data = (char *)(uintptr_t)data_ptr; 3081 3082 if (copyin_nofault(user_data, vaddr, size) != 0) { 3083 /* The physical object once assigned is fixed for the lifetime 3084 * of the obj, so we can safely drop the lock and continue 3085 * to access vaddr. 3086 */ 3087 DRM_UNLOCK(dev); 3088 ret = -copyin(user_data, vaddr, size); 3089 DRM_LOCK(dev); 3090 if (ret != 0) 3091 return (ret); 3092 } 3093 3094 intel_gtt_chipset_flush(); 3095 return (0); 3096 } 3097 3098 void 3099 i915_gem_release(struct drm_device *dev, struct drm_file *file) 3100 { 3101 struct drm_i915_file_private *file_priv; 3102 struct drm_i915_gem_request *request; 3103 3104 file_priv = file->driver_priv; 3105 3106 /* Clean up our request list when the client is going away, so that 3107 * later retire_requests won't dereference our soon-to-be-gone 3108 * file_priv. 3109 */ 3110 spin_lock(&file_priv->mm.lock); 3111 while (!list_empty(&file_priv->mm.request_list)) { 3112 request = list_first_entry(&file_priv->mm.request_list, 3113 struct drm_i915_gem_request, 3114 client_list); 3115 list_del(&request->client_list); 3116 request->file_priv = NULL; 3117 } 3118 spin_unlock(&file_priv->mm.lock); 3119 } 3120 3121 static int 3122 i915_gem_swap_io(struct drm_device *dev, struct drm_i915_gem_object *obj, 3123 uint64_t data_ptr, uint64_t size, uint64_t offset, enum uio_rw rw, 3124 struct drm_file *file) 3125 { 3126 vm_object_t vm_obj; 3127 vm_page_t m; 3128 struct sf_buf *sf; 3129 vm_offset_t mkva; 3130 vm_pindex_t obj_pi; 3131 int cnt, do_bit17_swizzling, length, obj_po, ret, swizzled_po; 3132 3133 if (obj->gtt_offset != 0 && rw == UIO_READ) 3134 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 3135 else 3136 do_bit17_swizzling = 0; 3137 3138 obj->dirty = 1; 3139 vm_obj = obj->base.vm_obj; 3140 ret = 0; 3141 3142 VM_OBJECT_LOCK(vm_obj); 3143 vm_object_pip_add(vm_obj, 1); 3144 while (size > 0) { 3145 obj_pi = OFF_TO_IDX(offset); 3146 obj_po = offset & PAGE_MASK; 3147 3148 m = i915_gem_wire_page(vm_obj, obj_pi); 3149 VM_OBJECT_UNLOCK(vm_obj); 3150 3151 sf = sf_buf_alloc(m); 3152 mkva = sf_buf_kva(sf); 3153 length = min(size, PAGE_SIZE - obj_po); 3154 while (length > 0) { 3155 if (do_bit17_swizzling && 3156 (VM_PAGE_TO_PHYS(m) & (1 << 17)) != 0) { 3157 cnt = roundup2(obj_po + 1, 64); 3158 cnt = min(cnt - obj_po, length); 3159 swizzled_po = obj_po ^ 64; 3160 } else { 3161 cnt = length; 3162 swizzled_po = obj_po; 3163 } 3164 if (rw == UIO_READ) 3165 ret = -copyout_nofault( 3166 (char *)mkva + swizzled_po, 3167 (void *)(uintptr_t)data_ptr, cnt); 3168 else 3169 ret = -copyin_nofault( 3170 (void *)(uintptr_t)data_ptr, 3171 (char *)mkva + swizzled_po, cnt); 3172 if (ret != 0) 3173 break; 3174 data_ptr += cnt; 3175 size -= cnt; 3176 length -= cnt; 3177 offset += cnt; 3178 obj_po += cnt; 3179 } 3180 sf_buf_free(sf); 3181 VM_OBJECT_LOCK(vm_obj); 3182 if (rw == UIO_WRITE) 3183 vm_page_dirty(m); 3184 vm_page_reference(m); 3185 vm_page_busy_wait(m, FALSE, "i915gem"); 3186 vm_page_unwire(m, 1); 3187 vm_page_wakeup(m); 3188 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 3189 3190 if (ret != 0) 3191 break; 3192 } 3193 vm_object_pip_wakeup(vm_obj); 3194 VM_OBJECT_UNLOCK(vm_obj); 3195 3196 return (ret); 3197 } 3198 3199 static int 3200 i915_gem_gtt_write(struct drm_device *dev, struct drm_i915_gem_object *obj, 3201 uint64_t data_ptr, uint64_t size, uint64_t offset, struct drm_file *file) 3202 { 3203 vm_offset_t mkva; 3204 int ret; 3205 3206 /* 3207 * Pass the unaligned physical address and size to pmap_mapdev_attr() 3208 * so it can properly calculate whether an extra page needs to be 3209 * mapped or not to cover the requested range. The function will 3210 * add the page offset into the returned mkva for us. 3211 */ 3212 mkva = (vm_offset_t)pmap_mapdev_attr(dev->agp->base + obj->gtt_offset + 3213 offset, size, PAT_WRITE_COMBINING); 3214 ret = -copyin_nofault((void *)(uintptr_t)data_ptr, (char *)mkva, size); 3215 pmap_unmapdev(mkva, size); 3216 return (ret); 3217 } 3218 3219 static int 3220 i915_gem_obj_io(struct drm_device *dev, uint32_t handle, uint64_t data_ptr, 3221 uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file) 3222 { 3223 struct drm_i915_gem_object *obj; 3224 vm_page_t *ma; 3225 vm_offset_t start, end; 3226 int npages, ret; 3227 3228 if (size == 0) 3229 return (0); 3230 start = trunc_page(data_ptr); 3231 end = round_page(data_ptr + size); 3232 npages = howmany(end - start, PAGE_SIZE); 3233 ma = kmalloc(npages * sizeof(vm_page_t), DRM_I915_GEM, M_WAITOK | 3234 M_ZERO); 3235 npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, 3236 (vm_offset_t)data_ptr, size, 3237 (rw == UIO_READ ? VM_PROT_WRITE : 0 ) | VM_PROT_READ, ma, npages); 3238 if (npages == -1) { 3239 ret = -EFAULT; 3240 goto free_ma; 3241 } 3242 3243 ret = i915_mutex_lock_interruptible(dev); 3244 if (ret != 0) 3245 goto unlocked; 3246 3247 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); 3248 if (&obj->base == NULL) { 3249 ret = -ENOENT; 3250 goto unlock; 3251 } 3252 if (offset > obj->base.size || size > obj->base.size - offset) { 3253 ret = -EINVAL; 3254 goto out; 3255 } 3256 3257 if (rw == UIO_READ) { 3258 ret = i915_gem_object_set_cpu_read_domain_range(obj, 3259 offset, size); 3260 if (ret != 0) 3261 goto out; 3262 ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset, 3263 UIO_READ, file); 3264 } else { 3265 if (obj->phys_obj) { 3266 ret = i915_gem_phys_pwrite(dev, obj, data_ptr, offset, 3267 size, file); 3268 } else if (obj->gtt_space && 3269 obj->base.write_domain != I915_GEM_DOMAIN_CPU) { 3270 ret = i915_gem_object_pin(obj, 0, true); 3271 if (ret != 0) 3272 goto out; 3273 ret = i915_gem_object_set_to_gtt_domain(obj, true); 3274 if (ret != 0) 3275 goto out_unpin; 3276 ret = i915_gem_object_put_fence(obj); 3277 if (ret != 0) 3278 goto out_unpin; 3279 ret = i915_gem_gtt_write(dev, obj, data_ptr, size, 3280 offset, file); 3281 out_unpin: 3282 i915_gem_object_unpin(obj); 3283 } else { 3284 ret = i915_gem_object_set_to_cpu_domain(obj, true); 3285 if (ret != 0) 3286 goto out; 3287 ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset, 3288 UIO_WRITE, file); 3289 } 3290 } 3291 out: 3292 drm_gem_object_unreference(&obj->base); 3293 unlock: 3294 DRM_UNLOCK(dev); 3295 unlocked: 3296 vm_page_unhold_pages(ma, npages); 3297 free_ma: 3298 drm_free(ma, DRM_I915_GEM); 3299 return (ret); 3300 } 3301 3302 static int 3303 i915_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 3304 vm_ooffset_t foff, struct ucred *cred, u_short *color) 3305 { 3306 3307 *color = 0; /* XXXKIB */ 3308 return (0); 3309 } 3310 3311 int i915_intr_pf; 3312 3313 static int 3314 i915_gem_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot, 3315 vm_page_t *mres) 3316 { 3317 struct drm_gem_object *gem_obj; 3318 struct drm_i915_gem_object *obj; 3319 struct drm_device *dev; 3320 drm_i915_private_t *dev_priv; 3321 vm_page_t m, oldm; 3322 int cause, ret; 3323 bool write; 3324 3325 gem_obj = vm_obj->handle; 3326 obj = to_intel_bo(gem_obj); 3327 dev = obj->base.dev; 3328 dev_priv = dev->dev_private; 3329 #if 0 3330 write = (prot & VM_PROT_WRITE) != 0; 3331 #else 3332 write = true; 3333 #endif 3334 vm_object_pip_add(vm_obj, 1); 3335 3336 /* 3337 * Remove the placeholder page inserted by vm_fault() from the 3338 * object before dropping the object lock. If 3339 * i915_gem_release_mmap() is active in parallel on this gem 3340 * object, then it owns the drm device sx and might find the 3341 * placeholder already. Then, since the page is busy, 3342 * i915_gem_release_mmap() sleeps waiting for the busy state 3343 * of the page cleared. We will be not able to acquire drm 3344 * device lock until i915_gem_release_mmap() is able to make a 3345 * progress. 3346 */ 3347 if (*mres != NULL) { 3348 oldm = *mres; 3349 vm_page_remove(oldm); 3350 *mres = NULL; 3351 } else 3352 oldm = NULL; 3353 retry: 3354 VM_OBJECT_UNLOCK(vm_obj); 3355 unlocked_vmobj: 3356 cause = ret = 0; 3357 m = NULL; 3358 3359 if (i915_intr_pf) { 3360 ret = i915_mutex_lock_interruptible(dev); 3361 if (ret != 0) { 3362 cause = 10; 3363 goto out; 3364 } 3365 } else 3366 DRM_LOCK(dev); 3367 3368 /* 3369 * Since the object lock was dropped, other thread might have 3370 * faulted on the same GTT address and instantiated the 3371 * mapping for the page. Recheck. 3372 */ 3373 VM_OBJECT_LOCK(vm_obj); 3374 m = vm_page_lookup(vm_obj, OFF_TO_IDX(offset)); 3375 if (m != NULL) { 3376 if ((m->flags & PG_BUSY) != 0) { 3377 DRM_UNLOCK(dev); 3378 #if 0 /* XXX */ 3379 vm_page_sleep(m, "915pee"); 3380 #endif 3381 goto retry; 3382 } 3383 goto have_page; 3384 } else 3385 VM_OBJECT_UNLOCK(vm_obj); 3386 3387 /* Now bind it into the GTT if needed */ 3388 if (!obj->map_and_fenceable) { 3389 ret = i915_gem_object_unbind(obj); 3390 if (ret != 0) { 3391 cause = 20; 3392 goto unlock; 3393 } 3394 } 3395 if (!obj->gtt_space) { 3396 ret = i915_gem_object_bind_to_gtt(obj, 0, true); 3397 if (ret != 0) { 3398 cause = 30; 3399 goto unlock; 3400 } 3401 3402 ret = i915_gem_object_set_to_gtt_domain(obj, write); 3403 if (ret != 0) { 3404 cause = 40; 3405 goto unlock; 3406 } 3407 } 3408 3409 if (obj->tiling_mode == I915_TILING_NONE) 3410 ret = i915_gem_object_put_fence(obj); 3411 else 3412 ret = i915_gem_object_get_fence(obj, NULL); 3413 if (ret != 0) { 3414 cause = 50; 3415 goto unlock; 3416 } 3417 3418 if (i915_gem_object_is_inactive(obj)) 3419 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 3420 3421 obj->fault_mappable = true; 3422 VM_OBJECT_LOCK(vm_obj); 3423 m = vm_phys_fictitious_to_vm_page(dev->agp->base + obj->gtt_offset + 3424 offset); 3425 if (m == NULL) { 3426 cause = 60; 3427 ret = -EFAULT; 3428 goto unlock; 3429 } 3430 KASSERT((m->flags & PG_FICTITIOUS) != 0, 3431 ("not fictitious %p", m)); 3432 KASSERT(m->wire_count == 1, ("wire_count not 1 %p", m)); 3433 3434 if ((m->flags & PG_BUSY) != 0) { 3435 DRM_UNLOCK(dev); 3436 #if 0 /* XXX */ 3437 vm_page_sleep(m, "915pbs"); 3438 #endif 3439 goto retry; 3440 } 3441 m->valid = VM_PAGE_BITS_ALL; 3442 vm_page_insert(m, vm_obj, OFF_TO_IDX(offset)); 3443 have_page: 3444 *mres = m; 3445 vm_page_busy_try(m, false); 3446 3447 DRM_UNLOCK(dev); 3448 if (oldm != NULL) { 3449 vm_page_free(oldm); 3450 } 3451 vm_object_pip_wakeup(vm_obj); 3452 return (VM_PAGER_OK); 3453 3454 unlock: 3455 DRM_UNLOCK(dev); 3456 out: 3457 KASSERT(ret != 0, ("i915_gem_pager_fault: wrong return")); 3458 if (ret == -EAGAIN || ret == -EIO || ret == -EINTR) { 3459 goto unlocked_vmobj; 3460 } 3461 VM_OBJECT_LOCK(vm_obj); 3462 vm_object_pip_wakeup(vm_obj); 3463 return (VM_PAGER_ERROR); 3464 } 3465 3466 static void 3467 i915_gem_pager_dtor(void *handle) 3468 { 3469 struct drm_gem_object *obj; 3470 struct drm_device *dev; 3471 3472 obj = handle; 3473 dev = obj->dev; 3474 3475 DRM_LOCK(dev); 3476 drm_gem_free_mmap_offset(obj); 3477 i915_gem_release_mmap(to_intel_bo(obj)); 3478 drm_gem_object_unreference(obj); 3479 DRM_UNLOCK(dev); 3480 } 3481 3482 struct cdev_pager_ops i915_gem_pager_ops = { 3483 .cdev_pg_fault = i915_gem_pager_fault, 3484 .cdev_pg_ctor = i915_gem_pager_ctor, 3485 .cdev_pg_dtor = i915_gem_pager_dtor 3486 }; 3487 3488 static int 3489 i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj, 3490 uint64_t offset, uint64_t size) 3491 { 3492 uint32_t old_read_domains; 3493 int i, ret; 3494 3495 if (offset == 0 && size == obj->base.size) 3496 return (i915_gem_object_set_to_cpu_domain(obj, 0)); 3497 3498 ret = i915_gem_object_flush_gpu_write_domain(obj); 3499 if (ret != 0) 3500 return (ret); 3501 ret = i915_gem_object_wait_rendering(obj); 3502 if (ret != 0) 3503 return (ret); 3504 3505 i915_gem_object_flush_gtt_write_domain(obj); 3506 3507 if (obj->page_cpu_valid == NULL && 3508 (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0) 3509 return (0); 3510 3511 if (obj->page_cpu_valid == NULL) { 3512 obj->page_cpu_valid = kmalloc(obj->base.size / PAGE_SIZE, 3513 DRM_I915_GEM, M_WAITOK | M_ZERO); 3514 } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) 3515 memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE); 3516 3517 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE; 3518 i++) { 3519 if (obj->page_cpu_valid[i]) 3520 continue; 3521 drm_clflush_pages(obj->pages + i, 1); 3522 obj->page_cpu_valid[i] = 1; 3523 } 3524 3525 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 3526 ("In gpu write domain")); 3527 3528 old_read_domains = obj->base.read_domains; 3529 obj->base.read_domains |= I915_GEM_DOMAIN_CPU; 3530 3531 return (0); 3532 } 3533 3534 #define GEM_PARANOID_CHECK_GTT 0 3535 #if GEM_PARANOID_CHECK_GTT 3536 static void 3537 i915_gem_assert_pages_not_mapped(struct drm_device *dev, vm_page_t *ma, 3538 int page_count) 3539 { 3540 struct drm_i915_private *dev_priv; 3541 vm_paddr_t pa; 3542 unsigned long start, end; 3543 u_int i; 3544 int j; 3545 3546 dev_priv = dev->dev_private; 3547 start = OFF_TO_IDX(dev_priv->mm.gtt_start); 3548 end = OFF_TO_IDX(dev_priv->mm.gtt_end); 3549 for (i = start; i < end; i++) { 3550 pa = intel_gtt_read_pte_paddr(i); 3551 for (j = 0; j < page_count; j++) { 3552 if (pa == VM_PAGE_TO_PHYS(ma[j])) { 3553 panic("Page %p in GTT pte index %d pte %x", 3554 ma[i], i, intel_gtt_read_pte(i)); 3555 } 3556 } 3557 } 3558 } 3559 #endif 3560 3561 static void 3562 i915_gem_process_flushing_list(struct intel_ring_buffer *ring, 3563 uint32_t flush_domains) 3564 { 3565 struct drm_i915_gem_object *obj, *next; 3566 uint32_t old_write_domain; 3567 3568 list_for_each_entry_safe(obj, next, &ring->gpu_write_list, 3569 gpu_write_list) { 3570 if (obj->base.write_domain & flush_domains) { 3571 old_write_domain = obj->base.write_domain; 3572 obj->base.write_domain = 0; 3573 list_del_init(&obj->gpu_write_list); 3574 i915_gem_object_move_to_active(obj, ring, 3575 i915_gem_next_request_seqno(ring)); 3576 } 3577 } 3578 } 3579 3580 #define VM_OBJECT_LOCK_ASSERT_OWNED(object) 3581 3582 static vm_page_t 3583 i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex) 3584 { 3585 vm_page_t m; 3586 int rv; 3587 3588 VM_OBJECT_LOCK_ASSERT_OWNED(object); 3589 m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_RETRY); 3590 if (m->valid != VM_PAGE_BITS_ALL) { 3591 if (vm_pager_has_page(object, pindex)) { 3592 rv = vm_pager_get_page(object, &m, 1); 3593 m = vm_page_lookup(object, pindex); 3594 if (m == NULL) 3595 return (NULL); 3596 if (rv != VM_PAGER_OK) { 3597 vm_page_free(m); 3598 return (NULL); 3599 } 3600 } else { 3601 pmap_zero_page(VM_PAGE_TO_PHYS(m)); 3602 m->valid = VM_PAGE_BITS_ALL; 3603 m->dirty = 0; 3604 } 3605 } 3606 vm_page_wire(m); 3607 vm_page_wakeup(m); 3608 atomic_add_long(&i915_gem_wired_pages_cnt, 1); 3609 return (m); 3610 } 3611 3612 int 3613 i915_gem_flush_ring(struct intel_ring_buffer *ring, uint32_t invalidate_domains, 3614 uint32_t flush_domains) 3615 { 3616 int ret; 3617 3618 if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0) 3619 return 0; 3620 3621 ret = ring->flush(ring, invalidate_domains, flush_domains); 3622 if (ret) 3623 return ret; 3624 3625 if (flush_domains & I915_GEM_GPU_DOMAINS) 3626 i915_gem_process_flushing_list(ring, flush_domains); 3627 return 0; 3628 } 3629 3630 u32 3631 i915_gem_next_request_seqno(struct intel_ring_buffer *ring) 3632 { 3633 if (ring->outstanding_lazy_request == 0) 3634 ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev); 3635 3636 return ring->outstanding_lazy_request; 3637 } 3638 3639 static void 3640 i915_gem_clear_fence_reg(struct drm_device *dev, struct drm_i915_fence_reg *reg) 3641 { 3642 drm_i915_private_t *dev_priv = dev->dev_private; 3643 uint32_t fence_reg = reg - dev_priv->fence_regs; 3644 3645 switch (INTEL_INFO(dev)->gen) { 3646 case 7: 3647 case 6: 3648 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0); 3649 break; 3650 case 5: 3651 case 4: 3652 I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0); 3653 break; 3654 case 3: 3655 if (fence_reg >= 8) 3656 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4; 3657 else 3658 case 2: 3659 fence_reg = FENCE_REG_830_0 + fence_reg * 4; 3660 3661 I915_WRITE(fence_reg, 0); 3662 break; 3663 } 3664 3665 list_del_init(®->lru_list); 3666 reg->obj = NULL; 3667 reg->setup_seqno = 0; 3668 reg->pin_count = 0; 3669 } 3670 3671 static int 3672 i915_gpu_is_active(struct drm_device *dev) 3673 { 3674 drm_i915_private_t *dev_priv; 3675 3676 dev_priv = dev->dev_private; 3677 return (!list_empty(&dev_priv->mm.flushing_list) || 3678 !list_empty(&dev_priv->mm.active_list)); 3679 } 3680 3681 static void 3682 i915_gem_lowmem(void *arg) 3683 { 3684 struct drm_device *dev; 3685 struct drm_i915_private *dev_priv; 3686 struct drm_i915_gem_object *obj, *next; 3687 int cnt, cnt_fail, cnt_total; 3688 3689 dev = arg; 3690 dev_priv = dev->dev_private; 3691 3692 if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT)) 3693 return; 3694 3695 rescan: 3696 /* first scan for clean buffers */ 3697 i915_gem_retire_requests(dev); 3698 3699 cnt_total = cnt_fail = cnt = 0; 3700 3701 list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list, 3702 mm_list) { 3703 if (i915_gem_object_is_purgeable(obj)) { 3704 if (i915_gem_object_unbind(obj) != 0) 3705 cnt_total++; 3706 } else 3707 cnt_total++; 3708 } 3709 3710 /* second pass, evict/count anything still on the inactive list */ 3711 list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list, 3712 mm_list) { 3713 if (i915_gem_object_unbind(obj) == 0) 3714 cnt++; 3715 else 3716 cnt_fail++; 3717 } 3718 3719 if (cnt_fail > cnt_total / 100 && i915_gpu_is_active(dev)) { 3720 /* 3721 * We are desperate for pages, so as a last resort, wait 3722 * for the GPU to finish and discard whatever we can. 3723 * This has a dramatic impact to reduce the number of 3724 * OOM-killer events whilst running the GPU aggressively. 3725 */ 3726 if (i915_gpu_idle(dev) == 0) 3727 goto rescan; 3728 } 3729 DRM_UNLOCK(dev); 3730 } 3731 3732 void 3733 i915_gem_unload(struct drm_device *dev) 3734 { 3735 struct drm_i915_private *dev_priv; 3736 3737 dev_priv = dev->dev_private; 3738 EVENTHANDLER_DEREGISTER(vm_lowmem, dev_priv->mm.i915_lowmem); 3739 } 3740