1 /************************************************************************** 2 * 3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 /* 28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29 */ 30 31 #define pr_fmt(fmt) "[TTM] " fmt 32 33 #include <drm/ttm/ttm_module.h> 34 #include <drm/ttm/ttm_bo_driver.h> 35 #include <drm/ttm/ttm_placement.h> 36 #include <linux/jiffies.h> 37 #include <linux/slab.h> 38 #include <linux/sched.h> 39 #include <linux/mm.h> 40 #include <linux/file.h> 41 #include <linux/module.h> 42 #include <linux/atomic.h> 43 #include <linux/reservation.h> 44 45 #define TTM_ASSERT_LOCKED(param) 46 #define TTM_DEBUG(fmt, arg...) 47 #define TTM_BO_HASH_ORDER 13 48 49 static int ttm_bo_swapout(struct ttm_mem_shrink *shrink); 50 static void ttm_bo_global_kobj_release(struct kobject *kobj); 51 52 static struct attribute ttm_bo_count = { 53 .name = "bo_count", 54 .mode = S_IRUGO 55 }; 56 57 static inline int ttm_mem_type_from_place(const struct ttm_place *place, 58 uint32_t *mem_type) 59 { 60 int i; 61 62 for (i = 0; i <= TTM_PL_PRIV5; i++) { 63 if (place->flags & (1 << i)) { 64 *mem_type = i; 65 return 0; 66 } 67 } 68 return -EINVAL; 69 } 70 71 static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type) 72 { 73 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 74 75 pr_err(" has_type: %d\n", man->has_type); 76 pr_err(" use_type: %d\n", man->use_type); 77 pr_err(" flags: 0x%08X\n", man->flags); 78 pr_err(" gpu_offset: 0x%08lX\n", man->gpu_offset); 79 pr_err(" size: %ju\n", man->size); 80 pr_err(" available_caching: 0x%08X\n", man->available_caching); 81 pr_err(" default_caching: 0x%08X\n", man->default_caching); 82 if (mem_type != TTM_PL_SYSTEM) 83 (*man->func->debug)(man, TTM_PFX); 84 } 85 86 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, 87 struct ttm_placement *placement) 88 { 89 int i, ret, mem_type; 90 91 pr_err("No space for %p (%lu pages, %luK, %luM)\n", 92 bo, bo->mem.num_pages, bo->mem.size >> 10, 93 bo->mem.size >> 20); 94 for (i = 0; i < placement->num_placement; i++) { 95 ret = ttm_mem_type_from_place(&placement->placement[i], 96 &mem_type); 97 if (ret) 98 return; 99 pr_err(" placement[%d]=0x%08X (%d)\n", 100 i, placement->placement[i].flags, mem_type); 101 ttm_mem_type_debug(bo->bdev, mem_type); 102 } 103 } 104 105 static ssize_t ttm_bo_global_show(struct kobject *kobj, 106 struct attribute *attr, 107 char *buffer) 108 { 109 struct ttm_bo_global *glob = 110 container_of(kobj, struct ttm_bo_global, kobj); 111 112 return snprintf(buffer, PAGE_SIZE, "%lu\n", 113 (unsigned long) atomic_read(&glob->bo_count)); 114 } 115 116 static struct attribute *ttm_bo_global_attrs[] = { 117 &ttm_bo_count, 118 NULL 119 }; 120 121 static const struct sysfs_ops ttm_bo_global_ops = { 122 .show = &ttm_bo_global_show 123 }; 124 125 static struct kobj_type ttm_bo_glob_kobj_type = { 126 .release = &ttm_bo_global_kobj_release, 127 .sysfs_ops = &ttm_bo_global_ops, 128 .default_attrs = ttm_bo_global_attrs 129 }; 130 131 132 static inline uint32_t ttm_bo_type_flags(unsigned type) 133 { 134 return 1 << (type); 135 } 136 137 static void ttm_bo_release_list(struct kref *list_kref) 138 { 139 struct ttm_buffer_object *bo = 140 container_of(list_kref, struct ttm_buffer_object, list_kref); 141 struct ttm_bo_device *bdev = bo->bdev; 142 size_t acc_size = bo->acc_size; 143 144 BUG_ON(atomic_read(&bo->list_kref.refcount)); 145 BUG_ON(atomic_read(&bo->kref.refcount)); 146 BUG_ON(atomic_read(&bo->cpu_writers)); 147 BUG_ON(bo->mem.mm_node != NULL); 148 BUG_ON(!list_empty(&bo->lru)); 149 BUG_ON(!list_empty(&bo->ddestroy)); 150 151 if (bo->ttm) 152 ttm_tt_destroy(bo->ttm); 153 atomic_dec(&bo->glob->bo_count); 154 if (bo->resv == &bo->ttm_resv) 155 reservation_object_fini(&bo->ttm_resv); 156 mutex_destroy(&bo->wu_mutex); 157 if (bo->destroy) 158 bo->destroy(bo); 159 else { 160 kfree(bo); 161 } 162 ttm_mem_global_free(bdev->glob->mem_glob, acc_size); 163 } 164 165 void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) 166 { 167 struct ttm_bo_device *bdev = bo->bdev; 168 struct ttm_mem_type_manager *man; 169 170 lockdep_assert_held(&bo->resv->lock.base); 171 172 if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { 173 174 BUG_ON(!list_empty(&bo->lru)); 175 176 man = &bdev->man[bo->mem.mem_type]; 177 list_add_tail(&bo->lru, &man->lru); 178 kref_get(&bo->list_kref); 179 180 if (bo->ttm != NULL) { 181 list_add_tail(&bo->swap, &bo->glob->swap_lru); 182 kref_get(&bo->list_kref); 183 } 184 } 185 } 186 EXPORT_SYMBOL(ttm_bo_add_to_lru); 187 188 int ttm_bo_del_from_lru(struct ttm_buffer_object *bo) 189 { 190 int put_count = 0; 191 192 if (!list_empty(&bo->swap)) { 193 list_del_init(&bo->swap); 194 ++put_count; 195 } 196 if (!list_empty(&bo->lru)) { 197 list_del_init(&bo->lru); 198 ++put_count; 199 } 200 201 /* 202 * TODO: Add a driver hook to delete from 203 * driver-specific LRU's here. 204 */ 205 206 return put_count; 207 } 208 209 static void ttm_bo_ref_bug(struct kref *list_kref) 210 { 211 BUG(); 212 } 213 214 void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count, 215 bool never_free) 216 { 217 kref_sub(&bo->list_kref, count, 218 (never_free) ? ttm_bo_ref_bug : ttm_bo_release_list); 219 } 220 221 void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo) 222 { 223 int put_count; 224 225 lockmgr(&bo->glob->lru_lock, LK_EXCLUSIVE); 226 put_count = ttm_bo_del_from_lru(bo); 227 lockmgr(&bo->glob->lru_lock, LK_RELEASE); 228 ttm_bo_list_ref_sub(bo, put_count, true); 229 } 230 EXPORT_SYMBOL(ttm_bo_del_sub_from_lru); 231 232 /* 233 * Call bo->mutex locked. 234 */ 235 static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) 236 { 237 struct ttm_bo_device *bdev = bo->bdev; 238 struct ttm_bo_global *glob = bo->glob; 239 int ret = 0; 240 uint32_t page_flags = 0; 241 242 TTM_ASSERT_LOCKED(&bo->mutex); 243 bo->ttm = NULL; 244 245 if (bdev->need_dma32) 246 page_flags |= TTM_PAGE_FLAG_DMA32; 247 248 switch (bo->type) { 249 case ttm_bo_type_device: 250 if (zero_alloc) 251 page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC; 252 case ttm_bo_type_kernel: 253 bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, 254 page_flags, glob->dummy_read_page); 255 if (unlikely(bo->ttm == NULL)) 256 ret = -ENOMEM; 257 break; 258 case ttm_bo_type_sg: 259 bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, 260 page_flags | TTM_PAGE_FLAG_SG, 261 glob->dummy_read_page); 262 if (unlikely(bo->ttm == NULL)) { 263 ret = -ENOMEM; 264 break; 265 } 266 bo->ttm->sg = bo->sg; 267 break; 268 default: 269 pr_err("Illegal buffer object type\n"); 270 ret = -EINVAL; 271 break; 272 } 273 274 return ret; 275 } 276 277 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, 278 struct ttm_mem_reg *mem, 279 bool evict, bool interruptible, 280 bool no_wait_gpu) 281 { 282 struct ttm_bo_device *bdev = bo->bdev; 283 bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); 284 bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem); 285 struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type]; 286 struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type]; 287 int ret = 0; 288 289 if (old_is_pci || new_is_pci || 290 ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) { 291 ret = ttm_mem_io_lock(old_man, true); 292 if (unlikely(ret != 0)) 293 goto out_err; 294 ttm_bo_unmap_virtual_locked(bo); 295 ttm_mem_io_unlock(old_man); 296 } 297 298 /* 299 * Create and bind a ttm if required. 300 */ 301 302 if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 303 if (bo->ttm == NULL) { 304 bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED); 305 ret = ttm_bo_add_ttm(bo, zero); 306 if (ret) 307 goto out_err; 308 } 309 310 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); 311 if (ret) 312 goto out_err; 313 314 if (mem->mem_type != TTM_PL_SYSTEM) { 315 ret = ttm_tt_bind(bo->ttm, mem); 316 if (ret) 317 goto out_err; 318 } 319 320 if (bo->mem.mem_type == TTM_PL_SYSTEM) { 321 if (bdev->driver->move_notify) 322 bdev->driver->move_notify(bo, mem); 323 bo->mem = *mem; 324 mem->mm_node = NULL; 325 goto moved; 326 } 327 } 328 329 if (bdev->driver->move_notify) 330 bdev->driver->move_notify(bo, mem); 331 332 if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && 333 !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) 334 ret = ttm_bo_move_ttm(bo, evict, no_wait_gpu, mem); 335 else if (bdev->driver->move) 336 ret = bdev->driver->move(bo, evict, interruptible, 337 no_wait_gpu, mem); 338 else 339 ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, mem); 340 341 if (ret) { 342 if (bdev->driver->move_notify) { 343 struct ttm_mem_reg tmp_mem = *mem; 344 *mem = bo->mem; 345 bo->mem = tmp_mem; 346 bdev->driver->move_notify(bo, mem); 347 bo->mem = *mem; 348 *mem = tmp_mem; 349 } 350 351 goto out_err; 352 } 353 354 moved: 355 if (bo->evicted) { 356 if (bdev->driver->invalidate_caches) { 357 ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement); 358 if (ret) 359 pr_err("Can not flush read caches\n"); 360 } 361 bo->evicted = false; 362 } 363 364 if (bo->mem.mm_node) { 365 bo->offset = (bo->mem.start << PAGE_SHIFT) + 366 bdev->man[bo->mem.mem_type].gpu_offset; 367 bo->cur_placement = bo->mem.placement; 368 } else 369 bo->offset = 0; 370 371 return 0; 372 373 out_err: 374 new_man = &bdev->man[bo->mem.mem_type]; 375 if ((new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm) { 376 ttm_tt_unbind(bo->ttm); 377 ttm_tt_destroy(bo->ttm); 378 bo->ttm = NULL; 379 } 380 381 return ret; 382 } 383 384 /** 385 * Call bo::reserved. 386 * Will release GPU memory type usage on destruction. 387 * This is the place to put in driver specific hooks to release 388 * driver private resources. 389 * Will release the bo::reserved lock. 390 */ 391 392 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) 393 { 394 if (bo->bdev->driver->move_notify) 395 bo->bdev->driver->move_notify(bo, NULL); 396 397 if (bo->ttm) { 398 ttm_tt_unbind(bo->ttm); 399 ttm_tt_destroy(bo->ttm); 400 bo->ttm = NULL; 401 } 402 ttm_bo_mem_put(bo, &bo->mem); 403 404 ww_mutex_unlock (&bo->resv->lock); 405 } 406 407 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) 408 { 409 struct reservation_object_list *fobj; 410 struct fence *fence; 411 int i; 412 413 fobj = reservation_object_get_list(bo->resv); 414 fence = reservation_object_get_excl(bo->resv); 415 if (fence && !fence->ops->signaled) 416 fence_enable_sw_signaling(fence); 417 418 for (i = 0; fobj && i < fobj->shared_count; ++i) { 419 fence = rcu_dereference_protected(fobj->shared[i], 420 reservation_object_held(bo->resv)); 421 422 if (!fence->ops->signaled) 423 fence_enable_sw_signaling(fence); 424 } 425 } 426 427 static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) 428 { 429 struct ttm_bo_device *bdev = bo->bdev; 430 struct ttm_bo_global *glob = bo->glob; 431 int put_count; 432 int ret; 433 434 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 435 ret = __ttm_bo_reserve(bo, false, true, false, NULL); 436 437 if (!ret) { 438 if (!ttm_bo_wait(bo, false, false, true)) { 439 put_count = ttm_bo_del_from_lru(bo); 440 441 lockmgr(&glob->lru_lock, LK_RELEASE); 442 ttm_bo_cleanup_memtype_use(bo); 443 444 ttm_bo_list_ref_sub(bo, put_count, true); 445 446 return; 447 } else 448 ttm_bo_flush_all_fences(bo); 449 450 /* 451 * Make NO_EVICT bos immediately available to 452 * shrinkers, now that they are queued for 453 * destruction. 454 */ 455 if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) { 456 bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT; 457 ttm_bo_add_to_lru(bo); 458 } 459 460 __ttm_bo_unreserve(bo); 461 } 462 463 kref_get(&bo->list_kref); 464 list_add_tail(&bo->ddestroy, &bdev->ddestroy); 465 lockmgr(&glob->lru_lock, LK_RELEASE); 466 467 schedule_delayed_work(&bdev->wq, 468 ((HZ / 100) < 1) ? 1 : HZ / 100); 469 } 470 471 /** 472 * function ttm_bo_cleanup_refs_and_unlock 473 * If bo idle, remove from delayed- and lru lists, and unref. 474 * If not idle, do nothing. 475 * 476 * Must be called with lru_lock and reservation held, this function 477 * will drop both before returning. 478 * 479 * @interruptible Any sleeps should occur interruptibly. 480 * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. 481 */ 482 483 static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, 484 bool interruptible, 485 bool no_wait_gpu) 486 { 487 struct ttm_bo_global *glob = bo->glob; 488 int put_count; 489 int ret; 490 491 ret = ttm_bo_wait(bo, false, false, true); 492 493 if (ret && !no_wait_gpu) { 494 long lret; 495 ww_mutex_unlock(&bo->resv->lock); 496 lockmgr(&glob->lru_lock, LK_RELEASE); 497 498 lret = reservation_object_wait_timeout_rcu(bo->resv, 499 true, 500 interruptible, 501 30 * HZ); 502 503 if (lret < 0) 504 return lret; 505 else if (lret == 0) 506 return -EBUSY; 507 508 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 509 ret = __ttm_bo_reserve(bo, false, true, false, NULL); 510 511 /* 512 * We raced, and lost, someone else holds the reservation now, 513 * and is probably busy in ttm_bo_cleanup_memtype_use. 514 * 515 * Even if it's not the case, because we finished waiting any 516 * delayed destruction would succeed, so just return success 517 * here. 518 */ 519 if (ret) { 520 lockmgr(&glob->lru_lock, LK_RELEASE); 521 return 0; 522 } 523 524 /* 525 * remove sync_obj with ttm_bo_wait, the wait should be 526 * finished, and no new wait object should have been added. 527 */ 528 ret = ttm_bo_wait(bo, false, false, true); 529 WARN_ON(ret); 530 } 531 532 if (ret || unlikely(list_empty(&bo->ddestroy))) { 533 __ttm_bo_unreserve(bo); 534 lockmgr(&glob->lru_lock, LK_RELEASE); 535 return ret; 536 } 537 538 put_count = ttm_bo_del_from_lru(bo); 539 list_del_init(&bo->ddestroy); 540 ++put_count; 541 542 lockmgr(&glob->lru_lock, LK_RELEASE); 543 ttm_bo_cleanup_memtype_use(bo); 544 545 ttm_bo_list_ref_sub(bo, put_count, true); 546 547 return 0; 548 } 549 550 /** 551 * Traverse the delayed list, and call ttm_bo_cleanup_refs on all 552 * encountered buffers. 553 */ 554 555 static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) 556 { 557 struct ttm_bo_global *glob = bdev->glob; 558 struct ttm_buffer_object *entry = NULL; 559 int ret = 0; 560 561 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 562 if (list_empty(&bdev->ddestroy)) 563 goto out_unlock; 564 565 entry = list_first_entry(&bdev->ddestroy, 566 struct ttm_buffer_object, ddestroy); 567 kref_get(&entry->list_kref); 568 569 for (;;) { 570 struct ttm_buffer_object *nentry = NULL; 571 572 if (entry->ddestroy.next != &bdev->ddestroy) { 573 nentry = list_first_entry(&entry->ddestroy, 574 struct ttm_buffer_object, ddestroy); 575 kref_get(&nentry->list_kref); 576 } 577 578 ret = __ttm_bo_reserve(entry, false, true, false, 0); 579 if (remove_all && ret) { 580 lockmgr(&glob->lru_lock, LK_RELEASE); 581 ret = __ttm_bo_reserve(entry, false, false, 582 false, 0); 583 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 584 } 585 586 if (!ret) 587 ret = ttm_bo_cleanup_refs_and_unlock(entry, false, 588 !remove_all); 589 else 590 lockmgr(&glob->lru_lock, LK_RELEASE); 591 592 kref_put(&entry->list_kref, ttm_bo_release_list); 593 entry = nentry; 594 595 if (ret || !entry) 596 goto out; 597 598 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 599 if (list_empty(&entry->ddestroy)) 600 break; 601 } 602 603 out_unlock: 604 lockmgr(&glob->lru_lock, LK_RELEASE); 605 out: 606 if (entry) 607 kref_put(&entry->list_kref, ttm_bo_release_list); 608 return ret; 609 } 610 611 static void ttm_bo_delayed_workqueue(struct work_struct *work) 612 { 613 struct ttm_bo_device *bdev = 614 container_of(work, struct ttm_bo_device, wq.work); 615 616 if (ttm_bo_delayed_delete(bdev, false)) { 617 schedule_delayed_work(&bdev->wq, 618 ((HZ / 100) < 1) ? 1 : HZ / 100); 619 } 620 } 621 622 static void ttm_bo_release(struct kref *kref) 623 { 624 struct ttm_buffer_object *bo = 625 container_of(kref, struct ttm_buffer_object, kref); 626 struct ttm_bo_device *bdev = bo->bdev; 627 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 628 629 drm_vma_offset_remove(&bdev->vma_manager, &bo->vma_node); 630 ttm_mem_io_lock(man, false); 631 ttm_mem_io_free_vm(bo); 632 ttm_mem_io_unlock(man); 633 ttm_bo_cleanup_refs_or_queue(bo); 634 kref_put(&bo->list_kref, ttm_bo_release_list); 635 } 636 637 void ttm_bo_unref(struct ttm_buffer_object **p_bo) 638 { 639 struct ttm_buffer_object *bo = *p_bo; 640 641 *p_bo = NULL; 642 kref_put(&bo->kref, ttm_bo_release); 643 } 644 EXPORT_SYMBOL(ttm_bo_unref); 645 646 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev) 647 { 648 return cancel_delayed_work_sync(&bdev->wq); 649 } 650 EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue); 651 652 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) 653 { 654 if (resched) 655 schedule_delayed_work(&bdev->wq, 656 ((HZ / 100) < 1) ? 1 : HZ / 100); 657 } 658 EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue); 659 660 static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, 661 bool no_wait_gpu) 662 { 663 struct ttm_bo_device *bdev = bo->bdev; 664 struct ttm_mem_reg evict_mem; 665 struct ttm_placement placement; 666 int ret = 0; 667 668 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 669 670 if (unlikely(ret != 0)) { 671 if (ret != -ERESTARTSYS) { 672 pr_err("Failed to expire sync object before buffer eviction\n"); 673 } 674 goto out; 675 } 676 677 lockdep_assert_held(&bo->resv->lock.base); 678 679 evict_mem = bo->mem; 680 evict_mem.mm_node = NULL; 681 evict_mem.bus.io_reserved_vm = false; 682 evict_mem.bus.io_reserved_count = 0; 683 684 placement.num_placement = 0; 685 placement.num_busy_placement = 0; 686 bdev->driver->evict_flags(bo, &placement); 687 ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible, 688 no_wait_gpu); 689 if (ret) { 690 if (ret != -ERESTARTSYS) { 691 pr_err("Failed to find memory space for buffer 0x%p eviction\n", 692 bo); 693 ttm_bo_mem_space_debug(bo, &placement); 694 } 695 goto out; 696 } 697 698 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible, 699 no_wait_gpu); 700 if (ret) { 701 if (ret != -ERESTARTSYS) 702 pr_err("Buffer eviction failed\n"); 703 ttm_bo_mem_put(bo, &evict_mem); 704 goto out; 705 } 706 bo->evicted = true; 707 out: 708 return ret; 709 } 710 711 static int ttm_mem_evict_first(struct ttm_bo_device *bdev, 712 uint32_t mem_type, 713 const struct ttm_place *place, 714 bool interruptible, 715 bool no_wait_gpu) 716 { 717 struct ttm_bo_global *glob = bdev->glob; 718 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 719 struct ttm_buffer_object *bo; 720 int ret = -EBUSY, put_count; 721 722 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 723 list_for_each_entry(bo, &man->lru, lru) { 724 ret = __ttm_bo_reserve(bo, false, true, false, NULL); 725 if (!ret) { 726 if (place && (place->fpfn || place->lpfn)) { 727 /* Don't evict this BO if it's outside of the 728 * requested placement range 729 */ 730 if (place->fpfn >= (bo->mem.start + bo->mem.size) || 731 (place->lpfn && place->lpfn <= bo->mem.start)) { 732 __ttm_bo_unreserve(bo); 733 ret = -EBUSY; 734 continue; 735 } 736 } 737 738 break; 739 } 740 } 741 742 if (ret) { 743 lockmgr(&glob->lru_lock, LK_RELEASE); 744 return ret; 745 } 746 747 kref_get(&bo->list_kref); 748 749 if (!list_empty(&bo->ddestroy)) { 750 ret = ttm_bo_cleanup_refs_and_unlock(bo, interruptible, 751 no_wait_gpu); 752 kref_put(&bo->list_kref, ttm_bo_release_list); 753 return ret; 754 } 755 756 put_count = ttm_bo_del_from_lru(bo); 757 lockmgr(&glob->lru_lock, LK_RELEASE); 758 759 BUG_ON(ret != 0); 760 761 ttm_bo_list_ref_sub(bo, put_count, true); 762 763 ret = ttm_bo_evict(bo, interruptible, no_wait_gpu); 764 ttm_bo_unreserve(bo); 765 766 kref_put(&bo->list_kref, ttm_bo_release_list); 767 return ret; 768 } 769 770 void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) 771 { 772 struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type]; 773 774 if (mem->mm_node) 775 (*man->func->put_node)(man, mem); 776 } 777 EXPORT_SYMBOL(ttm_bo_mem_put); 778 779 /** 780 * Repeatedly evict memory from the LRU for @mem_type until we create enough 781 * space, or we've evicted everything and there isn't enough space. 782 */ 783 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, 784 uint32_t mem_type, 785 const struct ttm_place *place, 786 struct ttm_mem_reg *mem, 787 bool interruptible, 788 bool no_wait_gpu) 789 { 790 struct ttm_bo_device *bdev = bo->bdev; 791 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 792 int ret; 793 794 do { 795 ret = (*man->func->get_node)(man, bo, place, mem); 796 if (unlikely(ret != 0)) 797 return ret; 798 if (mem->mm_node) 799 break; 800 ret = ttm_mem_evict_first(bdev, mem_type, place, 801 interruptible, no_wait_gpu); 802 if (unlikely(ret != 0)) 803 return ret; 804 } while (1); 805 if (mem->mm_node == NULL) 806 return -ENOMEM; 807 mem->mem_type = mem_type; 808 return 0; 809 } 810 811 static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man, 812 uint32_t cur_placement, 813 uint32_t proposed_placement) 814 { 815 uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING; 816 uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING; 817 818 /** 819 * Keep current caching if possible. 820 */ 821 822 if ((cur_placement & caching) != 0) 823 result |= (cur_placement & caching); 824 else if ((man->default_caching & caching) != 0) 825 result |= man->default_caching; 826 else if ((TTM_PL_FLAG_CACHED & caching) != 0) 827 result |= TTM_PL_FLAG_CACHED; 828 else if ((TTM_PL_FLAG_WC & caching) != 0) 829 result |= TTM_PL_FLAG_WC; 830 else if ((TTM_PL_FLAG_UNCACHED & caching) != 0) 831 result |= TTM_PL_FLAG_UNCACHED; 832 833 return result; 834 } 835 836 static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, 837 uint32_t mem_type, 838 const struct ttm_place *place, 839 uint32_t *masked_placement) 840 { 841 uint32_t cur_flags = ttm_bo_type_flags(mem_type); 842 843 if ((cur_flags & place->flags & TTM_PL_MASK_MEM) == 0) 844 return false; 845 846 if ((place->flags & man->available_caching) == 0) 847 return false; 848 849 cur_flags |= (place->flags & man->available_caching); 850 851 *masked_placement = cur_flags; 852 return true; 853 } 854 855 /** 856 * Creates space for memory region @mem according to its type. 857 * 858 * This function first searches for free space in compatible memory types in 859 * the priority order defined by the driver. If free space isn't found, then 860 * ttm_bo_mem_force_space is attempted in priority order to evict and find 861 * space. 862 */ 863 int ttm_bo_mem_space(struct ttm_buffer_object *bo, 864 struct ttm_placement *placement, 865 struct ttm_mem_reg *mem, 866 bool interruptible, 867 bool no_wait_gpu) 868 { 869 struct ttm_bo_device *bdev = bo->bdev; 870 struct ttm_mem_type_manager *man; 871 uint32_t mem_type = TTM_PL_SYSTEM; 872 uint32_t cur_flags = 0; 873 bool type_found = false; 874 bool type_ok = false; 875 bool has_erestartsys = false; 876 int i, ret; 877 878 mem->mm_node = NULL; 879 for (i = 0; i < placement->num_placement; ++i) { 880 const struct ttm_place *place = &placement->placement[i]; 881 882 ret = ttm_mem_type_from_place(place, &mem_type); 883 if (ret) 884 return ret; 885 man = &bdev->man[mem_type]; 886 887 type_ok = ttm_bo_mt_compatible(man, mem_type, place, 888 &cur_flags); 889 890 if (!type_ok) 891 continue; 892 893 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 894 cur_flags); 895 /* 896 * Use the access and other non-mapping-related flag bits from 897 * the memory placement flags to the current flags 898 */ 899 ttm_flag_masked(&cur_flags, place->flags, 900 ~TTM_PL_MASK_MEMTYPE); 901 902 if (mem_type == TTM_PL_SYSTEM) 903 break; 904 905 if (man->has_type && man->use_type) { 906 type_found = true; 907 ret = (*man->func->get_node)(man, bo, place, mem); 908 if (unlikely(ret)) 909 return ret; 910 } 911 if (mem->mm_node) 912 break; 913 } 914 915 if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) { 916 mem->mem_type = mem_type; 917 mem->placement = cur_flags; 918 return 0; 919 } 920 921 if (!type_found) 922 return -EINVAL; 923 924 for (i = 0; i < placement->num_busy_placement; ++i) { 925 const struct ttm_place *place = &placement->busy_placement[i]; 926 927 ret = ttm_mem_type_from_place(place, &mem_type); 928 if (ret) 929 return ret; 930 man = &bdev->man[mem_type]; 931 if (!man->has_type) 932 continue; 933 if (!ttm_bo_mt_compatible(man, mem_type, place, &cur_flags)) 934 continue; 935 936 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 937 cur_flags); 938 /* 939 * Use the access and other non-mapping-related flag bits from 940 * the memory placement flags to the current flags 941 */ 942 ttm_flag_masked(&cur_flags, place->flags, 943 ~TTM_PL_MASK_MEMTYPE); 944 945 if (mem_type == TTM_PL_SYSTEM) { 946 mem->mem_type = mem_type; 947 mem->placement = cur_flags; 948 mem->mm_node = NULL; 949 return 0; 950 } 951 952 ret = ttm_bo_mem_force_space(bo, mem_type, place, mem, 953 interruptible, no_wait_gpu); 954 if (ret == 0 && mem->mm_node) { 955 mem->placement = cur_flags; 956 return 0; 957 } 958 if (ret == -ERESTARTSYS) 959 has_erestartsys = true; 960 } 961 ret = (has_erestartsys) ? -ERESTARTSYS : -ENOMEM; 962 return ret; 963 } 964 EXPORT_SYMBOL(ttm_bo_mem_space); 965 966 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, 967 struct ttm_placement *placement, 968 bool interruptible, 969 bool no_wait_gpu) 970 { 971 int ret = 0; 972 struct ttm_mem_reg mem; 973 974 lockdep_assert_held(&bo->resv->lock.base); 975 976 /* 977 * FIXME: It's possible to pipeline buffer moves. 978 * Have the driver move function wait for idle when necessary, 979 * instead of doing it here. 980 */ 981 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 982 if (ret) 983 return ret; 984 mem.num_pages = bo->num_pages; 985 mem.size = mem.num_pages << PAGE_SHIFT; 986 mem.page_alignment = bo->mem.page_alignment; 987 mem.bus.io_reserved_vm = false; 988 mem.bus.io_reserved_count = 0; 989 /* 990 * Determine where to move the buffer. 991 */ 992 ret = ttm_bo_mem_space(bo, placement, &mem, 993 interruptible, no_wait_gpu); 994 if (ret) 995 goto out_unlock; 996 ret = ttm_bo_handle_move_mem(bo, &mem, false, 997 interruptible, no_wait_gpu); 998 out_unlock: 999 if (ret && mem.mm_node) 1000 ttm_bo_mem_put(bo, &mem); 1001 return ret; 1002 } 1003 1004 static bool ttm_bo_mem_compat(struct ttm_placement *placement, 1005 struct ttm_mem_reg *mem, 1006 uint32_t *new_flags) 1007 { 1008 int i; 1009 1010 for (i = 0; i < placement->num_placement; i++) { 1011 const struct ttm_place *heap = &placement->placement[i]; 1012 if (mem->mm_node && 1013 (mem->start < heap->fpfn || 1014 (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn))) 1015 continue; 1016 1017 *new_flags = heap->flags; 1018 if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && 1019 (*new_flags & mem->placement & TTM_PL_MASK_MEM)) 1020 return true; 1021 } 1022 1023 for (i = 0; i < placement->num_busy_placement; i++) { 1024 const struct ttm_place *heap = &placement->busy_placement[i]; 1025 if (mem->mm_node && 1026 (mem->start < heap->fpfn || 1027 (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn))) 1028 continue; 1029 1030 *new_flags = heap->flags; 1031 if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && 1032 (*new_flags & mem->placement & TTM_PL_MASK_MEM)) 1033 return true; 1034 } 1035 1036 return false; 1037 } 1038 1039 int ttm_bo_validate(struct ttm_buffer_object *bo, 1040 struct ttm_placement *placement, 1041 bool interruptible, 1042 bool no_wait_gpu) 1043 { 1044 int ret; 1045 uint32_t new_flags; 1046 1047 lockdep_assert_held(&bo->resv->lock.base); 1048 /* 1049 * Check whether we need to move buffer. 1050 */ 1051 if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) { 1052 ret = ttm_bo_move_buffer(bo, placement, interruptible, 1053 no_wait_gpu); 1054 if (ret) 1055 return ret; 1056 } else { 1057 /* 1058 * Use the access and other non-mapping-related flag bits from 1059 * the compatible memory placement flags to the active flags 1060 */ 1061 ttm_flag_masked(&bo->mem.placement, new_flags, 1062 ~TTM_PL_MASK_MEMTYPE); 1063 } 1064 /* 1065 * We might need to add a TTM. 1066 */ 1067 if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { 1068 ret = ttm_bo_add_ttm(bo, true); 1069 if (ret) 1070 return ret; 1071 } 1072 return 0; 1073 } 1074 EXPORT_SYMBOL(ttm_bo_validate); 1075 1076 int ttm_bo_init(struct ttm_bo_device *bdev, 1077 struct ttm_buffer_object *bo, 1078 unsigned long size, 1079 enum ttm_bo_type type, 1080 struct ttm_placement *placement, 1081 uint32_t page_alignment, 1082 bool interruptible, 1083 struct vm_object *persistent_swap_storage, 1084 size_t acc_size, 1085 struct sg_table *sg, 1086 struct reservation_object *resv, 1087 void (*destroy) (struct ttm_buffer_object *)) 1088 { 1089 int ret = 0; 1090 unsigned long num_pages; 1091 struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; 1092 bool locked; 1093 1094 ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false); 1095 if (ret) { 1096 pr_err("Out of kernel memory\n"); 1097 if (destroy) 1098 (*destroy)(bo); 1099 else 1100 kfree(bo); 1101 return -ENOMEM; 1102 } 1103 1104 num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 1105 if (num_pages == 0) { 1106 pr_err("Illegal buffer object size\n"); 1107 if (destroy) 1108 (*destroy)(bo); 1109 else 1110 kfree(bo); 1111 ttm_mem_global_free(mem_glob, acc_size); 1112 return -EINVAL; 1113 } 1114 bo->destroy = destroy; 1115 1116 kref_init(&bo->kref); 1117 kref_init(&bo->list_kref); 1118 atomic_set(&bo->cpu_writers, 0); 1119 INIT_LIST_HEAD(&bo->lru); 1120 INIT_LIST_HEAD(&bo->ddestroy); 1121 INIT_LIST_HEAD(&bo->swap); 1122 INIT_LIST_HEAD(&bo->io_reserve_lru); 1123 lockinit(&bo->wu_mutex, "ttmbwm", 0, LK_CANRECURSE); 1124 bo->bdev = bdev; 1125 bo->glob = bdev->glob; 1126 bo->type = type; 1127 bo->num_pages = num_pages; 1128 bo->mem.size = num_pages << PAGE_SHIFT; 1129 bo->mem.mem_type = TTM_PL_SYSTEM; 1130 bo->mem.num_pages = bo->num_pages; 1131 bo->mem.mm_node = NULL; 1132 bo->mem.page_alignment = page_alignment; 1133 bo->mem.bus.io_reserved_vm = false; 1134 bo->mem.bus.io_reserved_count = 0; 1135 bo->priv_flags = 0; 1136 bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); 1137 bo->persistent_swap_storage = persistent_swap_storage; 1138 bo->acc_size = acc_size; 1139 bo->sg = sg; 1140 if (resv) { 1141 bo->resv = resv; 1142 lockdep_assert_held(&bo->resv->lock.base); 1143 } else { 1144 bo->resv = &bo->ttm_resv; 1145 reservation_object_init(&bo->ttm_resv); 1146 } 1147 atomic_inc(&bo->glob->bo_count); 1148 drm_vma_node_reset(&bo->vma_node); 1149 1150 /* 1151 * For ttm_bo_type_device buffers, allocate 1152 * address space from the device. 1153 */ 1154 if (bo->type == ttm_bo_type_device || 1155 bo->type == ttm_bo_type_sg) 1156 ret = drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node, 1157 bo->mem.num_pages); 1158 1159 /* passed reservation objects should already be locked, 1160 * since otherwise lockdep will be angered in radeon. 1161 */ 1162 if (!resv) { 1163 locked = ww_mutex_trylock(&bo->resv->lock); 1164 WARN_ON(!locked); 1165 } 1166 1167 if (likely(!ret)) 1168 ret = ttm_bo_validate(bo, placement, interruptible, false); 1169 1170 if (!resv) 1171 ttm_bo_unreserve(bo); 1172 1173 if (unlikely(ret)) 1174 ttm_bo_unref(&bo); 1175 1176 return ret; 1177 } 1178 EXPORT_SYMBOL(ttm_bo_init); 1179 1180 size_t ttm_bo_acc_size(struct ttm_bo_device *bdev, 1181 unsigned long bo_size, 1182 unsigned struct_size) 1183 { 1184 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1185 size_t size = 0; 1186 1187 size += ttm_round_pot(struct_size); 1188 size += PAGE_ALIGN(npages * sizeof(void *)); 1189 size += ttm_round_pot(sizeof(struct ttm_tt)); 1190 return size; 1191 } 1192 EXPORT_SYMBOL(ttm_bo_acc_size); 1193 1194 size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, 1195 unsigned long bo_size, 1196 unsigned struct_size) 1197 { 1198 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1199 size_t size = 0; 1200 1201 size += ttm_round_pot(struct_size); 1202 size += PAGE_ALIGN(npages * sizeof(void *)); 1203 size += PAGE_ALIGN(npages * sizeof(dma_addr_t)); 1204 size += ttm_round_pot(sizeof(struct ttm_dma_tt)); 1205 return size; 1206 } 1207 EXPORT_SYMBOL(ttm_bo_dma_acc_size); 1208 1209 int ttm_bo_create(struct ttm_bo_device *bdev, 1210 unsigned long size, 1211 enum ttm_bo_type type, 1212 struct ttm_placement *placement, 1213 uint32_t page_alignment, 1214 bool interruptible, 1215 struct vm_object *persistent_swap_storage, 1216 struct ttm_buffer_object **p_bo) 1217 { 1218 struct ttm_buffer_object *bo; 1219 size_t acc_size; 1220 int ret; 1221 1222 bo = kzalloc(sizeof(*bo), GFP_KERNEL); 1223 if (unlikely(bo == NULL)) 1224 return -ENOMEM; 1225 1226 acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); 1227 ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, 1228 interruptible, persistent_swap_storage, acc_size, 1229 NULL, NULL, NULL); 1230 if (likely(ret == 0)) 1231 *p_bo = bo; 1232 1233 return ret; 1234 } 1235 EXPORT_SYMBOL(ttm_bo_create); 1236 1237 static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, 1238 unsigned mem_type, bool allow_errors) 1239 { 1240 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1241 struct ttm_bo_global *glob = bdev->glob; 1242 int ret; 1243 1244 /* 1245 * Can't use standard list traversal since we're unlocking. 1246 */ 1247 1248 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 1249 while (!list_empty(&man->lru)) { 1250 lockmgr(&glob->lru_lock, LK_RELEASE); 1251 ret = ttm_mem_evict_first(bdev, mem_type, NULL, false, false); 1252 if (ret) { 1253 if (allow_errors) { 1254 return ret; 1255 } else { 1256 pr_err("Cleanup eviction failed\n"); 1257 } 1258 } 1259 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 1260 } 1261 lockmgr(&glob->lru_lock, LK_RELEASE); 1262 return 0; 1263 } 1264 1265 int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1266 { 1267 struct ttm_mem_type_manager *man; 1268 int ret = -EINVAL; 1269 1270 if (mem_type >= TTM_NUM_MEM_TYPES) { 1271 pr_err("Illegal memory type %d\n", mem_type); 1272 return ret; 1273 } 1274 man = &bdev->man[mem_type]; 1275 1276 if (!man->has_type) { 1277 pr_err("Trying to take down uninitialized memory manager type %u\n", 1278 mem_type); 1279 return ret; 1280 } 1281 1282 man->use_type = false; 1283 man->has_type = false; 1284 1285 ret = 0; 1286 if (mem_type > 0) { 1287 ttm_bo_force_list_clean(bdev, mem_type, false); 1288 1289 ret = (*man->func->takedown)(man); 1290 } 1291 1292 return ret; 1293 } 1294 EXPORT_SYMBOL(ttm_bo_clean_mm); 1295 1296 int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1297 { 1298 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1299 1300 if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) { 1301 pr_err("Illegal memory manager memory type %u\n", mem_type); 1302 return -EINVAL; 1303 } 1304 1305 if (!man->has_type) { 1306 pr_err("Memory type %u has not been initialized\n", mem_type); 1307 return 0; 1308 } 1309 1310 return ttm_bo_force_list_clean(bdev, mem_type, true); 1311 } 1312 EXPORT_SYMBOL(ttm_bo_evict_mm); 1313 1314 int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, 1315 unsigned long p_size) 1316 { 1317 int ret = -EINVAL; 1318 struct ttm_mem_type_manager *man; 1319 1320 BUG_ON(type >= TTM_NUM_MEM_TYPES); 1321 man = &bdev->man[type]; 1322 BUG_ON(man->has_type); 1323 man->io_reserve_fastpath = true; 1324 man->use_io_reserve_lru = false; 1325 lockinit(&man->io_reserve_mutex, "ttmior", 0, 0); 1326 INIT_LIST_HEAD(&man->io_reserve_lru); 1327 1328 ret = bdev->driver->init_mem_type(bdev, type, man); 1329 if (ret) 1330 return ret; 1331 man->bdev = bdev; 1332 1333 ret = 0; 1334 if (type != TTM_PL_SYSTEM) { 1335 ret = (*man->func->init)(man, p_size); 1336 if (ret) 1337 return ret; 1338 } 1339 man->has_type = true; 1340 man->use_type = true; 1341 man->size = p_size; 1342 1343 INIT_LIST_HEAD(&man->lru); 1344 1345 return 0; 1346 } 1347 EXPORT_SYMBOL(ttm_bo_init_mm); 1348 1349 static void ttm_bo_global_kobj_release(struct kobject *kobj) 1350 { 1351 struct ttm_bo_global *glob = 1352 container_of(kobj, struct ttm_bo_global, kobj); 1353 1354 ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink); 1355 __free_page(glob->dummy_read_page); 1356 kfree(glob); 1357 } 1358 1359 void ttm_bo_global_release(struct drm_global_reference *ref) 1360 { 1361 struct ttm_bo_global *glob = ref->object; 1362 1363 kobject_del(&glob->kobj); 1364 kobject_put(&glob->kobj); 1365 } 1366 EXPORT_SYMBOL(ttm_bo_global_release); 1367 1368 int ttm_bo_global_init(struct drm_global_reference *ref) 1369 { 1370 struct ttm_bo_global_ref *bo_ref = 1371 container_of(ref, struct ttm_bo_global_ref, ref); 1372 struct ttm_bo_global *glob = ref->object; 1373 int ret; 1374 1375 lockinit(&glob->device_list_mutex, "ttmdlm", 0, 0); 1376 lockinit(&glob->lru_lock, "ttmlru", 0, 0); 1377 glob->mem_glob = bo_ref->mem_glob; 1378 glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32); 1379 1380 if (unlikely(glob->dummy_read_page == NULL)) { 1381 ret = -ENOMEM; 1382 goto out_no_drp; 1383 } 1384 1385 INIT_LIST_HEAD(&glob->swap_lru); 1386 INIT_LIST_HEAD(&glob->device_list); 1387 1388 ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout); 1389 ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink); 1390 if (unlikely(ret != 0)) { 1391 pr_err("Could not register buffer object swapout\n"); 1392 goto out_no_shrink; 1393 } 1394 1395 atomic_set(&glob->bo_count, 0); 1396 1397 ret = kobject_init_and_add( 1398 &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects"); 1399 if (unlikely(ret != 0)) 1400 kobject_put(&glob->kobj); 1401 return ret; 1402 out_no_shrink: 1403 __free_page(glob->dummy_read_page); 1404 out_no_drp: 1405 kfree(glob); 1406 return ret; 1407 } 1408 EXPORT_SYMBOL(ttm_bo_global_init); 1409 1410 1411 int ttm_bo_device_release(struct ttm_bo_device *bdev) 1412 { 1413 int ret = 0; 1414 unsigned i = TTM_NUM_MEM_TYPES; 1415 struct ttm_mem_type_manager *man; 1416 struct ttm_bo_global *glob = bdev->glob; 1417 1418 while (i--) { 1419 man = &bdev->man[i]; 1420 if (man->has_type) { 1421 man->use_type = false; 1422 if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) { 1423 ret = -EBUSY; 1424 pr_err("DRM memory manager type %d is not clean\n", 1425 i); 1426 } 1427 man->has_type = false; 1428 } 1429 } 1430 1431 mutex_lock(&glob->device_list_mutex); 1432 list_del(&bdev->device_list); 1433 mutex_unlock(&glob->device_list_mutex); 1434 1435 cancel_delayed_work_sync(&bdev->wq); 1436 1437 while (ttm_bo_delayed_delete(bdev, true)) 1438 ; 1439 1440 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 1441 if (list_empty(&bdev->ddestroy)) 1442 TTM_DEBUG("Delayed destroy list was clean\n"); 1443 1444 if (list_empty(&bdev->man[0].lru)) 1445 TTM_DEBUG("Swap list was clean\n"); 1446 lockmgr(&glob->lru_lock, LK_RELEASE); 1447 1448 drm_vma_offset_manager_destroy(&bdev->vma_manager); 1449 1450 return ret; 1451 } 1452 EXPORT_SYMBOL(ttm_bo_device_release); 1453 1454 int ttm_bo_device_init(struct ttm_bo_device *bdev, 1455 struct ttm_bo_global *glob, 1456 struct ttm_bo_driver *driver, 1457 struct address_space *mapping, 1458 uint64_t file_page_offset, 1459 bool need_dma32) 1460 { 1461 int ret = -EINVAL; 1462 1463 bdev->driver = driver; 1464 1465 memset(bdev->man, 0, sizeof(bdev->man)); 1466 1467 /* 1468 * Initialize the system memory buffer type. 1469 * Other types need to be driver / IOCTL initialized. 1470 */ 1471 ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0); 1472 if (unlikely(ret != 0)) 1473 goto out_no_sys; 1474 1475 drm_vma_offset_manager_init(&bdev->vma_manager, file_page_offset, 1476 0x10000000); 1477 INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); 1478 INIT_LIST_HEAD(&bdev->ddestroy); 1479 /* 1480 * XXX DRAGONFLY - dev_mapping NULL atm, find other XXX DRAGONFLY 1481 * lines and fix when it no longer is in later API change. 1482 */ 1483 bdev->dev_mapping = mapping; 1484 bdev->glob = glob; 1485 bdev->need_dma32 = need_dma32; 1486 bdev->val_seq = 0; 1487 mutex_lock(&glob->device_list_mutex); 1488 list_add_tail(&bdev->device_list, &glob->device_list); 1489 mutex_unlock(&glob->device_list_mutex); 1490 1491 return 0; 1492 out_no_sys: 1493 return ret; 1494 } 1495 EXPORT_SYMBOL(ttm_bo_device_init); 1496 1497 /* 1498 * buffer object vm functions. 1499 */ 1500 1501 bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 1502 { 1503 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 1504 1505 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 1506 if (mem->mem_type == TTM_PL_SYSTEM) 1507 return false; 1508 1509 if (man->flags & TTM_MEMTYPE_FLAG_CMA) 1510 return false; 1511 1512 if (mem->placement & TTM_PL_FLAG_CACHED) 1513 return false; 1514 } 1515 return true; 1516 } 1517 1518 #ifdef __DragonFly__ 1519 1520 /* 1521 * XXX DRAGONFLY - device_mapping not yet implemented so 1522 * file_mapping is basically always NULL. We have to properly 1523 * release the mmap, etc. 1524 */ 1525 void ttm_bo_release_mmap(struct ttm_buffer_object *bo); 1526 1527 /** 1528 * drm_vma_node_unmap() - Unmap offset node 1529 * @node: Offset node 1530 * @file_mapping: Address space to unmap @node from 1531 * 1532 * Unmap all userspace mappings for a given offset node. The mappings must be 1533 * associated with the @file_mapping address-space. If no offset exists or 1534 * the address-space is invalid, nothing is done. 1535 * 1536 * This call is unlocked. The caller must guarantee that drm_vma_offset_remove() 1537 * is not called on this node concurrently. 1538 */ 1539 static inline void drm_vma_node_unmap(struct drm_vma_offset_node *node, 1540 struct address_space *file_mapping) 1541 { 1542 struct ttm_buffer_object *bo = container_of(node, struct ttm_buffer_object, vma_node); 1543 1544 if (drm_vma_node_has_offset(node)) 1545 unmap_mapping_range(file_mapping, 1546 drm_vma_node_offset_addr(node), 1547 drm_vma_node_size(node) << PAGE_SHIFT, 1); 1548 ttm_bo_release_mmap(bo); 1549 } 1550 #endif 1551 1552 void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) 1553 { 1554 struct ttm_bo_device *bdev = bo->bdev; 1555 1556 drm_vma_node_unmap(&bo->vma_node, bdev->dev_mapping); 1557 ttm_mem_io_free_vm(bo); 1558 } 1559 1560 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) 1561 { 1562 struct ttm_bo_device *bdev = bo->bdev; 1563 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 1564 1565 ttm_mem_io_lock(man, false); 1566 ttm_bo_unmap_virtual_locked(bo); 1567 ttm_mem_io_unlock(man); 1568 } 1569 1570 1571 EXPORT_SYMBOL(ttm_bo_unmap_virtual); 1572 1573 int ttm_bo_wait(struct ttm_buffer_object *bo, 1574 bool lazy, bool interruptible, bool no_wait) 1575 { 1576 struct reservation_object_list *fobj; 1577 struct reservation_object *resv; 1578 struct fence *excl; 1579 long timeout = 15 * HZ; 1580 int i; 1581 1582 resv = bo->resv; 1583 fobj = reservation_object_get_list(resv); 1584 excl = reservation_object_get_excl(resv); 1585 if (excl) { 1586 if (!fence_is_signaled(excl)) { 1587 if (no_wait) 1588 return -EBUSY; 1589 1590 timeout = fence_wait_timeout(excl, 1591 interruptible, timeout); 1592 } 1593 } 1594 1595 for (i = 0; fobj && timeout > 0 && i < fobj->shared_count; ++i) { 1596 struct fence *fence; 1597 fence = rcu_dereference_protected(fobj->shared[i], 1598 reservation_object_held(resv)); 1599 1600 if (!fence_is_signaled(fence)) { 1601 if (no_wait) 1602 return -EBUSY; 1603 1604 timeout = fence_wait_timeout(fence, 1605 interruptible, timeout); 1606 } 1607 } 1608 1609 if (timeout < 0) 1610 return timeout; 1611 1612 if (timeout == 0) 1613 return -EBUSY; 1614 1615 reservation_object_add_excl_fence(resv, NULL); 1616 clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); 1617 return 0; 1618 } 1619 EXPORT_SYMBOL(ttm_bo_wait); 1620 1621 int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait) 1622 { 1623 int ret = 0; 1624 1625 /* 1626 * Using ttm_bo_reserve makes sure the lru lists are updated. 1627 */ 1628 1629 ret = ttm_bo_reserve(bo, true, no_wait, false, 0); 1630 if (unlikely(ret != 0)) 1631 return ret; 1632 ret = ttm_bo_wait(bo, false, true, no_wait); 1633 if (likely(ret == 0)) 1634 atomic_inc(&bo->cpu_writers); 1635 ttm_bo_unreserve(bo); 1636 return ret; 1637 } 1638 EXPORT_SYMBOL(ttm_bo_synccpu_write_grab); 1639 1640 void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo) 1641 { 1642 atomic_dec(&bo->cpu_writers); 1643 } 1644 EXPORT_SYMBOL(ttm_bo_synccpu_write_release); 1645 1646 /** 1647 * A buffer object shrink method that tries to swap out the first 1648 * buffer object on the bo_global::swap_lru list. 1649 */ 1650 1651 static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) 1652 { 1653 struct ttm_bo_global *glob = 1654 container_of(shrink, struct ttm_bo_global, shrink); 1655 struct ttm_buffer_object *bo; 1656 int ret = -EBUSY; 1657 int put_count; 1658 uint32_t swap_placement = (TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM); 1659 1660 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 1661 list_for_each_entry(bo, &glob->swap_lru, swap) { 1662 ret = __ttm_bo_reserve(bo, false, true, false, 0); 1663 if (!ret) 1664 break; 1665 } 1666 1667 if (ret) { 1668 lockmgr(&glob->lru_lock, LK_RELEASE); 1669 return ret; 1670 } 1671 1672 kref_get(&bo->list_kref); 1673 1674 if (!list_empty(&bo->ddestroy)) { 1675 ret = ttm_bo_cleanup_refs_and_unlock(bo, false, false); 1676 kref_put(&bo->list_kref, ttm_bo_release_list); 1677 return ret; 1678 } 1679 1680 put_count = ttm_bo_del_from_lru(bo); 1681 lockmgr(&glob->lru_lock, LK_RELEASE); 1682 1683 ttm_bo_list_ref_sub(bo, put_count, true); 1684 1685 /** 1686 * Wait for GPU, then move to system cached. 1687 */ 1688 1689 ret = ttm_bo_wait(bo, false, false, false); 1690 1691 if (unlikely(ret != 0)) 1692 goto out; 1693 1694 if ((bo->mem.placement & swap_placement) != swap_placement) { 1695 struct ttm_mem_reg evict_mem; 1696 1697 evict_mem = bo->mem; 1698 evict_mem.mm_node = NULL; 1699 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED; 1700 evict_mem.mem_type = TTM_PL_SYSTEM; 1701 1702 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, 1703 false, false); 1704 if (unlikely(ret != 0)) 1705 goto out; 1706 } 1707 1708 ttm_bo_unmap_virtual(bo); 1709 1710 /** 1711 * Swap out. Buffer will be swapped in again as soon as 1712 * anyone tries to access a ttm page. 1713 */ 1714 1715 if (bo->bdev->driver->swap_notify) 1716 bo->bdev->driver->swap_notify(bo); 1717 1718 ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage); 1719 out: 1720 1721 /** 1722 * 1723 * Unreserve without putting on LRU to avoid swapping out an 1724 * already swapped buffer. 1725 */ 1726 1727 __ttm_bo_unreserve(bo); 1728 kref_put(&bo->list_kref, ttm_bo_release_list); 1729 return ret; 1730 } 1731 1732 void ttm_bo_swapout_all(struct ttm_bo_device *bdev) 1733 { 1734 while (ttm_bo_swapout(&bdev->glob->shrink) == 0) 1735 ; 1736 } 1737 EXPORT_SYMBOL(ttm_bo_swapout_all); 1738 1739 /** 1740 * ttm_bo_wait_unreserved - interruptible wait for a buffer object to become 1741 * unreserved 1742 * 1743 * @bo: Pointer to buffer 1744 */ 1745 int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo) 1746 { 1747 int ret; 1748 1749 /* 1750 * In the absense of a wait_unlocked API, 1751 * Use the bo::wu_mutex to avoid triggering livelocks due to 1752 * concurrent use of this function. Note that this use of 1753 * bo::wu_mutex can go away if we change locking order to 1754 * mmap_sem -> bo::reserve. 1755 */ 1756 ret = mutex_lock_interruptible(&bo->wu_mutex); 1757 if (unlikely(ret != 0)) 1758 return -ERESTARTSYS; 1759 if (!ww_mutex_is_locked(&bo->resv->lock)) 1760 goto out_unlock; 1761 ret = __ttm_bo_reserve(bo, true, false, false, NULL); 1762 if (unlikely(ret != 0)) 1763 goto out_unlock; 1764 __ttm_bo_unreserve(bo); 1765 1766 out_unlock: 1767 mutex_unlock(&bo->wu_mutex); 1768 return ret; 1769 } 1770