1 /************************************************************************** 2 * 3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 /* 28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29 */ 30 31 #define pr_fmt(fmt) "[TTM] " fmt 32 33 #include <drm/ttm/ttm_module.h> 34 #include <drm/ttm/ttm_bo_driver.h> 35 #include <drm/ttm/ttm_placement.h> 36 #include <linux/jiffies.h> 37 #include <linux/slab.h> 38 #include <linux/sched.h> 39 #include <linux/mm.h> 40 #include <linux/file.h> 41 #include <linux/module.h> 42 #include <linux/atomic.h> 43 #include <linux/reservation.h> 44 45 #define TTM_ASSERT_LOCKED(param) 46 #define TTM_DEBUG(fmt, arg...) 47 #define TTM_BO_HASH_ORDER 13 48 49 static void ttm_bo_global_kobj_release(struct kobject *kobj); 50 51 static struct attribute ttm_bo_count = { 52 .name = "bo_count", 53 .mode = S_IRUGO 54 }; 55 56 /* default destructor */ 57 static void ttm_bo_default_destroy(struct ttm_buffer_object *bo) 58 { 59 kfree(bo); 60 } 61 62 static inline int ttm_mem_type_from_place(const struct ttm_place *place, 63 uint32_t *mem_type) 64 { 65 int pos; 66 67 pos = ffs(place->flags & TTM_PL_MASK_MEM); 68 if (unlikely(!pos)) 69 return -EINVAL; 70 71 *mem_type = pos - 1; 72 return 0; 73 } 74 75 static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type) 76 { 77 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 78 struct drm_printer p = drm_debug_printer(TTM_PFX); 79 80 pr_err(" has_type: %d\n", man->has_type); 81 pr_err(" use_type: %d\n", man->use_type); 82 pr_err(" flags: 0x%08X\n", man->flags); 83 pr_err(" gpu_offset: 0x%08lX\n", man->gpu_offset); 84 pr_err(" size: %ju\n", man->size); 85 pr_err(" available_caching: 0x%08X\n", man->available_caching); 86 pr_err(" default_caching: 0x%08X\n", man->default_caching); 87 if (mem_type != TTM_PL_SYSTEM) 88 (*man->func->debug)(man, &p); 89 } 90 91 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, 92 struct ttm_placement *placement) 93 { 94 int i, ret, mem_type; 95 96 pr_err("No space for %p (%lu pages, %luK, %luM)\n", 97 bo, bo->mem.num_pages, bo->mem.size >> 10, 98 bo->mem.size >> 20); 99 for (i = 0; i < placement->num_placement; i++) { 100 ret = ttm_mem_type_from_place(&placement->placement[i], 101 &mem_type); 102 if (ret) 103 return; 104 pr_err(" placement[%d]=0x%08X (%d)\n", 105 i, placement->placement[i].flags, mem_type); 106 ttm_mem_type_debug(bo->bdev, mem_type); 107 } 108 } 109 110 static ssize_t ttm_bo_global_show(struct kobject *kobj, 111 struct attribute *attr, 112 char *buffer) 113 { 114 struct ttm_bo_global *glob = 115 container_of(kobj, struct ttm_bo_global, kobj); 116 117 return snprintf(buffer, PAGE_SIZE, "%d\n", 118 atomic_read(&glob->bo_count)); 119 } 120 121 static struct attribute *ttm_bo_global_attrs[] = { 122 &ttm_bo_count, 123 NULL 124 }; 125 126 static const struct sysfs_ops ttm_bo_global_ops = { 127 .show = &ttm_bo_global_show 128 }; 129 130 static struct kobj_type ttm_bo_glob_kobj_type = { 131 .release = &ttm_bo_global_kobj_release, 132 .sysfs_ops = &ttm_bo_global_ops, 133 .default_attrs = ttm_bo_global_attrs 134 }; 135 136 137 static inline uint32_t ttm_bo_type_flags(unsigned type) 138 { 139 return 1 << (type); 140 } 141 142 static void ttm_bo_release_list(struct kref *list_kref) 143 { 144 struct ttm_buffer_object *bo = 145 container_of(list_kref, struct ttm_buffer_object, list_kref); 146 struct ttm_bo_device *bdev = bo->bdev; 147 size_t acc_size = bo->acc_size; 148 149 BUG_ON(kref_read(&bo->list_kref)); 150 BUG_ON(kref_read(&bo->kref)); 151 BUG_ON(atomic_read(&bo->cpu_writers)); 152 BUG_ON(bo->mem.mm_node != NULL); 153 BUG_ON(!list_empty(&bo->lru)); 154 BUG_ON(!list_empty(&bo->ddestroy)); 155 ttm_tt_destroy(bo->ttm); 156 atomic_dec(&bo->bdev->glob->bo_count); 157 dma_fence_put(bo->moving); 158 reservation_object_fini(&bo->ttm_resv); 159 mutex_destroy(&bo->wu_mutex); 160 bo->destroy(bo); 161 ttm_mem_global_free(bdev->glob->mem_glob, acc_size); 162 } 163 164 void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) 165 { 166 struct ttm_bo_device *bdev = bo->bdev; 167 struct ttm_mem_type_manager *man; 168 169 lockdep_assert_held(&bo->resv->lock.base); 170 171 if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { 172 173 #ifdef __DragonFly__ 174 if (WARN_ON(!list_empty(&bo->lru))) 175 return; 176 #endif 177 178 man = &bdev->man[bo->mem.mem_type]; 179 list_add_tail(&bo->lru, &man->lru[bo->priority]); 180 kref_get(&bo->list_kref); 181 182 if (bo->ttm && !(bo->ttm->page_flags & 183 (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED))) { 184 list_add_tail(&bo->swap, 185 &bdev->glob->swap_lru[bo->priority]); 186 kref_get(&bo->list_kref); 187 } 188 } 189 } 190 EXPORT_SYMBOL(ttm_bo_add_to_lru); 191 192 static void ttm_bo_ref_bug(struct kref *list_kref) 193 { 194 BUG(); 195 } 196 197 void ttm_bo_del_from_lru(struct ttm_buffer_object *bo) 198 { 199 if (!list_empty(&bo->swap)) { 200 list_del_init(&bo->swap); 201 kref_put(&bo->list_kref, ttm_bo_ref_bug); 202 } 203 if (!list_empty(&bo->lru)) { 204 list_del_init(&bo->lru); 205 kref_put(&bo->list_kref, ttm_bo_ref_bug); 206 } 207 208 /* 209 * TODO: Add a driver hook to delete from 210 * driver-specific LRU's here. 211 */ 212 } 213 214 void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo) 215 { 216 struct ttm_bo_global *glob = bo->bdev->glob; 217 218 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 219 ttm_bo_del_from_lru(bo); 220 lockmgr(&glob->lru_lock, LK_RELEASE); 221 } 222 EXPORT_SYMBOL(ttm_bo_del_sub_from_lru); 223 224 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo) 225 { 226 lockdep_assert_held(&bo->resv->lock.base); 227 228 ttm_bo_del_from_lru(bo); 229 ttm_bo_add_to_lru(bo); 230 } 231 EXPORT_SYMBOL(ttm_bo_move_to_lru_tail); 232 233 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, 234 struct ttm_mem_reg *mem, bool evict, 235 struct ttm_operation_ctx *ctx) 236 { 237 struct ttm_bo_device *bdev = bo->bdev; 238 bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); 239 bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem); 240 struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type]; 241 struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type]; 242 int ret = 0; 243 244 if (old_is_pci || new_is_pci || 245 ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) { 246 ret = ttm_mem_io_lock(old_man, true); 247 if (unlikely(ret != 0)) 248 goto out_err; 249 ttm_bo_unmap_virtual_locked(bo); 250 ttm_mem_io_unlock(old_man); 251 } 252 253 /* 254 * Create and bind a ttm if required. 255 */ 256 257 if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 258 if (bo->ttm == NULL) { 259 bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED); 260 ret = ttm_tt_create(bo, zero); 261 if (ret) 262 goto out_err; 263 } 264 265 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); 266 if (ret) 267 goto out_err; 268 269 if (mem->mem_type != TTM_PL_SYSTEM) { 270 ret = ttm_tt_bind(bo->ttm, mem, ctx); 271 if (ret) 272 goto out_err; 273 } 274 275 if (bo->mem.mem_type == TTM_PL_SYSTEM) { 276 if (bdev->driver->move_notify) 277 bdev->driver->move_notify(bo, evict, mem); 278 bo->mem = *mem; 279 mem->mm_node = NULL; 280 goto moved; 281 } 282 } 283 284 if (bdev->driver->move_notify) 285 bdev->driver->move_notify(bo, evict, mem); 286 287 if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && 288 !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) 289 ret = ttm_bo_move_ttm(bo, ctx, mem); 290 else if (bdev->driver->move) 291 ret = bdev->driver->move(bo, evict, ctx, mem); 292 else 293 ret = ttm_bo_move_memcpy(bo, ctx, mem); 294 295 if (ret) { 296 if (bdev->driver->move_notify) { 297 swap(*mem, bo->mem); 298 bdev->driver->move_notify(bo, false, mem); 299 swap(*mem, bo->mem); 300 } 301 302 goto out_err; 303 } 304 305 moved: 306 if (bo->evicted) { 307 if (bdev->driver->invalidate_caches) { 308 ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement); 309 if (ret) 310 pr_err("Can not flush read caches\n"); 311 } 312 bo->evicted = false; 313 } 314 315 if (bo->mem.mm_node) 316 bo->offset = (bo->mem.start << PAGE_SHIFT) + 317 bdev->man[bo->mem.mem_type].gpu_offset; 318 else 319 bo->offset = 0; 320 321 ctx->bytes_moved += bo->num_pages << PAGE_SHIFT; 322 return 0; 323 324 out_err: 325 new_man = &bdev->man[bo->mem.mem_type]; 326 if (new_man->flags & TTM_MEMTYPE_FLAG_FIXED) { 327 ttm_tt_destroy(bo->ttm); 328 bo->ttm = NULL; 329 } 330 331 return ret; 332 } 333 334 /** 335 * Call bo::reserved. 336 * Will release GPU memory type usage on destruction. 337 * This is the place to put in driver specific hooks to release 338 * driver private resources. 339 * Will release the bo::reserved lock. 340 */ 341 342 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) 343 { 344 if (bo->bdev->driver->move_notify) 345 bo->bdev->driver->move_notify(bo, false, NULL); 346 347 ttm_tt_destroy(bo->ttm); 348 bo->ttm = NULL; 349 ttm_bo_mem_put(bo, &bo->mem); 350 } 351 352 static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo) 353 { 354 int r; 355 356 if (bo->resv == &bo->ttm_resv) 357 return 0; 358 359 BUG_ON(!reservation_object_trylock(&bo->ttm_resv)); 360 361 r = reservation_object_copy_fences(&bo->ttm_resv, bo->resv); 362 if (r) 363 reservation_object_unlock(&bo->ttm_resv); 364 365 return r; 366 } 367 368 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) 369 { 370 struct reservation_object_list *fobj; 371 struct dma_fence *fence; 372 int i; 373 374 fobj = reservation_object_get_list(&bo->ttm_resv); 375 fence = reservation_object_get_excl(&bo->ttm_resv); 376 if (fence && !fence->ops->signaled) 377 dma_fence_enable_sw_signaling(fence); 378 379 for (i = 0; fobj && i < fobj->shared_count; ++i) { 380 fence = rcu_dereference_protected(fobj->shared[i], 381 reservation_object_held(bo->resv)); 382 383 if (!fence->ops->signaled) 384 dma_fence_enable_sw_signaling(fence); 385 } 386 } 387 388 static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) 389 { 390 struct ttm_bo_device *bdev = bo->bdev; 391 struct ttm_bo_global *glob = bdev->glob; 392 int ret; 393 394 ret = ttm_bo_individualize_resv(bo); 395 if (ret) { 396 /* Last resort, if we fail to allocate memory for the 397 * fences block for the BO to become idle 398 */ 399 reservation_object_wait_timeout_rcu(bo->resv, true, false, 400 30 * HZ); 401 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 402 goto error; 403 } 404 405 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 406 ret = __ttm_bo_reserve(bo, false, true, NULL); 407 if (!ret) { 408 if (reservation_object_test_signaled_rcu(&bo->ttm_resv, true)) { 409 ttm_bo_del_from_lru(bo); 410 lockmgr(&glob->lru_lock, LK_RELEASE); 411 if (bo->resv != &bo->ttm_resv) 412 reservation_object_unlock(&bo->ttm_resv); 413 414 ttm_bo_cleanup_memtype_use(bo); 415 __ttm_bo_unreserve(bo); 416 return; 417 } 418 419 ttm_bo_flush_all_fences(bo); 420 421 /* 422 * Make NO_EVICT bos immediately available to 423 * shrinkers, now that they are queued for 424 * destruction. 425 */ 426 if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) { 427 bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT; 428 ttm_bo_add_to_lru(bo); 429 } 430 431 __ttm_bo_unreserve(bo); 432 } 433 if (bo->resv != &bo->ttm_resv) 434 reservation_object_unlock(&bo->ttm_resv); 435 436 error: 437 kref_get(&bo->list_kref); 438 list_add_tail(&bo->ddestroy, &bdev->ddestroy); 439 lockmgr(&glob->lru_lock, LK_RELEASE); 440 441 schedule_delayed_work(&bdev->wq, 442 ((HZ / 100) < 1) ? 1 : HZ / 100); 443 } 444 445 /** 446 * function ttm_bo_cleanup_refs 447 * If bo idle, remove from delayed- and lru lists, and unref. 448 * If not idle, do nothing. 449 * 450 * Must be called with lru_lock and reservation held, this function 451 * will drop the lru lock and optionally the reservation lock before returning. 452 * 453 * @interruptible Any sleeps should occur interruptibly. 454 * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. 455 * @unlock_resv Unlock the reservation lock as well. 456 */ 457 458 static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, 459 bool interruptible, bool no_wait_gpu, 460 bool unlock_resv) 461 { 462 struct ttm_bo_global *glob = bo->bdev->glob; 463 struct reservation_object *resv; 464 int ret; 465 466 if (unlikely(list_empty(&bo->ddestroy))) 467 resv = bo->resv; 468 else 469 resv = &bo->ttm_resv; 470 471 if (reservation_object_test_signaled_rcu(resv, true)) 472 ret = 0; 473 else 474 ret = -EBUSY; 475 476 if (ret && !no_wait_gpu) { 477 long lret; 478 479 if (unlock_resv) 480 ww_mutex_unlock(&bo->resv->lock); 481 lockmgr(&glob->lru_lock, LK_RELEASE); 482 483 lret = reservation_object_wait_timeout_rcu(resv, true, 484 interruptible, 485 30 * HZ); 486 487 if (lret < 0) 488 return lret; 489 else if (lret == 0) 490 return -EBUSY; 491 492 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 493 if (unlock_resv && __ttm_bo_reserve(bo, false, true, NULL)) { 494 /* 495 * We raced, and lost, someone else holds the reservation now, 496 * and is probably busy in ttm_bo_cleanup_memtype_use. 497 * 498 * Even if it's not the case, because we finished waiting any 499 * delayed destruction would succeed, so just return success 500 * here. 501 */ 502 lockmgr(&glob->lru_lock, LK_RELEASE); 503 return 0; 504 } 505 ret = 0; 506 } 507 508 if (ret || unlikely(list_empty(&bo->ddestroy))) { 509 if (unlock_resv) 510 __ttm_bo_unreserve(bo); 511 lockmgr(&glob->lru_lock, LK_RELEASE); 512 return ret; 513 } 514 515 ttm_bo_del_from_lru(bo); 516 list_del_init(&bo->ddestroy); 517 kref_put(&bo->list_kref, ttm_bo_ref_bug); 518 519 lockmgr(&glob->lru_lock, LK_RELEASE); 520 ttm_bo_cleanup_memtype_use(bo); 521 522 if (unlock_resv) 523 __ttm_bo_unreserve(bo); 524 525 return 0; 526 } 527 528 /** 529 * Traverse the delayed list, and call ttm_bo_cleanup_refs on all 530 * encountered buffers. 531 */ 532 static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) 533 { 534 struct ttm_bo_global *glob = bdev->glob; 535 struct list_head removed; 536 bool empty; 537 538 INIT_LIST_HEAD(&removed); 539 540 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 541 while (!list_empty(&bdev->ddestroy)) { 542 struct ttm_buffer_object *bo; 543 544 bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object, 545 ddestroy); 546 kref_get(&bo->list_kref); 547 list_move_tail(&bo->ddestroy, &removed); 548 549 if (remove_all || bo->resv != &bo->ttm_resv) { 550 lockmgr(&glob->lru_lock, LK_RELEASE); 551 reservation_object_lock(bo->resv, NULL); 552 553 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 554 ttm_bo_cleanup_refs(bo, false, !remove_all, true); 555 556 } else if (reservation_object_trylock(bo->resv)) { 557 ttm_bo_cleanup_refs(bo, false, !remove_all, true); 558 } else { 559 lockmgr(&glob->lru_lock, LK_RELEASE); 560 } 561 562 kref_put(&bo->list_kref, ttm_bo_release_list); 563 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 564 } 565 list_splice_tail(&removed, &bdev->ddestroy); 566 empty = list_empty(&bdev->ddestroy); 567 lockmgr(&glob->lru_lock, LK_RELEASE); 568 569 return empty; 570 } 571 572 static void ttm_bo_delayed_workqueue(struct work_struct *work) 573 { 574 struct ttm_bo_device *bdev = 575 container_of(work, struct ttm_bo_device, wq.work); 576 577 if (!ttm_bo_delayed_delete(bdev, false)) 578 schedule_delayed_work(&bdev->wq, 579 ((HZ / 100) < 1) ? 1 : HZ / 100); 580 } 581 582 static void ttm_bo_release(struct kref *kref) 583 { 584 struct ttm_buffer_object *bo = 585 container_of(kref, struct ttm_buffer_object, kref); 586 struct ttm_bo_device *bdev = bo->bdev; 587 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 588 589 drm_vma_offset_remove(&bdev->vma_manager, &bo->vma_node); 590 ttm_mem_io_lock(man, false); 591 ttm_mem_io_free_vm(bo); 592 ttm_mem_io_unlock(man); 593 ttm_bo_cleanup_refs_or_queue(bo); 594 kref_put(&bo->list_kref, ttm_bo_release_list); 595 } 596 597 void ttm_bo_put(struct ttm_buffer_object *bo) 598 { 599 kref_put(&bo->kref, ttm_bo_release); 600 } 601 EXPORT_SYMBOL(ttm_bo_put); 602 603 void ttm_bo_unref(struct ttm_buffer_object **p_bo) 604 { 605 struct ttm_buffer_object *bo = *p_bo; 606 607 *p_bo = NULL; 608 ttm_bo_put(bo); 609 } 610 EXPORT_SYMBOL(ttm_bo_unref); 611 612 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev) 613 { 614 return cancel_delayed_work_sync(&bdev->wq); 615 } 616 EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue); 617 618 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) 619 { 620 if (resched) 621 schedule_delayed_work(&bdev->wq, 622 ((HZ / 100) < 1) ? 1 : HZ / 100); 623 } 624 EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue); 625 626 static int ttm_bo_evict(struct ttm_buffer_object *bo, 627 struct ttm_operation_ctx *ctx) 628 { 629 struct ttm_bo_device *bdev = bo->bdev; 630 struct ttm_mem_reg evict_mem; 631 struct ttm_placement placement; 632 int ret = 0; 633 634 lockdep_assert_held(&bo->resv->lock.base); 635 636 placement.num_placement = 0; 637 placement.num_busy_placement = 0; 638 bdev->driver->evict_flags(bo, &placement); 639 640 if (!placement.num_placement && !placement.num_busy_placement) { 641 ret = ttm_bo_pipeline_gutting(bo); 642 if (ret) 643 return ret; 644 645 return ttm_tt_create(bo, false); 646 } 647 648 evict_mem = bo->mem; 649 evict_mem.mm_node = NULL; 650 evict_mem.bus.io_reserved_vm = false; 651 evict_mem.bus.io_reserved_count = 0; 652 653 ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx); 654 if (ret) { 655 if (ret != -ERESTARTSYS) { 656 pr_err("Failed to find memory space for buffer 0x%p eviction\n", 657 bo); 658 ttm_bo_mem_space_debug(bo, &placement); 659 } 660 goto out; 661 } 662 663 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, ctx); 664 if (unlikely(ret)) { 665 if (ret != -ERESTARTSYS) 666 pr_err("Buffer eviction failed\n"); 667 ttm_bo_mem_put(bo, &evict_mem); 668 goto out; 669 } 670 bo->evicted = true; 671 out: 672 return ret; 673 } 674 675 bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, 676 const struct ttm_place *place) 677 { 678 /* Don't evict this BO if it's outside of the 679 * requested placement range 680 */ 681 if (place->fpfn >= (bo->mem.start + bo->mem.num_pages) || 682 (place->lpfn && place->lpfn <= bo->mem.start)) 683 return false; 684 685 return true; 686 } 687 EXPORT_SYMBOL(ttm_bo_eviction_valuable); 688 689 /** 690 * Check the target bo is allowable to be evicted or swapout, including cases: 691 * 692 * a. if share same reservation object with ctx->resv, have assumption 693 * reservation objects should already be locked, so not lock again and 694 * return true directly when either the opreation allow_reserved_eviction 695 * or the target bo already is in delayed free list; 696 * 697 * b. Otherwise, trylock it. 698 */ 699 static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, 700 struct ttm_operation_ctx *ctx, bool *locked) 701 { 702 bool ret = false; 703 704 *locked = false; 705 if (bo->resv == ctx->resv) { 706 lockdep_assert_held(&bo->resv->lock.base); 707 if (ctx->flags & TTM_OPT_FLAG_ALLOW_RES_EVICT 708 || !list_empty(&bo->ddestroy)) 709 ret = true; 710 } else { 711 *locked = ww_mutex_trylock(&bo->resv->lock); 712 ret = *locked; 713 } 714 715 return ret; 716 } 717 718 static int ttm_mem_evict_first(struct ttm_bo_device *bdev, 719 uint32_t mem_type, 720 const struct ttm_place *place, 721 struct ttm_operation_ctx *ctx) 722 { 723 struct ttm_bo_global *glob = bdev->glob; 724 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 725 struct ttm_buffer_object *bo = NULL; 726 bool locked = false; 727 unsigned i; 728 int ret = -EBUSY; 729 730 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 731 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { 732 list_for_each_entry(bo, &man->lru[i], lru) { 733 if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked)) 734 continue; 735 736 if (place && !bdev->driver->eviction_valuable(bo, 737 place)) { 738 if (locked) 739 __ttm_bo_unreserve(bo); 740 continue; 741 } 742 break; 743 } 744 745 /* If the inner loop terminated early, we have our candidate */ 746 if (&bo->lru != &man->lru[i]) 747 break; 748 749 bo = NULL; 750 } 751 752 if (!bo) { 753 lockmgr(&glob->lru_lock, LK_RELEASE); 754 return -EBUSY; 755 } 756 757 kref_get(&bo->list_kref); 758 759 if (!list_empty(&bo->ddestroy)) { 760 ret = ttm_bo_cleanup_refs(bo, ctx->interruptible, 761 ctx->no_wait_gpu, locked); 762 kref_put(&bo->list_kref, ttm_bo_release_list); 763 return ret; 764 } 765 766 ttm_bo_del_from_lru(bo); 767 lockmgr(&glob->lru_lock, LK_RELEASE); 768 769 ret = ttm_bo_evict(bo, ctx); 770 if (locked) { 771 ttm_bo_unreserve(bo); 772 } else { 773 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 774 ttm_bo_add_to_lru(bo); 775 lockmgr(&glob->lru_lock, LK_RELEASE); 776 } 777 778 kref_put(&bo->list_kref, ttm_bo_release_list); 779 return ret; 780 } 781 782 void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) 783 { 784 struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type]; 785 786 if (mem->mm_node) 787 (*man->func->put_node)(man, mem); 788 } 789 EXPORT_SYMBOL(ttm_bo_mem_put); 790 791 /** 792 * Add the last move fence to the BO and reserve a new shared slot. 793 */ 794 static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo, 795 struct ttm_mem_type_manager *man, 796 struct ttm_mem_reg *mem) 797 { 798 struct dma_fence *fence; 799 int ret; 800 801 lockmgr(&man->move_lock, LK_EXCLUSIVE); 802 fence = dma_fence_get(man->move); 803 lockmgr(&man->move_lock, LK_RELEASE); 804 805 if (fence) { 806 reservation_object_add_shared_fence(bo->resv, fence); 807 808 ret = reservation_object_reserve_shared(bo->resv); 809 if (unlikely(ret)) 810 return ret; 811 812 dma_fence_put(bo->moving); 813 bo->moving = fence; 814 } 815 816 return 0; 817 } 818 819 /** 820 * Repeatedly evict memory from the LRU for @mem_type until we create enough 821 * space, or we've evicted everything and there isn't enough space. 822 */ 823 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, 824 uint32_t mem_type, 825 const struct ttm_place *place, 826 struct ttm_mem_reg *mem, 827 struct ttm_operation_ctx *ctx) 828 { 829 struct ttm_bo_device *bdev = bo->bdev; 830 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 831 int ret; 832 833 do { 834 ret = (*man->func->get_node)(man, bo, place, mem); 835 if (unlikely(ret != 0)) 836 return ret; 837 if (mem->mm_node) 838 break; 839 ret = ttm_mem_evict_first(bdev, mem_type, place, ctx); 840 if (unlikely(ret != 0)) 841 return ret; 842 } while (1); 843 mem->mem_type = mem_type; 844 return ttm_bo_add_move_fence(bo, man, mem); 845 } 846 847 static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man, 848 uint32_t cur_placement, 849 uint32_t proposed_placement) 850 { 851 uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING; 852 uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING; 853 854 /** 855 * Keep current caching if possible. 856 */ 857 858 if ((cur_placement & caching) != 0) 859 result |= (cur_placement & caching); 860 else if ((man->default_caching & caching) != 0) 861 result |= man->default_caching; 862 else if ((TTM_PL_FLAG_CACHED & caching) != 0) 863 result |= TTM_PL_FLAG_CACHED; 864 else if ((TTM_PL_FLAG_WC & caching) != 0) 865 result |= TTM_PL_FLAG_WC; 866 else if ((TTM_PL_FLAG_UNCACHED & caching) != 0) 867 result |= TTM_PL_FLAG_UNCACHED; 868 869 return result; 870 } 871 872 static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, 873 uint32_t mem_type, 874 const struct ttm_place *place, 875 uint32_t *masked_placement) 876 { 877 uint32_t cur_flags = ttm_bo_type_flags(mem_type); 878 879 if ((cur_flags & place->flags & TTM_PL_MASK_MEM) == 0) 880 return false; 881 882 if ((place->flags & man->available_caching) == 0) 883 return false; 884 885 cur_flags |= (place->flags & man->available_caching); 886 887 *masked_placement = cur_flags; 888 return true; 889 } 890 891 /** 892 * Creates space for memory region @mem according to its type. 893 * 894 * This function first searches for free space in compatible memory types in 895 * the priority order defined by the driver. If free space isn't found, then 896 * ttm_bo_mem_force_space is attempted in priority order to evict and find 897 * space. 898 */ 899 int ttm_bo_mem_space(struct ttm_buffer_object *bo, 900 struct ttm_placement *placement, 901 struct ttm_mem_reg *mem, 902 struct ttm_operation_ctx *ctx) 903 { 904 struct ttm_bo_device *bdev = bo->bdev; 905 struct ttm_mem_type_manager *man; 906 uint32_t mem_type = TTM_PL_SYSTEM; 907 uint32_t cur_flags = 0; 908 bool type_found = false; 909 bool type_ok = false; 910 bool has_erestartsys = false; 911 int i, ret; 912 913 ret = reservation_object_reserve_shared(bo->resv); 914 if (unlikely(ret)) 915 return ret; 916 917 mem->mm_node = NULL; 918 for (i = 0; i < placement->num_placement; ++i) { 919 const struct ttm_place *place = &placement->placement[i]; 920 921 ret = ttm_mem_type_from_place(place, &mem_type); 922 if (ret) 923 return ret; 924 man = &bdev->man[mem_type]; 925 if (!man->has_type || !man->use_type) 926 continue; 927 928 type_ok = ttm_bo_mt_compatible(man, mem_type, place, 929 &cur_flags); 930 931 if (!type_ok) 932 continue; 933 934 type_found = true; 935 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 936 cur_flags); 937 /* 938 * Use the access and other non-mapping-related flag bits from 939 * the memory placement flags to the current flags 940 */ 941 ttm_flag_masked(&cur_flags, place->flags, 942 ~TTM_PL_MASK_MEMTYPE); 943 944 if (mem_type == TTM_PL_SYSTEM) 945 break; 946 947 ret = (*man->func->get_node)(man, bo, place, mem); 948 if (unlikely(ret)) 949 return ret; 950 951 if (mem->mm_node) { 952 ret = ttm_bo_add_move_fence(bo, man, mem); 953 if (unlikely(ret)) { 954 (*man->func->put_node)(man, mem); 955 return ret; 956 } 957 break; 958 } 959 } 960 961 if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) { 962 mem->mem_type = mem_type; 963 mem->placement = cur_flags; 964 return 0; 965 } 966 967 for (i = 0; i < placement->num_busy_placement; ++i) { 968 const struct ttm_place *place = &placement->busy_placement[i]; 969 970 ret = ttm_mem_type_from_place(place, &mem_type); 971 if (ret) 972 return ret; 973 man = &bdev->man[mem_type]; 974 if (!man->has_type || !man->use_type) 975 continue; 976 if (!ttm_bo_mt_compatible(man, mem_type, place, &cur_flags)) 977 continue; 978 979 type_found = true; 980 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 981 cur_flags); 982 /* 983 * Use the access and other non-mapping-related flag bits from 984 * the memory placement flags to the current flags 985 */ 986 ttm_flag_masked(&cur_flags, place->flags, 987 ~TTM_PL_MASK_MEMTYPE); 988 989 if (mem_type == TTM_PL_SYSTEM) { 990 mem->mem_type = mem_type; 991 mem->placement = cur_flags; 992 mem->mm_node = NULL; 993 return 0; 994 } 995 996 ret = ttm_bo_mem_force_space(bo, mem_type, place, mem, ctx); 997 if (ret == 0 && mem->mm_node) { 998 mem->placement = cur_flags; 999 return 0; 1000 } 1001 if (ret == -ERESTARTSYS) 1002 has_erestartsys = true; 1003 } 1004 1005 if (!type_found) { 1006 pr_err(TTM_PFX "No compatible memory type found\n"); 1007 return -EINVAL; 1008 } 1009 1010 return (has_erestartsys) ? -ERESTARTSYS : -ENOMEM; 1011 } 1012 EXPORT_SYMBOL(ttm_bo_mem_space); 1013 1014 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, 1015 struct ttm_placement *placement, 1016 struct ttm_operation_ctx *ctx) 1017 { 1018 int ret = 0; 1019 struct ttm_mem_reg mem; 1020 1021 lockdep_assert_held(&bo->resv->lock.base); 1022 1023 mem.num_pages = bo->num_pages; 1024 mem.size = mem.num_pages << PAGE_SHIFT; 1025 mem.page_alignment = bo->mem.page_alignment; 1026 mem.bus.io_reserved_vm = false; 1027 mem.bus.io_reserved_count = 0; 1028 /* 1029 * Determine where to move the buffer. 1030 */ 1031 ret = ttm_bo_mem_space(bo, placement, &mem, ctx); 1032 if (ret) 1033 goto out_unlock; 1034 ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx); 1035 out_unlock: 1036 if (ret && mem.mm_node) 1037 ttm_bo_mem_put(bo, &mem); 1038 return ret; 1039 } 1040 1041 static bool ttm_bo_places_compat(const struct ttm_place *places, 1042 unsigned num_placement, 1043 struct ttm_mem_reg *mem, 1044 uint32_t *new_flags) 1045 { 1046 unsigned i; 1047 1048 for (i = 0; i < num_placement; i++) { 1049 const struct ttm_place *heap = &places[i]; 1050 1051 if (mem->mm_node && (mem->start < heap->fpfn || 1052 (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn))) 1053 continue; 1054 1055 *new_flags = heap->flags; 1056 if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && 1057 (*new_flags & mem->placement & TTM_PL_MASK_MEM) && 1058 (!(*new_flags & TTM_PL_FLAG_CONTIGUOUS) || 1059 (mem->placement & TTM_PL_FLAG_CONTIGUOUS))) 1060 return true; 1061 } 1062 return false; 1063 } 1064 1065 bool ttm_bo_mem_compat(struct ttm_placement *placement, 1066 struct ttm_mem_reg *mem, 1067 uint32_t *new_flags) 1068 { 1069 if (ttm_bo_places_compat(placement->placement, placement->num_placement, 1070 mem, new_flags)) 1071 return true; 1072 1073 if ((placement->busy_placement != placement->placement || 1074 placement->num_busy_placement > placement->num_placement) && 1075 ttm_bo_places_compat(placement->busy_placement, 1076 placement->num_busy_placement, 1077 mem, new_flags)) 1078 return true; 1079 1080 return false; 1081 } 1082 EXPORT_SYMBOL(ttm_bo_mem_compat); 1083 1084 int ttm_bo_validate(struct ttm_buffer_object *bo, 1085 struct ttm_placement *placement, 1086 struct ttm_operation_ctx *ctx) 1087 { 1088 int ret; 1089 uint32_t new_flags; 1090 1091 lockdep_assert_held(&bo->resv->lock.base); 1092 /* 1093 * Check whether we need to move buffer. 1094 */ 1095 if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) { 1096 ret = ttm_bo_move_buffer(bo, placement, ctx); 1097 if (ret) 1098 return ret; 1099 } else { 1100 /* 1101 * Use the access and other non-mapping-related flag bits from 1102 * the compatible memory placement flags to the active flags 1103 */ 1104 ttm_flag_masked(&bo->mem.placement, new_flags, 1105 ~TTM_PL_MASK_MEMTYPE); 1106 } 1107 /* 1108 * We might need to add a TTM. 1109 */ 1110 if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { 1111 ret = ttm_tt_create(bo, true); 1112 if (ret) 1113 return ret; 1114 } 1115 return 0; 1116 } 1117 EXPORT_SYMBOL(ttm_bo_validate); 1118 1119 int ttm_bo_init_reserved(struct ttm_bo_device *bdev, 1120 struct ttm_buffer_object *bo, 1121 unsigned long size, 1122 enum ttm_bo_type type, 1123 struct ttm_placement *placement, 1124 uint32_t page_alignment, 1125 struct ttm_operation_ctx *ctx, 1126 size_t acc_size, 1127 struct sg_table *sg, 1128 struct reservation_object *resv, 1129 void (*destroy) (struct ttm_buffer_object *)) 1130 { 1131 int ret = 0; 1132 unsigned long num_pages; 1133 struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; 1134 bool locked; 1135 1136 ret = ttm_mem_global_alloc(mem_glob, acc_size, ctx); 1137 if (ret) { 1138 pr_err("Out of kernel memory\n"); 1139 if (destroy) 1140 (*destroy)(bo); 1141 else 1142 kfree(bo); 1143 return -ENOMEM; 1144 } 1145 1146 num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 1147 if (num_pages == 0) { 1148 pr_err("Illegal buffer object size\n"); 1149 if (destroy) 1150 (*destroy)(bo); 1151 else 1152 kfree(bo); 1153 ttm_mem_global_free(mem_glob, acc_size); 1154 return -EINVAL; 1155 } 1156 bo->destroy = destroy ? destroy : ttm_bo_default_destroy; 1157 1158 kref_init(&bo->kref); 1159 kref_init(&bo->list_kref); 1160 atomic_set(&bo->cpu_writers, 0); 1161 INIT_LIST_HEAD(&bo->lru); 1162 INIT_LIST_HEAD(&bo->ddestroy); 1163 INIT_LIST_HEAD(&bo->swap); 1164 INIT_LIST_HEAD(&bo->io_reserve_lru); 1165 lockinit(&bo->wu_mutex, "ttmbwm", 0, LK_CANRECURSE); 1166 bo->bdev = bdev; 1167 bo->type = type; 1168 bo->num_pages = num_pages; 1169 bo->mem.size = num_pages << PAGE_SHIFT; 1170 bo->mem.mem_type = TTM_PL_SYSTEM; 1171 bo->mem.num_pages = bo->num_pages; 1172 bo->mem.mm_node = NULL; 1173 bo->mem.page_alignment = page_alignment; 1174 bo->mem.bus.io_reserved_vm = false; 1175 bo->mem.bus.io_reserved_count = 0; 1176 bo->moving = NULL; 1177 bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); 1178 bo->acc_size = acc_size; 1179 bo->sg = sg; 1180 if (resv) { 1181 bo->resv = resv; 1182 lockdep_assert_held(&bo->resv->lock.base); 1183 } else { 1184 bo->resv = &bo->ttm_resv; 1185 } 1186 reservation_object_init(&bo->ttm_resv); 1187 atomic_inc(&bo->bdev->glob->bo_count); 1188 drm_vma_node_reset(&bo->vma_node); 1189 bo->priority = 0; 1190 1191 /* 1192 * For ttm_bo_type_device buffers, allocate 1193 * address space from the device. 1194 */ 1195 if (bo->type == ttm_bo_type_device || 1196 bo->type == ttm_bo_type_sg) 1197 ret = drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node, 1198 bo->mem.num_pages); 1199 1200 /* passed reservation objects should already be locked, 1201 * since otherwise lockdep will be angered in radeon. 1202 */ 1203 if (!resv) { 1204 locked = ww_mutex_trylock(&bo->resv->lock); 1205 WARN_ON(!locked); 1206 } 1207 1208 if (likely(!ret)) 1209 ret = ttm_bo_validate(bo, placement, ctx); 1210 1211 if (unlikely(ret)) { 1212 if (!resv) 1213 ttm_bo_unreserve(bo); 1214 1215 ttm_bo_put(bo); 1216 return ret; 1217 } 1218 1219 if (resv && !(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { 1220 lockmgr(&bdev->glob->lru_lock, LK_EXCLUSIVE); 1221 ttm_bo_add_to_lru(bo); 1222 lockmgr(&bdev->glob->lru_lock, LK_RELEASE); 1223 } 1224 1225 return ret; 1226 } 1227 EXPORT_SYMBOL(ttm_bo_init_reserved); 1228 1229 int ttm_bo_init(struct ttm_bo_device *bdev, 1230 struct ttm_buffer_object *bo, 1231 unsigned long size, 1232 enum ttm_bo_type type, 1233 struct ttm_placement *placement, 1234 uint32_t page_alignment, 1235 bool interruptible, 1236 size_t acc_size, 1237 struct sg_table *sg, 1238 struct reservation_object *resv, 1239 void (*destroy) (struct ttm_buffer_object *)) 1240 { 1241 struct ttm_operation_ctx ctx = { interruptible, false }; 1242 int ret; 1243 1244 ret = ttm_bo_init_reserved(bdev, bo, size, type, placement, 1245 page_alignment, &ctx, acc_size, 1246 sg, resv, destroy); 1247 if (ret) 1248 return ret; 1249 1250 if (!resv) 1251 ttm_bo_unreserve(bo); 1252 1253 return 0; 1254 } 1255 EXPORT_SYMBOL(ttm_bo_init); 1256 1257 size_t ttm_bo_acc_size(struct ttm_bo_device *bdev, 1258 unsigned long bo_size, 1259 unsigned struct_size) 1260 { 1261 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1262 size_t size = 0; 1263 1264 size += ttm_round_pot(struct_size); 1265 size += ttm_round_pot(npages * sizeof(void *)); 1266 size += ttm_round_pot(sizeof(struct ttm_tt)); 1267 return size; 1268 } 1269 EXPORT_SYMBOL(ttm_bo_acc_size); 1270 1271 size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, 1272 unsigned long bo_size, 1273 unsigned struct_size) 1274 { 1275 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1276 size_t size = 0; 1277 1278 size += ttm_round_pot(struct_size); 1279 size += ttm_round_pot(npages * (2*sizeof(void *) + sizeof(dma_addr_t))); 1280 size += ttm_round_pot(sizeof(struct ttm_dma_tt)); 1281 return size; 1282 } 1283 EXPORT_SYMBOL(ttm_bo_dma_acc_size); 1284 1285 int ttm_bo_create(struct ttm_bo_device *bdev, 1286 unsigned long size, 1287 enum ttm_bo_type type, 1288 struct ttm_placement *placement, 1289 uint32_t page_alignment, 1290 bool interruptible, 1291 struct ttm_buffer_object **p_bo) 1292 { 1293 struct ttm_buffer_object *bo; 1294 size_t acc_size; 1295 int ret; 1296 1297 bo = kzalloc(sizeof(*bo), GFP_KERNEL); 1298 if (unlikely(bo == NULL)) 1299 return -ENOMEM; 1300 1301 acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); 1302 ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, 1303 interruptible, acc_size, 1304 NULL, NULL, NULL); 1305 if (likely(ret == 0)) 1306 *p_bo = bo; 1307 1308 return ret; 1309 } 1310 EXPORT_SYMBOL(ttm_bo_create); 1311 1312 static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, 1313 unsigned mem_type) 1314 { 1315 struct ttm_operation_ctx ctx = { 1316 .interruptible = false, 1317 .no_wait_gpu = false, 1318 .flags = TTM_OPT_FLAG_FORCE_ALLOC 1319 }; 1320 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1321 struct ttm_bo_global *glob = bdev->glob; 1322 struct dma_fence *fence; 1323 int ret; 1324 unsigned i; 1325 1326 /* 1327 * Can't use standard list traversal since we're unlocking. 1328 */ 1329 1330 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 1331 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { 1332 while (!list_empty(&man->lru[i])) { 1333 lockmgr(&glob->lru_lock, LK_RELEASE); 1334 ret = ttm_mem_evict_first(bdev, mem_type, NULL, &ctx); 1335 if (ret) 1336 return ret; 1337 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 1338 } 1339 } 1340 lockmgr(&glob->lru_lock, LK_RELEASE); 1341 1342 lockmgr(&man->move_lock, LK_EXCLUSIVE); 1343 fence = dma_fence_get(man->move); 1344 lockmgr(&man->move_lock, LK_RELEASE); 1345 1346 if (fence) { 1347 ret = dma_fence_wait(fence, false); 1348 dma_fence_put(fence); 1349 if (ret) 1350 return ret; 1351 } 1352 1353 return 0; 1354 } 1355 1356 int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1357 { 1358 struct ttm_mem_type_manager *man; 1359 int ret = -EINVAL; 1360 1361 if (mem_type >= TTM_NUM_MEM_TYPES) { 1362 pr_err("Illegal memory type %d\n", mem_type); 1363 return ret; 1364 } 1365 man = &bdev->man[mem_type]; 1366 1367 if (!man->has_type) { 1368 pr_err("Trying to take down uninitialized memory manager type %u\n", 1369 mem_type); 1370 return ret; 1371 } 1372 1373 man->use_type = false; 1374 man->has_type = false; 1375 1376 ret = 0; 1377 if (mem_type > 0) { 1378 ret = ttm_bo_force_list_clean(bdev, mem_type); 1379 if (ret) { 1380 pr_err("Cleanup eviction failed\n"); 1381 return ret; 1382 } 1383 1384 ret = (*man->func->takedown)(man); 1385 } 1386 1387 dma_fence_put(man->move); 1388 man->move = NULL; 1389 1390 return ret; 1391 } 1392 EXPORT_SYMBOL(ttm_bo_clean_mm); 1393 1394 int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1395 { 1396 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1397 1398 if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) { 1399 pr_err("Illegal memory manager memory type %u\n", mem_type); 1400 return -EINVAL; 1401 } 1402 1403 if (!man->has_type) { 1404 pr_err("Memory type %u has not been initialized\n", mem_type); 1405 return 0; 1406 } 1407 1408 return ttm_bo_force_list_clean(bdev, mem_type); 1409 } 1410 EXPORT_SYMBOL(ttm_bo_evict_mm); 1411 1412 int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, 1413 unsigned long p_size) 1414 { 1415 int ret; 1416 struct ttm_mem_type_manager *man; 1417 unsigned i; 1418 1419 BUG_ON(type >= TTM_NUM_MEM_TYPES); 1420 man = &bdev->man[type]; 1421 BUG_ON(man->has_type); 1422 man->io_reserve_fastpath = true; 1423 man->use_io_reserve_lru = false; 1424 lockinit(&man->io_reserve_mutex, "ttmior", 0, 0); 1425 lockinit(&man->move_lock, "ttmml", 0, 0); 1426 INIT_LIST_HEAD(&man->io_reserve_lru); 1427 1428 ret = bdev->driver->init_mem_type(bdev, type, man); 1429 if (ret) 1430 return ret; 1431 man->bdev = bdev; 1432 1433 if (type != TTM_PL_SYSTEM) { 1434 ret = (*man->func->init)(man, p_size); 1435 if (ret) 1436 return ret; 1437 } 1438 man->has_type = true; 1439 man->use_type = true; 1440 man->size = p_size; 1441 1442 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) 1443 INIT_LIST_HEAD(&man->lru[i]); 1444 man->move = NULL; 1445 1446 return 0; 1447 } 1448 EXPORT_SYMBOL(ttm_bo_init_mm); 1449 1450 static void ttm_bo_global_kobj_release(struct kobject *kobj) 1451 { 1452 struct ttm_bo_global *glob = 1453 container_of(kobj, struct ttm_bo_global, kobj); 1454 1455 __free_page(glob->dummy_read_page); 1456 } 1457 1458 void ttm_bo_global_release(struct drm_global_reference *ref) 1459 { 1460 struct ttm_bo_global *glob = ref->object; 1461 1462 kobject_del(&glob->kobj); 1463 kobject_put(&glob->kobj); 1464 } 1465 EXPORT_SYMBOL(ttm_bo_global_release); 1466 1467 int ttm_bo_global_init(struct drm_global_reference *ref) 1468 { 1469 struct ttm_bo_global_ref *bo_ref = 1470 container_of(ref, struct ttm_bo_global_ref, ref); 1471 struct ttm_bo_global *glob = ref->object; 1472 int ret; 1473 unsigned i; 1474 1475 lockinit(&glob->device_list_mutex, "ttmdlm", 0, 0); 1476 lockinit(&glob->lru_lock, "ttmlru", 0, 0); 1477 glob->mem_glob = bo_ref->mem_glob; 1478 glob->mem_glob->bo_glob = glob; 1479 glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32); 1480 1481 if (unlikely(glob->dummy_read_page == NULL)) { 1482 ret = -ENOMEM; 1483 goto out_no_drp; 1484 } 1485 1486 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) 1487 INIT_LIST_HEAD(&glob->swap_lru[i]); 1488 INIT_LIST_HEAD(&glob->device_list); 1489 atomic_set(&glob->bo_count, 0); 1490 1491 ret = kobject_init_and_add( 1492 &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects"); 1493 if (unlikely(ret != 0)) 1494 kobject_put(&glob->kobj); 1495 return ret; 1496 out_no_drp: 1497 kfree(glob); 1498 return ret; 1499 } 1500 EXPORT_SYMBOL(ttm_bo_global_init); 1501 1502 1503 int ttm_bo_device_release(struct ttm_bo_device *bdev) 1504 { 1505 int ret = 0; 1506 unsigned i = TTM_NUM_MEM_TYPES; 1507 struct ttm_mem_type_manager *man; 1508 struct ttm_bo_global *glob = bdev->glob; 1509 1510 while (i--) { 1511 man = &bdev->man[i]; 1512 if (man->has_type) { 1513 man->use_type = false; 1514 if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) { 1515 ret = -EBUSY; 1516 pr_err("DRM memory manager type %d is not clean\n", 1517 i); 1518 } 1519 man->has_type = false; 1520 } 1521 } 1522 1523 mutex_lock(&glob->device_list_mutex); 1524 list_del(&bdev->device_list); 1525 mutex_unlock(&glob->device_list_mutex); 1526 1527 cancel_delayed_work_sync(&bdev->wq); 1528 1529 if (ttm_bo_delayed_delete(bdev, true)) 1530 pr_debug("Delayed destroy list was clean\n"); 1531 1532 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 1533 if (list_empty(&bdev->ddestroy)) 1534 TTM_DEBUG("Delayed destroy list was clean\n"); 1535 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) 1536 if (list_empty(&bdev->man[0].lru[0])) 1537 pr_debug("Swap list %d was clean\n", i); 1538 lockmgr(&glob->lru_lock, LK_RELEASE); 1539 1540 drm_vma_offset_manager_destroy(&bdev->vma_manager); 1541 1542 return ret; 1543 } 1544 EXPORT_SYMBOL(ttm_bo_device_release); 1545 1546 int ttm_bo_device_init(struct ttm_bo_device *bdev, 1547 struct ttm_bo_global *glob, 1548 struct ttm_bo_driver *driver, 1549 struct address_space *mapping, 1550 uint64_t file_page_offset, 1551 bool need_dma32) 1552 { 1553 int ret = -EINVAL; 1554 1555 bdev->driver = driver; 1556 1557 memset(bdev->man, 0, sizeof(bdev->man)); 1558 1559 /* 1560 * Initialize the system memory buffer type. 1561 * Other types need to be driver / IOCTL initialized. 1562 */ 1563 ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0); 1564 if (unlikely(ret != 0)) 1565 goto out_no_sys; 1566 1567 drm_vma_offset_manager_init(&bdev->vma_manager, file_page_offset, 1568 0x10000000); 1569 INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); 1570 INIT_LIST_HEAD(&bdev->ddestroy); 1571 /* 1572 * XXX DRAGONFLY - dev_mapping NULL atm, find other XXX DRAGONFLY 1573 * lines and fix when it no longer is in later API change. 1574 */ 1575 bdev->dev_mapping = mapping; 1576 bdev->glob = glob; 1577 bdev->need_dma32 = need_dma32; 1578 mutex_lock(&glob->device_list_mutex); 1579 list_add_tail(&bdev->device_list, &glob->device_list); 1580 mutex_unlock(&glob->device_list_mutex); 1581 1582 return 0; 1583 out_no_sys: 1584 return ret; 1585 } 1586 EXPORT_SYMBOL(ttm_bo_device_init); 1587 1588 /* 1589 * buffer object vm functions. 1590 */ 1591 1592 bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 1593 { 1594 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 1595 1596 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 1597 if (mem->mem_type == TTM_PL_SYSTEM) 1598 return false; 1599 1600 if (man->flags & TTM_MEMTYPE_FLAG_CMA) 1601 return false; 1602 1603 if (mem->placement & TTM_PL_FLAG_CACHED) 1604 return false; 1605 } 1606 return true; 1607 } 1608 1609 #ifdef __DragonFly__ 1610 1611 /* 1612 * XXX DRAGONFLY - device_mapping not yet implemented so 1613 * file_mapping is basically always NULL. We have to properly 1614 * release the mmap, etc. 1615 */ 1616 void ttm_bo_release_mmap(struct ttm_buffer_object *bo); 1617 1618 #endif 1619 1620 void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) 1621 { 1622 // drm_vma_node_unmap(&bo->vma_node, bdev->dev_mapping); 1623 ttm_bo_release_mmap(bo); 1624 ttm_mem_io_free_vm(bo); 1625 } 1626 1627 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) 1628 { 1629 struct ttm_bo_device *bdev = bo->bdev; 1630 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 1631 1632 ttm_mem_io_lock(man, false); 1633 ttm_bo_unmap_virtual_locked(bo); 1634 ttm_mem_io_unlock(man); 1635 } 1636 1637 1638 EXPORT_SYMBOL(ttm_bo_unmap_virtual); 1639 1640 int ttm_bo_wait(struct ttm_buffer_object *bo, 1641 bool interruptible, bool no_wait) 1642 { 1643 long timeout = 15 * HZ; 1644 1645 if (no_wait) { 1646 if (reservation_object_test_signaled_rcu(bo->resv, true)) 1647 return 0; 1648 else 1649 return -EBUSY; 1650 } 1651 1652 timeout = reservation_object_wait_timeout_rcu(bo->resv, true, 1653 interruptible, timeout); 1654 if (timeout < 0) 1655 return timeout; 1656 1657 if (timeout == 0) 1658 return -EBUSY; 1659 1660 reservation_object_add_excl_fence(bo->resv, NULL); 1661 return 0; 1662 } 1663 EXPORT_SYMBOL(ttm_bo_wait); 1664 1665 int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait) 1666 { 1667 int ret = 0; 1668 1669 /* 1670 * Using ttm_bo_reserve makes sure the lru lists are updated. 1671 */ 1672 1673 ret = ttm_bo_reserve(bo, true, no_wait, NULL); 1674 if (unlikely(ret != 0)) 1675 return ret; 1676 ret = ttm_bo_wait(bo, true, no_wait); 1677 if (likely(ret == 0)) 1678 atomic_inc(&bo->cpu_writers); 1679 ttm_bo_unreserve(bo); 1680 return ret; 1681 } 1682 EXPORT_SYMBOL(ttm_bo_synccpu_write_grab); 1683 1684 void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo) 1685 { 1686 atomic_dec(&bo->cpu_writers); 1687 } 1688 EXPORT_SYMBOL(ttm_bo_synccpu_write_release); 1689 1690 /** 1691 * A buffer object shrink method that tries to swap out the first 1692 * buffer object on the bo_global::swap_lru list. 1693 */ 1694 int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx) 1695 { 1696 struct ttm_buffer_object *bo; 1697 int ret = -EBUSY; 1698 bool locked; 1699 unsigned i; 1700 1701 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 1702 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { 1703 list_for_each_entry(bo, &glob->swap_lru[i], swap) { 1704 if (ttm_bo_evict_swapout_allowable(bo, ctx, &locked)) { 1705 ret = 0; 1706 break; 1707 } 1708 } 1709 if (!ret) 1710 break; 1711 } 1712 1713 if (ret) { 1714 lockmgr(&glob->lru_lock, LK_RELEASE); 1715 return ret; 1716 } 1717 1718 kref_get(&bo->list_kref); 1719 1720 if (!list_empty(&bo->ddestroy)) { 1721 ret = ttm_bo_cleanup_refs(bo, false, false, locked); 1722 kref_put(&bo->list_kref, ttm_bo_release_list); 1723 return ret; 1724 } 1725 1726 ttm_bo_del_from_lru(bo); 1727 lockmgr(&glob->lru_lock, LK_RELEASE); 1728 1729 /** 1730 * Move to system cached 1731 */ 1732 1733 if (bo->mem.mem_type != TTM_PL_SYSTEM || 1734 bo->ttm->caching_state != tt_cached) { 1735 struct ttm_operation_ctx ctx = { false, false }; 1736 struct ttm_mem_reg evict_mem; 1737 1738 evict_mem = bo->mem; 1739 evict_mem.mm_node = NULL; 1740 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED; 1741 evict_mem.mem_type = TTM_PL_SYSTEM; 1742 1743 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx); 1744 if (unlikely(ret != 0)) 1745 goto out; 1746 } 1747 1748 /** 1749 * Make sure BO is idle. 1750 */ 1751 1752 ret = ttm_bo_wait(bo, false, false); 1753 if (unlikely(ret != 0)) 1754 goto out; 1755 1756 ttm_bo_unmap_virtual(bo); 1757 1758 /** 1759 * Swap out. Buffer will be swapped in again as soon as 1760 * anyone tries to access a ttm page. 1761 */ 1762 1763 if (bo->bdev->driver->swap_notify) 1764 bo->bdev->driver->swap_notify(bo); 1765 1766 ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage); 1767 out: 1768 1769 /** 1770 * 1771 * Unreserve without putting on LRU to avoid swapping out an 1772 * already swapped buffer. 1773 */ 1774 if (locked) 1775 __ttm_bo_unreserve(bo); 1776 kref_put(&bo->list_kref, ttm_bo_release_list); 1777 return ret; 1778 } 1779 EXPORT_SYMBOL(ttm_bo_swapout); 1780 1781 void ttm_bo_swapout_all(struct ttm_bo_device *bdev) 1782 { 1783 struct ttm_operation_ctx ctx = { 1784 .interruptible = false, 1785 .no_wait_gpu = false 1786 }; 1787 1788 while (ttm_bo_swapout(bdev->glob, &ctx) == 0) 1789 ; 1790 } 1791 EXPORT_SYMBOL(ttm_bo_swapout_all); 1792 1793 /** 1794 * ttm_bo_wait_unreserved - interruptible wait for a buffer object to become 1795 * unreserved 1796 * 1797 * @bo: Pointer to buffer 1798 */ 1799 int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo) 1800 { 1801 int ret; 1802 1803 /* 1804 * In the absense of a wait_unlocked API, 1805 * Use the bo::wu_mutex to avoid triggering livelocks due to 1806 * concurrent use of this function. Note that this use of 1807 * bo::wu_mutex can go away if we change locking order to 1808 * mmap_sem -> bo::reserve. 1809 */ 1810 ret = mutex_lock_interruptible(&bo->wu_mutex); 1811 if (unlikely(ret != 0)) 1812 return -ERESTARTSYS; 1813 if (!ww_mutex_is_locked(&bo->resv->lock)) 1814 goto out_unlock; 1815 ret = __ttm_bo_reserve(bo, true, false, NULL); 1816 if (unlikely(ret != 0)) 1817 goto out_unlock; 1818 __ttm_bo_unreserve(bo); 1819 1820 out_unlock: 1821 mutex_unlock(&bo->wu_mutex); 1822 return ret; 1823 } 1824