1 /************************************************************************** 2 * 3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 /* 28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29 */ 30 31 #define pr_fmt(fmt) "[TTM] " fmt 32 33 #include <drm/ttm/ttm_module.h> 34 #include <drm/ttm/ttm_bo_driver.h> 35 #include <drm/ttm/ttm_placement.h> 36 #include <linux/jiffies.h> 37 #include <linux/slab.h> 38 #include <linux/sched.h> 39 #include <linux/mm.h> 40 #include <linux/file.h> 41 #include <linux/module.h> 42 #include <linux/atomic.h> 43 #include <linux/reservation.h> 44 45 #define TTM_ASSERT_LOCKED(param) 46 #define TTM_DEBUG(fmt, arg...) 47 #define TTM_BO_HASH_ORDER 13 48 49 static int ttm_bo_swapout(struct ttm_mem_shrink *shrink); 50 static void ttm_bo_global_kobj_release(struct kobject *kobj); 51 52 static struct attribute ttm_bo_count = { 53 .name = "bo_count", 54 .mode = S_IRUGO 55 }; 56 57 static inline int ttm_mem_type_from_place(const struct ttm_place *place, 58 uint32_t *mem_type) 59 { 60 int pos; 61 62 pos = ffs(place->flags & TTM_PL_MASK_MEM); 63 if (unlikely(!pos)) 64 return -EINVAL; 65 66 *mem_type = pos - 1; 67 return 0; 68 } 69 70 static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type) 71 { 72 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 73 74 pr_err(" has_type: %d\n", man->has_type); 75 pr_err(" use_type: %d\n", man->use_type); 76 pr_err(" flags: 0x%08X\n", man->flags); 77 pr_err(" gpu_offset: 0x%08lX\n", man->gpu_offset); 78 pr_err(" size: %ju\n", man->size); 79 pr_err(" available_caching: 0x%08X\n", man->available_caching); 80 pr_err(" default_caching: 0x%08X\n", man->default_caching); 81 if (mem_type != TTM_PL_SYSTEM) 82 (*man->func->debug)(man, TTM_PFX); 83 } 84 85 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, 86 struct ttm_placement *placement) 87 { 88 int i, ret, mem_type; 89 90 pr_err("No space for %p (%lu pages, %luK, %luM)\n", 91 bo, bo->mem.num_pages, bo->mem.size >> 10, 92 bo->mem.size >> 20); 93 for (i = 0; i < placement->num_placement; i++) { 94 ret = ttm_mem_type_from_place(&placement->placement[i], 95 &mem_type); 96 if (ret) 97 return; 98 pr_err(" placement[%d]=0x%08X (%d)\n", 99 i, placement->placement[i].flags, mem_type); 100 ttm_mem_type_debug(bo->bdev, mem_type); 101 } 102 } 103 104 static ssize_t ttm_bo_global_show(struct kobject *kobj, 105 struct attribute *attr, 106 char *buffer) 107 { 108 struct ttm_bo_global *glob = 109 container_of(kobj, struct ttm_bo_global, kobj); 110 111 return snprintf(buffer, PAGE_SIZE, "%lu\n", 112 (unsigned long) atomic_read(&glob->bo_count)); 113 } 114 115 static struct attribute *ttm_bo_global_attrs[] = { 116 &ttm_bo_count, 117 NULL 118 }; 119 120 static const struct sysfs_ops ttm_bo_global_ops = { 121 .show = &ttm_bo_global_show 122 }; 123 124 static struct kobj_type ttm_bo_glob_kobj_type = { 125 .release = &ttm_bo_global_kobj_release, 126 .sysfs_ops = &ttm_bo_global_ops, 127 .default_attrs = ttm_bo_global_attrs 128 }; 129 130 131 static inline uint32_t ttm_bo_type_flags(unsigned type) 132 { 133 return 1 << (type); 134 } 135 136 static void ttm_bo_release_list(struct kref *list_kref) 137 { 138 struct ttm_buffer_object *bo = 139 container_of(list_kref, struct ttm_buffer_object, list_kref); 140 struct ttm_bo_device *bdev = bo->bdev; 141 size_t acc_size = bo->acc_size; 142 143 BUG_ON(kref_read(&bo->list_kref)); 144 BUG_ON(kref_read(&bo->kref)); 145 BUG_ON(atomic_read(&bo->cpu_writers)); 146 BUG_ON(bo->mem.mm_node != NULL); 147 BUG_ON(!list_empty(&bo->lru)); 148 BUG_ON(!list_empty(&bo->ddestroy)); 149 ttm_tt_destroy(bo->ttm); 150 atomic_dec(&bo->glob->bo_count); 151 dma_fence_put(bo->moving); 152 if (bo->resv == &bo->ttm_resv) 153 reservation_object_fini(&bo->ttm_resv); 154 mutex_destroy(&bo->wu_mutex); 155 if (bo->destroy) 156 bo->destroy(bo); 157 else { 158 kfree(bo); 159 } 160 ttm_mem_global_free(bdev->glob->mem_glob, acc_size); 161 } 162 163 void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) 164 { 165 struct ttm_bo_device *bdev = bo->bdev; 166 struct ttm_mem_type_manager *man; 167 168 lockdep_assert_held(&bo->resv->lock.base); 169 170 if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { 171 172 #ifdef __DragonFly__ 173 if (WARN_ON(!list_empty(&bo->lru))) 174 return; 175 #endif 176 177 man = &bdev->man[bo->mem.mem_type]; 178 list_add_tail(&bo->lru, &man->lru[bo->priority]); 179 kref_get(&bo->list_kref); 180 181 if (bo->ttm && !(bo->ttm->page_flags & TTM_PAGE_FLAG_SG)) { 182 list_add_tail(&bo->swap, 183 &bo->glob->swap_lru[bo->priority]); 184 kref_get(&bo->list_kref); 185 } 186 } 187 } 188 EXPORT_SYMBOL(ttm_bo_add_to_lru); 189 190 static void ttm_bo_ref_bug(struct kref *list_kref) 191 { 192 BUG(); 193 } 194 195 void ttm_bo_del_from_lru(struct ttm_buffer_object *bo) 196 { 197 if (!list_empty(&bo->swap)) { 198 list_del_init(&bo->swap); 199 kref_put(&bo->list_kref, ttm_bo_ref_bug); 200 } 201 if (!list_empty(&bo->lru)) { 202 list_del_init(&bo->lru); 203 kref_put(&bo->list_kref, ttm_bo_ref_bug); 204 } 205 206 /* 207 * TODO: Add a driver hook to delete from 208 * driver-specific LRU's here. 209 */ 210 } 211 212 void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo) 213 { 214 lockmgr(&bo->glob->lru_lock, LK_EXCLUSIVE); 215 ttm_bo_del_from_lru(bo); 216 lockmgr(&bo->glob->lru_lock, LK_RELEASE); 217 } 218 EXPORT_SYMBOL(ttm_bo_del_sub_from_lru); 219 220 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo) 221 { 222 lockdep_assert_held(&bo->resv->lock.base); 223 224 ttm_bo_del_from_lru(bo); 225 ttm_bo_add_to_lru(bo); 226 } 227 EXPORT_SYMBOL(ttm_bo_move_to_lru_tail); 228 229 /* 230 * Call bo->mutex locked. 231 */ 232 static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) 233 { 234 struct ttm_bo_device *bdev = bo->bdev; 235 struct ttm_bo_global *glob = bo->glob; 236 int ret = 0; 237 uint32_t page_flags = 0; 238 239 TTM_ASSERT_LOCKED(&bo->mutex); 240 bo->ttm = NULL; 241 242 if (bdev->need_dma32) 243 page_flags |= TTM_PAGE_FLAG_DMA32; 244 245 switch (bo->type) { 246 case ttm_bo_type_device: 247 if (zero_alloc) 248 page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC; 249 case ttm_bo_type_kernel: 250 bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, 251 page_flags, glob->dummy_read_page); 252 if (unlikely(bo->ttm == NULL)) 253 ret = -ENOMEM; 254 break; 255 case ttm_bo_type_sg: 256 bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, 257 page_flags | TTM_PAGE_FLAG_SG, 258 glob->dummy_read_page); 259 if (unlikely(bo->ttm == NULL)) { 260 ret = -ENOMEM; 261 break; 262 } 263 bo->ttm->sg = bo->sg; 264 break; 265 default: 266 pr_err("Illegal buffer object type\n"); 267 ret = -EINVAL; 268 break; 269 } 270 271 return ret; 272 } 273 274 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, 275 struct ttm_mem_reg *mem, 276 bool evict, bool interruptible, 277 bool no_wait_gpu) 278 { 279 struct ttm_bo_device *bdev = bo->bdev; 280 bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); 281 bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem); 282 struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type]; 283 struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type]; 284 int ret = 0; 285 286 if (old_is_pci || new_is_pci || 287 ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) { 288 ret = ttm_mem_io_lock(old_man, true); 289 if (unlikely(ret != 0)) 290 goto out_err; 291 ttm_bo_unmap_virtual_locked(bo); 292 ttm_mem_io_unlock(old_man); 293 } 294 295 /* 296 * Create and bind a ttm if required. 297 */ 298 299 if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 300 if (bo->ttm == NULL) { 301 bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED); 302 ret = ttm_bo_add_ttm(bo, zero); 303 if (ret) 304 goto out_err; 305 } 306 307 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); 308 if (ret) 309 goto out_err; 310 311 if (mem->mem_type != TTM_PL_SYSTEM) { 312 ret = ttm_tt_bind(bo->ttm, mem); 313 if (ret) 314 goto out_err; 315 } 316 317 if (bo->mem.mem_type == TTM_PL_SYSTEM) { 318 if (bdev->driver->move_notify) 319 bdev->driver->move_notify(bo, evict, mem); 320 bo->mem = *mem; 321 mem->mm_node = NULL; 322 goto moved; 323 } 324 } 325 326 if (bdev->driver->move_notify) 327 bdev->driver->move_notify(bo, evict, mem); 328 329 if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && 330 !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) 331 ret = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, mem); 332 else if (bdev->driver->move) 333 ret = bdev->driver->move(bo, evict, interruptible, 334 no_wait_gpu, mem); 335 else 336 ret = ttm_bo_move_memcpy(bo, interruptible, no_wait_gpu, mem); 337 338 if (ret) { 339 if (bdev->driver->move_notify) { 340 struct ttm_mem_reg tmp_mem = *mem; 341 *mem = bo->mem; 342 bo->mem = tmp_mem; 343 bdev->driver->move_notify(bo, false, mem); 344 bo->mem = *mem; 345 *mem = tmp_mem; 346 } 347 348 goto out_err; 349 } 350 351 moved: 352 if (bo->evicted) { 353 if (bdev->driver->invalidate_caches) { 354 ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement); 355 if (ret) 356 pr_err("Can not flush read caches\n"); 357 } 358 bo->evicted = false; 359 } 360 361 if (bo->mem.mm_node) { 362 bo->offset = (bo->mem.start << PAGE_SHIFT) + 363 bdev->man[bo->mem.mem_type].gpu_offset; 364 bo->cur_placement = bo->mem.placement; 365 } else 366 bo->offset = 0; 367 368 return 0; 369 370 out_err: 371 new_man = &bdev->man[bo->mem.mem_type]; 372 if (new_man->flags & TTM_MEMTYPE_FLAG_FIXED) { 373 ttm_tt_destroy(bo->ttm); 374 bo->ttm = NULL; 375 } 376 377 return ret; 378 } 379 380 /** 381 * Call bo::reserved. 382 * Will release GPU memory type usage on destruction. 383 * This is the place to put in driver specific hooks to release 384 * driver private resources. 385 * Will release the bo::reserved lock. 386 */ 387 388 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) 389 { 390 if (bo->bdev->driver->move_notify) 391 bo->bdev->driver->move_notify(bo, false, NULL); 392 393 ttm_tt_destroy(bo->ttm); 394 bo->ttm = NULL; 395 ttm_bo_mem_put(bo, &bo->mem); 396 397 ww_mutex_unlock (&bo->resv->lock); 398 } 399 400 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) 401 { 402 struct reservation_object_list *fobj; 403 struct dma_fence *fence; 404 int i; 405 406 fobj = reservation_object_get_list(bo->resv); 407 fence = reservation_object_get_excl(bo->resv); 408 if (fence && !fence->ops->signaled) 409 dma_fence_enable_sw_signaling(fence); 410 411 for (i = 0; fobj && i < fobj->shared_count; ++i) { 412 fence = rcu_dereference_protected(fobj->shared[i], 413 reservation_object_held(bo->resv)); 414 415 if (!fence->ops->signaled) 416 dma_fence_enable_sw_signaling(fence); 417 } 418 } 419 420 static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) 421 { 422 struct ttm_bo_device *bdev = bo->bdev; 423 struct ttm_bo_global *glob = bo->glob; 424 int ret; 425 426 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 427 ret = __ttm_bo_reserve(bo, false, true, NULL); 428 429 if (!ret) { 430 if (!ttm_bo_wait(bo, false, true)) { 431 ttm_bo_del_from_lru(bo); 432 lockmgr(&glob->lru_lock, LK_RELEASE); 433 ttm_bo_cleanup_memtype_use(bo); 434 435 return; 436 } else 437 ttm_bo_flush_all_fences(bo); 438 439 /* 440 * Make NO_EVICT bos immediately available to 441 * shrinkers, now that they are queued for 442 * destruction. 443 */ 444 if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) { 445 bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT; 446 ttm_bo_add_to_lru(bo); 447 } 448 449 __ttm_bo_unreserve(bo); 450 } 451 452 kref_get(&bo->list_kref); 453 list_add_tail(&bo->ddestroy, &bdev->ddestroy); 454 lockmgr(&glob->lru_lock, LK_RELEASE); 455 456 schedule_delayed_work(&bdev->wq, 457 ((HZ / 100) < 1) ? 1 : HZ / 100); 458 } 459 460 /** 461 * function ttm_bo_cleanup_refs_and_unlock 462 * If bo idle, remove from delayed- and lru lists, and unref. 463 * If not idle, do nothing. 464 * 465 * Must be called with lru_lock and reservation held, this function 466 * will drop both before returning. 467 * 468 * @interruptible Any sleeps should occur interruptibly. 469 * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. 470 */ 471 472 static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, 473 bool interruptible, 474 bool no_wait_gpu) 475 { 476 struct ttm_bo_global *glob = bo->glob; 477 int ret; 478 479 ret = ttm_bo_wait(bo, false, true); 480 481 if (ret && !no_wait_gpu) { 482 long lret; 483 ww_mutex_unlock(&bo->resv->lock); 484 lockmgr(&glob->lru_lock, LK_RELEASE); 485 486 lret = reservation_object_wait_timeout_rcu(bo->resv, 487 true, 488 interruptible, 489 30 * HZ); 490 491 if (lret < 0) 492 return lret; 493 else if (lret == 0) 494 return -EBUSY; 495 496 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 497 ret = __ttm_bo_reserve(bo, false, true, NULL); 498 499 /* 500 * We raced, and lost, someone else holds the reservation now, 501 * and is probably busy in ttm_bo_cleanup_memtype_use. 502 * 503 * Even if it's not the case, because we finished waiting any 504 * delayed destruction would succeed, so just return success 505 * here. 506 */ 507 if (ret) { 508 lockmgr(&glob->lru_lock, LK_RELEASE); 509 return 0; 510 } 511 512 /* 513 * remove sync_obj with ttm_bo_wait, the wait should be 514 * finished, and no new wait object should have been added. 515 */ 516 ret = ttm_bo_wait(bo, false, true); 517 WARN_ON(ret); 518 } 519 520 if (ret || unlikely(list_empty(&bo->ddestroy))) { 521 __ttm_bo_unreserve(bo); 522 lockmgr(&glob->lru_lock, LK_RELEASE); 523 return ret; 524 } 525 526 ttm_bo_del_from_lru(bo); 527 list_del_init(&bo->ddestroy); 528 kref_put(&bo->list_kref, ttm_bo_ref_bug); 529 530 lockmgr(&glob->lru_lock, LK_RELEASE); 531 ttm_bo_cleanup_memtype_use(bo); 532 533 return 0; 534 } 535 536 /** 537 * Traverse the delayed list, and call ttm_bo_cleanup_refs on all 538 * encountered buffers. 539 */ 540 541 static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) 542 { 543 struct ttm_bo_global *glob = bdev->glob; 544 struct ttm_buffer_object *entry = NULL; 545 int ret = 0; 546 547 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 548 if (list_empty(&bdev->ddestroy)) 549 goto out_unlock; 550 551 entry = list_first_entry(&bdev->ddestroy, 552 struct ttm_buffer_object, ddestroy); 553 kref_get(&entry->list_kref); 554 555 for (;;) { 556 struct ttm_buffer_object *nentry = NULL; 557 558 if (entry->ddestroy.next != &bdev->ddestroy) { 559 nentry = list_first_entry(&entry->ddestroy, 560 struct ttm_buffer_object, ddestroy); 561 kref_get(&nentry->list_kref); 562 } 563 564 ret = __ttm_bo_reserve(entry, false, true, NULL); 565 if (remove_all && ret) { 566 lockmgr(&glob->lru_lock, LK_RELEASE); 567 ret = __ttm_bo_reserve(entry, false, false, NULL); 568 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 569 } 570 571 if (!ret) 572 ret = ttm_bo_cleanup_refs_and_unlock(entry, false, 573 !remove_all); 574 else 575 lockmgr(&glob->lru_lock, LK_RELEASE); 576 577 kref_put(&entry->list_kref, ttm_bo_release_list); 578 entry = nentry; 579 580 if (ret || !entry) 581 goto out; 582 583 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 584 if (list_empty(&entry->ddestroy)) 585 break; 586 } 587 588 out_unlock: 589 lockmgr(&glob->lru_lock, LK_RELEASE); 590 out: 591 if (entry) 592 kref_put(&entry->list_kref, ttm_bo_release_list); 593 return ret; 594 } 595 596 static void ttm_bo_delayed_workqueue(struct work_struct *work) 597 { 598 struct ttm_bo_device *bdev = 599 container_of(work, struct ttm_bo_device, wq.work); 600 601 if (ttm_bo_delayed_delete(bdev, false)) { 602 schedule_delayed_work(&bdev->wq, 603 ((HZ / 100) < 1) ? 1 : HZ / 100); 604 } 605 } 606 607 static void ttm_bo_release(struct kref *kref) 608 { 609 struct ttm_buffer_object *bo = 610 container_of(kref, struct ttm_buffer_object, kref); 611 struct ttm_bo_device *bdev = bo->bdev; 612 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 613 614 drm_vma_offset_remove(&bdev->vma_manager, &bo->vma_node); 615 ttm_mem_io_lock(man, false); 616 ttm_mem_io_free_vm(bo); 617 ttm_mem_io_unlock(man); 618 ttm_bo_cleanup_refs_or_queue(bo); 619 kref_put(&bo->list_kref, ttm_bo_release_list); 620 } 621 622 void ttm_bo_unref(struct ttm_buffer_object **p_bo) 623 { 624 struct ttm_buffer_object *bo = *p_bo; 625 626 *p_bo = NULL; 627 kref_put(&bo->kref, ttm_bo_release); 628 } 629 EXPORT_SYMBOL(ttm_bo_unref); 630 631 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev) 632 { 633 return cancel_delayed_work_sync(&bdev->wq); 634 } 635 EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue); 636 637 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) 638 { 639 if (resched) 640 schedule_delayed_work(&bdev->wq, 641 ((HZ / 100) < 1) ? 1 : HZ / 100); 642 } 643 EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue); 644 645 static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, 646 bool no_wait_gpu) 647 { 648 struct ttm_bo_device *bdev = bo->bdev; 649 struct ttm_mem_reg evict_mem; 650 struct ttm_placement placement; 651 int ret = 0; 652 653 lockdep_assert_held(&bo->resv->lock.base); 654 655 evict_mem = bo->mem; 656 evict_mem.mm_node = NULL; 657 evict_mem.bus.io_reserved_vm = false; 658 evict_mem.bus.io_reserved_count = 0; 659 660 placement.num_placement = 0; 661 placement.num_busy_placement = 0; 662 bdev->driver->evict_flags(bo, &placement); 663 ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible, 664 no_wait_gpu); 665 if (ret) { 666 if (ret != -ERESTARTSYS) { 667 pr_err("Failed to find memory space for buffer 0x%p eviction\n", 668 bo); 669 ttm_bo_mem_space_debug(bo, &placement); 670 } 671 goto out; 672 } 673 674 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible, 675 no_wait_gpu); 676 if (unlikely(ret)) { 677 if (ret != -ERESTARTSYS) 678 pr_err("Buffer eviction failed\n"); 679 ttm_bo_mem_put(bo, &evict_mem); 680 goto out; 681 } 682 bo->evicted = true; 683 out: 684 return ret; 685 } 686 687 bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, 688 const struct ttm_place *place) 689 { 690 /* Don't evict this BO if it's outside of the 691 * requested placement range 692 */ 693 if (place->fpfn >= (bo->mem.start + bo->mem.size) || 694 (place->lpfn && place->lpfn <= bo->mem.start)) 695 return false; 696 697 return true; 698 } 699 EXPORT_SYMBOL(ttm_bo_eviction_valuable); 700 701 static int ttm_mem_evict_first(struct ttm_bo_device *bdev, 702 uint32_t mem_type, 703 const struct ttm_place *place, 704 bool interruptible, 705 bool no_wait_gpu) 706 { 707 struct ttm_bo_global *glob = bdev->glob; 708 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 709 struct ttm_buffer_object *bo; 710 int ret = -EBUSY; 711 unsigned i; 712 713 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 714 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { 715 list_for_each_entry(bo, &man->lru[i], lru) { 716 ret = __ttm_bo_reserve(bo, false, true, NULL); 717 if (ret) 718 continue; 719 720 if (place && !bdev->driver->eviction_valuable(bo, 721 place)) { 722 __ttm_bo_unreserve(bo); 723 ret = -EBUSY; 724 continue; 725 } 726 727 break; 728 } 729 730 if (!ret) 731 break; 732 } 733 734 if (ret) { 735 lockmgr(&glob->lru_lock, LK_RELEASE); 736 return ret; 737 } 738 739 kref_get(&bo->list_kref); 740 741 if (!list_empty(&bo->ddestroy)) { 742 ret = ttm_bo_cleanup_refs_and_unlock(bo, interruptible, 743 no_wait_gpu); 744 kref_put(&bo->list_kref, ttm_bo_release_list); 745 return ret; 746 } 747 748 ttm_bo_del_from_lru(bo); 749 lockmgr(&glob->lru_lock, LK_RELEASE); 750 751 BUG_ON(ret != 0); 752 753 ret = ttm_bo_evict(bo, interruptible, no_wait_gpu); 754 ttm_bo_unreserve(bo); 755 756 kref_put(&bo->list_kref, ttm_bo_release_list); 757 return ret; 758 } 759 760 void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) 761 { 762 struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type]; 763 764 if (mem->mm_node) 765 (*man->func->put_node)(man, mem); 766 } 767 EXPORT_SYMBOL(ttm_bo_mem_put); 768 769 /** 770 * Add the last move fence to the BO and reserve a new shared slot. 771 */ 772 static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo, 773 struct ttm_mem_type_manager *man, 774 struct ttm_mem_reg *mem) 775 { 776 struct dma_fence *fence; 777 int ret; 778 779 lockmgr(&man->move_lock, LK_EXCLUSIVE); 780 fence = dma_fence_get(man->move); 781 lockmgr(&man->move_lock, LK_RELEASE); 782 783 if (fence) { 784 reservation_object_add_shared_fence(bo->resv, fence); 785 786 ret = reservation_object_reserve_shared(bo->resv); 787 if (unlikely(ret)) 788 return ret; 789 790 dma_fence_put(bo->moving); 791 bo->moving = fence; 792 } 793 794 return 0; 795 } 796 797 /** 798 * Repeatedly evict memory from the LRU for @mem_type until we create enough 799 * space, or we've evicted everything and there isn't enough space. 800 */ 801 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, 802 uint32_t mem_type, 803 const struct ttm_place *place, 804 struct ttm_mem_reg *mem, 805 bool interruptible, 806 bool no_wait_gpu) 807 { 808 struct ttm_bo_device *bdev = bo->bdev; 809 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 810 int ret; 811 812 do { 813 ret = (*man->func->get_node)(man, bo, place, mem); 814 if (unlikely(ret != 0)) 815 return ret; 816 if (mem->mm_node) 817 break; 818 ret = ttm_mem_evict_first(bdev, mem_type, place, 819 interruptible, no_wait_gpu); 820 if (unlikely(ret != 0)) 821 return ret; 822 } while (1); 823 mem->mem_type = mem_type; 824 return ttm_bo_add_move_fence(bo, man, mem); 825 } 826 827 static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man, 828 uint32_t cur_placement, 829 uint32_t proposed_placement) 830 { 831 uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING; 832 uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING; 833 834 /** 835 * Keep current caching if possible. 836 */ 837 838 if ((cur_placement & caching) != 0) 839 result |= (cur_placement & caching); 840 else if ((man->default_caching & caching) != 0) 841 result |= man->default_caching; 842 else if ((TTM_PL_FLAG_CACHED & caching) != 0) 843 result |= TTM_PL_FLAG_CACHED; 844 else if ((TTM_PL_FLAG_WC & caching) != 0) 845 result |= TTM_PL_FLAG_WC; 846 else if ((TTM_PL_FLAG_UNCACHED & caching) != 0) 847 result |= TTM_PL_FLAG_UNCACHED; 848 849 return result; 850 } 851 852 static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, 853 uint32_t mem_type, 854 const struct ttm_place *place, 855 uint32_t *masked_placement) 856 { 857 uint32_t cur_flags = ttm_bo_type_flags(mem_type); 858 859 if ((cur_flags & place->flags & TTM_PL_MASK_MEM) == 0) 860 return false; 861 862 if ((place->flags & man->available_caching) == 0) 863 return false; 864 865 cur_flags |= (place->flags & man->available_caching); 866 867 *masked_placement = cur_flags; 868 return true; 869 } 870 871 /** 872 * Creates space for memory region @mem according to its type. 873 * 874 * This function first searches for free space in compatible memory types in 875 * the priority order defined by the driver. If free space isn't found, then 876 * ttm_bo_mem_force_space is attempted in priority order to evict and find 877 * space. 878 */ 879 int ttm_bo_mem_space(struct ttm_buffer_object *bo, 880 struct ttm_placement *placement, 881 struct ttm_mem_reg *mem, 882 bool interruptible, 883 bool no_wait_gpu) 884 { 885 struct ttm_bo_device *bdev = bo->bdev; 886 struct ttm_mem_type_manager *man; 887 uint32_t mem_type = TTM_PL_SYSTEM; 888 uint32_t cur_flags = 0; 889 bool type_found = false; 890 bool type_ok = false; 891 bool has_erestartsys = false; 892 int i, ret; 893 894 ret = reservation_object_reserve_shared(bo->resv); 895 if (unlikely(ret)) 896 return ret; 897 898 mem->mm_node = NULL; 899 for (i = 0; i < placement->num_placement; ++i) { 900 const struct ttm_place *place = &placement->placement[i]; 901 902 ret = ttm_mem_type_from_place(place, &mem_type); 903 if (ret) 904 return ret; 905 man = &bdev->man[mem_type]; 906 if (!man->has_type || !man->use_type) 907 continue; 908 909 type_ok = ttm_bo_mt_compatible(man, mem_type, place, 910 &cur_flags); 911 912 if (!type_ok) 913 continue; 914 915 type_found = true; 916 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 917 cur_flags); 918 /* 919 * Use the access and other non-mapping-related flag bits from 920 * the memory placement flags to the current flags 921 */ 922 ttm_flag_masked(&cur_flags, place->flags, 923 ~TTM_PL_MASK_MEMTYPE); 924 925 if (mem_type == TTM_PL_SYSTEM) 926 break; 927 928 ret = (*man->func->get_node)(man, bo, place, mem); 929 if (unlikely(ret)) 930 return ret; 931 932 if (mem->mm_node) { 933 ret = ttm_bo_add_move_fence(bo, man, mem); 934 if (unlikely(ret)) { 935 (*man->func->put_node)(man, mem); 936 return ret; 937 } 938 break; 939 } 940 } 941 942 if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) { 943 mem->mem_type = mem_type; 944 mem->placement = cur_flags; 945 return 0; 946 } 947 948 for (i = 0; i < placement->num_busy_placement; ++i) { 949 const struct ttm_place *place = &placement->busy_placement[i]; 950 951 ret = ttm_mem_type_from_place(place, &mem_type); 952 if (ret) 953 return ret; 954 man = &bdev->man[mem_type]; 955 if (!man->has_type || !man->use_type) 956 continue; 957 if (!ttm_bo_mt_compatible(man, mem_type, place, &cur_flags)) 958 continue; 959 960 type_found = true; 961 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 962 cur_flags); 963 /* 964 * Use the access and other non-mapping-related flag bits from 965 * the memory placement flags to the current flags 966 */ 967 ttm_flag_masked(&cur_flags, place->flags, 968 ~TTM_PL_MASK_MEMTYPE); 969 970 if (mem_type == TTM_PL_SYSTEM) { 971 mem->mem_type = mem_type; 972 mem->placement = cur_flags; 973 mem->mm_node = NULL; 974 return 0; 975 } 976 977 ret = ttm_bo_mem_force_space(bo, mem_type, place, mem, 978 interruptible, no_wait_gpu); 979 if (ret == 0 && mem->mm_node) { 980 mem->placement = cur_flags; 981 return 0; 982 } 983 if (ret == -ERESTARTSYS) 984 has_erestartsys = true; 985 } 986 987 if (!type_found) { 988 pr_err(TTM_PFX "No compatible memory type found\n"); 989 return -EINVAL; 990 } 991 992 return (has_erestartsys) ? -ERESTARTSYS : -ENOMEM; 993 } 994 EXPORT_SYMBOL(ttm_bo_mem_space); 995 996 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, 997 struct ttm_placement *placement, 998 bool interruptible, 999 bool no_wait_gpu) 1000 { 1001 int ret = 0; 1002 struct ttm_mem_reg mem; 1003 1004 lockdep_assert_held(&bo->resv->lock.base); 1005 1006 mem.num_pages = bo->num_pages; 1007 mem.size = mem.num_pages << PAGE_SHIFT; 1008 mem.page_alignment = bo->mem.page_alignment; 1009 mem.bus.io_reserved_vm = false; 1010 mem.bus.io_reserved_count = 0; 1011 /* 1012 * Determine where to move the buffer. 1013 */ 1014 ret = ttm_bo_mem_space(bo, placement, &mem, 1015 interruptible, no_wait_gpu); 1016 if (ret) 1017 goto out_unlock; 1018 ret = ttm_bo_handle_move_mem(bo, &mem, false, 1019 interruptible, no_wait_gpu); 1020 out_unlock: 1021 if (ret && mem.mm_node) 1022 ttm_bo_mem_put(bo, &mem); 1023 return ret; 1024 } 1025 1026 static bool ttm_bo_places_compat(const struct ttm_place *places, 1027 unsigned num_placement, 1028 struct ttm_mem_reg *mem, 1029 uint32_t *new_flags) 1030 { 1031 unsigned i; 1032 1033 for (i = 0; i < num_placement; i++) { 1034 const struct ttm_place *heap = &places[i]; 1035 1036 if (mem->mm_node && (mem->start < heap->fpfn || 1037 (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn))) 1038 continue; 1039 1040 *new_flags = heap->flags; 1041 if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && 1042 (*new_flags & mem->placement & TTM_PL_MASK_MEM) && 1043 (!(*new_flags & TTM_PL_FLAG_CONTIGUOUS) || 1044 (mem->placement & TTM_PL_FLAG_CONTIGUOUS))) 1045 return true; 1046 } 1047 return false; 1048 } 1049 1050 bool ttm_bo_mem_compat(struct ttm_placement *placement, 1051 struct ttm_mem_reg *mem, 1052 uint32_t *new_flags) 1053 { 1054 if (ttm_bo_places_compat(placement->placement, placement->num_placement, 1055 mem, new_flags)) 1056 return true; 1057 1058 if ((placement->busy_placement != placement->placement || 1059 placement->num_busy_placement > placement->num_placement) && 1060 ttm_bo_places_compat(placement->busy_placement, 1061 placement->num_busy_placement, 1062 mem, new_flags)) 1063 return true; 1064 1065 return false; 1066 } 1067 EXPORT_SYMBOL(ttm_bo_mem_compat); 1068 1069 int ttm_bo_validate(struct ttm_buffer_object *bo, 1070 struct ttm_placement *placement, 1071 bool interruptible, 1072 bool no_wait_gpu) 1073 { 1074 int ret; 1075 uint32_t new_flags; 1076 1077 lockdep_assert_held(&bo->resv->lock.base); 1078 /* 1079 * Check whether we need to move buffer. 1080 */ 1081 if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) { 1082 ret = ttm_bo_move_buffer(bo, placement, interruptible, 1083 no_wait_gpu); 1084 if (ret) 1085 return ret; 1086 } else { 1087 /* 1088 * Use the access and other non-mapping-related flag bits from 1089 * the compatible memory placement flags to the active flags 1090 */ 1091 ttm_flag_masked(&bo->mem.placement, new_flags, 1092 ~TTM_PL_MASK_MEMTYPE); 1093 } 1094 /* 1095 * We might need to add a TTM. 1096 */ 1097 if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { 1098 ret = ttm_bo_add_ttm(bo, true); 1099 if (ret) 1100 return ret; 1101 } 1102 return 0; 1103 } 1104 EXPORT_SYMBOL(ttm_bo_validate); 1105 1106 int ttm_bo_init_reserved(struct ttm_bo_device *bdev, 1107 struct ttm_buffer_object *bo, 1108 unsigned long size, 1109 enum ttm_bo_type type, 1110 struct ttm_placement *placement, 1111 uint32_t page_alignment, 1112 bool interruptible, 1113 struct vm_object *persistent_swap_storage, 1114 size_t acc_size, 1115 struct sg_table *sg, 1116 struct reservation_object *resv, 1117 void (*destroy) (struct ttm_buffer_object *)) 1118 { 1119 int ret = 0; 1120 unsigned long num_pages; 1121 struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; 1122 bool locked; 1123 1124 ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false); 1125 if (ret) { 1126 pr_err("Out of kernel memory\n"); 1127 if (destroy) 1128 (*destroy)(bo); 1129 else 1130 kfree(bo); 1131 return -ENOMEM; 1132 } 1133 1134 num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 1135 if (num_pages == 0) { 1136 pr_err("Illegal buffer object size\n"); 1137 if (destroy) 1138 (*destroy)(bo); 1139 else 1140 kfree(bo); 1141 ttm_mem_global_free(mem_glob, acc_size); 1142 return -EINVAL; 1143 } 1144 bo->destroy = destroy; 1145 1146 kref_init(&bo->kref); 1147 kref_init(&bo->list_kref); 1148 atomic_set(&bo->cpu_writers, 0); 1149 INIT_LIST_HEAD(&bo->lru); 1150 INIT_LIST_HEAD(&bo->ddestroy); 1151 INIT_LIST_HEAD(&bo->swap); 1152 INIT_LIST_HEAD(&bo->io_reserve_lru); 1153 lockinit(&bo->wu_mutex, "ttmbwm", 0, LK_CANRECURSE); 1154 bo->bdev = bdev; 1155 bo->glob = bdev->glob; 1156 bo->type = type; 1157 bo->num_pages = num_pages; 1158 bo->mem.size = num_pages << PAGE_SHIFT; 1159 bo->mem.mem_type = TTM_PL_SYSTEM; 1160 bo->mem.num_pages = bo->num_pages; 1161 bo->mem.mm_node = NULL; 1162 bo->mem.page_alignment = page_alignment; 1163 bo->mem.bus.io_reserved_vm = false; 1164 bo->mem.bus.io_reserved_count = 0; 1165 bo->moving = NULL; 1166 bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); 1167 bo->persistent_swap_storage = persistent_swap_storage; 1168 bo->acc_size = acc_size; 1169 bo->sg = sg; 1170 if (resv) { 1171 bo->resv = resv; 1172 lockdep_assert_held(&bo->resv->lock.base); 1173 } else { 1174 bo->resv = &bo->ttm_resv; 1175 reservation_object_init(&bo->ttm_resv); 1176 } 1177 atomic_inc(&bo->glob->bo_count); 1178 drm_vma_node_reset(&bo->vma_node); 1179 bo->priority = 0; 1180 1181 /* 1182 * For ttm_bo_type_device buffers, allocate 1183 * address space from the device. 1184 */ 1185 if (bo->type == ttm_bo_type_device || 1186 bo->type == ttm_bo_type_sg) 1187 ret = drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node, 1188 bo->mem.num_pages); 1189 1190 /* passed reservation objects should already be locked, 1191 * since otherwise lockdep will be angered in radeon. 1192 */ 1193 if (!resv) { 1194 locked = ww_mutex_trylock(&bo->resv->lock); 1195 WARN_ON(!locked); 1196 } 1197 1198 if (likely(!ret)) 1199 ret = ttm_bo_validate(bo, placement, interruptible, false); 1200 1201 if (unlikely(ret)) { 1202 if (!resv) 1203 ttm_bo_unreserve(bo); 1204 1205 ttm_bo_unref(&bo); 1206 return ret; 1207 } 1208 1209 if (resv && !(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { 1210 lockmgr(&bo->glob->lru_lock, LK_EXCLUSIVE); 1211 ttm_bo_add_to_lru(bo); 1212 lockmgr(&bo->glob->lru_lock, LK_RELEASE); 1213 } 1214 1215 return ret; 1216 } 1217 EXPORT_SYMBOL(ttm_bo_init_reserved); 1218 1219 int ttm_bo_init(struct ttm_bo_device *bdev, 1220 struct ttm_buffer_object *bo, 1221 unsigned long size, 1222 enum ttm_bo_type type, 1223 struct ttm_placement *placement, 1224 uint32_t page_alignment, 1225 bool interruptible, 1226 struct vm_object *persistent_swap_storage, 1227 size_t acc_size, 1228 struct sg_table *sg, 1229 struct reservation_object *resv, 1230 void (*destroy) (struct ttm_buffer_object *)) 1231 { 1232 int ret; 1233 1234 ret = ttm_bo_init_reserved(bdev, bo, size, type, placement, 1235 page_alignment, interruptible, 1236 persistent_swap_storage, acc_size, 1237 sg, resv, destroy); 1238 if (ret) 1239 return ret; 1240 1241 if (!resv) 1242 ttm_bo_unreserve(bo); 1243 1244 return 0; 1245 } 1246 EXPORT_SYMBOL(ttm_bo_init); 1247 1248 size_t ttm_bo_acc_size(struct ttm_bo_device *bdev, 1249 unsigned long bo_size, 1250 unsigned struct_size) 1251 { 1252 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1253 size_t size = 0; 1254 1255 size += ttm_round_pot(struct_size); 1256 size += ttm_round_pot(npages * sizeof(void *)); 1257 size += ttm_round_pot(sizeof(struct ttm_tt)); 1258 return size; 1259 } 1260 EXPORT_SYMBOL(ttm_bo_acc_size); 1261 1262 size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, 1263 unsigned long bo_size, 1264 unsigned struct_size) 1265 { 1266 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1267 size_t size = 0; 1268 1269 size += ttm_round_pot(struct_size); 1270 size += ttm_round_pot(npages * (2*sizeof(void *) + sizeof(dma_addr_t))); 1271 size += ttm_round_pot(sizeof(struct ttm_dma_tt)); 1272 return size; 1273 } 1274 EXPORT_SYMBOL(ttm_bo_dma_acc_size); 1275 1276 int ttm_bo_create(struct ttm_bo_device *bdev, 1277 unsigned long size, 1278 enum ttm_bo_type type, 1279 struct ttm_placement *placement, 1280 uint32_t page_alignment, 1281 bool interruptible, 1282 struct vm_object *persistent_swap_storage, 1283 struct ttm_buffer_object **p_bo) 1284 { 1285 struct ttm_buffer_object *bo; 1286 size_t acc_size; 1287 int ret; 1288 1289 bo = kzalloc(sizeof(*bo), GFP_KERNEL); 1290 if (unlikely(bo == NULL)) 1291 return -ENOMEM; 1292 1293 acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); 1294 ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, 1295 interruptible, persistent_swap_storage, acc_size, 1296 NULL, NULL, NULL); 1297 if (likely(ret == 0)) 1298 *p_bo = bo; 1299 1300 return ret; 1301 } 1302 EXPORT_SYMBOL(ttm_bo_create); 1303 1304 static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, 1305 unsigned mem_type) 1306 { 1307 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1308 struct ttm_bo_global *glob = bdev->glob; 1309 struct dma_fence *fence; 1310 int ret; 1311 unsigned i; 1312 1313 /* 1314 * Can't use standard list traversal since we're unlocking. 1315 */ 1316 1317 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 1318 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { 1319 while (!list_empty(&man->lru[i])) { 1320 lockmgr(&glob->lru_lock, LK_RELEASE); 1321 ret = ttm_mem_evict_first(bdev, mem_type, NULL, false, false); 1322 if (ret) 1323 return ret; 1324 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 1325 } 1326 } 1327 lockmgr(&glob->lru_lock, LK_RELEASE); 1328 1329 lockmgr(&man->move_lock, LK_EXCLUSIVE); 1330 fence = dma_fence_get(man->move); 1331 lockmgr(&man->move_lock, LK_RELEASE); 1332 1333 if (fence) { 1334 ret = dma_fence_wait(fence, false); 1335 dma_fence_put(fence); 1336 if (ret) 1337 return ret; 1338 } 1339 1340 return 0; 1341 } 1342 1343 int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1344 { 1345 struct ttm_mem_type_manager *man; 1346 int ret = -EINVAL; 1347 1348 if (mem_type >= TTM_NUM_MEM_TYPES) { 1349 pr_err("Illegal memory type %d\n", mem_type); 1350 return ret; 1351 } 1352 man = &bdev->man[mem_type]; 1353 1354 if (!man->has_type) { 1355 pr_err("Trying to take down uninitialized memory manager type %u\n", 1356 mem_type); 1357 return ret; 1358 } 1359 1360 man->use_type = false; 1361 man->has_type = false; 1362 1363 ret = 0; 1364 if (mem_type > 0) { 1365 ret = ttm_bo_force_list_clean(bdev, mem_type); 1366 if (ret) { 1367 pr_err("Cleanup eviction failed\n"); 1368 return ret; 1369 } 1370 1371 ret = (*man->func->takedown)(man); 1372 } 1373 1374 dma_fence_put(man->move); 1375 man->move = NULL; 1376 1377 return ret; 1378 } 1379 EXPORT_SYMBOL(ttm_bo_clean_mm); 1380 1381 int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1382 { 1383 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1384 1385 if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) { 1386 pr_err("Illegal memory manager memory type %u\n", mem_type); 1387 return -EINVAL; 1388 } 1389 1390 if (!man->has_type) { 1391 pr_err("Memory type %u has not been initialized\n", mem_type); 1392 return 0; 1393 } 1394 1395 return ttm_bo_force_list_clean(bdev, mem_type); 1396 } 1397 EXPORT_SYMBOL(ttm_bo_evict_mm); 1398 1399 int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, 1400 unsigned long p_size) 1401 { 1402 int ret; 1403 struct ttm_mem_type_manager *man; 1404 unsigned i; 1405 1406 BUG_ON(type >= TTM_NUM_MEM_TYPES); 1407 man = &bdev->man[type]; 1408 BUG_ON(man->has_type); 1409 man->io_reserve_fastpath = true; 1410 man->use_io_reserve_lru = false; 1411 lockinit(&man->io_reserve_mutex, "ttmior", 0, 0); 1412 lockinit(&man->move_lock, "ttmml", 0, 0); 1413 INIT_LIST_HEAD(&man->io_reserve_lru); 1414 1415 ret = bdev->driver->init_mem_type(bdev, type, man); 1416 if (ret) 1417 return ret; 1418 man->bdev = bdev; 1419 1420 if (type != TTM_PL_SYSTEM) { 1421 ret = (*man->func->init)(man, p_size); 1422 if (ret) 1423 return ret; 1424 } 1425 man->has_type = true; 1426 man->use_type = true; 1427 man->size = p_size; 1428 1429 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) 1430 INIT_LIST_HEAD(&man->lru[i]); 1431 man->move = NULL; 1432 1433 return 0; 1434 } 1435 EXPORT_SYMBOL(ttm_bo_init_mm); 1436 1437 static void ttm_bo_global_kobj_release(struct kobject *kobj) 1438 { 1439 struct ttm_bo_global *glob = 1440 container_of(kobj, struct ttm_bo_global, kobj); 1441 1442 ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink); 1443 __free_page(glob->dummy_read_page); 1444 kfree(glob); 1445 } 1446 1447 void ttm_bo_global_release(struct drm_global_reference *ref) 1448 { 1449 struct ttm_bo_global *glob = ref->object; 1450 1451 kobject_del(&glob->kobj); 1452 kobject_put(&glob->kobj); 1453 } 1454 EXPORT_SYMBOL(ttm_bo_global_release); 1455 1456 int ttm_bo_global_init(struct drm_global_reference *ref) 1457 { 1458 struct ttm_bo_global_ref *bo_ref = 1459 container_of(ref, struct ttm_bo_global_ref, ref); 1460 struct ttm_bo_global *glob = ref->object; 1461 int ret; 1462 unsigned i; 1463 1464 lockinit(&glob->device_list_mutex, "ttmdlm", 0, 0); 1465 lockinit(&glob->lru_lock, "ttmlru", 0, 0); 1466 glob->mem_glob = bo_ref->mem_glob; 1467 glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32); 1468 1469 if (unlikely(glob->dummy_read_page == NULL)) { 1470 ret = -ENOMEM; 1471 goto out_no_drp; 1472 } 1473 1474 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) 1475 INIT_LIST_HEAD(&glob->swap_lru[i]); 1476 INIT_LIST_HEAD(&glob->device_list); 1477 1478 ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout); 1479 ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink); 1480 if (unlikely(ret != 0)) { 1481 pr_err("Could not register buffer object swapout\n"); 1482 goto out_no_shrink; 1483 } 1484 1485 atomic_set(&glob->bo_count, 0); 1486 1487 ret = kobject_init_and_add( 1488 &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects"); 1489 if (unlikely(ret != 0)) 1490 kobject_put(&glob->kobj); 1491 return ret; 1492 out_no_shrink: 1493 __free_page(glob->dummy_read_page); 1494 out_no_drp: 1495 kfree(glob); 1496 return ret; 1497 } 1498 EXPORT_SYMBOL(ttm_bo_global_init); 1499 1500 1501 int ttm_bo_device_release(struct ttm_bo_device *bdev) 1502 { 1503 int ret = 0; 1504 unsigned i = TTM_NUM_MEM_TYPES; 1505 struct ttm_mem_type_manager *man; 1506 struct ttm_bo_global *glob = bdev->glob; 1507 1508 while (i--) { 1509 man = &bdev->man[i]; 1510 if (man->has_type) { 1511 man->use_type = false; 1512 if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) { 1513 ret = -EBUSY; 1514 pr_err("DRM memory manager type %d is not clean\n", 1515 i); 1516 } 1517 man->has_type = false; 1518 } 1519 } 1520 1521 mutex_lock(&glob->device_list_mutex); 1522 list_del(&bdev->device_list); 1523 mutex_unlock(&glob->device_list_mutex); 1524 1525 cancel_delayed_work_sync(&bdev->wq); 1526 1527 while (ttm_bo_delayed_delete(bdev, true)) 1528 ; 1529 1530 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 1531 if (list_empty(&bdev->ddestroy)) 1532 TTM_DEBUG("Delayed destroy list was clean\n"); 1533 1534 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) 1535 if (list_empty(&bdev->man[0].lru[0])) 1536 TTM_DEBUG("Swap list %d was clean\n", i); 1537 lockmgr(&glob->lru_lock, LK_RELEASE); 1538 1539 drm_vma_offset_manager_destroy(&bdev->vma_manager); 1540 1541 return ret; 1542 } 1543 EXPORT_SYMBOL(ttm_bo_device_release); 1544 1545 int ttm_bo_device_init(struct ttm_bo_device *bdev, 1546 struct ttm_bo_global *glob, 1547 struct ttm_bo_driver *driver, 1548 struct address_space *mapping, 1549 uint64_t file_page_offset, 1550 bool need_dma32) 1551 { 1552 int ret = -EINVAL; 1553 1554 bdev->driver = driver; 1555 1556 memset(bdev->man, 0, sizeof(bdev->man)); 1557 1558 /* 1559 * Initialize the system memory buffer type. 1560 * Other types need to be driver / IOCTL initialized. 1561 */ 1562 ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0); 1563 if (unlikely(ret != 0)) 1564 goto out_no_sys; 1565 1566 drm_vma_offset_manager_init(&bdev->vma_manager, file_page_offset, 1567 0x10000000); 1568 INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); 1569 INIT_LIST_HEAD(&bdev->ddestroy); 1570 /* 1571 * XXX DRAGONFLY - dev_mapping NULL atm, find other XXX DRAGONFLY 1572 * lines and fix when it no longer is in later API change. 1573 */ 1574 bdev->dev_mapping = mapping; 1575 bdev->glob = glob; 1576 bdev->need_dma32 = need_dma32; 1577 mutex_lock(&glob->device_list_mutex); 1578 list_add_tail(&bdev->device_list, &glob->device_list); 1579 mutex_unlock(&glob->device_list_mutex); 1580 1581 return 0; 1582 out_no_sys: 1583 return ret; 1584 } 1585 EXPORT_SYMBOL(ttm_bo_device_init); 1586 1587 /* 1588 * buffer object vm functions. 1589 */ 1590 1591 bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 1592 { 1593 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 1594 1595 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 1596 if (mem->mem_type == TTM_PL_SYSTEM) 1597 return false; 1598 1599 if (man->flags & TTM_MEMTYPE_FLAG_CMA) 1600 return false; 1601 1602 if (mem->placement & TTM_PL_FLAG_CACHED) 1603 return false; 1604 } 1605 return true; 1606 } 1607 1608 #ifdef __DragonFly__ 1609 1610 /* 1611 * XXX DRAGONFLY - device_mapping not yet implemented so 1612 * file_mapping is basically always NULL. We have to properly 1613 * release the mmap, etc. 1614 */ 1615 void ttm_bo_release_mmap(struct ttm_buffer_object *bo); 1616 1617 /** 1618 * drm_vma_node_unmap() - Unmap offset node 1619 * @node: Offset node 1620 * @file_mapping: Address space to unmap @node from 1621 * 1622 * Unmap all userspace mappings for a given offset node. The mappings must be 1623 * associated with the @file_mapping address-space. If no offset exists or 1624 * the address-space is invalid, nothing is done. 1625 * 1626 * This call is unlocked. The caller must guarantee that drm_vma_offset_remove() 1627 * is not called on this node concurrently. 1628 */ 1629 static inline void drm_vma_node_unmap(struct drm_vma_offset_node *node, 1630 struct address_space *file_mapping) 1631 { 1632 struct ttm_buffer_object *bo = container_of(node, struct ttm_buffer_object, vma_node); 1633 1634 if (drm_vma_node_has_offset(node)) 1635 unmap_mapping_range(file_mapping, 1636 drm_vma_node_offset_addr(node), 1637 drm_vma_node_size(node) << PAGE_SHIFT, 1); 1638 ttm_bo_release_mmap(bo); 1639 } 1640 #endif 1641 1642 void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) 1643 { 1644 struct ttm_bo_device *bdev = bo->bdev; 1645 1646 drm_vma_node_unmap(&bo->vma_node, bdev->dev_mapping); 1647 ttm_mem_io_free_vm(bo); 1648 } 1649 1650 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) 1651 { 1652 struct ttm_bo_device *bdev = bo->bdev; 1653 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 1654 1655 ttm_mem_io_lock(man, false); 1656 ttm_bo_unmap_virtual_locked(bo); 1657 ttm_mem_io_unlock(man); 1658 } 1659 1660 1661 EXPORT_SYMBOL(ttm_bo_unmap_virtual); 1662 1663 int ttm_bo_wait(struct ttm_buffer_object *bo, 1664 bool interruptible, bool no_wait) 1665 { 1666 long timeout = 15 * HZ; 1667 1668 if (no_wait) { 1669 if (reservation_object_test_signaled_rcu(bo->resv, true)) 1670 return 0; 1671 else 1672 return -EBUSY; 1673 } 1674 1675 timeout = reservation_object_wait_timeout_rcu(bo->resv, true, 1676 interruptible, timeout); 1677 if (timeout < 0) 1678 return timeout; 1679 1680 if (timeout == 0) 1681 return -EBUSY; 1682 1683 reservation_object_add_excl_fence(bo->resv, NULL); 1684 return 0; 1685 } 1686 EXPORT_SYMBOL(ttm_bo_wait); 1687 1688 int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait) 1689 { 1690 int ret = 0; 1691 1692 /* 1693 * Using ttm_bo_reserve makes sure the lru lists are updated. 1694 */ 1695 1696 ret = ttm_bo_reserve(bo, true, no_wait, NULL); 1697 if (unlikely(ret != 0)) 1698 return ret; 1699 ret = ttm_bo_wait(bo, true, no_wait); 1700 if (likely(ret == 0)) 1701 atomic_inc(&bo->cpu_writers); 1702 ttm_bo_unreserve(bo); 1703 return ret; 1704 } 1705 EXPORT_SYMBOL(ttm_bo_synccpu_write_grab); 1706 1707 void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo) 1708 { 1709 atomic_dec(&bo->cpu_writers); 1710 } 1711 EXPORT_SYMBOL(ttm_bo_synccpu_write_release); 1712 1713 /** 1714 * A buffer object shrink method that tries to swap out the first 1715 * buffer object on the bo_global::swap_lru list. 1716 */ 1717 1718 static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) 1719 { 1720 struct ttm_bo_global *glob = 1721 container_of(shrink, struct ttm_bo_global, shrink); 1722 struct ttm_buffer_object *bo; 1723 int ret = -EBUSY; 1724 unsigned i; 1725 1726 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 1727 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { 1728 list_for_each_entry(bo, &glob->swap_lru[i], swap) { 1729 ret = __ttm_bo_reserve(bo, false, true, NULL); 1730 if (!ret) 1731 break; 1732 } 1733 if (!ret) 1734 break; 1735 } 1736 1737 if (ret) { 1738 lockmgr(&glob->lru_lock, LK_RELEASE); 1739 return ret; 1740 } 1741 1742 kref_get(&bo->list_kref); 1743 1744 if (!list_empty(&bo->ddestroy)) { 1745 ret = ttm_bo_cleanup_refs_and_unlock(bo, false, false); 1746 kref_put(&bo->list_kref, ttm_bo_release_list); 1747 return ret; 1748 } 1749 1750 ttm_bo_del_from_lru(bo); 1751 lockmgr(&glob->lru_lock, LK_RELEASE); 1752 1753 /** 1754 * Move to system cached 1755 */ 1756 1757 if (bo->mem.mem_type != TTM_PL_SYSTEM || 1758 bo->ttm->caching_state != tt_cached) { 1759 struct ttm_mem_reg evict_mem; 1760 1761 evict_mem = bo->mem; 1762 evict_mem.mm_node = NULL; 1763 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED; 1764 evict_mem.mem_type = TTM_PL_SYSTEM; 1765 1766 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, 1767 false, false); 1768 if (unlikely(ret != 0)) 1769 goto out; 1770 } 1771 1772 /** 1773 * Make sure BO is idle. 1774 */ 1775 1776 ret = ttm_bo_wait(bo, false, false); 1777 if (unlikely(ret != 0)) 1778 goto out; 1779 1780 ttm_bo_unmap_virtual(bo); 1781 1782 /** 1783 * Swap out. Buffer will be swapped in again as soon as 1784 * anyone tries to access a ttm page. 1785 */ 1786 1787 if (bo->bdev->driver->swap_notify) 1788 bo->bdev->driver->swap_notify(bo); 1789 1790 ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage); 1791 out: 1792 1793 /** 1794 * 1795 * Unreserve without putting on LRU to avoid swapping out an 1796 * already swapped buffer. 1797 */ 1798 1799 __ttm_bo_unreserve(bo); 1800 kref_put(&bo->list_kref, ttm_bo_release_list); 1801 return ret; 1802 } 1803 1804 void ttm_bo_swapout_all(struct ttm_bo_device *bdev) 1805 { 1806 while (ttm_bo_swapout(&bdev->glob->shrink) == 0) 1807 ; 1808 } 1809 EXPORT_SYMBOL(ttm_bo_swapout_all); 1810 1811 /** 1812 * ttm_bo_wait_unreserved - interruptible wait for a buffer object to become 1813 * unreserved 1814 * 1815 * @bo: Pointer to buffer 1816 */ 1817 int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo) 1818 { 1819 int ret; 1820 1821 /* 1822 * In the absense of a wait_unlocked API, 1823 * Use the bo::wu_mutex to avoid triggering livelocks due to 1824 * concurrent use of this function. Note that this use of 1825 * bo::wu_mutex can go away if we change locking order to 1826 * mmap_sem -> bo::reserve. 1827 */ 1828 ret = mutex_lock_interruptible(&bo->wu_mutex); 1829 if (unlikely(ret != 0)) 1830 return -ERESTARTSYS; 1831 if (!ww_mutex_is_locked(&bo->resv->lock)) 1832 goto out_unlock; 1833 ret = __ttm_bo_reserve(bo, true, false, NULL); 1834 if (unlikely(ret != 0)) 1835 goto out_unlock; 1836 __ttm_bo_unreserve(bo); 1837 1838 out_unlock: 1839 mutex_unlock(&bo->wu_mutex); 1840 return ret; 1841 } 1842