1 /************************************************************************** 2 * 3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 /* 28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29 */ 30 31 #define pr_fmt(fmt) "[TTM] " fmt 32 33 #include <drm/ttm/ttm_module.h> 34 #include <drm/ttm/ttm_bo_driver.h> 35 #include <drm/ttm/ttm_placement.h> 36 #include <linux/jiffies.h> 37 #include <linux/slab.h> 38 #include <linux/sched.h> 39 #include <linux/mm.h> 40 #include <linux/file.h> 41 #include <linux/module.h> 42 #include <linux/atomic.h> 43 44 #define TTM_ASSERT_LOCKED(param) 45 #define TTM_DEBUG(fmt, arg...) 46 #define TTM_BO_HASH_ORDER 13 47 48 static int ttm_bo_setup_vm(struct ttm_buffer_object *bo); 49 static int ttm_bo_swapout(struct ttm_mem_shrink *shrink); 50 static void ttm_bo_global_kobj_release(struct kobject *kobj); 51 52 static struct attribute ttm_bo_count = { 53 .name = "bo_count", 54 .mode = S_IRUGO 55 }; 56 57 static inline int ttm_mem_type_from_place(const struct ttm_place *place, 58 uint32_t *mem_type) 59 { 60 int i; 61 62 for (i = 0; i <= TTM_PL_PRIV5; i++) { 63 if (place->flags & (1 << i)) { 64 *mem_type = i; 65 return 0; 66 } 67 } 68 return -EINVAL; 69 } 70 71 static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type) 72 { 73 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 74 75 pr_err(" has_type: %d\n", man->has_type); 76 pr_err(" use_type: %d\n", man->use_type); 77 pr_err(" flags: 0x%08X\n", man->flags); 78 pr_err(" gpu_offset: 0x%08lX\n", man->gpu_offset); 79 pr_err(" size: %ju\n", man->size); 80 pr_err(" available_caching: 0x%08X\n", man->available_caching); 81 pr_err(" default_caching: 0x%08X\n", man->default_caching); 82 if (mem_type != TTM_PL_SYSTEM) 83 (*man->func->debug)(man, TTM_PFX); 84 } 85 86 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, 87 struct ttm_placement *placement) 88 { 89 int i, ret, mem_type; 90 91 pr_err("No space for %p (%lu pages, %luK, %luM)\n", 92 bo, bo->mem.num_pages, bo->mem.size >> 10, 93 bo->mem.size >> 20); 94 for (i = 0; i < placement->num_placement; i++) { 95 ret = ttm_mem_type_from_place(&placement->placement[i], 96 &mem_type); 97 if (ret) 98 return; 99 pr_err(" placement[%d]=0x%08X (%d)\n", 100 i, placement->placement[i].flags, mem_type); 101 ttm_mem_type_debug(bo->bdev, mem_type); 102 } 103 } 104 105 static ssize_t ttm_bo_global_show(struct kobject *kobj, 106 struct attribute *attr, 107 char *buffer) 108 { 109 struct ttm_bo_global *glob = 110 container_of(kobj, struct ttm_bo_global, kobj); 111 112 return snprintf(buffer, PAGE_SIZE, "%lu\n", 113 (unsigned long) atomic_read(&glob->bo_count)); 114 } 115 116 static struct attribute *ttm_bo_global_attrs[] = { 117 &ttm_bo_count, 118 NULL 119 }; 120 121 static const struct sysfs_ops ttm_bo_global_ops = { 122 .show = &ttm_bo_global_show 123 }; 124 125 static struct kobj_type ttm_bo_glob_kobj_type = { 126 .release = &ttm_bo_global_kobj_release, 127 .sysfs_ops = &ttm_bo_global_ops, 128 .default_attrs = ttm_bo_global_attrs 129 }; 130 131 132 static inline uint32_t ttm_bo_type_flags(unsigned type) 133 { 134 return 1 << (type); 135 } 136 137 static void ttm_bo_release_list(struct kref *list_kref) 138 { 139 struct ttm_buffer_object *bo = 140 container_of(list_kref, struct ttm_buffer_object, list_kref); 141 struct ttm_bo_device *bdev = bo->bdev; 142 size_t acc_size = bo->acc_size; 143 144 BUG_ON(atomic_read(&bo->list_kref.refcount)); 145 BUG_ON(atomic_read(&bo->kref.refcount)); 146 BUG_ON(atomic_read(&bo->cpu_writers)); 147 BUG_ON(bo->sync_obj != NULL); 148 BUG_ON(bo->mem.mm_node != NULL); 149 BUG_ON(!list_empty(&bo->lru)); 150 BUG_ON(!list_empty(&bo->ddestroy)); 151 152 if (bo->ttm) 153 ttm_tt_destroy(bo->ttm); 154 atomic_dec(&bo->glob->bo_count); 155 if (bo->resv == &bo->ttm_resv) 156 reservation_object_fini(&bo->ttm_resv); 157 158 if (bo->destroy) 159 bo->destroy(bo); 160 else { 161 kfree(bo); 162 } 163 ttm_mem_global_free(bdev->glob->mem_glob, acc_size); 164 } 165 166 void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) 167 { 168 struct ttm_bo_device *bdev = bo->bdev; 169 struct ttm_mem_type_manager *man; 170 171 lockdep_assert_held(&bo->resv->lock.base); 172 173 if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { 174 175 BUG_ON(!list_empty(&bo->lru)); 176 177 man = &bdev->man[bo->mem.mem_type]; 178 list_add_tail(&bo->lru, &man->lru); 179 kref_get(&bo->list_kref); 180 181 if (bo->ttm != NULL) { 182 list_add_tail(&bo->swap, &bo->glob->swap_lru); 183 kref_get(&bo->list_kref); 184 } 185 } 186 } 187 EXPORT_SYMBOL(ttm_bo_add_to_lru); 188 189 int ttm_bo_del_from_lru(struct ttm_buffer_object *bo) 190 { 191 int put_count = 0; 192 193 if (!list_empty(&bo->swap)) { 194 list_del_init(&bo->swap); 195 ++put_count; 196 } 197 if (!list_empty(&bo->lru)) { 198 list_del_init(&bo->lru); 199 ++put_count; 200 } 201 202 /* 203 * TODO: Add a driver hook to delete from 204 * driver-specific LRU's here. 205 */ 206 207 return put_count; 208 } 209 210 static void ttm_bo_ref_bug(struct kref *list_kref) 211 { 212 BUG(); 213 } 214 215 void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count, 216 bool never_free) 217 { 218 kref_sub(&bo->list_kref, count, 219 (never_free) ? ttm_bo_ref_bug : ttm_bo_release_list); 220 } 221 222 void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo) 223 { 224 int put_count; 225 226 lockmgr(&bo->glob->lru_lock, LK_EXCLUSIVE); 227 put_count = ttm_bo_del_from_lru(bo); 228 lockmgr(&bo->glob->lru_lock, LK_RELEASE); 229 ttm_bo_list_ref_sub(bo, put_count, true); 230 } 231 EXPORT_SYMBOL(ttm_bo_del_sub_from_lru); 232 233 /* 234 * Call bo->mutex locked. 235 */ 236 static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) 237 { 238 struct ttm_bo_device *bdev = bo->bdev; 239 struct ttm_bo_global *glob = bo->glob; 240 int ret = 0; 241 uint32_t page_flags = 0; 242 243 TTM_ASSERT_LOCKED(&bo->mutex); 244 bo->ttm = NULL; 245 246 if (bdev->need_dma32) 247 page_flags |= TTM_PAGE_FLAG_DMA32; 248 249 switch (bo->type) { 250 case ttm_bo_type_device: 251 if (zero_alloc) 252 page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC; 253 case ttm_bo_type_kernel: 254 bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, 255 page_flags, glob->dummy_read_page); 256 if (unlikely(bo->ttm == NULL)) 257 ret = -ENOMEM; 258 break; 259 case ttm_bo_type_sg: 260 bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, 261 page_flags | TTM_PAGE_FLAG_SG, 262 glob->dummy_read_page); 263 if (unlikely(bo->ttm == NULL)) { 264 ret = -ENOMEM; 265 break; 266 } 267 bo->ttm->sg = bo->sg; 268 break; 269 default: 270 pr_err("Illegal buffer object type\n"); 271 ret = -EINVAL; 272 break; 273 } 274 275 return ret; 276 } 277 278 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, 279 struct ttm_mem_reg *mem, 280 bool evict, bool interruptible, 281 bool no_wait_gpu) 282 { 283 struct ttm_bo_device *bdev = bo->bdev; 284 bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); 285 bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem); 286 struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type]; 287 struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type]; 288 int ret = 0; 289 290 if (old_is_pci || new_is_pci || 291 ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) { 292 ret = ttm_mem_io_lock(old_man, true); 293 if (unlikely(ret != 0)) 294 goto out_err; 295 ttm_bo_unmap_virtual_locked(bo); 296 ttm_mem_io_unlock(old_man); 297 } 298 299 /* 300 * Create and bind a ttm if required. 301 */ 302 303 if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 304 if (bo->ttm == NULL) { 305 bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED); 306 ret = ttm_bo_add_ttm(bo, zero); 307 if (ret) 308 goto out_err; 309 } 310 311 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); 312 if (ret) 313 goto out_err; 314 315 if (mem->mem_type != TTM_PL_SYSTEM) { 316 ret = ttm_tt_bind(bo->ttm, mem); 317 if (ret) 318 goto out_err; 319 } 320 321 if (bo->mem.mem_type == TTM_PL_SYSTEM) { 322 if (bdev->driver->move_notify) 323 bdev->driver->move_notify(bo, mem); 324 bo->mem = *mem; 325 mem->mm_node = NULL; 326 goto moved; 327 } 328 } 329 330 if (bdev->driver->move_notify) 331 bdev->driver->move_notify(bo, mem); 332 333 if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && 334 !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) 335 ret = ttm_bo_move_ttm(bo, evict, no_wait_gpu, mem); 336 else if (bdev->driver->move) 337 ret = bdev->driver->move(bo, evict, interruptible, 338 no_wait_gpu, mem); 339 else 340 ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, mem); 341 342 if (ret) { 343 if (bdev->driver->move_notify) { 344 struct ttm_mem_reg tmp_mem = *mem; 345 *mem = bo->mem; 346 bo->mem = tmp_mem; 347 bdev->driver->move_notify(bo, mem); 348 bo->mem = *mem; 349 *mem = tmp_mem; 350 } 351 352 goto out_err; 353 } 354 355 moved: 356 if (bo->evicted) { 357 ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement); 358 if (ret) 359 pr_err("Can not flush read caches\n"); 360 bo->evicted = false; 361 } 362 363 if (bo->mem.mm_node) { 364 bo->offset = (bo->mem.start << PAGE_SHIFT) + 365 bdev->man[bo->mem.mem_type].gpu_offset; 366 bo->cur_placement = bo->mem.placement; 367 } else 368 bo->offset = 0; 369 370 return 0; 371 372 out_err: 373 new_man = &bdev->man[bo->mem.mem_type]; 374 if ((new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm) { 375 ttm_tt_unbind(bo->ttm); 376 ttm_tt_destroy(bo->ttm); 377 bo->ttm = NULL; 378 } 379 380 return ret; 381 } 382 383 /** 384 * Call bo::reserved. 385 * Will release GPU memory type usage on destruction. 386 * This is the place to put in driver specific hooks to release 387 * driver private resources. 388 * Will release the bo::reserved lock. 389 */ 390 391 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) 392 { 393 if (bo->bdev->driver->move_notify) 394 bo->bdev->driver->move_notify(bo, NULL); 395 396 if (bo->ttm) { 397 ttm_tt_unbind(bo->ttm); 398 ttm_tt_destroy(bo->ttm); 399 bo->ttm = NULL; 400 } 401 ttm_bo_mem_put(bo, &bo->mem); 402 403 ww_mutex_unlock (&bo->resv->lock); 404 } 405 406 static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) 407 { 408 struct ttm_bo_device *bdev = bo->bdev; 409 struct ttm_bo_global *glob = bo->glob; 410 struct ttm_bo_driver *driver = bdev->driver; 411 void *sync_obj = NULL; 412 int put_count; 413 int ret; 414 415 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 416 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); 417 418 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 419 (void) ttm_bo_wait(bo, false, false, true); 420 if (!ret && !bo->sync_obj) { 421 lockmgr(&bdev->fence_lock, LK_RELEASE); 422 put_count = ttm_bo_del_from_lru(bo); 423 424 lockmgr(&glob->lru_lock, LK_RELEASE); 425 ttm_bo_cleanup_memtype_use(bo); 426 427 ttm_bo_list_ref_sub(bo, put_count, true); 428 429 return; 430 } 431 if (bo->sync_obj) 432 sync_obj = driver->sync_obj_ref(bo->sync_obj); 433 lockmgr(&bdev->fence_lock, LK_RELEASE); 434 435 if (!ret) 436 ww_mutex_unlock(&bo->resv->lock); 437 438 kref_get(&bo->list_kref); 439 list_add_tail(&bo->ddestroy, &bdev->ddestroy); 440 lockmgr(&glob->lru_lock, LK_RELEASE); 441 442 if (sync_obj) { 443 driver->sync_obj_flush(sync_obj); 444 driver->sync_obj_unref(&sync_obj); 445 } 446 schedule_delayed_work(&bdev->wq, 447 ((HZ / 100) < 1) ? 1 : HZ / 100); 448 } 449 450 /** 451 * function ttm_bo_cleanup_refs_and_unlock 452 * If bo idle, remove from delayed- and lru lists, and unref. 453 * If not idle, do nothing. 454 * 455 * Must be called with lru_lock and reservation held, this function 456 * will drop both before returning. 457 * 458 * @interruptible Any sleeps should occur interruptibly. 459 * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. 460 */ 461 462 static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, 463 bool interruptible, 464 bool no_wait_gpu) 465 { 466 struct ttm_bo_device *bdev = bo->bdev; 467 struct ttm_bo_driver *driver = bdev->driver; 468 struct ttm_bo_global *glob = bo->glob; 469 int put_count; 470 int ret; 471 472 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 473 ret = ttm_bo_wait(bo, false, false, true); 474 475 if (ret && !no_wait_gpu) { 476 void *sync_obj; 477 478 /* 479 * Take a reference to the fence and unreserve, 480 * at this point the buffer should be dead, so 481 * no new sync objects can be attached. 482 */ 483 sync_obj = driver->sync_obj_ref(bo->sync_obj); 484 lockmgr(&bdev->fence_lock, LK_RELEASE); 485 486 ww_mutex_unlock(&bo->resv->lock); 487 lockmgr(&glob->lru_lock, LK_RELEASE); 488 489 ret = driver->sync_obj_wait(sync_obj, false, interruptible); 490 driver->sync_obj_unref(&sync_obj); 491 if (ret) 492 return ret; 493 494 /* 495 * remove sync_obj with ttm_bo_wait, the wait should be 496 * finished, and no new wait object should have been added. 497 */ 498 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 499 ret = ttm_bo_wait(bo, false, false, true); 500 WARN_ON(ret); 501 lockmgr(&bdev->fence_lock, LK_RELEASE); 502 if (ret) 503 return ret; 504 505 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 506 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); 507 508 /* 509 * We raced, and lost, someone else holds the reservation now, 510 * and is probably busy in ttm_bo_cleanup_memtype_use. 511 * 512 * Even if it's not the case, because we finished waiting any 513 * delayed destruction would succeed, so just return success 514 * here. 515 */ 516 if (ret) { 517 lockmgr(&glob->lru_lock, LK_RELEASE); 518 return 0; 519 } 520 } else 521 lockmgr(&bdev->fence_lock, LK_RELEASE); 522 523 if (ret || unlikely(list_empty(&bo->ddestroy))) { 524 ww_mutex_unlock(&bo->resv->lock); 525 lockmgr(&glob->lru_lock, LK_RELEASE); 526 return ret; 527 } 528 529 put_count = ttm_bo_del_from_lru(bo); 530 list_del_init(&bo->ddestroy); 531 ++put_count; 532 533 lockmgr(&glob->lru_lock, LK_RELEASE); 534 ttm_bo_cleanup_memtype_use(bo); 535 536 ttm_bo_list_ref_sub(bo, put_count, true); 537 538 return 0; 539 } 540 541 /** 542 * Traverse the delayed list, and call ttm_bo_cleanup_refs on all 543 * encountered buffers. 544 */ 545 546 static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) 547 { 548 struct ttm_bo_global *glob = bdev->glob; 549 struct ttm_buffer_object *entry = NULL; 550 int ret = 0; 551 552 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 553 if (list_empty(&bdev->ddestroy)) 554 goto out_unlock; 555 556 entry = list_first_entry(&bdev->ddestroy, 557 struct ttm_buffer_object, ddestroy); 558 kref_get(&entry->list_kref); 559 560 for (;;) { 561 struct ttm_buffer_object *nentry = NULL; 562 563 if (entry->ddestroy.next != &bdev->ddestroy) { 564 nentry = list_first_entry(&entry->ddestroy, 565 struct ttm_buffer_object, ddestroy); 566 kref_get(&nentry->list_kref); 567 } 568 569 ret = ttm_bo_reserve_nolru(entry, false, true, false, 0); 570 if (remove_all && ret) { 571 lockmgr(&glob->lru_lock, LK_RELEASE); 572 ret = ttm_bo_reserve_nolru(entry, false, false, 573 false, 0); 574 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 575 } 576 577 if (!ret) 578 ret = ttm_bo_cleanup_refs_and_unlock(entry, false, 579 !remove_all); 580 else 581 lockmgr(&glob->lru_lock, LK_RELEASE); 582 583 kref_put(&entry->list_kref, ttm_bo_release_list); 584 entry = nentry; 585 586 if (ret || !entry) 587 goto out; 588 589 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 590 if (list_empty(&entry->ddestroy)) 591 break; 592 } 593 594 out_unlock: 595 lockmgr(&glob->lru_lock, LK_RELEASE); 596 out: 597 if (entry) 598 kref_put(&entry->list_kref, ttm_bo_release_list); 599 return ret; 600 } 601 602 static void ttm_bo_delayed_workqueue(struct work_struct *work) 603 { 604 struct ttm_bo_device *bdev = 605 container_of(work, struct ttm_bo_device, wq.work); 606 607 if (ttm_bo_delayed_delete(bdev, false)) { 608 schedule_delayed_work(&bdev->wq, 609 ((HZ / 100) < 1) ? 1 : HZ / 100); 610 } 611 } 612 613 static void ttm_bo_release(struct kref *kref) 614 { 615 struct ttm_buffer_object *bo = 616 container_of(kref, struct ttm_buffer_object, kref); 617 struct ttm_bo_device *bdev = bo->bdev; 618 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 619 620 drm_vma_offset_remove(&bdev->vma_manager, &bo->vma_node); 621 ttm_mem_io_lock(man, false); 622 ttm_mem_io_free_vm(bo); 623 ttm_mem_io_unlock(man); 624 ttm_bo_cleanup_refs_or_queue(bo); 625 kref_put(&bo->list_kref, ttm_bo_release_list); 626 } 627 628 void ttm_bo_unref(struct ttm_buffer_object **p_bo) 629 { 630 struct ttm_buffer_object *bo = *p_bo; 631 632 *p_bo = NULL; 633 kref_put(&bo->kref, ttm_bo_release); 634 } 635 EXPORT_SYMBOL(ttm_bo_unref); 636 637 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev) 638 { 639 return cancel_delayed_work_sync(&bdev->wq); 640 } 641 EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue); 642 643 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) 644 { 645 if (resched) 646 schedule_delayed_work(&bdev->wq, 647 ((HZ / 100) < 1) ? 1 : HZ / 100); 648 } 649 EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue); 650 651 static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, 652 bool no_wait_gpu) 653 { 654 struct ttm_bo_device *bdev = bo->bdev; 655 struct ttm_mem_reg evict_mem; 656 struct ttm_placement placement; 657 int ret = 0; 658 659 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 660 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 661 lockmgr(&bdev->fence_lock, LK_RELEASE); 662 663 if (unlikely(ret != 0)) { 664 if (ret != -ERESTARTSYS) { 665 pr_err("Failed to expire sync object before buffer eviction\n"); 666 } 667 goto out; 668 } 669 670 lockdep_assert_held(&bo->resv->lock.base); 671 672 evict_mem = bo->mem; 673 evict_mem.mm_node = NULL; 674 evict_mem.bus.io_reserved_vm = false; 675 evict_mem.bus.io_reserved_count = 0; 676 677 placement.fpfn = 0; 678 placement.lpfn = 0; 679 placement.num_placement = 0; 680 placement.num_busy_placement = 0; 681 bdev->driver->evict_flags(bo, &placement); 682 ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible, 683 no_wait_gpu); 684 if (ret) { 685 if (ret != -ERESTARTSYS) { 686 pr_err("Failed to find memory space for buffer 0x%p eviction\n", 687 bo); 688 ttm_bo_mem_space_debug(bo, &placement); 689 } 690 goto out; 691 } 692 693 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible, 694 no_wait_gpu); 695 if (ret) { 696 if (ret != -ERESTARTSYS) 697 pr_err("Buffer eviction failed\n"); 698 ttm_bo_mem_put(bo, &evict_mem); 699 goto out; 700 } 701 bo->evicted = true; 702 out: 703 return ret; 704 } 705 706 static int ttm_mem_evict_first(struct ttm_bo_device *bdev, 707 uint32_t mem_type, 708 bool interruptible, 709 bool no_wait_gpu) 710 { 711 struct ttm_bo_global *glob = bdev->glob; 712 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 713 struct ttm_buffer_object *bo; 714 int ret = -EBUSY, put_count; 715 716 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 717 list_for_each_entry(bo, &man->lru, lru) { 718 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); 719 if (!ret) 720 break; 721 } 722 723 if (ret) { 724 lockmgr(&glob->lru_lock, LK_RELEASE); 725 return ret; 726 } 727 728 kref_get(&bo->list_kref); 729 730 if (!list_empty(&bo->ddestroy)) { 731 ret = ttm_bo_cleanup_refs_and_unlock(bo, interruptible, 732 no_wait_gpu); 733 kref_put(&bo->list_kref, ttm_bo_release_list); 734 return ret; 735 } 736 737 put_count = ttm_bo_del_from_lru(bo); 738 lockmgr(&glob->lru_lock, LK_RELEASE); 739 740 BUG_ON(ret != 0); 741 742 ttm_bo_list_ref_sub(bo, put_count, true); 743 744 ret = ttm_bo_evict(bo, interruptible, no_wait_gpu); 745 ttm_bo_unreserve(bo); 746 747 kref_put(&bo->list_kref, ttm_bo_release_list); 748 return ret; 749 } 750 751 void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) 752 { 753 struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type]; 754 755 if (mem->mm_node) 756 (*man->func->put_node)(man, mem); 757 } 758 EXPORT_SYMBOL(ttm_bo_mem_put); 759 760 /** 761 * Repeatedly evict memory from the LRU for @mem_type until we create enough 762 * space, or we've evicted everything and there isn't enough space. 763 */ 764 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, 765 uint32_t mem_type, 766 struct ttm_placement *placement, 767 struct ttm_mem_reg *mem, 768 bool interruptible, 769 bool no_wait_gpu) 770 { 771 struct ttm_bo_device *bdev = bo->bdev; 772 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 773 int ret; 774 775 do { 776 ret = (*man->func->get_node)(man, bo, placement, mem); 777 if (unlikely(ret != 0)) 778 return ret; 779 if (mem->mm_node) 780 break; 781 ret = ttm_mem_evict_first(bdev, mem_type, 782 interruptible, no_wait_gpu); 783 if (unlikely(ret != 0)) 784 return ret; 785 } while (1); 786 if (mem->mm_node == NULL) 787 return -ENOMEM; 788 mem->mem_type = mem_type; 789 return 0; 790 } 791 792 static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man, 793 uint32_t cur_placement, 794 uint32_t proposed_placement) 795 { 796 uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING; 797 uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING; 798 799 /** 800 * Keep current caching if possible. 801 */ 802 803 if ((cur_placement & caching) != 0) 804 result |= (cur_placement & caching); 805 else if ((man->default_caching & caching) != 0) 806 result |= man->default_caching; 807 else if ((TTM_PL_FLAG_CACHED & caching) != 0) 808 result |= TTM_PL_FLAG_CACHED; 809 else if ((TTM_PL_FLAG_WC & caching) != 0) 810 result |= TTM_PL_FLAG_WC; 811 else if ((TTM_PL_FLAG_UNCACHED & caching) != 0) 812 result |= TTM_PL_FLAG_UNCACHED; 813 814 return result; 815 } 816 817 static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, 818 uint32_t mem_type, 819 const struct ttm_place *place, 820 uint32_t *masked_placement) 821 { 822 uint32_t cur_flags = ttm_bo_type_flags(mem_type); 823 824 if ((cur_flags & place->flags & TTM_PL_MASK_MEM) == 0) 825 return false; 826 827 if ((place->flags & man->available_caching) == 0) 828 return false; 829 830 cur_flags |= (place->flags & man->available_caching); 831 832 *masked_placement = cur_flags; 833 return true; 834 } 835 836 /** 837 * Creates space for memory region @mem according to its type. 838 * 839 * This function first searches for free space in compatible memory types in 840 * the priority order defined by the driver. If free space isn't found, then 841 * ttm_bo_mem_force_space is attempted in priority order to evict and find 842 * space. 843 */ 844 int ttm_bo_mem_space(struct ttm_buffer_object *bo, 845 struct ttm_placement *placement, 846 struct ttm_mem_reg *mem, 847 bool interruptible, 848 bool no_wait_gpu) 849 { 850 struct ttm_bo_device *bdev = bo->bdev; 851 struct ttm_mem_type_manager *man; 852 uint32_t mem_type = TTM_PL_SYSTEM; 853 uint32_t cur_flags = 0; 854 bool type_found = false; 855 bool type_ok = false; 856 bool has_erestartsys = false; 857 int i, ret; 858 859 mem->mm_node = NULL; 860 for (i = 0; i < placement->num_placement; ++i) { 861 ret = ttm_mem_type_from_place(&placement->placement[i], 862 &mem_type); 863 if (ret) 864 return ret; 865 man = &bdev->man[mem_type]; 866 867 type_ok = ttm_bo_mt_compatible(man, 868 mem_type, 869 &placement->placement[i], 870 &cur_flags); 871 872 if (!type_ok) 873 continue; 874 875 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 876 cur_flags); 877 /* 878 * Use the access and other non-mapping-related flag bits from 879 * the memory placement flags to the current flags 880 */ 881 ttm_flag_masked(&cur_flags, placement->placement[i].flags, 882 ~TTM_PL_MASK_MEMTYPE); 883 884 if (mem_type == TTM_PL_SYSTEM) 885 break; 886 887 if (man->has_type && man->use_type) { 888 type_found = true; 889 ret = (*man->func->get_node)(man, bo, placement, mem); 890 if (unlikely(ret)) 891 return ret; 892 } 893 if (mem->mm_node) 894 break; 895 } 896 897 if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) { 898 mem->mem_type = mem_type; 899 mem->placement = cur_flags; 900 return 0; 901 } 902 903 if (!type_found) 904 return -EINVAL; 905 906 for (i = 0; i < placement->num_busy_placement; ++i) { 907 ret = ttm_mem_type_from_place(&placement->busy_placement[i], 908 &mem_type); 909 if (ret) 910 return ret; 911 man = &bdev->man[mem_type]; 912 if (!man->has_type) 913 continue; 914 if (!ttm_bo_mt_compatible(man, 915 mem_type, 916 &placement->busy_placement[i], 917 &cur_flags)) 918 continue; 919 920 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 921 cur_flags); 922 /* 923 * Use the access and other non-mapping-related flag bits from 924 * the memory placement flags to the current flags 925 */ 926 ttm_flag_masked(&cur_flags, placement->busy_placement[i].flags, 927 ~TTM_PL_MASK_MEMTYPE); 928 929 930 if (mem_type == TTM_PL_SYSTEM) { 931 mem->mem_type = mem_type; 932 mem->placement = cur_flags; 933 mem->mm_node = NULL; 934 return 0; 935 } 936 937 ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem, 938 interruptible, no_wait_gpu); 939 if (ret == 0 && mem->mm_node) { 940 mem->placement = cur_flags; 941 return 0; 942 } 943 if (ret == -ERESTARTSYS) 944 has_erestartsys = true; 945 } 946 ret = (has_erestartsys) ? -ERESTARTSYS : -ENOMEM; 947 return ret; 948 } 949 EXPORT_SYMBOL(ttm_bo_mem_space); 950 951 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, 952 struct ttm_placement *placement, 953 bool interruptible, 954 bool no_wait_gpu) 955 { 956 int ret = 0; 957 struct ttm_mem_reg mem; 958 struct ttm_bo_device *bdev = bo->bdev; 959 960 lockdep_assert_held(&bo->resv->lock.base); 961 962 /* 963 * FIXME: It's possible to pipeline buffer moves. 964 * Have the driver move function wait for idle when necessary, 965 * instead of doing it here. 966 */ 967 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 968 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 969 lockmgr(&bdev->fence_lock, LK_RELEASE); 970 if (ret) 971 return ret; 972 mem.num_pages = bo->num_pages; 973 mem.size = mem.num_pages << PAGE_SHIFT; 974 mem.page_alignment = bo->mem.page_alignment; 975 mem.bus.io_reserved_vm = false; 976 mem.bus.io_reserved_count = 0; 977 /* 978 * Determine where to move the buffer. 979 */ 980 ret = ttm_bo_mem_space(bo, placement, &mem, 981 interruptible, no_wait_gpu); 982 if (ret) 983 goto out_unlock; 984 ret = ttm_bo_handle_move_mem(bo, &mem, false, 985 interruptible, no_wait_gpu); 986 out_unlock: 987 if (ret && mem.mm_node) 988 ttm_bo_mem_put(bo, &mem); 989 return ret; 990 } 991 992 static int ttm_bo_mem_compat(struct ttm_placement *placement, 993 struct ttm_mem_reg *mem) 994 { 995 int i; 996 997 if (mem->mm_node && placement->lpfn != 0 && 998 (mem->start < placement->fpfn || 999 mem->start + mem->num_pages > placement->lpfn)) 1000 return -1; 1001 1002 for (i = 0; i < placement->num_placement; i++) { 1003 if ((placement->placement[i].flags & mem->placement & 1004 TTM_PL_MASK_CACHING) && 1005 (placement->placement[i].flags & mem->placement & 1006 TTM_PL_MASK_MEM)) 1007 return i; 1008 } 1009 return -1; 1010 } 1011 1012 int ttm_bo_validate(struct ttm_buffer_object *bo, 1013 struct ttm_placement *placement, 1014 bool interruptible, 1015 bool no_wait_gpu) 1016 { 1017 int ret; 1018 1019 lockdep_assert_held(&bo->resv->lock.base); 1020 /* Check that range is valid */ 1021 if (placement->lpfn || placement->fpfn) 1022 if (placement->fpfn > placement->lpfn || 1023 (placement->lpfn - placement->fpfn) < bo->num_pages) 1024 return -EINVAL; 1025 /* 1026 * Check whether we need to move buffer. 1027 */ 1028 ret = ttm_bo_mem_compat(placement, &bo->mem); 1029 if (ret < 0) { 1030 ret = ttm_bo_move_buffer(bo, placement, interruptible, 1031 no_wait_gpu); 1032 if (ret) 1033 return ret; 1034 } else { 1035 /* 1036 * Use the access and other non-mapping-related flag bits from 1037 * the compatible memory placement flags to the active flags 1038 */ 1039 ttm_flag_masked(&bo->mem.placement, 1040 placement->placement[ret].flags, 1041 ~TTM_PL_MASK_MEMTYPE); 1042 } 1043 /* 1044 * We might need to add a TTM. 1045 */ 1046 if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { 1047 ret = ttm_bo_add_ttm(bo, true); 1048 if (ret) 1049 return ret; 1050 } 1051 return 0; 1052 } 1053 EXPORT_SYMBOL(ttm_bo_validate); 1054 1055 int ttm_bo_check_placement(struct ttm_buffer_object *bo, 1056 struct ttm_placement *placement) 1057 { 1058 BUG_ON((placement->fpfn || placement->lpfn) && 1059 (bo->mem.num_pages > (placement->lpfn - placement->fpfn))); 1060 1061 return 0; 1062 } 1063 1064 int ttm_bo_init(struct ttm_bo_device *bdev, 1065 struct ttm_buffer_object *bo, 1066 unsigned long size, 1067 enum ttm_bo_type type, 1068 struct ttm_placement *placement, 1069 uint32_t page_alignment, 1070 bool interruptible, 1071 struct vm_object *persistent_swap_storage, 1072 size_t acc_size, 1073 struct sg_table *sg, 1074 void (*destroy) (struct ttm_buffer_object *)) 1075 { 1076 int ret = 0; 1077 unsigned long num_pages; 1078 struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; 1079 bool locked; 1080 1081 ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false); 1082 if (ret) { 1083 pr_err("Out of kernel memory\n"); 1084 if (destroy) 1085 (*destroy)(bo); 1086 else 1087 kfree(bo); 1088 return -ENOMEM; 1089 } 1090 1091 num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 1092 if (num_pages == 0) { 1093 pr_err("Illegal buffer object size\n"); 1094 if (destroy) 1095 (*destroy)(bo); 1096 else 1097 kfree(bo); 1098 ttm_mem_global_free(mem_glob, acc_size); 1099 return -EINVAL; 1100 } 1101 bo->destroy = destroy; 1102 1103 kref_init(&bo->kref); 1104 kref_init(&bo->list_kref); 1105 atomic_set(&bo->cpu_writers, 0); 1106 INIT_LIST_HEAD(&bo->lru); 1107 INIT_LIST_HEAD(&bo->ddestroy); 1108 INIT_LIST_HEAD(&bo->swap); 1109 INIT_LIST_HEAD(&bo->io_reserve_lru); 1110 bo->bdev = bdev; 1111 bo->glob = bdev->glob; 1112 bo->type = type; 1113 bo->num_pages = num_pages; 1114 bo->mem.size = num_pages << PAGE_SHIFT; 1115 bo->mem.mem_type = TTM_PL_SYSTEM; 1116 bo->mem.num_pages = bo->num_pages; 1117 bo->mem.mm_node = NULL; 1118 bo->mem.page_alignment = page_alignment; 1119 bo->mem.bus.io_reserved_vm = false; 1120 bo->mem.bus.io_reserved_count = 0; 1121 bo->priv_flags = 0; 1122 bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); 1123 bo->persistent_swap_storage = persistent_swap_storage; 1124 bo->acc_size = acc_size; 1125 bo->sg = sg; 1126 bo->resv = &bo->ttm_resv; 1127 reservation_object_init(bo->resv); 1128 atomic_inc(&bo->glob->bo_count); 1129 drm_vma_node_reset(&bo->vma_node); 1130 1131 ret = ttm_bo_check_placement(bo, placement); 1132 1133 /* 1134 * For ttm_bo_type_device buffers, allocate 1135 * address space from the device. 1136 */ 1137 if (likely(!ret) && 1138 (bo->type == ttm_bo_type_device || 1139 bo->type == ttm_bo_type_sg)) 1140 ret = ttm_bo_setup_vm(bo); 1141 1142 locked = ww_mutex_trylock(&bo->resv->lock); 1143 WARN_ON(!locked); 1144 1145 if (likely(!ret)) 1146 ret = ttm_bo_validate(bo, placement, interruptible, false); 1147 1148 ttm_bo_unreserve(bo); 1149 1150 if (unlikely(ret)) 1151 ttm_bo_unref(&bo); 1152 1153 return ret; 1154 } 1155 EXPORT_SYMBOL(ttm_bo_init); 1156 1157 size_t ttm_bo_acc_size(struct ttm_bo_device *bdev, 1158 unsigned long bo_size, 1159 unsigned struct_size) 1160 { 1161 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1162 size_t size = 0; 1163 1164 size += ttm_round_pot(struct_size); 1165 size += PAGE_ALIGN(npages * sizeof(void *)); 1166 size += ttm_round_pot(sizeof(struct ttm_tt)); 1167 return size; 1168 } 1169 EXPORT_SYMBOL(ttm_bo_acc_size); 1170 1171 size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, 1172 unsigned long bo_size, 1173 unsigned struct_size) 1174 { 1175 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1176 size_t size = 0; 1177 1178 size += ttm_round_pot(struct_size); 1179 size += PAGE_ALIGN(npages * sizeof(void *)); 1180 size += PAGE_ALIGN(npages * sizeof(dma_addr_t)); 1181 size += ttm_round_pot(sizeof(struct ttm_dma_tt)); 1182 return size; 1183 } 1184 EXPORT_SYMBOL(ttm_bo_dma_acc_size); 1185 1186 int ttm_bo_create(struct ttm_bo_device *bdev, 1187 unsigned long size, 1188 enum ttm_bo_type type, 1189 struct ttm_placement *placement, 1190 uint32_t page_alignment, 1191 bool interruptible, 1192 struct vm_object *persistent_swap_storage, 1193 struct ttm_buffer_object **p_bo) 1194 { 1195 struct ttm_buffer_object *bo; 1196 size_t acc_size; 1197 int ret; 1198 1199 bo = kzalloc(sizeof(*bo), GFP_KERNEL); 1200 if (unlikely(bo == NULL)) 1201 return -ENOMEM; 1202 1203 acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); 1204 ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, 1205 interruptible, persistent_swap_storage, acc_size, 1206 NULL, NULL); 1207 if (likely(ret == 0)) 1208 *p_bo = bo; 1209 1210 return ret; 1211 } 1212 EXPORT_SYMBOL(ttm_bo_create); 1213 1214 static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, 1215 unsigned mem_type, bool allow_errors) 1216 { 1217 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1218 struct ttm_bo_global *glob = bdev->glob; 1219 int ret; 1220 1221 /* 1222 * Can't use standard list traversal since we're unlocking. 1223 */ 1224 1225 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 1226 while (!list_empty(&man->lru)) { 1227 lockmgr(&glob->lru_lock, LK_RELEASE); 1228 ret = ttm_mem_evict_first(bdev, mem_type, false, false); 1229 if (ret) { 1230 if (allow_errors) { 1231 return ret; 1232 } else { 1233 pr_err("Cleanup eviction failed\n"); 1234 } 1235 } 1236 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 1237 } 1238 lockmgr(&glob->lru_lock, LK_RELEASE); 1239 return 0; 1240 } 1241 1242 int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1243 { 1244 struct ttm_mem_type_manager *man; 1245 int ret = -EINVAL; 1246 1247 if (mem_type >= TTM_NUM_MEM_TYPES) { 1248 pr_err("Illegal memory type %d\n", mem_type); 1249 return ret; 1250 } 1251 man = &bdev->man[mem_type]; 1252 1253 if (!man->has_type) { 1254 pr_err("Trying to take down uninitialized memory manager type %u\n", 1255 mem_type); 1256 return ret; 1257 } 1258 1259 man->use_type = false; 1260 man->has_type = false; 1261 1262 ret = 0; 1263 if (mem_type > 0) { 1264 ttm_bo_force_list_clean(bdev, mem_type, false); 1265 1266 ret = (*man->func->takedown)(man); 1267 } 1268 1269 return ret; 1270 } 1271 EXPORT_SYMBOL(ttm_bo_clean_mm); 1272 1273 int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1274 { 1275 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1276 1277 if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) { 1278 pr_err("Illegal memory manager memory type %u\n", mem_type); 1279 return -EINVAL; 1280 } 1281 1282 if (!man->has_type) { 1283 pr_err("Memory type %u has not been initialized\n", mem_type); 1284 return 0; 1285 } 1286 1287 return ttm_bo_force_list_clean(bdev, mem_type, true); 1288 } 1289 EXPORT_SYMBOL(ttm_bo_evict_mm); 1290 1291 int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, 1292 unsigned long p_size) 1293 { 1294 int ret = -EINVAL; 1295 struct ttm_mem_type_manager *man; 1296 1297 BUG_ON(type >= TTM_NUM_MEM_TYPES); 1298 man = &bdev->man[type]; 1299 BUG_ON(man->has_type); 1300 man->io_reserve_fastpath = true; 1301 man->use_io_reserve_lru = false; 1302 lockinit(&man->io_reserve_mutex, "ttmior", 0, 0); 1303 INIT_LIST_HEAD(&man->io_reserve_lru); 1304 1305 ret = bdev->driver->init_mem_type(bdev, type, man); 1306 if (ret) 1307 return ret; 1308 man->bdev = bdev; 1309 1310 ret = 0; 1311 if (type != TTM_PL_SYSTEM) { 1312 ret = (*man->func->init)(man, p_size); 1313 if (ret) 1314 return ret; 1315 } 1316 man->has_type = true; 1317 man->use_type = true; 1318 man->size = p_size; 1319 1320 INIT_LIST_HEAD(&man->lru); 1321 1322 return 0; 1323 } 1324 EXPORT_SYMBOL(ttm_bo_init_mm); 1325 1326 static void ttm_bo_global_kobj_release(struct kobject *kobj) 1327 { 1328 struct ttm_bo_global *glob = 1329 container_of(kobj, struct ttm_bo_global, kobj); 1330 1331 ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink); 1332 __free_page(glob->dummy_read_page); 1333 kfree(glob); 1334 } 1335 1336 void ttm_bo_global_release(struct drm_global_reference *ref) 1337 { 1338 struct ttm_bo_global *glob = ref->object; 1339 1340 kobject_del(&glob->kobj); 1341 kobject_put(&glob->kobj); 1342 } 1343 EXPORT_SYMBOL(ttm_bo_global_release); 1344 1345 int ttm_bo_global_init(struct drm_global_reference *ref) 1346 { 1347 struct ttm_bo_global_ref *bo_ref = 1348 container_of(ref, struct ttm_bo_global_ref, ref); 1349 struct ttm_bo_global *glob = ref->object; 1350 int ret; 1351 1352 lockinit(&glob->device_list_mutex, "ttmdlm", 0, 0); 1353 lockinit(&glob->lru_lock, "ttmlru", 0, 0); 1354 glob->mem_glob = bo_ref->mem_glob; 1355 glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32); 1356 1357 if (unlikely(glob->dummy_read_page == NULL)) { 1358 ret = -ENOMEM; 1359 goto out_no_drp; 1360 } 1361 1362 INIT_LIST_HEAD(&glob->swap_lru); 1363 INIT_LIST_HEAD(&glob->device_list); 1364 1365 ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout); 1366 ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink); 1367 if (unlikely(ret != 0)) { 1368 pr_err("Could not register buffer object swapout\n"); 1369 goto out_no_shrink; 1370 } 1371 1372 atomic_set(&glob->bo_count, 0); 1373 1374 ret = kobject_init_and_add( 1375 &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects"); 1376 if (unlikely(ret != 0)) 1377 kobject_put(&glob->kobj); 1378 return ret; 1379 out_no_shrink: 1380 __free_page(glob->dummy_read_page); 1381 out_no_drp: 1382 kfree(glob); 1383 return ret; 1384 } 1385 EXPORT_SYMBOL(ttm_bo_global_init); 1386 1387 1388 int ttm_bo_device_release(struct ttm_bo_device *bdev) 1389 { 1390 int ret = 0; 1391 unsigned i = TTM_NUM_MEM_TYPES; 1392 struct ttm_mem_type_manager *man; 1393 struct ttm_bo_global *glob = bdev->glob; 1394 1395 while (i--) { 1396 man = &bdev->man[i]; 1397 if (man->has_type) { 1398 man->use_type = false; 1399 if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) { 1400 ret = -EBUSY; 1401 pr_err("DRM memory manager type %d is not clean\n", 1402 i); 1403 } 1404 man->has_type = false; 1405 } 1406 } 1407 1408 lockmgr(&glob->device_list_mutex, LK_EXCLUSIVE); 1409 list_del(&bdev->device_list); 1410 lockmgr(&glob->device_list_mutex, LK_RELEASE); 1411 1412 cancel_delayed_work_sync(&bdev->wq); 1413 1414 while (ttm_bo_delayed_delete(bdev, true)) 1415 ; 1416 1417 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 1418 if (list_empty(&bdev->ddestroy)) 1419 TTM_DEBUG("Delayed destroy list was clean\n"); 1420 1421 if (list_empty(&bdev->man[0].lru)) 1422 TTM_DEBUG("Swap list was clean\n"); 1423 lockmgr(&glob->lru_lock, LK_RELEASE); 1424 1425 drm_vma_offset_manager_destroy(&bdev->vma_manager); 1426 1427 return ret; 1428 } 1429 EXPORT_SYMBOL(ttm_bo_device_release); 1430 1431 int ttm_bo_device_init(struct ttm_bo_device *bdev, 1432 struct ttm_bo_global *glob, 1433 struct ttm_bo_driver *driver, 1434 uint64_t file_page_offset, 1435 bool need_dma32) 1436 { 1437 int ret = -EINVAL; 1438 1439 bdev->driver = driver; 1440 1441 memset(bdev->man, 0, sizeof(bdev->man)); 1442 1443 /* 1444 * Initialize the system memory buffer type. 1445 * Other types need to be driver / IOCTL initialized. 1446 */ 1447 ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0); 1448 if (unlikely(ret != 0)) 1449 goto out_no_sys; 1450 1451 drm_vma_offset_manager_init(&bdev->vma_manager, file_page_offset, 1452 0x10000000); 1453 INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); 1454 INIT_LIST_HEAD(&bdev->ddestroy); 1455 /* 1456 * XXX DRAGONFLY - dev_mapping NULL atm, find other XXX DRAGONFLY 1457 * lines and fix when it no longer is in later API change. 1458 */ 1459 bdev->dev_mapping = NULL; 1460 bdev->glob = glob; 1461 bdev->need_dma32 = need_dma32; 1462 bdev->val_seq = 0; 1463 lockinit(&bdev->fence_lock, "ttmfnc", 0, 0); 1464 lockmgr(&glob->device_list_mutex, LK_EXCLUSIVE); 1465 list_add_tail(&bdev->device_list, &glob->device_list); 1466 lockmgr(&glob->device_list_mutex, LK_RELEASE); 1467 1468 return 0; 1469 out_no_sys: 1470 return ret; 1471 } 1472 EXPORT_SYMBOL(ttm_bo_device_init); 1473 1474 /* 1475 * buffer object vm functions. 1476 */ 1477 1478 bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 1479 { 1480 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 1481 1482 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 1483 if (mem->mem_type == TTM_PL_SYSTEM) 1484 return false; 1485 1486 if (man->flags & TTM_MEMTYPE_FLAG_CMA) 1487 return false; 1488 1489 if (mem->placement & TTM_PL_FLAG_CACHED) 1490 return false; 1491 } 1492 return true; 1493 } 1494 1495 void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) 1496 { 1497 struct ttm_bo_device *bdev = bo->bdev; 1498 loff_t offset, holelen; 1499 1500 if (!bdev->dev_mapping) { 1501 /* 1502 * XXX DRAGONFLY - device_mapping not yet implemented so 1503 * dev_mapping is basically always NULL. We have to properly 1504 * release the mmap, etc. 1505 */ 1506 ttm_bo_release_mmap(bo); 1507 ttm_mem_io_free_vm(bo); 1508 return; 1509 } 1510 1511 if (drm_vma_node_has_offset(&bo->vma_node)) { 1512 offset = (loff_t) drm_vma_node_offset_addr(&bo->vma_node); 1513 holelen = ((loff_t) bo->mem.num_pages) << PAGE_SHIFT; 1514 1515 unmap_mapping_range(bdev->dev_mapping, offset, holelen, 1); 1516 } 1517 ttm_bo_release_mmap(bo); /* for DragonFly VM interface */ 1518 ttm_mem_io_free_vm(bo); 1519 } 1520 1521 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) 1522 { 1523 struct ttm_bo_device *bdev = bo->bdev; 1524 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 1525 1526 ttm_mem_io_lock(man, false); 1527 ttm_bo_unmap_virtual_locked(bo); 1528 ttm_mem_io_unlock(man); 1529 } 1530 1531 1532 EXPORT_SYMBOL(ttm_bo_unmap_virtual); 1533 1534 /** 1535 * ttm_bo_setup_vm: 1536 * 1537 * @bo: the buffer to allocate address space for 1538 * 1539 * Allocate address space in the drm device so that applications 1540 * can mmap the buffer and access the contents. This only 1541 * applies to ttm_bo_type_device objects as others are not 1542 * placed in the drm device address space. 1543 */ 1544 1545 static int ttm_bo_setup_vm(struct ttm_buffer_object *bo) 1546 { 1547 struct ttm_bo_device *bdev = bo->bdev; 1548 1549 return drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node, 1550 bo->mem.num_pages); 1551 } 1552 1553 int ttm_bo_wait(struct ttm_buffer_object *bo, 1554 bool lazy, bool interruptible, bool no_wait) 1555 { 1556 struct ttm_bo_driver *driver = bo->bdev->driver; 1557 struct ttm_bo_device *bdev = bo->bdev; 1558 void *sync_obj; 1559 int ret = 0; 1560 1561 if (likely(bo->sync_obj == NULL)) 1562 return 0; 1563 1564 while (bo->sync_obj) { 1565 1566 if (driver->sync_obj_signaled(bo->sync_obj)) { 1567 void *tmp_obj = bo->sync_obj; 1568 bo->sync_obj = NULL; 1569 clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); 1570 lockmgr(&bdev->fence_lock, LK_RELEASE); 1571 driver->sync_obj_unref(&tmp_obj); 1572 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 1573 continue; 1574 } 1575 1576 if (no_wait) 1577 return -EBUSY; 1578 1579 sync_obj = driver->sync_obj_ref(bo->sync_obj); 1580 lockmgr(&bdev->fence_lock, LK_RELEASE); 1581 ret = driver->sync_obj_wait(sync_obj, 1582 lazy, interruptible); 1583 if (unlikely(ret != 0)) { 1584 driver->sync_obj_unref(&sync_obj); 1585 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 1586 return ret; 1587 } 1588 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 1589 if (likely(bo->sync_obj == sync_obj)) { 1590 void *tmp_obj = bo->sync_obj; 1591 bo->sync_obj = NULL; 1592 clear_bit(TTM_BO_PRIV_FLAG_MOVING, 1593 &bo->priv_flags); 1594 lockmgr(&bdev->fence_lock, LK_RELEASE); 1595 driver->sync_obj_unref(&sync_obj); 1596 driver->sync_obj_unref(&tmp_obj); 1597 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 1598 } else { 1599 lockmgr(&bdev->fence_lock, LK_RELEASE); 1600 driver->sync_obj_unref(&sync_obj); 1601 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 1602 } 1603 } 1604 return 0; 1605 } 1606 EXPORT_SYMBOL(ttm_bo_wait); 1607 1608 int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait) 1609 { 1610 struct ttm_bo_device *bdev = bo->bdev; 1611 int ret = 0; 1612 1613 /* 1614 * Using ttm_bo_reserve makes sure the lru lists are updated. 1615 */ 1616 1617 ret = ttm_bo_reserve(bo, true, no_wait, false, 0); 1618 if (unlikely(ret != 0)) 1619 return ret; 1620 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 1621 ret = ttm_bo_wait(bo, false, true, no_wait); 1622 lockmgr(&bdev->fence_lock, LK_RELEASE); 1623 if (likely(ret == 0)) 1624 atomic_inc(&bo->cpu_writers); 1625 ttm_bo_unreserve(bo); 1626 return ret; 1627 } 1628 EXPORT_SYMBOL(ttm_bo_synccpu_write_grab); 1629 1630 void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo) 1631 { 1632 atomic_dec(&bo->cpu_writers); 1633 } 1634 EXPORT_SYMBOL(ttm_bo_synccpu_write_release); 1635 1636 /** 1637 * A buffer object shrink method that tries to swap out the first 1638 * buffer object on the bo_global::swap_lru list. 1639 */ 1640 1641 static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) 1642 { 1643 struct ttm_bo_global *glob = 1644 container_of(shrink, struct ttm_bo_global, shrink); 1645 struct ttm_buffer_object *bo; 1646 int ret = -EBUSY; 1647 int put_count; 1648 uint32_t swap_placement = (TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM); 1649 1650 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 1651 list_for_each_entry(bo, &glob->swap_lru, swap) { 1652 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); 1653 if (!ret) 1654 break; 1655 } 1656 1657 if (ret) { 1658 lockmgr(&glob->lru_lock, LK_RELEASE); 1659 return ret; 1660 } 1661 1662 kref_get(&bo->list_kref); 1663 1664 if (!list_empty(&bo->ddestroy)) { 1665 ret = ttm_bo_cleanup_refs_and_unlock(bo, false, false); 1666 kref_put(&bo->list_kref, ttm_bo_release_list); 1667 return ret; 1668 } 1669 1670 put_count = ttm_bo_del_from_lru(bo); 1671 lockmgr(&glob->lru_lock, LK_RELEASE); 1672 1673 ttm_bo_list_ref_sub(bo, put_count, true); 1674 1675 /** 1676 * Wait for GPU, then move to system cached. 1677 */ 1678 1679 lockmgr(&bo->bdev->fence_lock, LK_EXCLUSIVE); 1680 ret = ttm_bo_wait(bo, false, false, false); 1681 lockmgr(&bo->bdev->fence_lock, LK_RELEASE); 1682 1683 if (unlikely(ret != 0)) 1684 goto out; 1685 1686 if ((bo->mem.placement & swap_placement) != swap_placement) { 1687 struct ttm_mem_reg evict_mem; 1688 1689 evict_mem = bo->mem; 1690 evict_mem.mm_node = NULL; 1691 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED; 1692 evict_mem.mem_type = TTM_PL_SYSTEM; 1693 1694 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, 1695 false, false); 1696 if (unlikely(ret != 0)) 1697 goto out; 1698 } 1699 1700 ttm_bo_unmap_virtual(bo); 1701 1702 /** 1703 * Swap out. Buffer will be swapped in again as soon as 1704 * anyone tries to access a ttm page. 1705 */ 1706 1707 if (bo->bdev->driver->swap_notify) 1708 bo->bdev->driver->swap_notify(bo); 1709 1710 ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage); 1711 out: 1712 1713 /** 1714 * 1715 * Unreserve without putting on LRU to avoid swapping out an 1716 * already swapped buffer. 1717 */ 1718 1719 ww_mutex_unlock(&bo->resv->lock); 1720 kref_put(&bo->list_kref, ttm_bo_release_list); 1721 return ret; 1722 } 1723 1724 void ttm_bo_swapout_all(struct ttm_bo_device *bdev) 1725 { 1726 while (ttm_bo_swapout(&bdev->glob->shrink) == 0) 1727 ; 1728 } 1729 EXPORT_SYMBOL(ttm_bo_swapout_all); 1730