1 /************************************************************************** 2 * 3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 /* 28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29 * 30 * $FreeBSD: head/sys/dev/drm2/ttm/ttm_bo.c 248060 2013-03-08 18:11:02Z dumbbell $ 31 */ 32 33 #include <drm/drmP.h> 34 #include <dev/drm/ttm/ttm_module.h> 35 #include <dev/drm/ttm/ttm_bo_driver.h> 36 #include <dev/drm/ttm/ttm_placement.h> 37 38 #define TTM_ASSERT_LOCKED(param) 39 #define TTM_DEBUG(fmt, arg...) 40 #define TTM_BO_HASH_ORDER 13 41 42 static int ttm_bo_setup_vm(struct ttm_buffer_object *bo); 43 static int ttm_bo_swapout(struct ttm_mem_shrink *shrink); 44 static void ttm_bo_global_kobj_release(struct ttm_bo_global *glob); 45 46 MALLOC_DEFINE(M_TTM_BO, "ttm_bo", "TTM Buffer Objects"); 47 48 static inline int ttm_mem_type_from_flags(uint32_t flags, uint32_t *mem_type) 49 { 50 int i; 51 52 for (i = 0; i <= TTM_PL_PRIV5; i++) 53 if (flags & (1 << i)) { 54 *mem_type = i; 55 return 0; 56 } 57 return -EINVAL; 58 } 59 60 static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type) 61 { 62 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 63 64 kprintf(" has_type: %d\n", man->has_type); 65 kprintf(" use_type: %d\n", man->use_type); 66 kprintf(" flags: 0x%08X\n", man->flags); 67 kprintf(" gpu_offset: 0x%08lX\n", man->gpu_offset); 68 kprintf(" size: %ju\n", (uintmax_t)man->size); 69 kprintf(" available_caching: 0x%08X\n", man->available_caching); 70 kprintf(" default_caching: 0x%08X\n", man->default_caching); 71 if (mem_type != TTM_PL_SYSTEM) 72 (*man->func->debug)(man, TTM_PFX); 73 } 74 75 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, 76 struct ttm_placement *placement) 77 { 78 int i, ret, mem_type; 79 80 kprintf("No space for %p (%lu pages, %luK, %luM)\n", 81 bo, bo->mem.num_pages, bo->mem.size >> 10, 82 bo->mem.size >> 20); 83 for (i = 0; i < placement->num_placement; i++) { 84 ret = ttm_mem_type_from_flags(placement->placement[i], 85 &mem_type); 86 if (ret) 87 return; 88 kprintf(" placement[%d]=0x%08X (%d)\n", 89 i, placement->placement[i], mem_type); 90 ttm_mem_type_debug(bo->bdev, mem_type); 91 } 92 } 93 94 #if 0 95 static ssize_t ttm_bo_global_show(struct ttm_bo_global *glob, 96 char *buffer) 97 { 98 99 return snprintf(buffer, PAGE_SIZE, "%lu\n", 100 (unsigned long) atomic_read(&glob->bo_count)); 101 } 102 #endif 103 104 static inline uint32_t ttm_bo_type_flags(unsigned type) 105 { 106 return 1 << (type); 107 } 108 109 static void ttm_bo_release_list(struct kref *list_kref) 110 { 111 struct ttm_buffer_object *bo = 112 container_of(list_kref, struct ttm_buffer_object, list_kref); 113 struct ttm_bo_device *bdev = bo->bdev; 114 size_t acc_size = bo->acc_size; 115 116 BUG_ON(atomic_read(&bo->list_kref.refcount)); 117 BUG_ON(atomic_read(&bo->kref.refcount)); 118 BUG_ON(atomic_read(&bo->cpu_writers)); 119 BUG_ON(bo->sync_obj != NULL); 120 BUG_ON(bo->mem.mm_node != NULL); 121 BUG_ON(!list_empty(&bo->lru)); 122 BUG_ON(!list_empty(&bo->ddestroy)); 123 124 if (bo->ttm) 125 ttm_tt_destroy(bo->ttm); 126 atomic_dec(&bo->glob->bo_count); 127 if (bo->destroy) 128 bo->destroy(bo); 129 else { 130 kfree(bo, M_TTM_BO); 131 } 132 ttm_mem_global_free(bdev->glob->mem_glob, acc_size); 133 } 134 135 static int 136 ttm_bo_wait_unreserved_locked(struct ttm_buffer_object *bo, bool interruptible) 137 { 138 const char *wmsg; 139 int flags, ret; 140 141 ret = 0; 142 if (interruptible) { 143 flags = PCATCH; 144 wmsg = "ttbowi"; 145 } else { 146 flags = 0; 147 wmsg = "ttbowu"; 148 } 149 while (ttm_bo_is_reserved(bo)) { 150 ret = -lksleep(bo, &bo->glob->lru_lock, 0, wmsg, 0); 151 if (ret != 0) 152 break; 153 } 154 return (ret); 155 } 156 157 void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) 158 { 159 struct ttm_bo_device *bdev = bo->bdev; 160 struct ttm_mem_type_manager *man; 161 162 BUG_ON(!ttm_bo_is_reserved(bo)); 163 164 if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { 165 166 BUG_ON(!list_empty(&bo->lru)); 167 168 man = &bdev->man[bo->mem.mem_type]; 169 list_add_tail(&bo->lru, &man->lru); 170 kref_get(&bo->list_kref); 171 172 if (bo->ttm != NULL) { 173 list_add_tail(&bo->swap, &bo->glob->swap_lru); 174 kref_get(&bo->list_kref); 175 } 176 } 177 } 178 179 int ttm_bo_del_from_lru(struct ttm_buffer_object *bo) 180 { 181 int put_count = 0; 182 183 if (!list_empty(&bo->swap)) { 184 list_del_init(&bo->swap); 185 ++put_count; 186 } 187 if (!list_empty(&bo->lru)) { 188 list_del_init(&bo->lru); 189 ++put_count; 190 } 191 192 /* 193 * TODO: Add a driver hook to delete from 194 * driver-specific LRU's here. 195 */ 196 197 return put_count; 198 } 199 200 int ttm_bo_reserve_nolru(struct ttm_buffer_object *bo, 201 bool interruptible, 202 bool no_wait, bool use_sequence, uint32_t sequence) 203 { 204 int ret; 205 206 while (unlikely(atomic_read(&bo->reserved) != 0)) { 207 /** 208 * Deadlock avoidance for multi-bo reserving. 209 */ 210 if (use_sequence && bo->seq_valid) { 211 /** 212 * We've already reserved this one. 213 */ 214 if (unlikely(sequence == bo->val_seq)) 215 return -EDEADLK; 216 /** 217 * Already reserved by a thread that will not back 218 * off for us. We need to back off. 219 */ 220 if (unlikely(sequence - bo->val_seq < (1 << 31))) 221 return -EAGAIN; 222 } 223 224 if (no_wait) 225 return -EBUSY; 226 227 ret = ttm_bo_wait_unreserved_locked(bo, interruptible); 228 229 if (unlikely(ret)) 230 return ret; 231 } 232 233 if (use_sequence) { 234 bool wake_up = false; 235 /** 236 * Wake up waiters that may need to recheck for deadlock, 237 * if we decreased the sequence number. 238 */ 239 if (unlikely((bo->val_seq - sequence < (1 << 31)) 240 || !bo->seq_valid)) 241 wake_up = true; 242 243 /* 244 * In the worst case with memory ordering these values can be 245 * seen in the wrong order. However since we call wake_up_all 246 * in that case, this will hopefully not pose a problem, 247 * and the worst case would only cause someone to accidentally 248 * hit -EAGAIN in ttm_bo_reserve when they see old value of 249 * val_seq. However this would only happen if seq_valid was 250 * written before val_seq was, and just means some slightly 251 * increased cpu usage 252 */ 253 bo->val_seq = sequence; 254 bo->seq_valid = true; 255 if (wake_up) 256 wakeup(bo); 257 } else { 258 bo->seq_valid = false; 259 } 260 261 return 0; 262 } 263 264 static void ttm_bo_ref_bug(struct kref *list_kref) 265 { 266 BUG(); 267 } 268 269 void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count, 270 bool never_free) 271 { 272 kref_sub(&bo->list_kref, count, 273 (never_free) ? ttm_bo_ref_bug : ttm_bo_release_list); 274 } 275 276 int ttm_bo_reserve(struct ttm_buffer_object *bo, 277 bool interruptible, 278 bool no_wait, bool use_sequence, uint32_t sequence) 279 { 280 struct ttm_bo_global *glob = bo->glob; 281 int put_count = 0; 282 int ret; 283 284 lockmgr(&bo->glob->lru_lock, LK_EXCLUSIVE); 285 ret = ttm_bo_reserve_nolru(bo, interruptible, no_wait, use_sequence, 286 sequence); 287 if (likely(ret == 0)) { 288 put_count = ttm_bo_del_from_lru(bo); 289 lockmgr(&glob->lru_lock, LK_RELEASE); 290 ttm_bo_list_ref_sub(bo, put_count, true); 291 } else 292 lockmgr(&bo->glob->lru_lock, LK_RELEASE); 293 294 return ret; 295 } 296 297 int ttm_bo_reserve_slowpath_nolru(struct ttm_buffer_object *bo, 298 bool interruptible, uint32_t sequence) 299 { 300 bool wake_up = false; 301 int ret; 302 303 while (unlikely(atomic_xchg(&bo->reserved, 1) != 0)) { 304 if (bo->seq_valid && sequence == bo->val_seq) { 305 DRM_ERROR( 306 "%s: bo->seq_valid && sequence == bo->val_seq", 307 __func__); 308 } 309 310 ret = ttm_bo_wait_unreserved_locked(bo, interruptible); 311 312 if (unlikely(ret)) 313 return ret; 314 } 315 316 if ((bo->val_seq - sequence < (1 << 31)) || !bo->seq_valid) 317 wake_up = true; 318 319 /** 320 * Wake up waiters that may need to recheck for deadlock, 321 * if we decreased the sequence number. 322 */ 323 bo->val_seq = sequence; 324 bo->seq_valid = true; 325 if (wake_up) 326 wakeup(bo); 327 328 return 0; 329 } 330 331 int ttm_bo_reserve_slowpath(struct ttm_buffer_object *bo, 332 bool interruptible, uint32_t sequence) 333 { 334 struct ttm_bo_global *glob = bo->glob; 335 int put_count, ret; 336 337 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 338 ret = ttm_bo_reserve_slowpath_nolru(bo, interruptible, sequence); 339 if (likely(!ret)) { 340 put_count = ttm_bo_del_from_lru(bo); 341 lockmgr(&glob->lru_lock, LK_RELEASE); 342 ttm_bo_list_ref_sub(bo, put_count, true); 343 } else 344 lockmgr(&glob->lru_lock, LK_RELEASE); 345 return ret; 346 } 347 348 void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo) 349 { 350 ttm_bo_add_to_lru(bo); 351 atomic_set(&bo->reserved, 0); 352 wakeup(bo); 353 } 354 355 void ttm_bo_unreserve(struct ttm_buffer_object *bo) 356 { 357 struct ttm_bo_global *glob = bo->glob; 358 359 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 360 ttm_bo_unreserve_locked(bo); 361 lockmgr(&glob->lru_lock, LK_RELEASE); 362 } 363 364 /* 365 * Call bo->mutex locked. 366 */ 367 static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) 368 { 369 struct ttm_bo_device *bdev = bo->bdev; 370 struct ttm_bo_global *glob = bo->glob; 371 int ret = 0; 372 uint32_t page_flags = 0; 373 374 TTM_ASSERT_LOCKED(&bo->mutex); 375 bo->ttm = NULL; 376 377 if (bdev->need_dma32) 378 page_flags |= TTM_PAGE_FLAG_DMA32; 379 380 switch (bo->type) { 381 case ttm_bo_type_device: 382 if (zero_alloc) 383 page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC; 384 case ttm_bo_type_kernel: 385 bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, 386 page_flags, glob->dummy_read_page); 387 if (unlikely(bo->ttm == NULL)) 388 ret = -ENOMEM; 389 break; 390 case ttm_bo_type_sg: 391 bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, 392 page_flags | TTM_PAGE_FLAG_SG, 393 glob->dummy_read_page); 394 if (unlikely(bo->ttm == NULL)) { 395 ret = -ENOMEM; 396 break; 397 } 398 bo->ttm->sg = bo->sg; 399 break; 400 default: 401 kprintf("[TTM] Illegal buffer object type\n"); 402 ret = -EINVAL; 403 break; 404 } 405 406 return ret; 407 } 408 409 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, 410 struct ttm_mem_reg *mem, 411 bool evict, bool interruptible, 412 bool no_wait_gpu) 413 { 414 struct ttm_bo_device *bdev = bo->bdev; 415 bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); 416 bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem); 417 struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type]; 418 struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type]; 419 int ret = 0; 420 421 if (old_is_pci || new_is_pci || 422 ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) { 423 ret = ttm_mem_io_lock(old_man, true); 424 if (unlikely(ret != 0)) 425 goto out_err; 426 ttm_bo_unmap_virtual_locked(bo); 427 ttm_mem_io_unlock(old_man); 428 } 429 430 /* 431 * Create and bind a ttm if required. 432 */ 433 434 if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 435 if (bo->ttm == NULL) { 436 bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED); 437 ret = ttm_bo_add_ttm(bo, zero); 438 if (ret) 439 goto out_err; 440 } 441 442 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); 443 if (ret) 444 goto out_err; 445 446 if (mem->mem_type != TTM_PL_SYSTEM) { 447 ret = ttm_tt_bind(bo->ttm, mem); 448 if (ret) 449 goto out_err; 450 } 451 452 if (bo->mem.mem_type == TTM_PL_SYSTEM) { 453 if (bdev->driver->move_notify) 454 bdev->driver->move_notify(bo, mem); 455 bo->mem = *mem; 456 mem->mm_node = NULL; 457 goto moved; 458 } 459 } 460 461 if (bdev->driver->move_notify) 462 bdev->driver->move_notify(bo, mem); 463 464 if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && 465 !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) 466 ret = ttm_bo_move_ttm(bo, evict, no_wait_gpu, mem); 467 else if (bdev->driver->move) 468 ret = bdev->driver->move(bo, evict, interruptible, 469 no_wait_gpu, mem); 470 else 471 ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, mem); 472 473 if (ret) { 474 if (bdev->driver->move_notify) { 475 struct ttm_mem_reg tmp_mem = *mem; 476 *mem = bo->mem; 477 bo->mem = tmp_mem; 478 bdev->driver->move_notify(bo, mem); 479 bo->mem = *mem; 480 *mem = tmp_mem; 481 } 482 483 goto out_err; 484 } 485 486 moved: 487 if (bo->evicted) { 488 ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement); 489 if (ret) 490 kprintf("[TTM] Can not flush read caches\n"); 491 bo->evicted = false; 492 } 493 494 if (bo->mem.mm_node) { 495 bo->offset = (bo->mem.start << PAGE_SHIFT) + 496 bdev->man[bo->mem.mem_type].gpu_offset; 497 bo->cur_placement = bo->mem.placement; 498 } else 499 bo->offset = 0; 500 501 return 0; 502 503 out_err: 504 new_man = &bdev->man[bo->mem.mem_type]; 505 if ((new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm) { 506 ttm_tt_unbind(bo->ttm); 507 ttm_tt_destroy(bo->ttm); 508 bo->ttm = NULL; 509 } 510 511 return ret; 512 } 513 514 /** 515 * Call bo::reserved. 516 * Will release GPU memory type usage on destruction. 517 * This is the place to put in driver specific hooks to release 518 * driver private resources. 519 * Will release the bo::reserved lock. 520 */ 521 522 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) 523 { 524 if (bo->bdev->driver->move_notify) 525 bo->bdev->driver->move_notify(bo, NULL); 526 527 if (bo->ttm) { 528 ttm_tt_unbind(bo->ttm); 529 ttm_tt_destroy(bo->ttm); 530 bo->ttm = NULL; 531 } 532 ttm_bo_mem_put(bo, &bo->mem); 533 534 atomic_set(&bo->reserved, 0); 535 wakeup(&bo); 536 537 /* 538 * Since the final reference to this bo may not be dropped by 539 * the current task we have to put a memory barrier here to make 540 * sure the changes done in this function are always visible. 541 * 542 * This function only needs protection against the final kref_put. 543 */ 544 cpu_mfence(); 545 } 546 547 static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) 548 { 549 struct ttm_bo_device *bdev = bo->bdev; 550 struct ttm_bo_global *glob = bo->glob; 551 struct ttm_bo_driver *driver = bdev->driver; 552 void *sync_obj = NULL; 553 int put_count; 554 int ret; 555 556 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 557 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); 558 559 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 560 (void) ttm_bo_wait(bo, false, false, true); 561 if (!ret && !bo->sync_obj) { 562 lockmgr(&bdev->fence_lock, LK_RELEASE); 563 put_count = ttm_bo_del_from_lru(bo); 564 565 lockmgr(&glob->lru_lock, LK_RELEASE); 566 ttm_bo_cleanup_memtype_use(bo); 567 568 ttm_bo_list_ref_sub(bo, put_count, true); 569 570 return; 571 } 572 if (bo->sync_obj) 573 sync_obj = driver->sync_obj_ref(bo->sync_obj); 574 lockmgr(&bdev->fence_lock, LK_RELEASE); 575 576 if (!ret) { 577 atomic_set(&bo->reserved, 0); 578 wakeup(bo); 579 } 580 581 kref_get(&bo->list_kref); 582 list_add_tail(&bo->ddestroy, &bdev->ddestroy); 583 lockmgr(&glob->lru_lock, LK_RELEASE); 584 585 if (sync_obj) { 586 driver->sync_obj_flush(sync_obj); 587 driver->sync_obj_unref(&sync_obj); 588 } 589 taskqueue_enqueue_timeout(taskqueue_thread[mycpuid], &bdev->wq, 590 ((hz / 100) < 1) ? 1 : hz / 100); 591 } 592 593 /** 594 * function ttm_bo_cleanup_refs_and_unlock 595 * If bo idle, remove from delayed- and lru lists, and unref. 596 * If not idle, do nothing. 597 * 598 * Must be called with lru_lock and reservation held, this function 599 * will drop both before returning. 600 * 601 * @interruptible Any sleeps should occur interruptibly. 602 * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. 603 */ 604 605 static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, 606 bool interruptible, 607 bool no_wait_gpu) 608 { 609 struct ttm_bo_device *bdev = bo->bdev; 610 struct ttm_bo_driver *driver = bdev->driver; 611 struct ttm_bo_global *glob = bo->glob; 612 int put_count; 613 int ret; 614 615 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 616 ret = ttm_bo_wait(bo, false, false, true); 617 618 if (ret && !no_wait_gpu) { 619 void *sync_obj; 620 621 /* 622 * Take a reference to the fence and unreserve, 623 * at this point the buffer should be dead, so 624 * no new sync objects can be attached. 625 */ 626 sync_obj = driver->sync_obj_ref(bo->sync_obj); 627 lockmgr(&bdev->fence_lock, LK_RELEASE); 628 629 atomic_set(&bo->reserved, 0); 630 wakeup(bo); 631 lockmgr(&glob->lru_lock, LK_RELEASE); 632 633 ret = driver->sync_obj_wait(sync_obj, false, interruptible); 634 driver->sync_obj_unref(&sync_obj); 635 if (ret) 636 return ret; 637 638 /* 639 * remove sync_obj with ttm_bo_wait, the wait should be 640 * finished, and no new wait object should have been added. 641 */ 642 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 643 ret = ttm_bo_wait(bo, false, false, true); 644 lockmgr(&bdev->fence_lock, LK_RELEASE); 645 if (ret) 646 return ret; 647 648 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 649 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); 650 651 /* 652 * We raced, and lost, someone else holds the reservation now, 653 * and is probably busy in ttm_bo_cleanup_memtype_use. 654 * 655 * Even if it's not the case, because we finished waiting any 656 * delayed destruction would succeed, so just return success 657 * here. 658 */ 659 if (ret) { 660 lockmgr(&glob->lru_lock, LK_RELEASE); 661 return 0; 662 } 663 } else 664 lockmgr(&bdev->fence_lock, LK_RELEASE); 665 666 if (ret || unlikely(list_empty(&bo->ddestroy))) { 667 atomic_set(&bo->reserved, 0); 668 wakeup(bo); 669 lockmgr(&glob->lru_lock, LK_RELEASE); 670 return ret; 671 } 672 673 put_count = ttm_bo_del_from_lru(bo); 674 list_del_init(&bo->ddestroy); 675 ++put_count; 676 677 lockmgr(&glob->lru_lock, LK_RELEASE); 678 ttm_bo_cleanup_memtype_use(bo); 679 680 ttm_bo_list_ref_sub(bo, put_count, true); 681 682 return 0; 683 } 684 685 /** 686 * Traverse the delayed list, and call ttm_bo_cleanup_refs on all 687 * encountered buffers. 688 */ 689 690 static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) 691 { 692 struct ttm_bo_global *glob = bdev->glob; 693 struct ttm_buffer_object *entry = NULL; 694 int ret = 0; 695 696 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 697 if (list_empty(&bdev->ddestroy)) 698 goto out_unlock; 699 700 entry = list_first_entry(&bdev->ddestroy, 701 struct ttm_buffer_object, ddestroy); 702 kref_get(&entry->list_kref); 703 704 for (;;) { 705 struct ttm_buffer_object *nentry = NULL; 706 707 if (entry->ddestroy.next != &bdev->ddestroy) { 708 nentry = list_first_entry(&entry->ddestroy, 709 struct ttm_buffer_object, ddestroy); 710 kref_get(&nentry->list_kref); 711 } 712 713 ret = ttm_bo_reserve_nolru(entry, false, true, false, 0); 714 if (remove_all && ret) { 715 ret = ttm_bo_reserve_nolru(entry, false, false, 716 false, 0); 717 } 718 719 if (!ret) 720 ret = ttm_bo_cleanup_refs_and_unlock(entry, false, 721 !remove_all); 722 else 723 lockmgr(&glob->lru_lock, LK_RELEASE); 724 725 kref_put(&entry->list_kref, ttm_bo_release_list); 726 entry = nentry; 727 728 if (ret || !entry) 729 goto out; 730 731 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 732 if (list_empty(&entry->ddestroy)) 733 break; 734 } 735 736 out_unlock: 737 lockmgr(&glob->lru_lock, LK_RELEASE); 738 out: 739 if (entry) 740 kref_put(&entry->list_kref, ttm_bo_release_list); 741 return ret; 742 } 743 744 static void ttm_bo_delayed_workqueue(void *arg, int pending __unused) 745 { 746 struct ttm_bo_device *bdev = arg; 747 748 if (ttm_bo_delayed_delete(bdev, false)) { 749 taskqueue_enqueue_timeout(taskqueue_thread[mycpuid], &bdev->wq, 750 ((hz / 100) < 1) ? 1 : hz / 100); 751 } 752 } 753 754 static void ttm_bo_release(struct kref *kref) 755 { 756 struct ttm_buffer_object *bo = 757 container_of(kref, struct ttm_buffer_object, kref); 758 struct ttm_bo_device *bdev = bo->bdev; 759 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 760 761 lockmgr(&bdev->vm_lock, LK_EXCLUSIVE); 762 if (likely(bo->vm_node != NULL)) { 763 RB_REMOVE(ttm_bo_device_buffer_objects, 764 &bdev->addr_space_rb, bo); 765 drm_mm_put_block(bo->vm_node); 766 bo->vm_node = NULL; 767 } 768 lockmgr(&bdev->vm_lock, LK_RELEASE); 769 ttm_mem_io_lock(man, false); 770 ttm_mem_io_free_vm(bo); 771 ttm_mem_io_unlock(man); 772 ttm_bo_cleanup_refs_or_queue(bo); 773 kref_put(&bo->list_kref, ttm_bo_release_list); 774 } 775 776 void ttm_bo_unref(struct ttm_buffer_object **p_bo) 777 { 778 struct ttm_buffer_object *bo = *p_bo; 779 780 *p_bo = NULL; 781 kref_put(&bo->kref, ttm_bo_release); 782 } 783 784 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev) 785 { 786 int pending; 787 788 taskqueue_cancel_timeout(taskqueue_thread[mycpuid], &bdev->wq, &pending); 789 if (pending) 790 taskqueue_drain_timeout(taskqueue_thread[mycpuid], &bdev->wq); 791 return (pending); 792 } 793 794 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) 795 { 796 if (resched) { 797 taskqueue_enqueue_timeout(taskqueue_thread[mycpuid], &bdev->wq, 798 ((hz / 100) < 1) ? 1 : hz / 100); 799 } 800 } 801 802 static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, 803 bool no_wait_gpu) 804 { 805 struct ttm_bo_device *bdev = bo->bdev; 806 struct ttm_mem_reg evict_mem; 807 struct ttm_placement placement; 808 int ret = 0; 809 810 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 811 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 812 lockmgr(&bdev->fence_lock, LK_RELEASE); 813 814 if (unlikely(ret != 0)) { 815 if (ret != -ERESTART) { 816 kprintf("[TTM] Failed to expire sync object before buffer eviction\n"); 817 } 818 goto out; 819 } 820 821 KKASSERT(ttm_bo_is_reserved(bo)); 822 823 evict_mem = bo->mem; 824 evict_mem.mm_node = NULL; 825 evict_mem.bus.io_reserved_vm = false; 826 evict_mem.bus.io_reserved_count = 0; 827 828 placement.fpfn = 0; 829 placement.lpfn = 0; 830 placement.num_placement = 0; 831 placement.num_busy_placement = 0; 832 bdev->driver->evict_flags(bo, &placement); 833 ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible, 834 no_wait_gpu); 835 if (ret) { 836 if (ret != -ERESTART) { 837 kprintf("[TTM] Failed to find memory space for buffer 0x%p eviction\n", 838 bo); 839 ttm_bo_mem_space_debug(bo, &placement); 840 } 841 goto out; 842 } 843 844 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible, 845 no_wait_gpu); 846 if (ret) { 847 if (ret != -ERESTART) 848 kprintf("[TTM] Buffer eviction failed\n"); 849 ttm_bo_mem_put(bo, &evict_mem); 850 goto out; 851 } 852 bo->evicted = true; 853 out: 854 return ret; 855 } 856 857 static int ttm_mem_evict_first(struct ttm_bo_device *bdev, 858 uint32_t mem_type, 859 bool interruptible, 860 bool no_wait_gpu) 861 { 862 struct ttm_bo_global *glob = bdev->glob; 863 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 864 struct ttm_buffer_object *bo; 865 int ret = -EBUSY, put_count; 866 867 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 868 list_for_each_entry(bo, &man->lru, lru) { 869 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); 870 if (!ret) 871 break; 872 } 873 874 if (ret) { 875 lockmgr(&glob->lru_lock, LK_RELEASE); 876 return ret; 877 } 878 879 kref_get(&bo->list_kref); 880 881 if (!list_empty(&bo->ddestroy)) { 882 ret = ttm_bo_cleanup_refs_and_unlock(bo, interruptible, 883 no_wait_gpu); 884 kref_put(&bo->list_kref, ttm_bo_release_list); 885 return ret; 886 } 887 888 put_count = ttm_bo_del_from_lru(bo); 889 lockmgr(&glob->lru_lock, LK_RELEASE); 890 891 KKASSERT(ret == 0); 892 893 ttm_bo_list_ref_sub(bo, put_count, true); 894 895 ret = ttm_bo_evict(bo, interruptible, no_wait_gpu); 896 ttm_bo_unreserve(bo); 897 898 kref_put(&bo->list_kref, ttm_bo_release_list); 899 return ret; 900 } 901 902 void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) 903 { 904 struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type]; 905 906 if (mem->mm_node) 907 (*man->func->put_node)(man, mem); 908 } 909 910 /** 911 * Repeatedly evict memory from the LRU for @mem_type until we create enough 912 * space, or we've evicted everything and there isn't enough space. 913 */ 914 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, 915 uint32_t mem_type, 916 struct ttm_placement *placement, 917 struct ttm_mem_reg *mem, 918 bool interruptible, 919 bool no_wait_gpu) 920 { 921 struct ttm_bo_device *bdev = bo->bdev; 922 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 923 int ret; 924 925 do { 926 ret = (*man->func->get_node)(man, bo, placement, mem); 927 if (unlikely(ret != 0)) 928 return ret; 929 if (mem->mm_node) 930 break; 931 ret = ttm_mem_evict_first(bdev, mem_type, 932 interruptible, no_wait_gpu); 933 if (unlikely(ret != 0)) 934 return ret; 935 } while (1); 936 if (mem->mm_node == NULL) 937 return -ENOMEM; 938 mem->mem_type = mem_type; 939 return 0; 940 } 941 942 static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man, 943 uint32_t cur_placement, 944 uint32_t proposed_placement) 945 { 946 uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING; 947 uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING; 948 949 /** 950 * Keep current caching if possible. 951 */ 952 953 if ((cur_placement & caching) != 0) 954 result |= (cur_placement & caching); 955 else if ((man->default_caching & caching) != 0) 956 result |= man->default_caching; 957 else if ((TTM_PL_FLAG_CACHED & caching) != 0) 958 result |= TTM_PL_FLAG_CACHED; 959 else if ((TTM_PL_FLAG_WC & caching) != 0) 960 result |= TTM_PL_FLAG_WC; 961 else if ((TTM_PL_FLAG_UNCACHED & caching) != 0) 962 result |= TTM_PL_FLAG_UNCACHED; 963 964 return result; 965 } 966 967 static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, 968 uint32_t mem_type, 969 uint32_t proposed_placement, 970 uint32_t *masked_placement) 971 { 972 uint32_t cur_flags = ttm_bo_type_flags(mem_type); 973 974 if ((cur_flags & proposed_placement & TTM_PL_MASK_MEM) == 0) 975 return false; 976 977 if ((proposed_placement & man->available_caching) == 0) 978 return false; 979 980 cur_flags |= (proposed_placement & man->available_caching); 981 982 *masked_placement = cur_flags; 983 return true; 984 } 985 986 /** 987 * Creates space for memory region @mem according to its type. 988 * 989 * This function first searches for free space in compatible memory types in 990 * the priority order defined by the driver. If free space isn't found, then 991 * ttm_bo_mem_force_space is attempted in priority order to evict and find 992 * space. 993 */ 994 int ttm_bo_mem_space(struct ttm_buffer_object *bo, 995 struct ttm_placement *placement, 996 struct ttm_mem_reg *mem, 997 bool interruptible, 998 bool no_wait_gpu) 999 { 1000 struct ttm_bo_device *bdev = bo->bdev; 1001 struct ttm_mem_type_manager *man; 1002 uint32_t mem_type = TTM_PL_SYSTEM; 1003 uint32_t cur_flags = 0; 1004 bool type_found = false; 1005 bool type_ok = false; 1006 bool has_erestartsys = false; 1007 int i, ret; 1008 1009 mem->mm_node = NULL; 1010 for (i = 0; i < placement->num_placement; ++i) { 1011 ret = ttm_mem_type_from_flags(placement->placement[i], 1012 &mem_type); 1013 if (ret) 1014 return ret; 1015 man = &bdev->man[mem_type]; 1016 1017 type_ok = ttm_bo_mt_compatible(man, 1018 mem_type, 1019 placement->placement[i], 1020 &cur_flags); 1021 1022 if (!type_ok) 1023 continue; 1024 1025 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 1026 cur_flags); 1027 /* 1028 * Use the access and other non-mapping-related flag bits from 1029 * the memory placement flags to the current flags 1030 */ 1031 ttm_flag_masked(&cur_flags, placement->placement[i], 1032 ~TTM_PL_MASK_MEMTYPE); 1033 1034 if (mem_type == TTM_PL_SYSTEM) 1035 break; 1036 1037 if (man->has_type && man->use_type) { 1038 type_found = true; 1039 ret = (*man->func->get_node)(man, bo, placement, mem); 1040 if (unlikely(ret)) 1041 return ret; 1042 } 1043 if (mem->mm_node) 1044 break; 1045 } 1046 1047 if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) { 1048 mem->mem_type = mem_type; 1049 mem->placement = cur_flags; 1050 return 0; 1051 } 1052 1053 if (!type_found) 1054 return -EINVAL; 1055 1056 for (i = 0; i < placement->num_busy_placement; ++i) { 1057 ret = ttm_mem_type_from_flags(placement->busy_placement[i], 1058 &mem_type); 1059 if (ret) 1060 return ret; 1061 man = &bdev->man[mem_type]; 1062 if (!man->has_type) 1063 continue; 1064 if (!ttm_bo_mt_compatible(man, 1065 mem_type, 1066 placement->busy_placement[i], 1067 &cur_flags)) 1068 continue; 1069 1070 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 1071 cur_flags); 1072 /* 1073 * Use the access and other non-mapping-related flag bits from 1074 * the memory placement flags to the current flags 1075 */ 1076 ttm_flag_masked(&cur_flags, placement->busy_placement[i], 1077 ~TTM_PL_MASK_MEMTYPE); 1078 1079 1080 if (mem_type == TTM_PL_SYSTEM) { 1081 mem->mem_type = mem_type; 1082 mem->placement = cur_flags; 1083 mem->mm_node = NULL; 1084 return 0; 1085 } 1086 1087 ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem, 1088 interruptible, no_wait_gpu); 1089 if (ret == 0 && mem->mm_node) { 1090 mem->placement = cur_flags; 1091 return 0; 1092 } 1093 if (ret == -ERESTART) 1094 has_erestartsys = true; 1095 } 1096 ret = (has_erestartsys) ? -ERESTART : -ENOMEM; 1097 return ret; 1098 } 1099 1100 static 1101 int ttm_bo_move_buffer(struct ttm_buffer_object *bo, 1102 struct ttm_placement *placement, 1103 bool interruptible, 1104 bool no_wait_gpu) 1105 { 1106 int ret = 0; 1107 struct ttm_mem_reg mem; 1108 struct ttm_bo_device *bdev = bo->bdev; 1109 1110 KKASSERT(ttm_bo_is_reserved(bo)); 1111 1112 /* 1113 * FIXME: It's possible to pipeline buffer moves. 1114 * Have the driver move function wait for idle when necessary, 1115 * instead of doing it here. 1116 */ 1117 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 1118 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 1119 lockmgr(&bdev->fence_lock, LK_RELEASE); 1120 if (ret) 1121 return ret; 1122 mem.num_pages = bo->num_pages; 1123 mem.size = mem.num_pages << PAGE_SHIFT; 1124 mem.page_alignment = bo->mem.page_alignment; 1125 mem.bus.io_reserved_vm = false; 1126 mem.bus.io_reserved_count = 0; 1127 /* 1128 * Determine where to move the buffer. 1129 */ 1130 ret = ttm_bo_mem_space(bo, placement, &mem, 1131 interruptible, no_wait_gpu); 1132 if (ret) 1133 goto out_unlock; 1134 ret = ttm_bo_handle_move_mem(bo, &mem, false, 1135 interruptible, no_wait_gpu); 1136 out_unlock: 1137 if (ret && mem.mm_node) 1138 ttm_bo_mem_put(bo, &mem); 1139 return ret; 1140 } 1141 1142 static int ttm_bo_mem_compat(struct ttm_placement *placement, 1143 struct ttm_mem_reg *mem) 1144 { 1145 int i; 1146 1147 if (mem->mm_node && placement->lpfn != 0 && 1148 (mem->start < placement->fpfn || 1149 mem->start + mem->num_pages > placement->lpfn)) 1150 return -1; 1151 1152 for (i = 0; i < placement->num_placement; i++) { 1153 if ((placement->placement[i] & mem->placement & 1154 TTM_PL_MASK_CACHING) && 1155 (placement->placement[i] & mem->placement & 1156 TTM_PL_MASK_MEM)) 1157 return i; 1158 } 1159 return -1; 1160 } 1161 1162 int ttm_bo_validate(struct ttm_buffer_object *bo, 1163 struct ttm_placement *placement, 1164 bool interruptible, 1165 bool no_wait_gpu) 1166 { 1167 int ret; 1168 1169 KKASSERT(ttm_bo_is_reserved(bo)); 1170 /* Check that range is valid */ 1171 if (placement->lpfn || placement->fpfn) 1172 if (placement->fpfn > placement->lpfn || 1173 (placement->lpfn - placement->fpfn) < bo->num_pages) 1174 return -EINVAL; 1175 /* 1176 * Check whether we need to move buffer. 1177 */ 1178 ret = ttm_bo_mem_compat(placement, &bo->mem); 1179 if (ret < 0) { 1180 ret = ttm_bo_move_buffer(bo, placement, interruptible, 1181 no_wait_gpu); 1182 if (ret) 1183 return ret; 1184 } else { 1185 /* 1186 * Use the access and other non-mapping-related flag bits from 1187 * the compatible memory placement flags to the active flags 1188 */ 1189 ttm_flag_masked(&bo->mem.placement, placement->placement[ret], 1190 ~TTM_PL_MASK_MEMTYPE); 1191 } 1192 /* 1193 * We might need to add a TTM. 1194 */ 1195 if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { 1196 ret = ttm_bo_add_ttm(bo, true); 1197 if (ret) 1198 return ret; 1199 } 1200 return 0; 1201 } 1202 1203 int ttm_bo_check_placement(struct ttm_buffer_object *bo, 1204 struct ttm_placement *placement) 1205 { 1206 KKASSERT(!((placement->fpfn || placement->lpfn) && 1207 (bo->mem.num_pages > (placement->lpfn - placement->fpfn)))); 1208 1209 return 0; 1210 } 1211 1212 int ttm_bo_init(struct ttm_bo_device *bdev, 1213 struct ttm_buffer_object *bo, 1214 unsigned long size, 1215 enum ttm_bo_type type, 1216 struct ttm_placement *placement, 1217 uint32_t page_alignment, 1218 bool interruptible, 1219 struct vm_object *persistent_swap_storage, 1220 size_t acc_size, 1221 struct sg_table *sg, 1222 void (*destroy) (struct ttm_buffer_object *)) 1223 { 1224 int ret = 0; 1225 unsigned long num_pages; 1226 struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; 1227 1228 ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false); 1229 if (ret) { 1230 kprintf("[TTM] Out of kernel memory\n"); 1231 if (destroy) 1232 (*destroy)(bo); 1233 else 1234 drm_free(bo, M_TTM_BO); 1235 return -ENOMEM; 1236 } 1237 1238 num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 1239 if (num_pages == 0) { 1240 kprintf("[TTM] Illegal buffer object size\n"); 1241 if (destroy) 1242 (*destroy)(bo); 1243 else 1244 drm_free(bo, M_TTM_BO); 1245 ttm_mem_global_free(mem_glob, acc_size); 1246 return -EINVAL; 1247 } 1248 bo->destroy = destroy; 1249 1250 kref_init(&bo->kref); 1251 kref_init(&bo->list_kref); 1252 atomic_set(&bo->cpu_writers, 0); 1253 atomic_set(&bo->reserved, 1); 1254 INIT_LIST_HEAD(&bo->lru); 1255 INIT_LIST_HEAD(&bo->ddestroy); 1256 INIT_LIST_HEAD(&bo->swap); 1257 INIT_LIST_HEAD(&bo->io_reserve_lru); 1258 bo->bdev = bdev; 1259 bo->glob = bdev->glob; 1260 bo->type = type; 1261 bo->num_pages = num_pages; 1262 bo->mem.size = num_pages << PAGE_SHIFT; 1263 bo->mem.mem_type = TTM_PL_SYSTEM; 1264 bo->mem.num_pages = bo->num_pages; 1265 bo->mem.mm_node = NULL; 1266 bo->mem.page_alignment = page_alignment; 1267 bo->mem.bus.io_reserved_vm = false; 1268 bo->mem.bus.io_reserved_count = 0; 1269 bo->priv_flags = 0; 1270 bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); 1271 bo->seq_valid = false; 1272 bo->persistent_swap_storage = persistent_swap_storage; 1273 bo->acc_size = acc_size; 1274 bo->sg = sg; 1275 atomic_inc(&bo->glob->bo_count); 1276 1277 ret = ttm_bo_check_placement(bo, placement); 1278 if (unlikely(ret != 0)) 1279 goto out_err; 1280 1281 /* 1282 * For ttm_bo_type_device buffers, allocate 1283 * address space from the device. 1284 */ 1285 if (bo->type == ttm_bo_type_device || 1286 bo->type == ttm_bo_type_sg) { 1287 ret = ttm_bo_setup_vm(bo); 1288 if (ret) 1289 goto out_err; 1290 } 1291 1292 ret = ttm_bo_validate(bo, placement, interruptible, false); 1293 if (ret) 1294 goto out_err; 1295 1296 ttm_bo_unreserve(bo); 1297 return 0; 1298 1299 out_err: 1300 ttm_bo_unreserve(bo); 1301 ttm_bo_unref(&bo); 1302 1303 return ret; 1304 } 1305 1306 size_t ttm_bo_acc_size(struct ttm_bo_device *bdev, 1307 unsigned long bo_size, 1308 unsigned struct_size) 1309 { 1310 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1311 size_t size = 0; 1312 1313 size += ttm_round_pot(struct_size); 1314 size += PAGE_ALIGN(npages * sizeof(void *)); 1315 size += ttm_round_pot(sizeof(struct ttm_tt)); 1316 return size; 1317 } 1318 1319 size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, 1320 unsigned long bo_size, 1321 unsigned struct_size) 1322 { 1323 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1324 size_t size = 0; 1325 1326 size += ttm_round_pot(struct_size); 1327 size += PAGE_ALIGN(npages * sizeof(void *)); 1328 size += PAGE_ALIGN(npages * sizeof(dma_addr_t)); 1329 size += ttm_round_pot(sizeof(struct ttm_dma_tt)); 1330 return size; 1331 } 1332 1333 int ttm_bo_create(struct ttm_bo_device *bdev, 1334 unsigned long size, 1335 enum ttm_bo_type type, 1336 struct ttm_placement *placement, 1337 uint32_t page_alignment, 1338 bool interruptible, 1339 struct vm_object *persistent_swap_storage, 1340 struct ttm_buffer_object **p_bo) 1341 { 1342 struct ttm_buffer_object *bo; 1343 size_t acc_size; 1344 int ret; 1345 1346 bo = kmalloc(sizeof(*bo), M_TTM_BO, M_WAITOK | M_ZERO); 1347 acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); 1348 ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, 1349 interruptible, persistent_swap_storage, acc_size, 1350 NULL, NULL); 1351 if (likely(ret == 0)) 1352 *p_bo = bo; 1353 1354 return ret; 1355 } 1356 1357 static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, 1358 unsigned mem_type, bool allow_errors) 1359 { 1360 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1361 struct ttm_bo_global *glob = bdev->glob; 1362 int ret; 1363 1364 /* 1365 * Can't use standard list traversal since we're unlocking. 1366 */ 1367 1368 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 1369 while (!list_empty(&man->lru)) { 1370 lockmgr(&glob->lru_lock, LK_RELEASE); 1371 ret = ttm_mem_evict_first(bdev, mem_type, false, false); 1372 if (ret) { 1373 if (allow_errors) { 1374 return ret; 1375 } else { 1376 kprintf("[TTM] Cleanup eviction failed\n"); 1377 } 1378 } 1379 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 1380 } 1381 lockmgr(&glob->lru_lock, LK_RELEASE); 1382 return 0; 1383 } 1384 1385 int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1386 { 1387 struct ttm_mem_type_manager *man; 1388 int ret = -EINVAL; 1389 1390 if (mem_type >= TTM_NUM_MEM_TYPES) { 1391 kprintf("[TTM] Illegal memory type %d\n", mem_type); 1392 return ret; 1393 } 1394 man = &bdev->man[mem_type]; 1395 1396 if (!man->has_type) { 1397 kprintf("[TTM] Trying to take down uninitialized memory manager type %u\n", 1398 mem_type); 1399 return ret; 1400 } 1401 1402 man->use_type = false; 1403 man->has_type = false; 1404 1405 ret = 0; 1406 if (mem_type > 0) { 1407 ttm_bo_force_list_clean(bdev, mem_type, false); 1408 1409 ret = (*man->func->takedown)(man); 1410 } 1411 1412 return ret; 1413 } 1414 1415 int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1416 { 1417 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1418 1419 if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) { 1420 kprintf("[TTM] Illegal memory manager memory type %u\n", mem_type); 1421 return -EINVAL; 1422 } 1423 1424 if (!man->has_type) { 1425 kprintf("[TTM] Memory type %u has not been initialized\n", mem_type); 1426 return 0; 1427 } 1428 1429 return ttm_bo_force_list_clean(bdev, mem_type, true); 1430 } 1431 1432 int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, 1433 unsigned long p_size) 1434 { 1435 int ret = -EINVAL; 1436 struct ttm_mem_type_manager *man; 1437 1438 KKASSERT(type < TTM_NUM_MEM_TYPES); 1439 man = &bdev->man[type]; 1440 KKASSERT(!man->has_type); 1441 man->io_reserve_fastpath = true; 1442 man->use_io_reserve_lru = false; 1443 lockinit(&man->io_reserve_mutex, "ttmman", 0, LK_CANRECURSE); 1444 INIT_LIST_HEAD(&man->io_reserve_lru); 1445 1446 ret = bdev->driver->init_mem_type(bdev, type, man); 1447 if (ret) 1448 return ret; 1449 man->bdev = bdev; 1450 1451 ret = 0; 1452 if (type != TTM_PL_SYSTEM) { 1453 ret = (*man->func->init)(man, p_size); 1454 if (ret) 1455 return ret; 1456 } 1457 man->has_type = true; 1458 man->use_type = true; 1459 man->size = p_size; 1460 1461 INIT_LIST_HEAD(&man->lru); 1462 1463 return 0; 1464 } 1465 1466 static void ttm_bo_global_kobj_release(struct ttm_bo_global *glob) 1467 { 1468 1469 ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink); 1470 vm_page_free(glob->dummy_read_page); 1471 } 1472 1473 void ttm_bo_global_release(struct drm_global_reference *ref) 1474 { 1475 struct ttm_bo_global *glob = ref->object; 1476 1477 if (refcount_release(&glob->kobj_ref)) 1478 ttm_bo_global_kobj_release(glob); 1479 } 1480 1481 int ttm_bo_global_init(struct drm_global_reference *ref) 1482 { 1483 struct ttm_bo_global_ref *bo_ref = 1484 container_of(ref, struct ttm_bo_global_ref, ref); 1485 struct ttm_bo_global *glob = ref->object; 1486 int ret; 1487 1488 lockinit(&glob->device_list_mutex, "ttmdlm", 0, LK_CANRECURSE); 1489 lockinit(&glob->lru_lock, "ttmlru", 0, LK_CANRECURSE); 1490 glob->mem_glob = bo_ref->mem_glob; 1491 glob->dummy_read_page = vm_page_alloc_contig( 1492 0, VM_MAX_ADDRESS, PAGE_SIZE, 0, 1*PAGE_SIZE, VM_MEMATTR_UNCACHEABLE); 1493 1494 if (unlikely(glob->dummy_read_page == NULL)) { 1495 ret = -ENOMEM; 1496 goto out_no_drp; 1497 } 1498 1499 INIT_LIST_HEAD(&glob->swap_lru); 1500 INIT_LIST_HEAD(&glob->device_list); 1501 1502 ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout); 1503 ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink); 1504 if (unlikely(ret != 0)) { 1505 kprintf("[TTM] Could not register buffer object swapout\n"); 1506 goto out_no_shrink; 1507 } 1508 1509 atomic_set(&glob->bo_count, 0); 1510 1511 refcount_init(&glob->kobj_ref, 1); 1512 return (0); 1513 1514 out_no_shrink: 1515 vm_page_free(glob->dummy_read_page); 1516 out_no_drp: 1517 drm_free(glob, M_DRM_GLOBAL); 1518 return ret; 1519 } 1520 1521 int ttm_bo_device_release(struct ttm_bo_device *bdev) 1522 { 1523 int ret = 0; 1524 unsigned i = TTM_NUM_MEM_TYPES; 1525 struct ttm_mem_type_manager *man; 1526 struct ttm_bo_global *glob = bdev->glob; 1527 1528 while (i--) { 1529 man = &bdev->man[i]; 1530 if (man->has_type) { 1531 man->use_type = false; 1532 if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) { 1533 ret = -EBUSY; 1534 kprintf("[TTM] DRM memory manager type %d is not clean\n", 1535 i); 1536 } 1537 man->has_type = false; 1538 } 1539 } 1540 1541 lockmgr(&glob->device_list_mutex, LK_EXCLUSIVE); 1542 list_del(&bdev->device_list); 1543 lockmgr(&glob->device_list_mutex, LK_RELEASE); 1544 1545 if (taskqueue_cancel_timeout(taskqueue_thread[mycpuid], &bdev->wq, NULL)) 1546 taskqueue_drain_timeout(taskqueue_thread[mycpuid], &bdev->wq); 1547 1548 while (ttm_bo_delayed_delete(bdev, true)) 1549 ; 1550 1551 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 1552 if (list_empty(&bdev->ddestroy)) 1553 TTM_DEBUG("Delayed destroy list was clean\n"); 1554 1555 if (list_empty(&bdev->man[0].lru)) 1556 TTM_DEBUG("Swap list was clean\n"); 1557 lockmgr(&glob->lru_lock, LK_RELEASE); 1558 1559 KKASSERT(drm_mm_clean(&bdev->addr_space_mm)); 1560 lockmgr(&bdev->vm_lock, LK_EXCLUSIVE); 1561 drm_mm_takedown(&bdev->addr_space_mm); 1562 lockmgr(&bdev->vm_lock, LK_RELEASE); 1563 1564 return ret; 1565 } 1566 1567 int ttm_bo_device_init(struct ttm_bo_device *bdev, 1568 struct ttm_bo_global *glob, 1569 struct ttm_bo_driver *driver, 1570 uint64_t file_page_offset, 1571 bool need_dma32) 1572 { 1573 int ret = -EINVAL; 1574 1575 lockinit(&bdev->vm_lock, "ttmvml", 0, LK_CANRECURSE); 1576 bdev->driver = driver; 1577 1578 memset(bdev->man, 0, sizeof(bdev->man)); 1579 1580 /* 1581 * Initialize the system memory buffer type. 1582 * Other types need to be driver / IOCTL initialized. 1583 */ 1584 ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0); 1585 if (unlikely(ret != 0)) 1586 goto out_no_sys; 1587 1588 RB_INIT(&bdev->addr_space_rb); 1589 ret = drm_mm_init(&bdev->addr_space_mm, file_page_offset, 0x10000000); 1590 if (unlikely(ret != 0)) 1591 goto out_no_addr_mm; 1592 1593 TIMEOUT_TASK_INIT(taskqueue_thread[mycpuid], &bdev->wq, 0, 1594 ttm_bo_delayed_workqueue, bdev); 1595 INIT_LIST_HEAD(&bdev->ddestroy); 1596 bdev->dev_mapping = NULL; 1597 bdev->glob = glob; 1598 bdev->need_dma32 = need_dma32; 1599 bdev->val_seq = 0; 1600 lockinit(&bdev->fence_lock, "ttmfence", 0, LK_CANRECURSE); 1601 lockmgr(&glob->device_list_mutex, LK_EXCLUSIVE); 1602 list_add_tail(&bdev->device_list, &glob->device_list); 1603 lockmgr(&glob->device_list_mutex, LK_RELEASE); 1604 1605 return 0; 1606 out_no_addr_mm: 1607 ttm_bo_clean_mm(bdev, 0); 1608 out_no_sys: 1609 return ret; 1610 } 1611 1612 /* 1613 * buffer object vm functions. 1614 */ 1615 1616 bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 1617 { 1618 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 1619 1620 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 1621 if (mem->mem_type == TTM_PL_SYSTEM) 1622 return false; 1623 1624 if (man->flags & TTM_MEMTYPE_FLAG_CMA) 1625 return false; 1626 1627 if (mem->placement & TTM_PL_FLAG_CACHED) 1628 return false; 1629 } 1630 return true; 1631 } 1632 1633 void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) 1634 { 1635 1636 ttm_bo_release_mmap(bo); 1637 ttm_mem_io_free_vm(bo); 1638 } 1639 1640 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) 1641 { 1642 struct ttm_bo_device *bdev = bo->bdev; 1643 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 1644 1645 ttm_mem_io_lock(man, false); 1646 ttm_bo_unmap_virtual_locked(bo); 1647 ttm_mem_io_unlock(man); 1648 } 1649 1650 static void ttm_bo_vm_insert_rb(struct ttm_buffer_object *bo) 1651 { 1652 struct ttm_bo_device *bdev = bo->bdev; 1653 1654 /* The caller acquired bdev->vm_lock. */ 1655 RB_INSERT(ttm_bo_device_buffer_objects, &bdev->addr_space_rb, bo); 1656 } 1657 1658 /** 1659 * ttm_bo_setup_vm: 1660 * 1661 * @bo: the buffer to allocate address space for 1662 * 1663 * Allocate address space in the drm device so that applications 1664 * can mmap the buffer and access the contents. This only 1665 * applies to ttm_bo_type_device objects as others are not 1666 * placed in the drm device address space. 1667 */ 1668 1669 static int ttm_bo_setup_vm(struct ttm_buffer_object *bo) 1670 { 1671 struct ttm_bo_device *bdev = bo->bdev; 1672 int ret; 1673 1674 retry_pre_get: 1675 ret = drm_mm_pre_get(&bdev->addr_space_mm); 1676 if (unlikely(ret != 0)) 1677 return ret; 1678 1679 lockmgr(&bdev->vm_lock, LK_EXCLUSIVE); 1680 bo->vm_node = drm_mm_search_free(&bdev->addr_space_mm, 1681 bo->mem.num_pages, 0, 0); 1682 1683 if (unlikely(bo->vm_node == NULL)) { 1684 ret = -ENOMEM; 1685 goto out_unlock; 1686 } 1687 1688 bo->vm_node = drm_mm_get_block_atomic(bo->vm_node, 1689 bo->mem.num_pages, 0); 1690 1691 if (unlikely(bo->vm_node == NULL)) { 1692 lockmgr(&bdev->vm_lock, LK_RELEASE); 1693 goto retry_pre_get; 1694 } 1695 1696 ttm_bo_vm_insert_rb(bo); 1697 lockmgr(&bdev->vm_lock, LK_RELEASE); 1698 bo->addr_space_offset = ((uint64_t) bo->vm_node->start) << PAGE_SHIFT; 1699 1700 return 0; 1701 out_unlock: 1702 lockmgr(&bdev->vm_lock, LK_RELEASE); 1703 return ret; 1704 } 1705 1706 int ttm_bo_wait(struct ttm_buffer_object *bo, 1707 bool lazy, bool interruptible, bool no_wait) 1708 { 1709 struct ttm_bo_driver *driver = bo->bdev->driver; 1710 struct ttm_bo_device *bdev = bo->bdev; 1711 void *sync_obj; 1712 int ret = 0; 1713 1714 if (likely(bo->sync_obj == NULL)) 1715 return 0; 1716 1717 while (bo->sync_obj) { 1718 1719 if (driver->sync_obj_signaled(bo->sync_obj)) { 1720 void *tmp_obj = bo->sync_obj; 1721 bo->sync_obj = NULL; 1722 clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); 1723 lockmgr(&bdev->fence_lock, LK_RELEASE); 1724 driver->sync_obj_unref(&tmp_obj); 1725 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 1726 continue; 1727 } 1728 1729 if (no_wait) 1730 return -EBUSY; 1731 1732 sync_obj = driver->sync_obj_ref(bo->sync_obj); 1733 lockmgr(&bdev->fence_lock, LK_RELEASE); 1734 ret = driver->sync_obj_wait(sync_obj, 1735 lazy, interruptible); 1736 if (unlikely(ret != 0)) { 1737 driver->sync_obj_unref(&sync_obj); 1738 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 1739 return ret; 1740 } 1741 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 1742 if (likely(bo->sync_obj == sync_obj)) { 1743 void *tmp_obj = bo->sync_obj; 1744 bo->sync_obj = NULL; 1745 clear_bit(TTM_BO_PRIV_FLAG_MOVING, 1746 &bo->priv_flags); 1747 lockmgr(&bdev->fence_lock, LK_RELEASE); 1748 driver->sync_obj_unref(&sync_obj); 1749 driver->sync_obj_unref(&tmp_obj); 1750 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 1751 } else { 1752 lockmgr(&bdev->fence_lock, LK_RELEASE); 1753 driver->sync_obj_unref(&sync_obj); 1754 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 1755 } 1756 } 1757 return 0; 1758 } 1759 1760 int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait) 1761 { 1762 struct ttm_bo_device *bdev = bo->bdev; 1763 int ret = 0; 1764 1765 /* 1766 * Using ttm_bo_reserve makes sure the lru lists are updated. 1767 */ 1768 1769 ret = ttm_bo_reserve(bo, true, no_wait, false, 0); 1770 if (unlikely(ret != 0)) 1771 return ret; 1772 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 1773 ret = ttm_bo_wait(bo, false, true, no_wait); 1774 lockmgr(&bdev->fence_lock, LK_RELEASE); 1775 if (likely(ret == 0)) 1776 atomic_inc(&bo->cpu_writers); 1777 ttm_bo_unreserve(bo); 1778 return ret; 1779 } 1780 1781 void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo) 1782 { 1783 atomic_dec(&bo->cpu_writers); 1784 } 1785 1786 /** 1787 * A buffer object shrink method that tries to swap out the first 1788 * buffer object on the bo_global::swap_lru list. 1789 */ 1790 1791 static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) 1792 { 1793 struct ttm_bo_global *glob = 1794 container_of(shrink, struct ttm_bo_global, shrink); 1795 struct ttm_buffer_object *bo; 1796 int ret = -EBUSY; 1797 int put_count; 1798 uint32_t swap_placement = (TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM); 1799 1800 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 1801 list_for_each_entry(bo, &glob->swap_lru, swap) { 1802 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); 1803 if (!ret) 1804 break; 1805 } 1806 1807 if (ret) { 1808 lockmgr(&glob->lru_lock, LK_RELEASE); 1809 return ret; 1810 } 1811 1812 kref_get(&bo->list_kref); 1813 1814 if (!list_empty(&bo->ddestroy)) { 1815 ret = ttm_bo_cleanup_refs_and_unlock(bo, false, false); 1816 kref_put(&bo->list_kref, ttm_bo_release_list); 1817 return ret; 1818 } 1819 1820 put_count = ttm_bo_del_from_lru(bo); 1821 lockmgr(&glob->lru_lock, LK_RELEASE); 1822 1823 ttm_bo_list_ref_sub(bo, put_count, true); 1824 1825 /** 1826 * Wait for GPU, then move to system cached. 1827 */ 1828 1829 lockmgr(&bo->bdev->fence_lock, LK_EXCLUSIVE); 1830 ret = ttm_bo_wait(bo, false, false, false); 1831 lockmgr(&bo->bdev->fence_lock, LK_RELEASE); 1832 1833 if (unlikely(ret != 0)) 1834 goto out; 1835 1836 if ((bo->mem.placement & swap_placement) != swap_placement) { 1837 struct ttm_mem_reg evict_mem; 1838 1839 evict_mem = bo->mem; 1840 evict_mem.mm_node = NULL; 1841 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED; 1842 evict_mem.mem_type = TTM_PL_SYSTEM; 1843 1844 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, 1845 false, false); 1846 if (unlikely(ret != 0)) 1847 goto out; 1848 } 1849 1850 ttm_bo_unmap_virtual(bo); 1851 1852 /** 1853 * Swap out. Buffer will be swapped in again as soon as 1854 * anyone tries to access a ttm page. 1855 */ 1856 1857 if (bo->bdev->driver->swap_notify) 1858 bo->bdev->driver->swap_notify(bo); 1859 1860 ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage); 1861 out: 1862 1863 /** 1864 * 1865 * Unreserve without putting on LRU to avoid swapping out an 1866 * already swapped buffer. 1867 */ 1868 1869 atomic_set(&bo->reserved, 0); 1870 wakeup(bo); 1871 kref_put(&bo->list_kref, ttm_bo_release_list); 1872 return ret; 1873 } 1874 1875 void ttm_bo_swapout_all(struct ttm_bo_device *bdev) 1876 { 1877 while (ttm_bo_swapout(&bdev->glob->shrink) == 0) 1878 ; 1879 } 1880