1 /* 2 * Copyright 2009 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 /* 27 * Authors: 28 * Jerome Glisse <glisse@freedesktop.org> 29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> 30 * Dave Airlie 31 */ 32 #include <linux/list.h> 33 #include <linux/slab.h> 34 #include <drm/drmP.h> 35 #include <drm/amdgpu_drm.h> 36 #include <drm/drm_cache.h> 37 #include "amdgpu.h" 38 #include "amdgpu_trace.h" 39 #include "amdgpu_amdkfd.h" 40 41 /* undo vm namespace pollution */ 42 #undef min_offset 43 #undef max_offset 44 45 46 /** 47 * DOC: amdgpu_object 48 * 49 * This defines the interfaces to operate on an &amdgpu_bo buffer object which 50 * represents memory used by driver (VRAM, system memory, etc.). The driver 51 * provides DRM/GEM APIs to userspace. DRM/GEM APIs then use these interfaces 52 * to create/destroy/set buffer object which are then managed by the kernel TTM 53 * memory manager. 54 * The interfaces are also used internally by kernel clients, including gfx, 55 * uvd, etc. for kernel managed allocations used by the GPU. 56 * 57 */ 58 59 static bool amdgpu_bo_need_backup(struct amdgpu_device *adev) 60 { 61 if (adev->flags & AMD_IS_APU) 62 return false; 63 64 if (amdgpu_gpu_recovery == 0 || 65 (amdgpu_gpu_recovery == -1 && !amdgpu_sriov_vf(adev))) 66 return false; 67 68 return true; 69 } 70 71 /** 72 * amdgpu_bo_subtract_pin_size - Remove BO from pin_size accounting 73 * 74 * @bo: &amdgpu_bo buffer object 75 * 76 * This function is called when a BO stops being pinned, and updates the 77 * &amdgpu_device pin_size values accordingly. 78 */ 79 static void amdgpu_bo_subtract_pin_size(struct amdgpu_bo *bo) 80 { 81 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 82 83 if (bo->tbo.mem.mem_type == TTM_PL_VRAM) { 84 atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size); 85 atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo), 86 &adev->visible_pin_size); 87 } else if (bo->tbo.mem.mem_type == TTM_PL_TT) { 88 atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size); 89 } 90 } 91 92 static void amdgpu_bo_destroy(struct ttm_buffer_object *tbo) 93 { 94 struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev); 95 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); 96 97 if (bo->pin_count > 0) 98 amdgpu_bo_subtract_pin_size(bo); 99 100 if (bo->kfd_bo) 101 amdgpu_amdkfd_unreserve_system_memory_limit(bo); 102 103 amdgpu_bo_kunmap(bo); 104 105 if (bo->gem_base.import_attach) 106 drm_prime_gem_destroy(&bo->gem_base, bo->tbo.sg); 107 drm_gem_object_release(&bo->gem_base); 108 amdgpu_bo_unref(&bo->parent); 109 if (!list_empty(&bo->shadow_list)) { 110 mutex_lock(&adev->shadow_list_lock); 111 list_del_init(&bo->shadow_list); 112 mutex_unlock(&adev->shadow_list_lock); 113 } 114 kfree(bo->metadata); 115 kfree(bo); 116 } 117 118 /** 119 * amdgpu_bo_is_amdgpu_bo - check if the buffer object is an &amdgpu_bo 120 * @bo: buffer object to be checked 121 * 122 * Uses destroy function associated with the object to determine if this is 123 * an &amdgpu_bo. 124 * 125 * Returns: 126 * true if the object belongs to &amdgpu_bo, false if not. 127 */ 128 bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo) 129 { 130 if (bo->destroy == &amdgpu_bo_destroy) 131 return true; 132 return false; 133 } 134 135 /** 136 * amdgpu_bo_placement_from_domain - set buffer's placement 137 * @abo: &amdgpu_bo buffer object whose placement is to be set 138 * @domain: requested domain 139 * 140 * Sets buffer's placement according to requested domain and the buffer's 141 * flags. 142 */ 143 void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain) 144 { 145 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); 146 struct ttm_placement *placement = &abo->placement; 147 struct ttm_place *places = abo->placements; 148 u64 flags = abo->flags; 149 u32 c = 0; 150 151 if (domain & AMDGPU_GEM_DOMAIN_VRAM) { 152 unsigned visible_pfn = adev->gmc.visible_vram_size >> PAGE_SHIFT; 153 154 places[c].fpfn = 0; 155 places[c].lpfn = 0; 156 places[c].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED | 157 TTM_PL_FLAG_VRAM; 158 159 if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) 160 places[c].lpfn = visible_pfn; 161 else 162 places[c].flags |= TTM_PL_FLAG_TOPDOWN; 163 164 if (flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) 165 places[c].flags |= TTM_PL_FLAG_CONTIGUOUS; 166 c++; 167 } 168 169 if (domain & AMDGPU_GEM_DOMAIN_GTT) { 170 places[c].fpfn = 0; 171 if (flags & AMDGPU_GEM_CREATE_SHADOW) 172 places[c].lpfn = adev->gmc.gart_size >> PAGE_SHIFT; 173 else 174 places[c].lpfn = 0; 175 places[c].flags = TTM_PL_FLAG_TT; 176 if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) 177 places[c].flags |= TTM_PL_FLAG_WC | 178 TTM_PL_FLAG_UNCACHED; 179 else 180 places[c].flags |= TTM_PL_FLAG_CACHED; 181 c++; 182 } 183 184 if (domain & AMDGPU_GEM_DOMAIN_CPU) { 185 places[c].fpfn = 0; 186 places[c].lpfn = 0; 187 places[c].flags = TTM_PL_FLAG_SYSTEM; 188 if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) 189 places[c].flags |= TTM_PL_FLAG_WC | 190 TTM_PL_FLAG_UNCACHED; 191 else 192 places[c].flags |= TTM_PL_FLAG_CACHED; 193 c++; 194 } 195 196 if (domain & AMDGPU_GEM_DOMAIN_GDS) { 197 places[c].fpfn = 0; 198 places[c].lpfn = 0; 199 places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GDS; 200 c++; 201 } 202 203 if (domain & AMDGPU_GEM_DOMAIN_GWS) { 204 places[c].fpfn = 0; 205 places[c].lpfn = 0; 206 places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GWS; 207 c++; 208 } 209 210 if (domain & AMDGPU_GEM_DOMAIN_OA) { 211 places[c].fpfn = 0; 212 places[c].lpfn = 0; 213 places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_OA; 214 c++; 215 } 216 217 if (!c) { 218 places[c].fpfn = 0; 219 places[c].lpfn = 0; 220 places[c].flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM; 221 c++; 222 } 223 224 BUG_ON(c > AMDGPU_BO_MAX_PLACEMENTS); 225 226 placement->num_placement = c; 227 placement->placement = places; 228 229 placement->num_busy_placement = c; 230 placement->busy_placement = places; 231 } 232 233 /** 234 * amdgpu_bo_create_reserved - create reserved BO for kernel use 235 * 236 * @adev: amdgpu device object 237 * @size: size for the new BO 238 * @align: alignment for the new BO 239 * @domain: where to place it 240 * @bo_ptr: used to initialize BOs in structures 241 * @gpu_addr: GPU addr of the pinned BO 242 * @cpu_addr: optional CPU address mapping 243 * 244 * Allocates and pins a BO for kernel internal use, and returns it still 245 * reserved. 246 * 247 * Note: For bo_ptr new BO is only created if bo_ptr points to NULL. 248 * 249 * Returns: 250 * 0 on success, negative error code otherwise. 251 */ 252 int amdgpu_bo_create_reserved(struct amdgpu_device *adev, 253 unsigned long size, int align, 254 u32 domain, struct amdgpu_bo **bo_ptr, 255 u64 *gpu_addr, void **cpu_addr) 256 { 257 struct amdgpu_bo_param bp; 258 bool free = false; 259 int r; 260 261 memset(&bp, 0, sizeof(bp)); 262 bp.size = size; 263 bp.byte_align = align; 264 bp.domain = domain; 265 bp.flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED | 266 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; 267 bp.type = ttm_bo_type_kernel; 268 bp.resv = NULL; 269 270 if (!*bo_ptr) { 271 r = amdgpu_bo_create(adev, &bp, bo_ptr); 272 if (r) { 273 dev_err(adev->dev, "(%d) failed to allocate kernel bo\n", 274 r); 275 return r; 276 } 277 free = true; 278 } 279 280 r = amdgpu_bo_reserve(*bo_ptr, false); 281 if (r) { 282 dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r); 283 goto error_free; 284 } 285 286 r = amdgpu_bo_pin(*bo_ptr, domain); 287 if (r) { 288 dev_err(adev->dev, "(%d) kernel bo pin failed\n", r); 289 goto error_unreserve; 290 } 291 292 r = amdgpu_ttm_alloc_gart(&(*bo_ptr)->tbo); 293 if (r) { 294 dev_err(adev->dev, "%p bind failed\n", *bo_ptr); 295 goto error_unpin; 296 } 297 298 if (gpu_addr) 299 *gpu_addr = amdgpu_bo_gpu_offset(*bo_ptr); 300 301 if (cpu_addr) { 302 r = amdgpu_bo_kmap(*bo_ptr, cpu_addr); 303 if (r) { 304 dev_err(adev->dev, "(%d) kernel bo map failed\n", r); 305 goto error_unpin; 306 } 307 } 308 309 return 0; 310 311 error_unpin: 312 amdgpu_bo_unpin(*bo_ptr); 313 error_unreserve: 314 amdgpu_bo_unreserve(*bo_ptr); 315 316 error_free: 317 if (free) 318 amdgpu_bo_unref(bo_ptr); 319 320 return r; 321 } 322 323 /** 324 * amdgpu_bo_create_kernel - create BO for kernel use 325 * 326 * @adev: amdgpu device object 327 * @size: size for the new BO 328 * @align: alignment for the new BO 329 * @domain: where to place it 330 * @bo_ptr: used to initialize BOs in structures 331 * @gpu_addr: GPU addr of the pinned BO 332 * @cpu_addr: optional CPU address mapping 333 * 334 * Allocates and pins a BO for kernel internal use. 335 * 336 * Note: For bo_ptr new BO is only created if bo_ptr points to NULL. 337 * 338 * Returns: 339 * 0 on success, negative error code otherwise. 340 */ 341 int amdgpu_bo_create_kernel(struct amdgpu_device *adev, 342 unsigned long size, int align, 343 u32 domain, struct amdgpu_bo **bo_ptr, 344 u64 *gpu_addr, void **cpu_addr) 345 { 346 int r; 347 348 r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr, 349 gpu_addr, cpu_addr); 350 351 if (r) 352 return r; 353 354 amdgpu_bo_unreserve(*bo_ptr); 355 356 return 0; 357 } 358 359 /** 360 * amdgpu_bo_free_kernel - free BO for kernel use 361 * 362 * @bo: amdgpu BO to free 363 * @gpu_addr: pointer to where the BO's GPU memory space address was stored 364 * @cpu_addr: pointer to where the BO's CPU memory space address was stored 365 * 366 * unmaps and unpin a BO for kernel internal use. 367 */ 368 void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr, 369 void **cpu_addr) 370 { 371 if (*bo == NULL) 372 return; 373 374 if (likely(amdgpu_bo_reserve(*bo, true) == 0)) { 375 if (cpu_addr) 376 amdgpu_bo_kunmap(*bo); 377 378 amdgpu_bo_unpin(*bo); 379 amdgpu_bo_unreserve(*bo); 380 } 381 amdgpu_bo_unref(bo); 382 383 if (gpu_addr) 384 *gpu_addr = 0; 385 386 if (cpu_addr) 387 *cpu_addr = NULL; 388 } 389 390 /* Validate bo size is bit bigger then the request domain */ 391 static bool amdgpu_bo_validate_size(struct amdgpu_device *adev, 392 unsigned long size, u32 domain) 393 { 394 struct ttm_mem_type_manager *man = NULL; 395 396 /* 397 * If GTT is part of requested domains the check must succeed to 398 * allow fall back to GTT 399 */ 400 if (domain & AMDGPU_GEM_DOMAIN_GTT) { 401 man = &adev->mman.bdev.man[TTM_PL_TT]; 402 403 if (size < (man->size << PAGE_SHIFT)) 404 return true; 405 else 406 goto fail; 407 } 408 409 if (domain & AMDGPU_GEM_DOMAIN_VRAM) { 410 man = &adev->mman.bdev.man[TTM_PL_VRAM]; 411 412 if (size < (man->size << PAGE_SHIFT)) 413 return true; 414 else 415 goto fail; 416 } 417 418 419 /* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU */ 420 return true; 421 422 fail: 423 DRM_DEBUG("BO size %lu > total memory in domain: %lu\n", size, 424 man->size << PAGE_SHIFT); 425 return false; 426 } 427 428 static int amdgpu_bo_do_create(struct amdgpu_device *adev, 429 struct amdgpu_bo_param *bp, 430 struct amdgpu_bo **bo_ptr) 431 { 432 struct ttm_operation_ctx ctx = { 433 .interruptible = (bp->type != ttm_bo_type_kernel), 434 .no_wait_gpu = false, 435 .resv = bp->resv, 436 .flags = bp->type != ttm_bo_type_kernel ? 437 TTM_OPT_FLAG_ALLOW_RES_EVICT : 0 438 }; 439 struct amdgpu_bo *bo; 440 unsigned long page_align, size = bp->size; 441 size_t acc_size; 442 int r; 443 444 page_align = roundup(bp->byte_align, PAGE_SIZE) >> PAGE_SHIFT; 445 size = ALIGN(size, PAGE_SIZE); 446 447 if (!amdgpu_bo_validate_size(adev, size, bp->domain)) 448 return -ENOMEM; 449 450 *bo_ptr = NULL; 451 452 acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size, 453 sizeof(struct amdgpu_bo)); 454 455 bo = kzalloc(sizeof(struct amdgpu_bo), GFP_KERNEL); 456 if (bo == NULL) 457 return -ENOMEM; 458 drm_gem_private_object_init(adev->ddev, &bo->gem_base, size); 459 INIT_LIST_HEAD(&bo->shadow_list); 460 INIT_LIST_HEAD(&bo->va); 461 bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain : 462 bp->domain; 463 bo->allowed_domains = bo->preferred_domains; 464 if (bp->type != ttm_bo_type_kernel && 465 bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM) 466 bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT; 467 468 bo->flags = bp->flags; 469 470 #ifdef CONFIG_X86_32 471 /* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit 472 * See https://bugs.freedesktop.org/show_bug.cgi?id=84627 473 */ 474 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; 475 #elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT) 476 /* Don't try to enable write-combining when it can't work, or things 477 * may be slow 478 * See https://bugs.freedesktop.org/show_bug.cgi?id=88758 479 */ 480 481 #ifndef CONFIG_COMPILE_TEST 482 #warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \ 483 thanks to write-combining 484 #endif 485 486 if (bo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) 487 DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for " 488 "better performance thanks to write-combining\n"); 489 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; 490 #else 491 /* For architectures that don't support WC memory, 492 * mask out the WC flag from the BO 493 */ 494 if (!drm_arch_can_wc_memory()) 495 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; 496 #endif 497 498 bo->tbo.bdev = &adev->mman.bdev; 499 amdgpu_bo_placement_from_domain(bo, bp->domain); 500 if (bp->type == ttm_bo_type_kernel) 501 bo->tbo.priority = 1; 502 503 r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, bp->type, 504 &bo->placement, page_align, &ctx, acc_size, 505 NULL, bp->resv, &amdgpu_bo_destroy); 506 if (unlikely(r != 0)) 507 return r; 508 509 if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && 510 bo->tbo.mem.mem_type == TTM_PL_VRAM && 511 bo->tbo.mem.start < adev->gmc.visible_vram_size >> PAGE_SHIFT) 512 amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 513 ctx.bytes_moved); 514 else 515 amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0); 516 517 if (bp->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED && 518 bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) { 519 struct dma_fence *fence; 520 521 r = amdgpu_fill_buffer(bo, 0, bo->tbo.resv, &fence); 522 if (unlikely(r)) 523 goto fail_unreserve; 524 525 amdgpu_bo_fence(bo, fence, false); 526 dma_fence_put(bo->tbo.moving); 527 bo->tbo.moving = dma_fence_get(fence); 528 dma_fence_put(fence); 529 } 530 if (!bp->resv) 531 amdgpu_bo_unreserve(bo); 532 *bo_ptr = bo; 533 534 trace_amdgpu_bo_create(bo); 535 536 /* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */ 537 if (bp->type == ttm_bo_type_device) 538 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 539 540 return 0; 541 542 fail_unreserve: 543 if (!bp->resv) 544 ww_mutex_unlock(&bo->tbo.resv->lock); 545 amdgpu_bo_unref(&bo); 546 return r; 547 } 548 549 static int amdgpu_bo_create_shadow(struct amdgpu_device *adev, 550 unsigned long size, int byte_align, 551 struct amdgpu_bo *bo) 552 { 553 struct amdgpu_bo_param bp; 554 int r; 555 556 if (bo->shadow) 557 return 0; 558 559 memset(&bp, 0, sizeof(bp)); 560 bp.size = size; 561 bp.byte_align = byte_align; 562 bp.domain = AMDGPU_GEM_DOMAIN_GTT; 563 bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC | 564 AMDGPU_GEM_CREATE_SHADOW; 565 bp.type = ttm_bo_type_kernel; 566 bp.resv = bo->tbo.resv; 567 568 r = amdgpu_bo_do_create(adev, &bp, &bo->shadow); 569 if (!r) { 570 bo->shadow->parent = amdgpu_bo_ref(bo); 571 mutex_lock(&adev->shadow_list_lock); 572 list_add_tail(&bo->shadow_list, &adev->shadow_list); 573 mutex_unlock(&adev->shadow_list_lock); 574 } 575 576 return r; 577 } 578 579 /** 580 * amdgpu_bo_create - create an &amdgpu_bo buffer object 581 * @adev: amdgpu device object 582 * @bp: parameters to be used for the buffer object 583 * @bo_ptr: pointer to the buffer object pointer 584 * 585 * Creates an &amdgpu_bo buffer object; and if requested, also creates a 586 * shadow object. 587 * Shadow object is used to backup the original buffer object, and is always 588 * in GTT. 589 * 590 * Returns: 591 * 0 for success or a negative error code on failure. 592 */ 593 int amdgpu_bo_create(struct amdgpu_device *adev, 594 struct amdgpu_bo_param *bp, 595 struct amdgpu_bo **bo_ptr) 596 { 597 u64 flags = bp->flags; 598 int r; 599 600 bp->flags = bp->flags & ~AMDGPU_GEM_CREATE_SHADOW; 601 r = amdgpu_bo_do_create(adev, bp, bo_ptr); 602 if (r) 603 return r; 604 605 if ((flags & AMDGPU_GEM_CREATE_SHADOW) && amdgpu_bo_need_backup(adev)) { 606 if (!bp->resv) 607 WARN_ON(reservation_object_lock((*bo_ptr)->tbo.resv, 608 NULL)); 609 610 r = amdgpu_bo_create_shadow(adev, bp->size, bp->byte_align, (*bo_ptr)); 611 612 if (!bp->resv) 613 reservation_object_unlock((*bo_ptr)->tbo.resv); 614 615 if (r) 616 amdgpu_bo_unref(bo_ptr); 617 } 618 619 return r; 620 } 621 622 /** 623 * amdgpu_bo_backup_to_shadow - Backs up an &amdgpu_bo buffer object 624 * @adev: amdgpu device object 625 * @ring: amdgpu_ring for the engine handling the buffer operations 626 * @bo: &amdgpu_bo buffer to be backed up 627 * @resv: reservation object with embedded fence 628 * @fence: dma_fence associated with the operation 629 * @direct: whether to submit the job directly 630 * 631 * Copies an &amdgpu_bo buffer object to its shadow object. 632 * Not used for now. 633 * 634 * Returns: 635 * 0 for success or a negative error code on failure. 636 */ 637 int amdgpu_bo_backup_to_shadow(struct amdgpu_device *adev, 638 struct amdgpu_ring *ring, 639 struct amdgpu_bo *bo, 640 struct reservation_object *resv, 641 struct dma_fence **fence, 642 bool direct) 643 644 { 645 struct amdgpu_bo *shadow = bo->shadow; 646 uint64_t bo_addr, shadow_addr; 647 int r; 648 649 if (!shadow) 650 return -EINVAL; 651 652 bo_addr = amdgpu_bo_gpu_offset(bo); 653 shadow_addr = amdgpu_bo_gpu_offset(bo->shadow); 654 655 r = reservation_object_reserve_shared(bo->tbo.resv); 656 if (r) 657 goto err; 658 659 r = amdgpu_copy_buffer(ring, bo_addr, shadow_addr, 660 amdgpu_bo_size(bo), resv, fence, 661 direct, false); 662 if (!r) 663 amdgpu_bo_fence(bo, *fence, true); 664 665 err: 666 return r; 667 } 668 669 /** 670 * amdgpu_bo_validate - validate an &amdgpu_bo buffer object 671 * @bo: pointer to the buffer object 672 * 673 * Sets placement according to domain; and changes placement and caching 674 * policy of the buffer object according to the placement. 675 * This is used for validating shadow bos. It calls ttm_bo_validate() to 676 * make sure the buffer is resident where it needs to be. 677 * 678 * Returns: 679 * 0 for success or a negative error code on failure. 680 */ 681 int amdgpu_bo_validate(struct amdgpu_bo *bo) 682 { 683 struct ttm_operation_ctx ctx = { false, false }; 684 uint32_t domain; 685 int r; 686 687 if (bo->pin_count) 688 return 0; 689 690 domain = bo->preferred_domains; 691 692 retry: 693 amdgpu_bo_placement_from_domain(bo, domain); 694 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 695 if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) { 696 domain = bo->allowed_domains; 697 goto retry; 698 } 699 700 return r; 701 } 702 703 /** 704 * amdgpu_bo_restore_from_shadow - restore an &amdgpu_bo buffer object 705 * @adev: amdgpu device object 706 * @ring: amdgpu_ring for the engine handling the buffer operations 707 * @bo: &amdgpu_bo buffer to be restored 708 * @resv: reservation object with embedded fence 709 * @fence: dma_fence associated with the operation 710 * @direct: whether to submit the job directly 711 * 712 * Copies a buffer object's shadow content back to the object. 713 * This is used for recovering a buffer from its shadow in case of a gpu 714 * reset where vram context may be lost. 715 * 716 * Returns: 717 * 0 for success or a negative error code on failure. 718 */ 719 int amdgpu_bo_restore_from_shadow(struct amdgpu_device *adev, 720 struct amdgpu_ring *ring, 721 struct amdgpu_bo *bo, 722 struct reservation_object *resv, 723 struct dma_fence **fence, 724 bool direct) 725 726 { 727 struct amdgpu_bo *shadow = bo->shadow; 728 uint64_t bo_addr, shadow_addr; 729 int r; 730 731 if (!shadow) 732 return -EINVAL; 733 734 bo_addr = amdgpu_bo_gpu_offset(bo); 735 shadow_addr = amdgpu_bo_gpu_offset(bo->shadow); 736 737 r = reservation_object_reserve_shared(bo->tbo.resv); 738 if (r) 739 goto err; 740 741 r = amdgpu_copy_buffer(ring, shadow_addr, bo_addr, 742 amdgpu_bo_size(bo), resv, fence, 743 direct, false); 744 if (!r) 745 amdgpu_bo_fence(bo, *fence, true); 746 747 err: 748 return r; 749 } 750 751 /** 752 * amdgpu_bo_kmap - map an &amdgpu_bo buffer object 753 * @bo: &amdgpu_bo buffer object to be mapped 754 * @ptr: kernel virtual address to be returned 755 * 756 * Calls ttm_bo_kmap() to set up the kernel virtual mapping; calls 757 * amdgpu_bo_kptr() to get the kernel virtual address. 758 * 759 * Returns: 760 * 0 for success or a negative error code on failure. 761 */ 762 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr) 763 { 764 void *kptr; 765 long r; 766 767 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) 768 return -EPERM; 769 770 kptr = amdgpu_bo_kptr(bo); 771 if (kptr) { 772 if (ptr) 773 *ptr = kptr; 774 return 0; 775 } 776 777 r = reservation_object_wait_timeout_rcu(bo->tbo.resv, false, false, 778 MAX_SCHEDULE_TIMEOUT); 779 if (r < 0) 780 return r; 781 782 r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, &bo->kmap); 783 if (r) 784 return r; 785 786 if (ptr) 787 *ptr = amdgpu_bo_kptr(bo); 788 789 return 0; 790 } 791 792 /** 793 * amdgpu_bo_kptr - returns a kernel virtual address of the buffer object 794 * @bo: &amdgpu_bo buffer object 795 * 796 * Calls ttm_kmap_obj_virtual() to get the kernel virtual address 797 * 798 * Returns: 799 * the virtual address of a buffer object area. 800 */ 801 void *amdgpu_bo_kptr(struct amdgpu_bo *bo) 802 { 803 bool is_iomem; 804 805 return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem); 806 } 807 808 /** 809 * amdgpu_bo_kunmap - unmap an &amdgpu_bo buffer object 810 * @bo: &amdgpu_bo buffer object to be unmapped 811 * 812 * Unmaps a kernel map set up by amdgpu_bo_kmap(). 813 */ 814 void amdgpu_bo_kunmap(struct amdgpu_bo *bo) 815 { 816 if (bo->kmap.bo) 817 ttm_bo_kunmap(&bo->kmap); 818 } 819 820 /** 821 * amdgpu_bo_ref - reference an &amdgpu_bo buffer object 822 * @bo: &amdgpu_bo buffer object 823 * 824 * References the contained &ttm_buffer_object. 825 * 826 * Returns: 827 * a refcounted pointer to the &amdgpu_bo buffer object. 828 */ 829 struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo) 830 { 831 if (bo == NULL) 832 return NULL; 833 834 ttm_bo_get(&bo->tbo); 835 return bo; 836 } 837 838 /** 839 * amdgpu_bo_unref - unreference an &amdgpu_bo buffer object 840 * @bo: &amdgpu_bo buffer object 841 * 842 * Unreferences the contained &ttm_buffer_object and clear the pointer 843 */ 844 void amdgpu_bo_unref(struct amdgpu_bo **bo) 845 { 846 struct ttm_buffer_object *tbo; 847 848 if ((*bo) == NULL) 849 return; 850 851 tbo = &((*bo)->tbo); 852 ttm_bo_put(tbo); 853 *bo = NULL; 854 } 855 856 /** 857 * amdgpu_bo_pin_restricted - pin an &amdgpu_bo buffer object 858 * @bo: &amdgpu_bo buffer object to be pinned 859 * @domain: domain to be pinned to 860 * @min_offset: the start of requested address range 861 * @max_offset: the end of requested address range 862 * 863 * Pins the buffer object according to requested domain and address range. If 864 * the memory is unbound gart memory, binds the pages into gart table. Adjusts 865 * pin_count and pin_size accordingly. 866 * 867 * Pinning means to lock pages in memory along with keeping them at a fixed 868 * offset. It is required when a buffer can not be moved, for example, when 869 * a display buffer is being scanned out. 870 * 871 * Compared with amdgpu_bo_pin(), this function gives more flexibility on 872 * where to pin a buffer if there are specific restrictions on where a buffer 873 * must be located. 874 * 875 * Returns: 876 * 0 for success or a negative error code on failure. 877 */ 878 int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain, 879 u64 min_offset, u64 max_offset) 880 { 881 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 882 struct ttm_operation_ctx ctx = { false, false }; 883 int r, i; 884 885 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) 886 return -EPERM; 887 888 if (WARN_ON_ONCE(min_offset > max_offset)) 889 return -EINVAL; 890 891 /* A shared bo cannot be migrated to VRAM */ 892 if (bo->prime_shared_count) { 893 if (domain & AMDGPU_GEM_DOMAIN_GTT) 894 domain = AMDGPU_GEM_DOMAIN_GTT; 895 else 896 return -EINVAL; 897 } 898 899 /* This assumes only APU display buffers are pinned with (VRAM|GTT). 900 * See function amdgpu_display_supported_domains() 901 */ 902 domain = amdgpu_bo_get_preferred_pin_domain(adev, domain); 903 904 if (bo->pin_count) { 905 uint32_t mem_type = bo->tbo.mem.mem_type; 906 907 if (!(domain & amdgpu_mem_type_to_domain(mem_type))) 908 return -EINVAL; 909 910 bo->pin_count++; 911 912 if (max_offset != 0) { 913 u64 domain_start = bo->tbo.bdev->man[mem_type].gpu_offset; 914 WARN_ON_ONCE(max_offset < 915 (amdgpu_bo_gpu_offset(bo) - domain_start)); 916 } 917 918 return 0; 919 } 920 921 bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; 922 /* force to pin into visible video ram */ 923 if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) 924 bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 925 amdgpu_bo_placement_from_domain(bo, domain); 926 for (i = 0; i < bo->placement.num_placement; i++) { 927 unsigned fpfn, lpfn; 928 929 fpfn = min_offset >> PAGE_SHIFT; 930 lpfn = max_offset >> PAGE_SHIFT; 931 932 if (fpfn > bo->placements[i].fpfn) 933 bo->placements[i].fpfn = fpfn; 934 if (!bo->placements[i].lpfn || 935 (lpfn && lpfn < bo->placements[i].lpfn)) 936 bo->placements[i].lpfn = lpfn; 937 bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT; 938 } 939 940 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 941 if (unlikely(r)) { 942 dev_err(adev->dev, "%p pin failed\n", bo); 943 goto error; 944 } 945 946 bo->pin_count = 1; 947 948 domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type); 949 if (domain == AMDGPU_GEM_DOMAIN_VRAM) { 950 atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size); 951 atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo), 952 &adev->visible_pin_size); 953 } else if (domain == AMDGPU_GEM_DOMAIN_GTT) { 954 atomic64_add(amdgpu_bo_size(bo), &adev->gart_pin_size); 955 } 956 957 error: 958 return r; 959 } 960 961 /** 962 * amdgpu_bo_pin - pin an &amdgpu_bo buffer object 963 * @bo: &amdgpu_bo buffer object to be pinned 964 * @domain: domain to be pinned to 965 * 966 * A simple wrapper to amdgpu_bo_pin_restricted(). 967 * Provides a simpler API for buffers that do not have any strict restrictions 968 * on where a buffer must be located. 969 * 970 * Returns: 971 * 0 for success or a negative error code on failure. 972 */ 973 int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain) 974 { 975 return amdgpu_bo_pin_restricted(bo, domain, 0, 0); 976 } 977 978 /** 979 * amdgpu_bo_unpin - unpin an &amdgpu_bo buffer object 980 * @bo: &amdgpu_bo buffer object to be unpinned 981 * 982 * Decreases the pin_count, and clears the flags if pin_count reaches 0. 983 * Changes placement and pin size accordingly. 984 * 985 * Returns: 986 * 0 for success or a negative error code on failure. 987 */ 988 int amdgpu_bo_unpin(struct amdgpu_bo *bo) 989 { 990 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 991 struct ttm_operation_ctx ctx = { false, false }; 992 int r, i; 993 994 if (!bo->pin_count) { 995 dev_warn(adev->dev, "%p unpin not necessary\n", bo); 996 return 0; 997 } 998 bo->pin_count--; 999 if (bo->pin_count) 1000 return 0; 1001 1002 amdgpu_bo_subtract_pin_size(bo); 1003 1004 for (i = 0; i < bo->placement.num_placement; i++) { 1005 bo->placements[i].lpfn = 0; 1006 bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT; 1007 } 1008 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 1009 if (unlikely(r)) 1010 dev_err(adev->dev, "%p validate failed for unpin\n", bo); 1011 1012 return r; 1013 } 1014 1015 /** 1016 * amdgpu_bo_evict_vram - evict VRAM buffers 1017 * @adev: amdgpu device object 1018 * 1019 * Evicts all VRAM buffers on the lru list of the memory type. 1020 * Mainly used for evicting vram at suspend time. 1021 * 1022 * Returns: 1023 * 0 for success or a negative error code on failure. 1024 */ 1025 int amdgpu_bo_evict_vram(struct amdgpu_device *adev) 1026 { 1027 /* late 2.6.33 fix IGP hibernate - we need pm ops to do this correct */ 1028 if (0 && (adev->flags & AMD_IS_APU)) { 1029 /* Useless to evict on IGP chips */ 1030 return 0; 1031 } 1032 return ttm_bo_evict_mm(&adev->mman.bdev, TTM_PL_VRAM); 1033 } 1034 1035 static const char *amdgpu_vram_names[] = { 1036 "UNKNOWN", 1037 "GDDR1", 1038 "DDR2", 1039 "GDDR3", 1040 "GDDR4", 1041 "GDDR5", 1042 "HBM", 1043 "DDR3", 1044 "DDR4", 1045 }; 1046 1047 /** 1048 * amdgpu_bo_init - initialize memory manager 1049 * @adev: amdgpu device object 1050 * 1051 * Calls amdgpu_ttm_init() to initialize amdgpu memory manager. 1052 * 1053 * Returns: 1054 * 0 for success or a negative error code on failure. 1055 */ 1056 int amdgpu_bo_init(struct amdgpu_device *adev) 1057 { 1058 /* reserve PAT memory space to WC for VRAM */ 1059 arch_io_reserve_memtype_wc(adev->gmc.aper_base, 1060 adev->gmc.aper_size); 1061 1062 /* Add an MTRR for the VRAM */ 1063 adev->gmc.vram_mtrr = arch_phys_wc_add(adev->gmc.aper_base, 1064 adev->gmc.aper_size); 1065 DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n", 1066 adev->gmc.mc_vram_size >> 20, 1067 (unsigned long long)adev->gmc.aper_size >> 20); 1068 DRM_INFO("RAM width %dbits %s\n", 1069 adev->gmc.vram_width, amdgpu_vram_names[adev->gmc.vram_type]); 1070 return amdgpu_ttm_init(adev); 1071 } 1072 1073 /** 1074 * amdgpu_bo_late_init - late init 1075 * @adev: amdgpu device object 1076 * 1077 * Calls amdgpu_ttm_late_init() to free resources used earlier during 1078 * initialization. 1079 * 1080 * Returns: 1081 * 0 for success or a negative error code on failure. 1082 */ 1083 int amdgpu_bo_late_init(struct amdgpu_device *adev) 1084 { 1085 amdgpu_ttm_late_init(adev); 1086 1087 return 0; 1088 } 1089 1090 /** 1091 * amdgpu_bo_fini - tear down memory manager 1092 * @adev: amdgpu device object 1093 * 1094 * Reverses amdgpu_bo_init() to tear down memory manager. 1095 */ 1096 void amdgpu_bo_fini(struct amdgpu_device *adev) 1097 { 1098 amdgpu_ttm_fini(adev); 1099 arch_phys_wc_del(adev->gmc.vram_mtrr); 1100 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); 1101 } 1102 1103 /** 1104 * amdgpu_bo_fbdev_mmap - mmap fbdev memory 1105 * @bo: &amdgpu_bo buffer object 1106 * @vma: vma as input from the fbdev mmap method 1107 * 1108 * Calls ttm_fbdev_mmap() to mmap fbdev memory if it is backed by a bo. 1109 * 1110 * Returns: 1111 * 0 for success or a negative error code on failure. 1112 */ 1113 int amdgpu_bo_fbdev_mmap(struct amdgpu_bo *bo, 1114 struct vm_area_struct *vma) 1115 { 1116 return ttm_fbdev_mmap(vma, &bo->tbo); 1117 } 1118 1119 /** 1120 * amdgpu_bo_set_tiling_flags - set tiling flags 1121 * @bo: &amdgpu_bo buffer object 1122 * @tiling_flags: new flags 1123 * 1124 * Sets buffer object's tiling flags with the new one. Used by GEM ioctl or 1125 * kernel driver to set the tiling flags on a buffer. 1126 * 1127 * Returns: 1128 * 0 for success or a negative error code on failure. 1129 */ 1130 int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags) 1131 { 1132 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1133 1134 if (adev->family <= AMDGPU_FAMILY_CZ && 1135 AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6) 1136 return -EINVAL; 1137 1138 bo->tiling_flags = tiling_flags; 1139 return 0; 1140 } 1141 1142 /** 1143 * amdgpu_bo_get_tiling_flags - get tiling flags 1144 * @bo: &amdgpu_bo buffer object 1145 * @tiling_flags: returned flags 1146 * 1147 * Gets buffer object's tiling flags. Used by GEM ioctl or kernel driver to 1148 * set the tiling flags on a buffer. 1149 */ 1150 void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags) 1151 { 1152 lockdep_assert_held(&bo->tbo.resv->lock.base); 1153 1154 if (tiling_flags) 1155 *tiling_flags = bo->tiling_flags; 1156 } 1157 1158 /** 1159 * amdgpu_bo_set_metadata - set metadata 1160 * @bo: &amdgpu_bo buffer object 1161 * @metadata: new metadata 1162 * @metadata_size: size of the new metadata 1163 * @flags: flags of the new metadata 1164 * 1165 * Sets buffer object's metadata, its size and flags. 1166 * Used via GEM ioctl. 1167 * 1168 * Returns: 1169 * 0 for success or a negative error code on failure. 1170 */ 1171 int amdgpu_bo_set_metadata (struct amdgpu_bo *bo, void *metadata, 1172 uint32_t metadata_size, uint64_t flags) 1173 { 1174 void *buffer; 1175 1176 if (!metadata_size) { 1177 if (bo->metadata_size) { 1178 kfree(bo->metadata); 1179 bo->metadata = NULL; 1180 bo->metadata_size = 0; 1181 } 1182 return 0; 1183 } 1184 1185 if (metadata == NULL) 1186 return -EINVAL; 1187 1188 buffer = kmemdup(metadata, metadata_size, GFP_KERNEL); 1189 if (buffer == NULL) 1190 return -ENOMEM; 1191 1192 kfree(bo->metadata); 1193 bo->metadata_flags = flags; 1194 bo->metadata = buffer; 1195 bo->metadata_size = metadata_size; 1196 1197 return 0; 1198 } 1199 1200 /** 1201 * amdgpu_bo_get_metadata - get metadata 1202 * @bo: &amdgpu_bo buffer object 1203 * @buffer: returned metadata 1204 * @buffer_size: size of the buffer 1205 * @metadata_size: size of the returned metadata 1206 * @flags: flags of the returned metadata 1207 * 1208 * Gets buffer object's metadata, its size and flags. buffer_size shall not be 1209 * less than metadata_size. 1210 * Used via GEM ioctl. 1211 * 1212 * Returns: 1213 * 0 for success or a negative error code on failure. 1214 */ 1215 int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer, 1216 size_t buffer_size, uint32_t *metadata_size, 1217 uint64_t *flags) 1218 { 1219 if (!buffer && !metadata_size) 1220 return -EINVAL; 1221 1222 if (buffer) { 1223 if (buffer_size < bo->metadata_size) 1224 return -EINVAL; 1225 1226 if (bo->metadata_size) 1227 memcpy(buffer, bo->metadata, bo->metadata_size); 1228 } 1229 1230 if (metadata_size) 1231 *metadata_size = bo->metadata_size; 1232 if (flags) 1233 *flags = bo->metadata_flags; 1234 1235 return 0; 1236 } 1237 1238 /** 1239 * amdgpu_bo_move_notify - notification about a memory move 1240 * @bo: pointer to a buffer object 1241 * @evict: if this move is evicting the buffer from the graphics address space 1242 * @new_mem: new information of the bufer object 1243 * 1244 * Marks the corresponding &amdgpu_bo buffer object as invalid, also performs 1245 * bookkeeping. 1246 * TTM driver callback which is called when ttm moves a buffer. 1247 */ 1248 void amdgpu_bo_move_notify(struct ttm_buffer_object *bo, 1249 bool evict, 1250 struct ttm_mem_reg *new_mem) 1251 { 1252 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 1253 struct amdgpu_bo *abo; 1254 #if 0 1255 struct ttm_mem_reg *old_mem = &bo->mem; 1256 #endif 1257 1258 if (!amdgpu_bo_is_amdgpu_bo(bo)) 1259 return; 1260 1261 abo = ttm_to_amdgpu_bo(bo); 1262 amdgpu_vm_bo_invalidate(adev, abo, evict); 1263 1264 amdgpu_bo_kunmap(abo); 1265 1266 /* remember the eviction */ 1267 if (evict) 1268 atomic64_inc(&adev->num_evictions); 1269 1270 /* update statistics */ 1271 if (!new_mem) 1272 return; 1273 1274 /* move_notify is called before move happens */ 1275 trace_amdgpu_bo_move(abo, new_mem->mem_type, old_mem->mem_type); 1276 } 1277 1278 /** 1279 * amdgpu_bo_fault_reserve_notify - notification about a memory fault 1280 * @bo: pointer to a buffer object 1281 * 1282 * Notifies the driver we are taking a fault on this BO and have reserved it, 1283 * also performs bookkeeping. 1284 * TTM driver callback for dealing with vm faults. 1285 * 1286 * Returns: 1287 * 0 for success or a negative error code on failure. 1288 */ 1289 int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo) 1290 { 1291 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 1292 struct ttm_operation_ctx ctx = { false, false }; 1293 struct amdgpu_bo *abo; 1294 unsigned long offset, size; 1295 int r; 1296 1297 if (!amdgpu_bo_is_amdgpu_bo(bo)) 1298 return 0; 1299 1300 abo = ttm_to_amdgpu_bo(bo); 1301 1302 /* Remember that this BO was accessed by the CPU */ 1303 abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 1304 1305 if (bo->mem.mem_type != TTM_PL_VRAM) 1306 return 0; 1307 1308 size = bo->mem.num_pages << PAGE_SHIFT; 1309 offset = bo->mem.start << PAGE_SHIFT; 1310 if ((offset + size) <= adev->gmc.visible_vram_size) 1311 return 0; 1312 1313 /* Can't move a pinned BO to visible VRAM */ 1314 if (abo->pin_count > 0) 1315 return -EINVAL; 1316 1317 /* hurrah the memory is not visible ! */ 1318 atomic64_inc(&adev->num_vram_cpu_page_faults); 1319 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM | 1320 AMDGPU_GEM_DOMAIN_GTT); 1321 1322 /* Avoid costly evictions; only set GTT as a busy placement */ 1323 abo->placement.num_busy_placement = 1; 1324 abo->placement.busy_placement = &abo->placements[1]; 1325 1326 r = ttm_bo_validate(bo, &abo->placement, &ctx); 1327 if (unlikely(r != 0)) 1328 return r; 1329 1330 offset = bo->mem.start << PAGE_SHIFT; 1331 /* this should never happen */ 1332 if (bo->mem.mem_type == TTM_PL_VRAM && 1333 (offset + size) > adev->gmc.visible_vram_size) 1334 return -EINVAL; 1335 1336 return 0; 1337 } 1338 1339 /** 1340 * amdgpu_bo_fence - add fence to buffer object 1341 * 1342 * @bo: buffer object in question 1343 * @fence: fence to add 1344 * @shared: true if fence should be added shared 1345 * 1346 */ 1347 void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence, 1348 bool shared) 1349 { 1350 struct reservation_object *resv = bo->tbo.resv; 1351 1352 if (shared) 1353 reservation_object_add_shared_fence(resv, fence); 1354 else 1355 reservation_object_add_excl_fence(resv, fence); 1356 } 1357 1358 /** 1359 * amdgpu_bo_gpu_offset - return GPU offset of bo 1360 * @bo: amdgpu object for which we query the offset 1361 * 1362 * Note: object should either be pinned or reserved when calling this 1363 * function, it might be useful to add check for this for debugging. 1364 * 1365 * Returns: 1366 * current GPU offset of the object. 1367 */ 1368 u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo) 1369 { 1370 WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_SYSTEM); 1371 WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_TT && 1372 !amdgpu_gtt_mgr_has_gart_addr(&bo->tbo.mem)); 1373 WARN_ON_ONCE(!ww_mutex_is_locked(&bo->tbo.resv->lock) && 1374 !bo->pin_count); 1375 WARN_ON_ONCE(bo->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET); 1376 WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_VRAM && 1377 !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)); 1378 1379 return bo->tbo.offset; 1380 } 1381 1382 /** 1383 * amdgpu_bo_get_preferred_pin_domain - get preferred domain for scanout 1384 * @adev: amdgpu device object 1385 * @domain: allowed :ref:`memory domains <amdgpu_memory_domains>` 1386 * 1387 * Returns: 1388 * Which of the allowed domains is preferred for pinning the BO for scanout. 1389 */ 1390 uint32_t amdgpu_bo_get_preferred_pin_domain(struct amdgpu_device *adev, 1391 uint32_t domain) 1392 { 1393 if (domain == (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) { 1394 domain = AMDGPU_GEM_DOMAIN_VRAM; 1395 if (adev->gmc.real_vram_size <= AMDGPU_SG_THRESHOLD) 1396 domain = AMDGPU_GEM_DOMAIN_GTT; 1397 } 1398 return domain; 1399 } 1400