1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_bo.h" 7 8 #include <linux/dma-buf.h> 9 10 #include <drm/drm_drv.h> 11 #include <drm/drm_gem_ttm_helper.h> 12 #include <drm/drm_managed.h> 13 #include <drm/ttm/ttm_device.h> 14 #include <drm/ttm/ttm_placement.h> 15 #include <drm/ttm/ttm_tt.h> 16 #include <drm/xe_drm.h> 17 18 #include "xe_device.h" 19 #include "xe_dma_buf.h" 20 #include "xe_drm_client.h" 21 #include "xe_ggtt.h" 22 #include "xe_gt.h" 23 #include "xe_map.h" 24 #include "xe_migrate.h" 25 #include "xe_preempt_fence.h" 26 #include "xe_res_cursor.h" 27 #include "xe_trace.h" 28 #include "xe_ttm_stolen_mgr.h" 29 #include "xe_vm.h" 30 31 static const struct ttm_place sys_placement_flags = { 32 .fpfn = 0, 33 .lpfn = 0, 34 .mem_type = XE_PL_SYSTEM, 35 .flags = 0, 36 }; 37 38 static struct ttm_placement sys_placement = { 39 .num_placement = 1, 40 .placement = &sys_placement_flags, 41 }; 42 43 static const struct ttm_place tt_placement_flags[] = { 44 { 45 .fpfn = 0, 46 .lpfn = 0, 47 .mem_type = XE_PL_TT, 48 .flags = TTM_PL_FLAG_DESIRED, 49 }, 50 { 51 .fpfn = 0, 52 .lpfn = 0, 53 .mem_type = XE_PL_SYSTEM, 54 .flags = TTM_PL_FLAG_FALLBACK, 55 } 56 }; 57 58 static struct ttm_placement tt_placement = { 59 .num_placement = 2, 60 .placement = tt_placement_flags, 61 }; 62 63 bool mem_type_is_vram(u32 mem_type) 64 { 65 return mem_type >= XE_PL_VRAM0 && mem_type != XE_PL_STOLEN; 66 } 67 68 static bool resource_is_stolen_vram(struct xe_device *xe, struct ttm_resource *res) 69 { 70 return res->mem_type == XE_PL_STOLEN && IS_DGFX(xe); 71 } 72 73 static bool resource_is_vram(struct ttm_resource *res) 74 { 75 return mem_type_is_vram(res->mem_type); 76 } 77 78 bool xe_bo_is_vram(struct xe_bo *bo) 79 { 80 return resource_is_vram(bo->ttm.resource) || 81 resource_is_stolen_vram(xe_bo_device(bo), bo->ttm.resource); 82 } 83 84 bool xe_bo_is_stolen(struct xe_bo *bo) 85 { 86 return bo->ttm.resource->mem_type == XE_PL_STOLEN; 87 } 88 89 /** 90 * xe_bo_is_stolen_devmem - check if BO is of stolen type accessed via PCI BAR 91 * @bo: The BO 92 * 93 * The stolen memory is accessed through the PCI BAR for both DGFX and some 94 * integrated platforms that have a dedicated bit in the PTE for devmem (DM). 95 * 96 * Returns: true if it's stolen memory accessed via PCI BAR, false otherwise. 97 */ 98 bool xe_bo_is_stolen_devmem(struct xe_bo *bo) 99 { 100 return xe_bo_is_stolen(bo) && 101 GRAPHICS_VERx100(xe_bo_device(bo)) >= 1270; 102 } 103 104 static bool xe_bo_is_user(struct xe_bo *bo) 105 { 106 return bo->flags & XE_BO_CREATE_USER_BIT; 107 } 108 109 static struct xe_migrate * 110 mem_type_to_migrate(struct xe_device *xe, u32 mem_type) 111 { 112 struct xe_tile *tile; 113 114 xe_assert(xe, mem_type == XE_PL_STOLEN || mem_type_is_vram(mem_type)); 115 tile = &xe->tiles[mem_type == XE_PL_STOLEN ? 0 : (mem_type - XE_PL_VRAM0)]; 116 return tile->migrate; 117 } 118 119 static struct xe_mem_region *res_to_mem_region(struct ttm_resource *res) 120 { 121 struct xe_device *xe = ttm_to_xe_device(res->bo->bdev); 122 struct ttm_resource_manager *mgr; 123 124 xe_assert(xe, resource_is_vram(res)); 125 mgr = ttm_manager_type(&xe->ttm, res->mem_type); 126 return to_xe_ttm_vram_mgr(mgr)->vram; 127 } 128 129 static void try_add_system(struct xe_device *xe, struct xe_bo *bo, 130 u32 bo_flags, u32 *c) 131 { 132 if (bo_flags & XE_BO_CREATE_SYSTEM_BIT) { 133 xe_assert(xe, *c < ARRAY_SIZE(bo->placements)); 134 135 bo->placements[*c] = (struct ttm_place) { 136 .mem_type = XE_PL_TT, 137 }; 138 *c += 1; 139 140 if (bo->props.preferred_mem_type == XE_BO_PROPS_INVALID) 141 bo->props.preferred_mem_type = XE_PL_TT; 142 } 143 } 144 145 static void add_vram(struct xe_device *xe, struct xe_bo *bo, 146 struct ttm_place *places, u32 bo_flags, u32 mem_type, u32 *c) 147 { 148 struct ttm_place place = { .mem_type = mem_type }; 149 struct xe_mem_region *vram; 150 u64 io_size; 151 152 xe_assert(xe, *c < ARRAY_SIZE(bo->placements)); 153 154 vram = to_xe_ttm_vram_mgr(ttm_manager_type(&xe->ttm, mem_type))->vram; 155 xe_assert(xe, vram && vram->usable_size); 156 io_size = vram->io_size; 157 158 /* 159 * For eviction / restore on suspend / resume objects 160 * pinned in VRAM must be contiguous 161 */ 162 if (bo_flags & (XE_BO_CREATE_PINNED_BIT | 163 XE_BO_CREATE_GGTT_BIT)) 164 place.flags |= TTM_PL_FLAG_CONTIGUOUS; 165 166 if (io_size < vram->usable_size) { 167 if (bo_flags & XE_BO_NEEDS_CPU_ACCESS) { 168 place.fpfn = 0; 169 place.lpfn = io_size >> PAGE_SHIFT; 170 } else { 171 place.flags |= TTM_PL_FLAG_TOPDOWN; 172 } 173 } 174 places[*c] = place; 175 *c += 1; 176 177 if (bo->props.preferred_mem_type == XE_BO_PROPS_INVALID) 178 bo->props.preferred_mem_type = mem_type; 179 } 180 181 static void try_add_vram(struct xe_device *xe, struct xe_bo *bo, 182 u32 bo_flags, u32 *c) 183 { 184 if (bo->props.preferred_gt == XE_GT1) { 185 if (bo_flags & XE_BO_CREATE_VRAM1_BIT) 186 add_vram(xe, bo, bo->placements, bo_flags, XE_PL_VRAM1, c); 187 if (bo_flags & XE_BO_CREATE_VRAM0_BIT) 188 add_vram(xe, bo, bo->placements, bo_flags, XE_PL_VRAM0, c); 189 } else { 190 if (bo_flags & XE_BO_CREATE_VRAM0_BIT) 191 add_vram(xe, bo, bo->placements, bo_flags, XE_PL_VRAM0, c); 192 if (bo_flags & XE_BO_CREATE_VRAM1_BIT) 193 add_vram(xe, bo, bo->placements, bo_flags, XE_PL_VRAM1, c); 194 } 195 } 196 197 static void try_add_stolen(struct xe_device *xe, struct xe_bo *bo, 198 u32 bo_flags, u32 *c) 199 { 200 if (bo_flags & XE_BO_CREATE_STOLEN_BIT) { 201 xe_assert(xe, *c < ARRAY_SIZE(bo->placements)); 202 203 bo->placements[*c] = (struct ttm_place) { 204 .mem_type = XE_PL_STOLEN, 205 .flags = bo_flags & (XE_BO_CREATE_PINNED_BIT | 206 XE_BO_CREATE_GGTT_BIT) ? 207 TTM_PL_FLAG_CONTIGUOUS : 0, 208 }; 209 *c += 1; 210 } 211 } 212 213 static int __xe_bo_placement_for_flags(struct xe_device *xe, struct xe_bo *bo, 214 u32 bo_flags) 215 { 216 u32 c = 0; 217 218 bo->props.preferred_mem_type = XE_BO_PROPS_INVALID; 219 220 /* The order of placements should indicate preferred location */ 221 222 if (bo->props.preferred_mem_class == DRM_XE_MEM_REGION_CLASS_SYSMEM) { 223 try_add_system(xe, bo, bo_flags, &c); 224 try_add_vram(xe, bo, bo_flags, &c); 225 } else { 226 try_add_vram(xe, bo, bo_flags, &c); 227 try_add_system(xe, bo, bo_flags, &c); 228 } 229 try_add_stolen(xe, bo, bo_flags, &c); 230 231 if (!c) 232 return -EINVAL; 233 234 bo->placement = (struct ttm_placement) { 235 .num_placement = c, 236 .placement = bo->placements, 237 }; 238 239 return 0; 240 } 241 242 int xe_bo_placement_for_flags(struct xe_device *xe, struct xe_bo *bo, 243 u32 bo_flags) 244 { 245 xe_bo_assert_held(bo); 246 return __xe_bo_placement_for_flags(xe, bo, bo_flags); 247 } 248 249 static void xe_evict_flags(struct ttm_buffer_object *tbo, 250 struct ttm_placement *placement) 251 { 252 if (!xe_bo_is_xe_bo(tbo)) { 253 /* Don't handle scatter gather BOs */ 254 if (tbo->type == ttm_bo_type_sg) { 255 placement->num_placement = 0; 256 return; 257 } 258 259 *placement = sys_placement; 260 return; 261 } 262 263 /* 264 * For xe, sg bos that are evicted to system just triggers a 265 * rebind of the sg list upon subsequent validation to XE_PL_TT. 266 */ 267 switch (tbo->resource->mem_type) { 268 case XE_PL_VRAM0: 269 case XE_PL_VRAM1: 270 case XE_PL_STOLEN: 271 *placement = tt_placement; 272 break; 273 case XE_PL_TT: 274 default: 275 *placement = sys_placement; 276 break; 277 } 278 } 279 280 struct xe_ttm_tt { 281 struct ttm_tt ttm; 282 struct device *dev; 283 struct sg_table sgt; 284 struct sg_table *sg; 285 }; 286 287 static int xe_tt_map_sg(struct ttm_tt *tt) 288 { 289 struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm); 290 unsigned long num_pages = tt->num_pages; 291 int ret; 292 293 XE_WARN_ON(tt->page_flags & TTM_TT_FLAG_EXTERNAL); 294 295 if (xe_tt->sg) 296 return 0; 297 298 ret = sg_alloc_table_from_pages_segment(&xe_tt->sgt, tt->pages, 299 num_pages, 0, 300 (u64)num_pages << PAGE_SHIFT, 301 xe_sg_segment_size(xe_tt->dev), 302 GFP_KERNEL); 303 if (ret) 304 return ret; 305 306 xe_tt->sg = &xe_tt->sgt; 307 ret = dma_map_sgtable(xe_tt->dev, xe_tt->sg, DMA_BIDIRECTIONAL, 308 DMA_ATTR_SKIP_CPU_SYNC); 309 if (ret) { 310 sg_free_table(xe_tt->sg); 311 xe_tt->sg = NULL; 312 return ret; 313 } 314 315 return 0; 316 } 317 318 struct sg_table *xe_bo_sg(struct xe_bo *bo) 319 { 320 struct ttm_tt *tt = bo->ttm.ttm; 321 struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm); 322 323 return xe_tt->sg; 324 } 325 326 static struct ttm_tt *xe_ttm_tt_create(struct ttm_buffer_object *ttm_bo, 327 u32 page_flags) 328 { 329 struct xe_bo *bo = ttm_to_xe_bo(ttm_bo); 330 struct xe_device *xe = xe_bo_device(bo); 331 struct xe_ttm_tt *tt; 332 unsigned long extra_pages; 333 enum ttm_caching caching; 334 int err; 335 336 tt = kzalloc(sizeof(*tt), GFP_KERNEL); 337 if (!tt) 338 return NULL; 339 340 tt->dev = xe->drm.dev; 341 342 extra_pages = 0; 343 if (xe_bo_needs_ccs_pages(bo)) 344 extra_pages = DIV_ROUND_UP(xe_device_ccs_bytes(xe, bo->size), 345 PAGE_SIZE); 346 347 switch (bo->cpu_caching) { 348 case DRM_XE_GEM_CPU_CACHING_WC: 349 caching = ttm_write_combined; 350 break; 351 default: 352 caching = ttm_cached; 353 break; 354 } 355 356 WARN_ON((bo->flags & XE_BO_CREATE_USER_BIT) && !bo->cpu_caching); 357 358 /* 359 * Display scanout is always non-coherent with the CPU cache. 360 * 361 * For Xe_LPG and beyond, PPGTT PTE lookups are also non-coherent and 362 * require a CPU:WC mapping. 363 */ 364 if ((!bo->cpu_caching && bo->flags & XE_BO_SCANOUT_BIT) || 365 (xe->info.graphics_verx100 >= 1270 && bo->flags & XE_BO_PAGETABLE)) 366 caching = ttm_write_combined; 367 368 err = ttm_tt_init(&tt->ttm, &bo->ttm, page_flags, caching, extra_pages); 369 if (err) { 370 kfree(tt); 371 return NULL; 372 } 373 374 return &tt->ttm; 375 } 376 377 static int xe_ttm_tt_populate(struct ttm_device *ttm_dev, struct ttm_tt *tt, 378 struct ttm_operation_ctx *ctx) 379 { 380 int err; 381 382 /* 383 * dma-bufs are not populated with pages, and the dma- 384 * addresses are set up when moved to XE_PL_TT. 385 */ 386 if (tt->page_flags & TTM_TT_FLAG_EXTERNAL) 387 return 0; 388 389 err = ttm_pool_alloc(&ttm_dev->pool, tt, ctx); 390 if (err) 391 return err; 392 393 /* A follow up may move this xe_bo_move when BO is moved to XE_PL_TT */ 394 err = xe_tt_map_sg(tt); 395 if (err) 396 ttm_pool_free(&ttm_dev->pool, tt); 397 398 return err; 399 } 400 401 static void xe_ttm_tt_unpopulate(struct ttm_device *ttm_dev, struct ttm_tt *tt) 402 { 403 struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm); 404 405 if (tt->page_flags & TTM_TT_FLAG_EXTERNAL) 406 return; 407 408 if (xe_tt->sg) { 409 dma_unmap_sgtable(xe_tt->dev, xe_tt->sg, 410 DMA_BIDIRECTIONAL, 0); 411 sg_free_table(xe_tt->sg); 412 xe_tt->sg = NULL; 413 } 414 415 return ttm_pool_free(&ttm_dev->pool, tt); 416 } 417 418 static void xe_ttm_tt_destroy(struct ttm_device *ttm_dev, struct ttm_tt *tt) 419 { 420 ttm_tt_fini(tt); 421 kfree(tt); 422 } 423 424 static int xe_ttm_io_mem_reserve(struct ttm_device *bdev, 425 struct ttm_resource *mem) 426 { 427 struct xe_device *xe = ttm_to_xe_device(bdev); 428 429 switch (mem->mem_type) { 430 case XE_PL_SYSTEM: 431 case XE_PL_TT: 432 return 0; 433 case XE_PL_VRAM0: 434 case XE_PL_VRAM1: { 435 struct xe_ttm_vram_mgr_resource *vres = 436 to_xe_ttm_vram_mgr_resource(mem); 437 struct xe_mem_region *vram = res_to_mem_region(mem); 438 439 if (vres->used_visible_size < mem->size) 440 return -EINVAL; 441 442 mem->bus.offset = mem->start << PAGE_SHIFT; 443 444 if (vram->mapping && 445 mem->placement & TTM_PL_FLAG_CONTIGUOUS) 446 mem->bus.addr = (u8 __force *)vram->mapping + 447 mem->bus.offset; 448 449 mem->bus.offset += vram->io_start; 450 mem->bus.is_iomem = true; 451 452 #if !defined(CONFIG_X86) 453 mem->bus.caching = ttm_write_combined; 454 #endif 455 return 0; 456 } case XE_PL_STOLEN: 457 return xe_ttm_stolen_io_mem_reserve(xe, mem); 458 default: 459 return -EINVAL; 460 } 461 } 462 463 static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo, 464 const struct ttm_operation_ctx *ctx) 465 { 466 struct dma_resv_iter cursor; 467 struct dma_fence *fence; 468 struct drm_gem_object *obj = &bo->ttm.base; 469 struct drm_gpuvm_bo *vm_bo; 470 bool idle = false; 471 int ret = 0; 472 473 dma_resv_assert_held(bo->ttm.base.resv); 474 475 if (!list_empty(&bo->ttm.base.gpuva.list)) { 476 dma_resv_iter_begin(&cursor, bo->ttm.base.resv, 477 DMA_RESV_USAGE_BOOKKEEP); 478 dma_resv_for_each_fence_unlocked(&cursor, fence) 479 dma_fence_enable_sw_signaling(fence); 480 dma_resv_iter_end(&cursor); 481 } 482 483 drm_gem_for_each_gpuvm_bo(vm_bo, obj) { 484 struct xe_vm *vm = gpuvm_to_vm(vm_bo->vm); 485 struct drm_gpuva *gpuva; 486 487 if (!xe_vm_in_fault_mode(vm)) { 488 drm_gpuvm_bo_evict(vm_bo, true); 489 continue; 490 } 491 492 if (!idle) { 493 long timeout; 494 495 if (ctx->no_wait_gpu && 496 !dma_resv_test_signaled(bo->ttm.base.resv, 497 DMA_RESV_USAGE_BOOKKEEP)) 498 return -EBUSY; 499 500 timeout = dma_resv_wait_timeout(bo->ttm.base.resv, 501 DMA_RESV_USAGE_BOOKKEEP, 502 ctx->interruptible, 503 MAX_SCHEDULE_TIMEOUT); 504 if (!timeout) 505 return -ETIME; 506 if (timeout < 0) 507 return timeout; 508 509 idle = true; 510 } 511 512 drm_gpuvm_bo_for_each_va(gpuva, vm_bo) { 513 struct xe_vma *vma = gpuva_to_vma(gpuva); 514 515 trace_xe_vma_evict(vma); 516 ret = xe_vm_invalidate_vma(vma); 517 if (XE_WARN_ON(ret)) 518 return ret; 519 } 520 } 521 522 return ret; 523 } 524 525 /* 526 * The dma-buf map_attachment() / unmap_attachment() is hooked up here. 527 * Note that unmapping the attachment is deferred to the next 528 * map_attachment time, or to bo destroy (after idling) whichever comes first. 529 * This is to avoid syncing before unmap_attachment(), assuming that the 530 * caller relies on idling the reservation object before moving the 531 * backing store out. Should that assumption not hold, then we will be able 532 * to unconditionally call unmap_attachment() when moving out to system. 533 */ 534 static int xe_bo_move_dmabuf(struct ttm_buffer_object *ttm_bo, 535 struct ttm_resource *new_res) 536 { 537 struct dma_buf_attachment *attach = ttm_bo->base.import_attach; 538 struct xe_ttm_tt *xe_tt = container_of(ttm_bo->ttm, struct xe_ttm_tt, 539 ttm); 540 struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev); 541 struct sg_table *sg; 542 543 xe_assert(xe, attach); 544 xe_assert(xe, ttm_bo->ttm); 545 546 if (new_res->mem_type == XE_PL_SYSTEM) 547 goto out; 548 549 if (ttm_bo->sg) { 550 dma_buf_unmap_attachment(attach, ttm_bo->sg, DMA_BIDIRECTIONAL); 551 ttm_bo->sg = NULL; 552 } 553 554 sg = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL); 555 if (IS_ERR(sg)) 556 return PTR_ERR(sg); 557 558 ttm_bo->sg = sg; 559 xe_tt->sg = sg; 560 561 out: 562 ttm_bo_move_null(ttm_bo, new_res); 563 564 return 0; 565 } 566 567 /** 568 * xe_bo_move_notify - Notify subsystems of a pending move 569 * @bo: The buffer object 570 * @ctx: The struct ttm_operation_ctx controlling locking and waits. 571 * 572 * This function notifies subsystems of an upcoming buffer move. 573 * Upon receiving such a notification, subsystems should schedule 574 * halting access to the underlying pages and optionally add a fence 575 * to the buffer object's dma_resv object, that signals when access is 576 * stopped. The caller will wait on all dma_resv fences before 577 * starting the move. 578 * 579 * A subsystem may commence access to the object after obtaining 580 * bindings to the new backing memory under the object lock. 581 * 582 * Return: 0 on success, -EINTR or -ERESTARTSYS if interrupted in fault mode, 583 * negative error code on error. 584 */ 585 static int xe_bo_move_notify(struct xe_bo *bo, 586 const struct ttm_operation_ctx *ctx) 587 { 588 struct ttm_buffer_object *ttm_bo = &bo->ttm; 589 struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev); 590 int ret; 591 592 /* 593 * If this starts to call into many components, consider 594 * using a notification chain here. 595 */ 596 597 if (xe_bo_is_pinned(bo)) 598 return -EINVAL; 599 600 xe_bo_vunmap(bo); 601 ret = xe_bo_trigger_rebind(xe, bo, ctx); 602 if (ret) 603 return ret; 604 605 /* Don't call move_notify() for imported dma-bufs. */ 606 if (ttm_bo->base.dma_buf && !ttm_bo->base.import_attach) 607 dma_buf_move_notify(ttm_bo->base.dma_buf); 608 609 return 0; 610 } 611 612 static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict, 613 struct ttm_operation_ctx *ctx, 614 struct ttm_resource *new_mem, 615 struct ttm_place *hop) 616 { 617 struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev); 618 struct xe_bo *bo = ttm_to_xe_bo(ttm_bo); 619 struct ttm_resource *old_mem = ttm_bo->resource; 620 u32 old_mem_type = old_mem ? old_mem->mem_type : XE_PL_SYSTEM; 621 struct ttm_tt *ttm = ttm_bo->ttm; 622 struct xe_migrate *migrate = NULL; 623 struct dma_fence *fence; 624 bool move_lacks_source; 625 bool tt_has_data; 626 bool needs_clear; 627 bool handle_system_ccs = (!IS_DGFX(xe) && xe_bo_needs_ccs_pages(bo) && 628 ttm && ttm_tt_is_populated(ttm)) ? true : false; 629 int ret = 0; 630 /* Bo creation path, moving to system or TT. */ 631 if ((!old_mem && ttm) && !handle_system_ccs) { 632 ttm_bo_move_null(ttm_bo, new_mem); 633 return 0; 634 } 635 636 if (ttm_bo->type == ttm_bo_type_sg) { 637 ret = xe_bo_move_notify(bo, ctx); 638 if (!ret) 639 ret = xe_bo_move_dmabuf(ttm_bo, new_mem); 640 goto out; 641 } 642 643 tt_has_data = ttm && (ttm_tt_is_populated(ttm) || 644 (ttm->page_flags & TTM_TT_FLAG_SWAPPED)); 645 646 move_lacks_source = handle_system_ccs ? (!bo->ccs_cleared) : 647 (!mem_type_is_vram(old_mem_type) && !tt_has_data); 648 649 needs_clear = (ttm && ttm->page_flags & TTM_TT_FLAG_ZERO_ALLOC) || 650 (!ttm && ttm_bo->type == ttm_bo_type_device); 651 652 if ((move_lacks_source && !needs_clear)) { 653 ttm_bo_move_null(ttm_bo, new_mem); 654 goto out; 655 } 656 657 if (old_mem_type == XE_PL_SYSTEM && new_mem->mem_type == XE_PL_TT && !handle_system_ccs) { 658 ttm_bo_move_null(ttm_bo, new_mem); 659 goto out; 660 } 661 662 /* 663 * Failed multi-hop where the old_mem is still marked as 664 * TTM_PL_FLAG_TEMPORARY, should just be a dummy move. 665 */ 666 if (old_mem_type == XE_PL_TT && 667 new_mem->mem_type == XE_PL_TT) { 668 ttm_bo_move_null(ttm_bo, new_mem); 669 goto out; 670 } 671 672 if (!move_lacks_source && !xe_bo_is_pinned(bo)) { 673 ret = xe_bo_move_notify(bo, ctx); 674 if (ret) 675 goto out; 676 } 677 678 if (old_mem_type == XE_PL_TT && 679 new_mem->mem_type == XE_PL_SYSTEM) { 680 long timeout = dma_resv_wait_timeout(ttm_bo->base.resv, 681 DMA_RESV_USAGE_BOOKKEEP, 682 true, 683 MAX_SCHEDULE_TIMEOUT); 684 if (timeout < 0) { 685 ret = timeout; 686 goto out; 687 } 688 689 if (!handle_system_ccs) { 690 ttm_bo_move_null(ttm_bo, new_mem); 691 goto out; 692 } 693 } 694 695 if (!move_lacks_source && 696 ((old_mem_type == XE_PL_SYSTEM && resource_is_vram(new_mem)) || 697 (mem_type_is_vram(old_mem_type) && 698 new_mem->mem_type == XE_PL_SYSTEM))) { 699 hop->fpfn = 0; 700 hop->lpfn = 0; 701 hop->mem_type = XE_PL_TT; 702 hop->flags = TTM_PL_FLAG_TEMPORARY; 703 ret = -EMULTIHOP; 704 goto out; 705 } 706 707 if (bo->tile) 708 migrate = bo->tile->migrate; 709 else if (resource_is_vram(new_mem)) 710 migrate = mem_type_to_migrate(xe, new_mem->mem_type); 711 else if (mem_type_is_vram(old_mem_type)) 712 migrate = mem_type_to_migrate(xe, old_mem_type); 713 else 714 migrate = xe->tiles[0].migrate; 715 716 xe_assert(xe, migrate); 717 718 trace_xe_bo_move(bo); 719 xe_device_mem_access_get(xe); 720 721 if (xe_bo_is_pinned(bo) && !xe_bo_is_user(bo)) { 722 /* 723 * Kernel memory that is pinned should only be moved on suspend 724 * / resume, some of the pinned memory is required for the 725 * device to resume / use the GPU to move other evicted memory 726 * (user memory) around. This likely could be optimized a bit 727 * futher where we find the minimum set of pinned memory 728 * required for resume but for simplity doing a memcpy for all 729 * pinned memory. 730 */ 731 ret = xe_bo_vmap(bo); 732 if (!ret) { 733 ret = ttm_bo_move_memcpy(ttm_bo, ctx, new_mem); 734 735 /* Create a new VMAP once kernel BO back in VRAM */ 736 if (!ret && resource_is_vram(new_mem)) { 737 struct xe_mem_region *vram = res_to_mem_region(new_mem); 738 void __iomem *new_addr = vram->mapping + 739 (new_mem->start << PAGE_SHIFT); 740 741 if (XE_WARN_ON(new_mem->start == XE_BO_INVALID_OFFSET)) { 742 ret = -EINVAL; 743 xe_device_mem_access_put(xe); 744 goto out; 745 } 746 747 xe_assert(xe, new_mem->start == 748 bo->placements->fpfn); 749 750 iosys_map_set_vaddr_iomem(&bo->vmap, new_addr); 751 } 752 } 753 } else { 754 if (move_lacks_source) 755 fence = xe_migrate_clear(migrate, bo, new_mem); 756 else 757 fence = xe_migrate_copy(migrate, bo, bo, old_mem, 758 new_mem, handle_system_ccs); 759 if (IS_ERR(fence)) { 760 ret = PTR_ERR(fence); 761 xe_device_mem_access_put(xe); 762 goto out; 763 } 764 if (!move_lacks_source) { 765 ret = ttm_bo_move_accel_cleanup(ttm_bo, fence, evict, 766 true, new_mem); 767 if (ret) { 768 dma_fence_wait(fence, false); 769 ttm_bo_move_null(ttm_bo, new_mem); 770 ret = 0; 771 } 772 } else { 773 /* 774 * ttm_bo_move_accel_cleanup() may blow up if 775 * bo->resource == NULL, so just attach the 776 * fence and set the new resource. 777 */ 778 dma_resv_add_fence(ttm_bo->base.resv, fence, 779 DMA_RESV_USAGE_KERNEL); 780 ttm_bo_move_null(ttm_bo, new_mem); 781 } 782 783 dma_fence_put(fence); 784 } 785 786 xe_device_mem_access_put(xe); 787 788 out: 789 return ret; 790 791 } 792 793 /** 794 * xe_bo_evict_pinned() - Evict a pinned VRAM object to system memory 795 * @bo: The buffer object to move. 796 * 797 * On successful completion, the object memory will be moved to sytem memory. 798 * This function blocks until the object has been fully moved. 799 * 800 * This is needed to for special handling of pinned VRAM object during 801 * suspend-resume. 802 * 803 * Return: 0 on success. Negative error code on failure. 804 */ 805 int xe_bo_evict_pinned(struct xe_bo *bo) 806 { 807 struct ttm_place place = { 808 .mem_type = XE_PL_TT, 809 }; 810 struct ttm_placement placement = { 811 .placement = &place, 812 .num_placement = 1, 813 }; 814 struct ttm_operation_ctx ctx = { 815 .interruptible = false, 816 }; 817 struct ttm_resource *new_mem; 818 int ret; 819 820 xe_bo_assert_held(bo); 821 822 if (WARN_ON(!bo->ttm.resource)) 823 return -EINVAL; 824 825 if (WARN_ON(!xe_bo_is_pinned(bo))) 826 return -EINVAL; 827 828 if (WARN_ON(!xe_bo_is_vram(bo))) 829 return -EINVAL; 830 831 ret = ttm_bo_mem_space(&bo->ttm, &placement, &new_mem, &ctx); 832 if (ret) 833 return ret; 834 835 if (!bo->ttm.ttm) { 836 bo->ttm.ttm = xe_ttm_tt_create(&bo->ttm, 0); 837 if (!bo->ttm.ttm) { 838 ret = -ENOMEM; 839 goto err_res_free; 840 } 841 } 842 843 ret = ttm_tt_populate(bo->ttm.bdev, bo->ttm.ttm, &ctx); 844 if (ret) 845 goto err_res_free; 846 847 ret = dma_resv_reserve_fences(bo->ttm.base.resv, 1); 848 if (ret) 849 goto err_res_free; 850 851 ret = xe_bo_move(&bo->ttm, false, &ctx, new_mem, NULL); 852 if (ret) 853 goto err_res_free; 854 855 dma_resv_wait_timeout(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL, 856 false, MAX_SCHEDULE_TIMEOUT); 857 858 return 0; 859 860 err_res_free: 861 ttm_resource_free(&bo->ttm, &new_mem); 862 return ret; 863 } 864 865 /** 866 * xe_bo_restore_pinned() - Restore a pinned VRAM object 867 * @bo: The buffer object to move. 868 * 869 * On successful completion, the object memory will be moved back to VRAM. 870 * This function blocks until the object has been fully moved. 871 * 872 * This is needed to for special handling of pinned VRAM object during 873 * suspend-resume. 874 * 875 * Return: 0 on success. Negative error code on failure. 876 */ 877 int xe_bo_restore_pinned(struct xe_bo *bo) 878 { 879 struct ttm_operation_ctx ctx = { 880 .interruptible = false, 881 }; 882 struct ttm_resource *new_mem; 883 int ret; 884 885 xe_bo_assert_held(bo); 886 887 if (WARN_ON(!bo->ttm.resource)) 888 return -EINVAL; 889 890 if (WARN_ON(!xe_bo_is_pinned(bo))) 891 return -EINVAL; 892 893 if (WARN_ON(xe_bo_is_vram(bo) || !bo->ttm.ttm)) 894 return -EINVAL; 895 896 ret = ttm_bo_mem_space(&bo->ttm, &bo->placement, &new_mem, &ctx); 897 if (ret) 898 return ret; 899 900 ret = ttm_tt_populate(bo->ttm.bdev, bo->ttm.ttm, &ctx); 901 if (ret) 902 goto err_res_free; 903 904 ret = dma_resv_reserve_fences(bo->ttm.base.resv, 1); 905 if (ret) 906 goto err_res_free; 907 908 ret = xe_bo_move(&bo->ttm, false, &ctx, new_mem, NULL); 909 if (ret) 910 goto err_res_free; 911 912 dma_resv_wait_timeout(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL, 913 false, MAX_SCHEDULE_TIMEOUT); 914 915 return 0; 916 917 err_res_free: 918 ttm_resource_free(&bo->ttm, &new_mem); 919 return ret; 920 } 921 922 static unsigned long xe_ttm_io_mem_pfn(struct ttm_buffer_object *ttm_bo, 923 unsigned long page_offset) 924 { 925 struct xe_bo *bo = ttm_to_xe_bo(ttm_bo); 926 struct xe_res_cursor cursor; 927 struct xe_mem_region *vram; 928 929 if (ttm_bo->resource->mem_type == XE_PL_STOLEN) 930 return xe_ttm_stolen_io_offset(bo, page_offset << PAGE_SHIFT) >> PAGE_SHIFT; 931 932 vram = res_to_mem_region(ttm_bo->resource); 933 xe_res_first(ttm_bo->resource, (u64)page_offset << PAGE_SHIFT, 0, &cursor); 934 return (vram->io_start + cursor.start) >> PAGE_SHIFT; 935 } 936 937 static void __xe_bo_vunmap(struct xe_bo *bo); 938 939 /* 940 * TODO: Move this function to TTM so we don't rely on how TTM does its 941 * locking, thereby abusing TTM internals. 942 */ 943 static bool xe_ttm_bo_lock_in_destructor(struct ttm_buffer_object *ttm_bo) 944 { 945 struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev); 946 bool locked; 947 948 xe_assert(xe, !kref_read(&ttm_bo->kref)); 949 950 /* 951 * We can typically only race with TTM trylocking under the 952 * lru_lock, which will immediately be unlocked again since 953 * the ttm_bo refcount is zero at this point. So trylocking *should* 954 * always succeed here, as long as we hold the lru lock. 955 */ 956 spin_lock(&ttm_bo->bdev->lru_lock); 957 locked = dma_resv_trylock(ttm_bo->base.resv); 958 spin_unlock(&ttm_bo->bdev->lru_lock); 959 xe_assert(xe, locked); 960 961 return locked; 962 } 963 964 static void xe_ttm_bo_release_notify(struct ttm_buffer_object *ttm_bo) 965 { 966 struct dma_resv_iter cursor; 967 struct dma_fence *fence; 968 struct dma_fence *replacement = NULL; 969 struct xe_bo *bo; 970 971 if (!xe_bo_is_xe_bo(ttm_bo)) 972 return; 973 974 bo = ttm_to_xe_bo(ttm_bo); 975 xe_assert(xe_bo_device(bo), !(bo->created && kref_read(&ttm_bo->base.refcount))); 976 977 /* 978 * Corner case where TTM fails to allocate memory and this BOs resv 979 * still points the VMs resv 980 */ 981 if (ttm_bo->base.resv != &ttm_bo->base._resv) 982 return; 983 984 if (!xe_ttm_bo_lock_in_destructor(ttm_bo)) 985 return; 986 987 /* 988 * Scrub the preempt fences if any. The unbind fence is already 989 * attached to the resv. 990 * TODO: Don't do this for external bos once we scrub them after 991 * unbind. 992 */ 993 dma_resv_for_each_fence(&cursor, ttm_bo->base.resv, 994 DMA_RESV_USAGE_BOOKKEEP, fence) { 995 if (xe_fence_is_xe_preempt(fence) && 996 !dma_fence_is_signaled(fence)) { 997 if (!replacement) 998 replacement = dma_fence_get_stub(); 999 1000 dma_resv_replace_fences(ttm_bo->base.resv, 1001 fence->context, 1002 replacement, 1003 DMA_RESV_USAGE_BOOKKEEP); 1004 } 1005 } 1006 dma_fence_put(replacement); 1007 1008 dma_resv_unlock(ttm_bo->base.resv); 1009 } 1010 1011 static void xe_ttm_bo_delete_mem_notify(struct ttm_buffer_object *ttm_bo) 1012 { 1013 if (!xe_bo_is_xe_bo(ttm_bo)) 1014 return; 1015 1016 /* 1017 * Object is idle and about to be destroyed. Release the 1018 * dma-buf attachment. 1019 */ 1020 if (ttm_bo->type == ttm_bo_type_sg && ttm_bo->sg) { 1021 struct xe_ttm_tt *xe_tt = container_of(ttm_bo->ttm, 1022 struct xe_ttm_tt, ttm); 1023 1024 dma_buf_unmap_attachment(ttm_bo->base.import_attach, ttm_bo->sg, 1025 DMA_BIDIRECTIONAL); 1026 ttm_bo->sg = NULL; 1027 xe_tt->sg = NULL; 1028 } 1029 } 1030 1031 struct ttm_device_funcs xe_ttm_funcs = { 1032 .ttm_tt_create = xe_ttm_tt_create, 1033 .ttm_tt_populate = xe_ttm_tt_populate, 1034 .ttm_tt_unpopulate = xe_ttm_tt_unpopulate, 1035 .ttm_tt_destroy = xe_ttm_tt_destroy, 1036 .evict_flags = xe_evict_flags, 1037 .move = xe_bo_move, 1038 .io_mem_reserve = xe_ttm_io_mem_reserve, 1039 .io_mem_pfn = xe_ttm_io_mem_pfn, 1040 .release_notify = xe_ttm_bo_release_notify, 1041 .eviction_valuable = ttm_bo_eviction_valuable, 1042 .delete_mem_notify = xe_ttm_bo_delete_mem_notify, 1043 }; 1044 1045 static void xe_ttm_bo_destroy(struct ttm_buffer_object *ttm_bo) 1046 { 1047 struct xe_bo *bo = ttm_to_xe_bo(ttm_bo); 1048 struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev); 1049 1050 if (bo->ttm.base.import_attach) 1051 drm_prime_gem_destroy(&bo->ttm.base, NULL); 1052 drm_gem_object_release(&bo->ttm.base); 1053 1054 xe_assert(xe, list_empty(&ttm_bo->base.gpuva.list)); 1055 1056 if (bo->ggtt_node.size) 1057 xe_ggtt_remove_bo(bo->tile->mem.ggtt, bo); 1058 1059 #ifdef CONFIG_PROC_FS 1060 if (bo->client) 1061 xe_drm_client_remove_bo(bo); 1062 #endif 1063 1064 if (bo->vm && xe_bo_is_user(bo)) 1065 xe_vm_put(bo->vm); 1066 1067 kfree(bo); 1068 } 1069 1070 static void xe_gem_object_free(struct drm_gem_object *obj) 1071 { 1072 /* Our BO reference counting scheme works as follows: 1073 * 1074 * The gem object kref is typically used throughout the driver, 1075 * and the gem object holds a ttm_buffer_object refcount, so 1076 * that when the last gem object reference is put, which is when 1077 * we end up in this function, we put also that ttm_buffer_object 1078 * refcount. Anything using gem interfaces is then no longer 1079 * allowed to access the object in a way that requires a gem 1080 * refcount, including locking the object. 1081 * 1082 * driver ttm callbacks is allowed to use the ttm_buffer_object 1083 * refcount directly if needed. 1084 */ 1085 __xe_bo_vunmap(gem_to_xe_bo(obj)); 1086 ttm_bo_put(container_of(obj, struct ttm_buffer_object, base)); 1087 } 1088 1089 static void xe_gem_object_close(struct drm_gem_object *obj, 1090 struct drm_file *file_priv) 1091 { 1092 struct xe_bo *bo = gem_to_xe_bo(obj); 1093 1094 if (bo->vm && !xe_vm_in_fault_mode(bo->vm)) { 1095 xe_assert(xe_bo_device(bo), xe_bo_is_user(bo)); 1096 1097 xe_bo_lock(bo, false); 1098 ttm_bo_set_bulk_move(&bo->ttm, NULL); 1099 xe_bo_unlock(bo); 1100 } 1101 } 1102 1103 static bool should_migrate_to_system(struct xe_bo *bo) 1104 { 1105 struct xe_device *xe = xe_bo_device(bo); 1106 1107 return xe_device_in_fault_mode(xe) && bo->props.cpu_atomic; 1108 } 1109 1110 static vm_fault_t xe_gem_fault(struct vm_fault *vmf) 1111 { 1112 struct ttm_buffer_object *tbo = vmf->vma->vm_private_data; 1113 struct drm_device *ddev = tbo->base.dev; 1114 vm_fault_t ret; 1115 int idx, r = 0; 1116 1117 ret = ttm_bo_vm_reserve(tbo, vmf); 1118 if (ret) 1119 return ret; 1120 1121 if (drm_dev_enter(ddev, &idx)) { 1122 struct xe_bo *bo = ttm_to_xe_bo(tbo); 1123 1124 trace_xe_bo_cpu_fault(bo); 1125 1126 if (should_migrate_to_system(bo)) { 1127 r = xe_bo_migrate(bo, XE_PL_TT); 1128 if (r == -EBUSY || r == -ERESTARTSYS || r == -EINTR) 1129 ret = VM_FAULT_NOPAGE; 1130 else if (r) 1131 ret = VM_FAULT_SIGBUS; 1132 } 1133 if (!ret) 1134 ret = ttm_bo_vm_fault_reserved(vmf, 1135 vmf->vma->vm_page_prot, 1136 TTM_BO_VM_NUM_PREFAULT); 1137 drm_dev_exit(idx); 1138 } else { 1139 ret = ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot); 1140 } 1141 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) 1142 return ret; 1143 1144 dma_resv_unlock(tbo->base.resv); 1145 return ret; 1146 } 1147 1148 static const struct vm_operations_struct xe_gem_vm_ops = { 1149 .fault = xe_gem_fault, 1150 .open = ttm_bo_vm_open, 1151 .close = ttm_bo_vm_close, 1152 .access = ttm_bo_vm_access 1153 }; 1154 1155 static const struct drm_gem_object_funcs xe_gem_object_funcs = { 1156 .free = xe_gem_object_free, 1157 .close = xe_gem_object_close, 1158 .mmap = drm_gem_ttm_mmap, 1159 .export = xe_gem_prime_export, 1160 .vm_ops = &xe_gem_vm_ops, 1161 }; 1162 1163 /** 1164 * xe_bo_alloc - Allocate storage for a struct xe_bo 1165 * 1166 * This funcition is intended to allocate storage to be used for input 1167 * to __xe_bo_create_locked(), in the case a pointer to the bo to be 1168 * created is needed before the call to __xe_bo_create_locked(). 1169 * If __xe_bo_create_locked ends up never to be called, then the 1170 * storage allocated with this function needs to be freed using 1171 * xe_bo_free(). 1172 * 1173 * Return: A pointer to an uninitialized struct xe_bo on success, 1174 * ERR_PTR(-ENOMEM) on error. 1175 */ 1176 struct xe_bo *xe_bo_alloc(void) 1177 { 1178 struct xe_bo *bo = kzalloc(sizeof(*bo), GFP_KERNEL); 1179 1180 if (!bo) 1181 return ERR_PTR(-ENOMEM); 1182 1183 return bo; 1184 } 1185 1186 /** 1187 * xe_bo_free - Free storage allocated using xe_bo_alloc() 1188 * @bo: The buffer object storage. 1189 * 1190 * Refer to xe_bo_alloc() documentation for valid use-cases. 1191 */ 1192 void xe_bo_free(struct xe_bo *bo) 1193 { 1194 kfree(bo); 1195 } 1196 1197 struct xe_bo *___xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo, 1198 struct xe_tile *tile, struct dma_resv *resv, 1199 struct ttm_lru_bulk_move *bulk, size_t size, 1200 u16 cpu_caching, enum ttm_bo_type type, 1201 u32 flags) 1202 { 1203 struct ttm_operation_ctx ctx = { 1204 .interruptible = true, 1205 .no_wait_gpu = false, 1206 }; 1207 struct ttm_placement *placement; 1208 uint32_t alignment; 1209 size_t aligned_size; 1210 int err; 1211 1212 /* Only kernel objects should set GT */ 1213 xe_assert(xe, !tile || type == ttm_bo_type_kernel); 1214 1215 if (XE_WARN_ON(!size)) { 1216 xe_bo_free(bo); 1217 return ERR_PTR(-EINVAL); 1218 } 1219 1220 if (flags & (XE_BO_CREATE_VRAM_MASK | XE_BO_CREATE_STOLEN_BIT) && 1221 !(flags & XE_BO_CREATE_IGNORE_MIN_PAGE_SIZE_BIT) && 1222 xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) { 1223 aligned_size = ALIGN(size, SZ_64K); 1224 if (type != ttm_bo_type_device) 1225 size = ALIGN(size, SZ_64K); 1226 flags |= XE_BO_INTERNAL_64K; 1227 alignment = SZ_64K >> PAGE_SHIFT; 1228 1229 } else { 1230 aligned_size = ALIGN(size, SZ_4K); 1231 flags &= ~XE_BO_INTERNAL_64K; 1232 alignment = SZ_4K >> PAGE_SHIFT; 1233 } 1234 1235 if (type == ttm_bo_type_device && aligned_size != size) 1236 return ERR_PTR(-EINVAL); 1237 1238 if (!bo) { 1239 bo = xe_bo_alloc(); 1240 if (IS_ERR(bo)) 1241 return bo; 1242 } 1243 1244 bo->ccs_cleared = false; 1245 bo->tile = tile; 1246 bo->size = size; 1247 bo->flags = flags; 1248 bo->cpu_caching = cpu_caching; 1249 bo->ttm.base.funcs = &xe_gem_object_funcs; 1250 bo->props.preferred_mem_class = XE_BO_PROPS_INVALID; 1251 bo->props.preferred_gt = XE_BO_PROPS_INVALID; 1252 bo->props.preferred_mem_type = XE_BO_PROPS_INVALID; 1253 bo->ttm.priority = XE_BO_PRIORITY_NORMAL; 1254 INIT_LIST_HEAD(&bo->pinned_link); 1255 #ifdef CONFIG_PROC_FS 1256 INIT_LIST_HEAD(&bo->client_link); 1257 #endif 1258 1259 drm_gem_private_object_init(&xe->drm, &bo->ttm.base, size); 1260 1261 if (resv) { 1262 ctx.allow_res_evict = !(flags & XE_BO_CREATE_NO_RESV_EVICT); 1263 ctx.resv = resv; 1264 } 1265 1266 if (!(flags & XE_BO_FIXED_PLACEMENT_BIT)) { 1267 err = __xe_bo_placement_for_flags(xe, bo, bo->flags); 1268 if (WARN_ON(err)) { 1269 xe_ttm_bo_destroy(&bo->ttm); 1270 return ERR_PTR(err); 1271 } 1272 } 1273 1274 /* Defer populating type_sg bos */ 1275 placement = (type == ttm_bo_type_sg || 1276 bo->flags & XE_BO_DEFER_BACKING) ? &sys_placement : 1277 &bo->placement; 1278 err = ttm_bo_init_reserved(&xe->ttm, &bo->ttm, type, 1279 placement, alignment, 1280 &ctx, NULL, resv, xe_ttm_bo_destroy); 1281 if (err) 1282 return ERR_PTR(err); 1283 1284 /* 1285 * The VRAM pages underneath are potentially still being accessed by the 1286 * GPU, as per async GPU clearing and async evictions. However TTM makes 1287 * sure to add any corresponding move/clear fences into the objects 1288 * dma-resv using the DMA_RESV_USAGE_KERNEL slot. 1289 * 1290 * For KMD internal buffers we don't care about GPU clearing, however we 1291 * still need to handle async evictions, where the VRAM is still being 1292 * accessed by the GPU. Most internal callers are not expecting this, 1293 * since they are missing the required synchronisation before accessing 1294 * the memory. To keep things simple just sync wait any kernel fences 1295 * here, if the buffer is designated KMD internal. 1296 * 1297 * For normal userspace objects we should already have the required 1298 * pipelining or sync waiting elsewhere, since we already have to deal 1299 * with things like async GPU clearing. 1300 */ 1301 if (type == ttm_bo_type_kernel) { 1302 long timeout = dma_resv_wait_timeout(bo->ttm.base.resv, 1303 DMA_RESV_USAGE_KERNEL, 1304 ctx.interruptible, 1305 MAX_SCHEDULE_TIMEOUT); 1306 1307 if (timeout < 0) { 1308 if (!resv) 1309 dma_resv_unlock(bo->ttm.base.resv); 1310 xe_bo_put(bo); 1311 return ERR_PTR(timeout); 1312 } 1313 } 1314 1315 bo->created = true; 1316 if (bulk) 1317 ttm_bo_set_bulk_move(&bo->ttm, bulk); 1318 else 1319 ttm_bo_move_to_lru_tail_unlocked(&bo->ttm); 1320 1321 return bo; 1322 } 1323 1324 static int __xe_bo_fixed_placement(struct xe_device *xe, 1325 struct xe_bo *bo, 1326 u32 flags, 1327 u64 start, u64 end, u64 size) 1328 { 1329 struct ttm_place *place = bo->placements; 1330 1331 if (flags & (XE_BO_CREATE_USER_BIT|XE_BO_CREATE_SYSTEM_BIT)) 1332 return -EINVAL; 1333 1334 place->flags = TTM_PL_FLAG_CONTIGUOUS; 1335 place->fpfn = start >> PAGE_SHIFT; 1336 place->lpfn = end >> PAGE_SHIFT; 1337 1338 switch (flags & (XE_BO_CREATE_STOLEN_BIT | XE_BO_CREATE_VRAM_MASK)) { 1339 case XE_BO_CREATE_VRAM0_BIT: 1340 place->mem_type = XE_PL_VRAM0; 1341 break; 1342 case XE_BO_CREATE_VRAM1_BIT: 1343 place->mem_type = XE_PL_VRAM1; 1344 break; 1345 case XE_BO_CREATE_STOLEN_BIT: 1346 place->mem_type = XE_PL_STOLEN; 1347 break; 1348 1349 default: 1350 /* 0 or multiple of the above set */ 1351 return -EINVAL; 1352 } 1353 1354 bo->placement = (struct ttm_placement) { 1355 .num_placement = 1, 1356 .placement = place, 1357 }; 1358 1359 return 0; 1360 } 1361 1362 static struct xe_bo * 1363 __xe_bo_create_locked(struct xe_device *xe, 1364 struct xe_tile *tile, struct xe_vm *vm, 1365 size_t size, u64 start, u64 end, 1366 u16 cpu_caching, enum ttm_bo_type type, u32 flags) 1367 { 1368 struct xe_bo *bo = NULL; 1369 int err; 1370 1371 if (vm) 1372 xe_vm_assert_held(vm); 1373 1374 if (start || end != ~0ULL) { 1375 bo = xe_bo_alloc(); 1376 if (IS_ERR(bo)) 1377 return bo; 1378 1379 flags |= XE_BO_FIXED_PLACEMENT_BIT; 1380 err = __xe_bo_fixed_placement(xe, bo, flags, start, end, size); 1381 if (err) { 1382 xe_bo_free(bo); 1383 return ERR_PTR(err); 1384 } 1385 } 1386 1387 bo = ___xe_bo_create_locked(xe, bo, tile, vm ? xe_vm_resv(vm) : NULL, 1388 vm && !xe_vm_in_fault_mode(vm) && 1389 flags & XE_BO_CREATE_USER_BIT ? 1390 &vm->lru_bulk_move : NULL, size, 1391 cpu_caching, type, flags); 1392 if (IS_ERR(bo)) 1393 return bo; 1394 1395 /* 1396 * Note that instead of taking a reference no the drm_gpuvm_resv_bo(), 1397 * to ensure the shared resv doesn't disappear under the bo, the bo 1398 * will keep a reference to the vm, and avoid circular references 1399 * by having all the vm's bo refereferences released at vm close 1400 * time. 1401 */ 1402 if (vm && xe_bo_is_user(bo)) 1403 xe_vm_get(vm); 1404 bo->vm = vm; 1405 1406 if (bo->flags & XE_BO_CREATE_GGTT_BIT) { 1407 if (!tile && flags & XE_BO_CREATE_STOLEN_BIT) 1408 tile = xe_device_get_root_tile(xe); 1409 1410 xe_assert(xe, tile); 1411 1412 if (flags & XE_BO_FIXED_PLACEMENT_BIT) { 1413 err = xe_ggtt_insert_bo_at(tile->mem.ggtt, bo, 1414 start + bo->size, U64_MAX); 1415 } else { 1416 err = xe_ggtt_insert_bo(tile->mem.ggtt, bo); 1417 } 1418 if (err) 1419 goto err_unlock_put_bo; 1420 } 1421 1422 return bo; 1423 1424 err_unlock_put_bo: 1425 __xe_bo_unset_bulk_move(bo); 1426 xe_bo_unlock_vm_held(bo); 1427 xe_bo_put(bo); 1428 return ERR_PTR(err); 1429 } 1430 1431 struct xe_bo * 1432 xe_bo_create_locked_range(struct xe_device *xe, 1433 struct xe_tile *tile, struct xe_vm *vm, 1434 size_t size, u64 start, u64 end, 1435 enum ttm_bo_type type, u32 flags) 1436 { 1437 return __xe_bo_create_locked(xe, tile, vm, size, start, end, 0, type, flags); 1438 } 1439 1440 struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_tile *tile, 1441 struct xe_vm *vm, size_t size, 1442 enum ttm_bo_type type, u32 flags) 1443 { 1444 return __xe_bo_create_locked(xe, tile, vm, size, 0, ~0ULL, 0, type, flags); 1445 } 1446 1447 struct xe_bo *xe_bo_create_user(struct xe_device *xe, struct xe_tile *tile, 1448 struct xe_vm *vm, size_t size, 1449 u16 cpu_caching, 1450 enum ttm_bo_type type, 1451 u32 flags) 1452 { 1453 struct xe_bo *bo = __xe_bo_create_locked(xe, tile, vm, size, 0, ~0ULL, 1454 cpu_caching, type, 1455 flags | XE_BO_CREATE_USER_BIT); 1456 if (!IS_ERR(bo)) 1457 xe_bo_unlock_vm_held(bo); 1458 1459 return bo; 1460 } 1461 1462 struct xe_bo *xe_bo_create(struct xe_device *xe, struct xe_tile *tile, 1463 struct xe_vm *vm, size_t size, 1464 enum ttm_bo_type type, u32 flags) 1465 { 1466 struct xe_bo *bo = xe_bo_create_locked(xe, tile, vm, size, type, flags); 1467 1468 if (!IS_ERR(bo)) 1469 xe_bo_unlock_vm_held(bo); 1470 1471 return bo; 1472 } 1473 1474 struct xe_bo *xe_bo_create_pin_map_at(struct xe_device *xe, struct xe_tile *tile, 1475 struct xe_vm *vm, 1476 size_t size, u64 offset, 1477 enum ttm_bo_type type, u32 flags) 1478 { 1479 struct xe_bo *bo; 1480 int err; 1481 u64 start = offset == ~0ull ? 0 : offset; 1482 u64 end = offset == ~0ull ? offset : start + size; 1483 1484 if (flags & XE_BO_CREATE_STOLEN_BIT && 1485 xe_ttm_stolen_cpu_access_needs_ggtt(xe)) 1486 flags |= XE_BO_CREATE_GGTT_BIT; 1487 1488 bo = xe_bo_create_locked_range(xe, tile, vm, size, start, end, type, 1489 flags | XE_BO_NEEDS_CPU_ACCESS); 1490 if (IS_ERR(bo)) 1491 return bo; 1492 1493 err = xe_bo_pin(bo); 1494 if (err) 1495 goto err_put; 1496 1497 err = xe_bo_vmap(bo); 1498 if (err) 1499 goto err_unpin; 1500 1501 xe_bo_unlock_vm_held(bo); 1502 1503 return bo; 1504 1505 err_unpin: 1506 xe_bo_unpin(bo); 1507 err_put: 1508 xe_bo_unlock_vm_held(bo); 1509 xe_bo_put(bo); 1510 return ERR_PTR(err); 1511 } 1512 1513 struct xe_bo *xe_bo_create_pin_map(struct xe_device *xe, struct xe_tile *tile, 1514 struct xe_vm *vm, size_t size, 1515 enum ttm_bo_type type, u32 flags) 1516 { 1517 return xe_bo_create_pin_map_at(xe, tile, vm, size, ~0ull, type, flags); 1518 } 1519 1520 struct xe_bo *xe_bo_create_from_data(struct xe_device *xe, struct xe_tile *tile, 1521 const void *data, size_t size, 1522 enum ttm_bo_type type, u32 flags) 1523 { 1524 struct xe_bo *bo = xe_bo_create_pin_map(xe, tile, NULL, 1525 ALIGN(size, PAGE_SIZE), 1526 type, flags); 1527 if (IS_ERR(bo)) 1528 return bo; 1529 1530 xe_map_memcpy_to(xe, &bo->vmap, 0, data, size); 1531 1532 return bo; 1533 } 1534 1535 static void __xe_bo_unpin_map_no_vm(struct drm_device *drm, void *arg) 1536 { 1537 xe_bo_unpin_map_no_vm(arg); 1538 } 1539 1540 struct xe_bo *xe_managed_bo_create_pin_map(struct xe_device *xe, struct xe_tile *tile, 1541 size_t size, u32 flags) 1542 { 1543 struct xe_bo *bo; 1544 int ret; 1545 1546 bo = xe_bo_create_pin_map(xe, tile, NULL, size, ttm_bo_type_kernel, flags); 1547 if (IS_ERR(bo)) 1548 return bo; 1549 1550 ret = drmm_add_action_or_reset(&xe->drm, __xe_bo_unpin_map_no_vm, bo); 1551 if (ret) 1552 return ERR_PTR(ret); 1553 1554 return bo; 1555 } 1556 1557 struct xe_bo *xe_managed_bo_create_from_data(struct xe_device *xe, struct xe_tile *tile, 1558 const void *data, size_t size, u32 flags) 1559 { 1560 struct xe_bo *bo = xe_managed_bo_create_pin_map(xe, tile, ALIGN(size, PAGE_SIZE), flags); 1561 1562 if (IS_ERR(bo)) 1563 return bo; 1564 1565 xe_map_memcpy_to(xe, &bo->vmap, 0, data, size); 1566 1567 return bo; 1568 } 1569 1570 /* 1571 * XXX: This is in the VM bind data path, likely should calculate this once and 1572 * store, with a recalculation if the BO is moved. 1573 */ 1574 uint64_t vram_region_gpu_offset(struct ttm_resource *res) 1575 { 1576 struct xe_device *xe = ttm_to_xe_device(res->bo->bdev); 1577 1578 if (res->mem_type == XE_PL_STOLEN) 1579 return xe_ttm_stolen_gpu_offset(xe); 1580 1581 return res_to_mem_region(res)->dpa_base; 1582 } 1583 1584 /** 1585 * xe_bo_pin_external - pin an external BO 1586 * @bo: buffer object to be pinned 1587 * 1588 * Pin an external (not tied to a VM, can be exported via dma-buf / prime FD) 1589 * BO. Unique call compared to xe_bo_pin as this function has it own set of 1590 * asserts and code to ensure evict / restore on suspend / resume. 1591 * 1592 * Returns 0 for success, negative error code otherwise. 1593 */ 1594 int xe_bo_pin_external(struct xe_bo *bo) 1595 { 1596 struct xe_device *xe = xe_bo_device(bo); 1597 int err; 1598 1599 xe_assert(xe, !bo->vm); 1600 xe_assert(xe, xe_bo_is_user(bo)); 1601 1602 if (!xe_bo_is_pinned(bo)) { 1603 err = xe_bo_validate(bo, NULL, false); 1604 if (err) 1605 return err; 1606 1607 if (xe_bo_is_vram(bo)) { 1608 spin_lock(&xe->pinned.lock); 1609 list_add_tail(&bo->pinned_link, 1610 &xe->pinned.external_vram); 1611 spin_unlock(&xe->pinned.lock); 1612 } 1613 } 1614 1615 ttm_bo_pin(&bo->ttm); 1616 1617 /* 1618 * FIXME: If we always use the reserve / unreserve functions for locking 1619 * we do not need this. 1620 */ 1621 ttm_bo_move_to_lru_tail_unlocked(&bo->ttm); 1622 1623 return 0; 1624 } 1625 1626 int xe_bo_pin(struct xe_bo *bo) 1627 { 1628 struct xe_device *xe = xe_bo_device(bo); 1629 int err; 1630 1631 /* We currently don't expect user BO to be pinned */ 1632 xe_assert(xe, !xe_bo_is_user(bo)); 1633 1634 /* Pinned object must be in GGTT or have pinned flag */ 1635 xe_assert(xe, bo->flags & (XE_BO_CREATE_PINNED_BIT | 1636 XE_BO_CREATE_GGTT_BIT)); 1637 1638 /* 1639 * No reason we can't support pinning imported dma-bufs we just don't 1640 * expect to pin an imported dma-buf. 1641 */ 1642 xe_assert(xe, !bo->ttm.base.import_attach); 1643 1644 /* We only expect at most 1 pin */ 1645 xe_assert(xe, !xe_bo_is_pinned(bo)); 1646 1647 err = xe_bo_validate(bo, NULL, false); 1648 if (err) 1649 return err; 1650 1651 /* 1652 * For pinned objects in on DGFX, which are also in vram, we expect 1653 * these to be in contiguous VRAM memory. Required eviction / restore 1654 * during suspend / resume (force restore to same physical address). 1655 */ 1656 if (IS_DGFX(xe) && !(IS_ENABLED(CONFIG_DRM_XE_DEBUG) && 1657 bo->flags & XE_BO_INTERNAL_TEST)) { 1658 struct ttm_place *place = &(bo->placements[0]); 1659 1660 if (mem_type_is_vram(place->mem_type)) { 1661 xe_assert(xe, place->flags & TTM_PL_FLAG_CONTIGUOUS); 1662 1663 place->fpfn = (xe_bo_addr(bo, 0, PAGE_SIZE) - 1664 vram_region_gpu_offset(bo->ttm.resource)) >> PAGE_SHIFT; 1665 place->lpfn = place->fpfn + (bo->size >> PAGE_SHIFT); 1666 1667 spin_lock(&xe->pinned.lock); 1668 list_add_tail(&bo->pinned_link, &xe->pinned.kernel_bo_present); 1669 spin_unlock(&xe->pinned.lock); 1670 } 1671 } 1672 1673 ttm_bo_pin(&bo->ttm); 1674 1675 /* 1676 * FIXME: If we always use the reserve / unreserve functions for locking 1677 * we do not need this. 1678 */ 1679 ttm_bo_move_to_lru_tail_unlocked(&bo->ttm); 1680 1681 return 0; 1682 } 1683 1684 /** 1685 * xe_bo_unpin_external - unpin an external BO 1686 * @bo: buffer object to be unpinned 1687 * 1688 * Unpin an external (not tied to a VM, can be exported via dma-buf / prime FD) 1689 * BO. Unique call compared to xe_bo_unpin as this function has it own set of 1690 * asserts and code to ensure evict / restore on suspend / resume. 1691 * 1692 * Returns 0 for success, negative error code otherwise. 1693 */ 1694 void xe_bo_unpin_external(struct xe_bo *bo) 1695 { 1696 struct xe_device *xe = xe_bo_device(bo); 1697 1698 xe_assert(xe, !bo->vm); 1699 xe_assert(xe, xe_bo_is_pinned(bo)); 1700 xe_assert(xe, xe_bo_is_user(bo)); 1701 1702 if (bo->ttm.pin_count == 1 && !list_empty(&bo->pinned_link)) { 1703 spin_lock(&xe->pinned.lock); 1704 list_del_init(&bo->pinned_link); 1705 spin_unlock(&xe->pinned.lock); 1706 } 1707 1708 ttm_bo_unpin(&bo->ttm); 1709 1710 /* 1711 * FIXME: If we always use the reserve / unreserve functions for locking 1712 * we do not need this. 1713 */ 1714 ttm_bo_move_to_lru_tail_unlocked(&bo->ttm); 1715 } 1716 1717 void xe_bo_unpin(struct xe_bo *bo) 1718 { 1719 struct xe_device *xe = xe_bo_device(bo); 1720 1721 xe_assert(xe, !bo->ttm.base.import_attach); 1722 xe_assert(xe, xe_bo_is_pinned(bo)); 1723 1724 if (IS_DGFX(xe) && !(IS_ENABLED(CONFIG_DRM_XE_DEBUG) && 1725 bo->flags & XE_BO_INTERNAL_TEST)) { 1726 struct ttm_place *place = &(bo->placements[0]); 1727 1728 if (mem_type_is_vram(place->mem_type)) { 1729 xe_assert(xe, !list_empty(&bo->pinned_link)); 1730 1731 spin_lock(&xe->pinned.lock); 1732 list_del_init(&bo->pinned_link); 1733 spin_unlock(&xe->pinned.lock); 1734 } 1735 } 1736 1737 ttm_bo_unpin(&bo->ttm); 1738 } 1739 1740 /** 1741 * xe_bo_validate() - Make sure the bo is in an allowed placement 1742 * @bo: The bo, 1743 * @vm: Pointer to a the vm the bo shares a locked dma_resv object with, or 1744 * NULL. Used together with @allow_res_evict. 1745 * @allow_res_evict: Whether it's allowed to evict bos sharing @vm's 1746 * reservation object. 1747 * 1748 * Make sure the bo is in allowed placement, migrating it if necessary. If 1749 * needed, other bos will be evicted. If bos selected for eviction shares 1750 * the @vm's reservation object, they can be evicted iff @allow_res_evict is 1751 * set to true, otherwise they will be bypassed. 1752 * 1753 * Return: 0 on success, negative error code on failure. May return 1754 * -EINTR or -ERESTARTSYS if internal waits are interrupted by a signal. 1755 */ 1756 int xe_bo_validate(struct xe_bo *bo, struct xe_vm *vm, bool allow_res_evict) 1757 { 1758 struct ttm_operation_ctx ctx = { 1759 .interruptible = true, 1760 .no_wait_gpu = false, 1761 }; 1762 1763 if (vm) { 1764 lockdep_assert_held(&vm->lock); 1765 xe_vm_assert_held(vm); 1766 1767 ctx.allow_res_evict = allow_res_evict; 1768 ctx.resv = xe_vm_resv(vm); 1769 } 1770 1771 return ttm_bo_validate(&bo->ttm, &bo->placement, &ctx); 1772 } 1773 1774 bool xe_bo_is_xe_bo(struct ttm_buffer_object *bo) 1775 { 1776 if (bo->destroy == &xe_ttm_bo_destroy) 1777 return true; 1778 1779 return false; 1780 } 1781 1782 /* 1783 * Resolve a BO address. There is no assert to check if the proper lock is held 1784 * so it should only be used in cases where it is not fatal to get the wrong 1785 * address, such as printing debug information, but not in cases where memory is 1786 * written based on this result. 1787 */ 1788 dma_addr_t __xe_bo_addr(struct xe_bo *bo, u64 offset, size_t page_size) 1789 { 1790 struct xe_device *xe = xe_bo_device(bo); 1791 struct xe_res_cursor cur; 1792 u64 page; 1793 1794 xe_assert(xe, page_size <= PAGE_SIZE); 1795 page = offset >> PAGE_SHIFT; 1796 offset &= (PAGE_SIZE - 1); 1797 1798 if (!xe_bo_is_vram(bo) && !xe_bo_is_stolen(bo)) { 1799 xe_assert(xe, bo->ttm.ttm); 1800 1801 xe_res_first_sg(xe_bo_sg(bo), page << PAGE_SHIFT, 1802 page_size, &cur); 1803 return xe_res_dma(&cur) + offset; 1804 } else { 1805 struct xe_res_cursor cur; 1806 1807 xe_res_first(bo->ttm.resource, page << PAGE_SHIFT, 1808 page_size, &cur); 1809 return cur.start + offset + vram_region_gpu_offset(bo->ttm.resource); 1810 } 1811 } 1812 1813 dma_addr_t xe_bo_addr(struct xe_bo *bo, u64 offset, size_t page_size) 1814 { 1815 if (!READ_ONCE(bo->ttm.pin_count)) 1816 xe_bo_assert_held(bo); 1817 return __xe_bo_addr(bo, offset, page_size); 1818 } 1819 1820 int xe_bo_vmap(struct xe_bo *bo) 1821 { 1822 void *virtual; 1823 bool is_iomem; 1824 int ret; 1825 1826 xe_bo_assert_held(bo); 1827 1828 if (!(bo->flags & XE_BO_NEEDS_CPU_ACCESS)) 1829 return -EINVAL; 1830 1831 if (!iosys_map_is_null(&bo->vmap)) 1832 return 0; 1833 1834 /* 1835 * We use this more or less deprecated interface for now since 1836 * ttm_bo_vmap() doesn't offer the optimization of kmapping 1837 * single page bos, which is done here. 1838 * TODO: Fix up ttm_bo_vmap to do that, or fix up ttm_bo_kmap 1839 * to use struct iosys_map. 1840 */ 1841 ret = ttm_bo_kmap(&bo->ttm, 0, bo->size >> PAGE_SHIFT, &bo->kmap); 1842 if (ret) 1843 return ret; 1844 1845 virtual = ttm_kmap_obj_virtual(&bo->kmap, &is_iomem); 1846 if (is_iomem) 1847 iosys_map_set_vaddr_iomem(&bo->vmap, (void __iomem *)virtual); 1848 else 1849 iosys_map_set_vaddr(&bo->vmap, virtual); 1850 1851 return 0; 1852 } 1853 1854 static void __xe_bo_vunmap(struct xe_bo *bo) 1855 { 1856 if (!iosys_map_is_null(&bo->vmap)) { 1857 iosys_map_clear(&bo->vmap); 1858 ttm_bo_kunmap(&bo->kmap); 1859 } 1860 } 1861 1862 void xe_bo_vunmap(struct xe_bo *bo) 1863 { 1864 xe_bo_assert_held(bo); 1865 __xe_bo_vunmap(bo); 1866 } 1867 1868 int xe_gem_create_ioctl(struct drm_device *dev, void *data, 1869 struct drm_file *file) 1870 { 1871 struct xe_device *xe = to_xe_device(dev); 1872 struct xe_file *xef = to_xe_file(file); 1873 struct drm_xe_gem_create *args = data; 1874 struct xe_vm *vm = NULL; 1875 struct xe_bo *bo; 1876 unsigned int bo_flags; 1877 u32 handle; 1878 int err; 1879 1880 if (XE_IOCTL_DBG(xe, args->extensions) || 1881 XE_IOCTL_DBG(xe, args->pad[0] || args->pad[1] || args->pad[2]) || 1882 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 1883 return -EINVAL; 1884 1885 /* at least one valid memory placement must be specified */ 1886 if (XE_IOCTL_DBG(xe, (args->placement & ~xe->info.mem_region_mask) || 1887 !args->placement)) 1888 return -EINVAL; 1889 1890 if (XE_IOCTL_DBG(xe, args->flags & 1891 ~(DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING | 1892 DRM_XE_GEM_CREATE_FLAG_SCANOUT | 1893 DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM))) 1894 return -EINVAL; 1895 1896 if (XE_IOCTL_DBG(xe, args->handle)) 1897 return -EINVAL; 1898 1899 if (XE_IOCTL_DBG(xe, !args->size)) 1900 return -EINVAL; 1901 1902 if (XE_IOCTL_DBG(xe, args->size > SIZE_MAX)) 1903 return -EINVAL; 1904 1905 if (XE_IOCTL_DBG(xe, args->size & ~PAGE_MASK)) 1906 return -EINVAL; 1907 1908 bo_flags = 0; 1909 if (args->flags & DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING) 1910 bo_flags |= XE_BO_DEFER_BACKING; 1911 1912 if (args->flags & DRM_XE_GEM_CREATE_FLAG_SCANOUT) 1913 bo_flags |= XE_BO_SCANOUT_BIT; 1914 1915 bo_flags |= args->placement << (ffs(XE_BO_CREATE_SYSTEM_BIT) - 1); 1916 1917 if (args->flags & DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM) { 1918 if (XE_IOCTL_DBG(xe, !(bo_flags & XE_BO_CREATE_VRAM_MASK))) 1919 return -EINVAL; 1920 1921 bo_flags |= XE_BO_NEEDS_CPU_ACCESS; 1922 } 1923 1924 if (XE_IOCTL_DBG(xe, !args->cpu_caching || 1925 args->cpu_caching > DRM_XE_GEM_CPU_CACHING_WC)) 1926 return -EINVAL; 1927 1928 if (XE_IOCTL_DBG(xe, bo_flags & XE_BO_CREATE_VRAM_MASK && 1929 args->cpu_caching != DRM_XE_GEM_CPU_CACHING_WC)) 1930 return -EINVAL; 1931 1932 if (XE_IOCTL_DBG(xe, bo_flags & XE_BO_SCANOUT_BIT && 1933 args->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB)) 1934 return -EINVAL; 1935 1936 if (args->vm_id) { 1937 vm = xe_vm_lookup(xef, args->vm_id); 1938 if (XE_IOCTL_DBG(xe, !vm)) 1939 return -ENOENT; 1940 err = xe_vm_lock(vm, true); 1941 if (err) 1942 goto out_vm; 1943 } 1944 1945 bo = xe_bo_create_user(xe, NULL, vm, args->size, args->cpu_caching, 1946 ttm_bo_type_device, bo_flags); 1947 1948 if (vm) 1949 xe_vm_unlock(vm); 1950 1951 if (IS_ERR(bo)) { 1952 err = PTR_ERR(bo); 1953 goto out_vm; 1954 } 1955 1956 err = drm_gem_handle_create(file, &bo->ttm.base, &handle); 1957 if (err) 1958 goto out_bulk; 1959 1960 args->handle = handle; 1961 goto out_put; 1962 1963 out_bulk: 1964 if (vm && !xe_vm_in_fault_mode(vm)) { 1965 xe_vm_lock(vm, false); 1966 __xe_bo_unset_bulk_move(bo); 1967 xe_vm_unlock(vm); 1968 } 1969 out_put: 1970 xe_bo_put(bo); 1971 out_vm: 1972 if (vm) 1973 xe_vm_put(vm); 1974 1975 return err; 1976 } 1977 1978 int xe_gem_mmap_offset_ioctl(struct drm_device *dev, void *data, 1979 struct drm_file *file) 1980 { 1981 struct xe_device *xe = to_xe_device(dev); 1982 struct drm_xe_gem_mmap_offset *args = data; 1983 struct drm_gem_object *gem_obj; 1984 1985 if (XE_IOCTL_DBG(xe, args->extensions) || 1986 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 1987 return -EINVAL; 1988 1989 if (XE_IOCTL_DBG(xe, args->flags)) 1990 return -EINVAL; 1991 1992 gem_obj = drm_gem_object_lookup(file, args->handle); 1993 if (XE_IOCTL_DBG(xe, !gem_obj)) 1994 return -ENOENT; 1995 1996 /* The mmap offset was set up at BO allocation time. */ 1997 args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node); 1998 1999 xe_bo_put(gem_to_xe_bo(gem_obj)); 2000 return 0; 2001 } 2002 2003 /** 2004 * xe_bo_lock() - Lock the buffer object's dma_resv object 2005 * @bo: The struct xe_bo whose lock is to be taken 2006 * @intr: Whether to perform any wait interruptible 2007 * 2008 * Locks the buffer object's dma_resv object. If the buffer object is 2009 * pointing to a shared dma_resv object, that shared lock is locked. 2010 * 2011 * Return: 0 on success, -EINTR if @intr is true and the wait for a 2012 * contended lock was interrupted. If @intr is set to false, the 2013 * function always returns 0. 2014 */ 2015 int xe_bo_lock(struct xe_bo *bo, bool intr) 2016 { 2017 if (intr) 2018 return dma_resv_lock_interruptible(bo->ttm.base.resv, NULL); 2019 2020 dma_resv_lock(bo->ttm.base.resv, NULL); 2021 2022 return 0; 2023 } 2024 2025 /** 2026 * xe_bo_unlock() - Unlock the buffer object's dma_resv object 2027 * @bo: The struct xe_bo whose lock is to be released. 2028 * 2029 * Unlock a buffer object lock that was locked by xe_bo_lock(). 2030 */ 2031 void xe_bo_unlock(struct xe_bo *bo) 2032 { 2033 dma_resv_unlock(bo->ttm.base.resv); 2034 } 2035 2036 /** 2037 * xe_bo_can_migrate - Whether a buffer object likely can be migrated 2038 * @bo: The buffer object to migrate 2039 * @mem_type: The TTM memory type intended to migrate to 2040 * 2041 * Check whether the buffer object supports migration to the 2042 * given memory type. Note that pinning may affect the ability to migrate as 2043 * returned by this function. 2044 * 2045 * This function is primarily intended as a helper for checking the 2046 * possibility to migrate buffer objects and can be called without 2047 * the object lock held. 2048 * 2049 * Return: true if migration is possible, false otherwise. 2050 */ 2051 bool xe_bo_can_migrate(struct xe_bo *bo, u32 mem_type) 2052 { 2053 unsigned int cur_place; 2054 2055 if (bo->ttm.type == ttm_bo_type_kernel) 2056 return true; 2057 2058 if (bo->ttm.type == ttm_bo_type_sg) 2059 return false; 2060 2061 for (cur_place = 0; cur_place < bo->placement.num_placement; 2062 cur_place++) { 2063 if (bo->placements[cur_place].mem_type == mem_type) 2064 return true; 2065 } 2066 2067 return false; 2068 } 2069 2070 static void xe_place_from_ttm_type(u32 mem_type, struct ttm_place *place) 2071 { 2072 memset(place, 0, sizeof(*place)); 2073 place->mem_type = mem_type; 2074 } 2075 2076 /** 2077 * xe_bo_migrate - Migrate an object to the desired region id 2078 * @bo: The buffer object to migrate. 2079 * @mem_type: The TTM region type to migrate to. 2080 * 2081 * Attempt to migrate the buffer object to the desired memory region. The 2082 * buffer object may not be pinned, and must be locked. 2083 * On successful completion, the object memory type will be updated, 2084 * but an async migration task may not have completed yet, and to 2085 * accomplish that, the object's kernel fences must be signaled with 2086 * the object lock held. 2087 * 2088 * Return: 0 on success. Negative error code on failure. In particular may 2089 * return -EINTR or -ERESTARTSYS if signal pending. 2090 */ 2091 int xe_bo_migrate(struct xe_bo *bo, u32 mem_type) 2092 { 2093 struct xe_device *xe = ttm_to_xe_device(bo->ttm.bdev); 2094 struct ttm_operation_ctx ctx = { 2095 .interruptible = true, 2096 .no_wait_gpu = false, 2097 }; 2098 struct ttm_placement placement; 2099 struct ttm_place requested; 2100 2101 xe_bo_assert_held(bo); 2102 2103 if (bo->ttm.resource->mem_type == mem_type) 2104 return 0; 2105 2106 if (xe_bo_is_pinned(bo)) 2107 return -EBUSY; 2108 2109 if (!xe_bo_can_migrate(bo, mem_type)) 2110 return -EINVAL; 2111 2112 xe_place_from_ttm_type(mem_type, &requested); 2113 placement.num_placement = 1; 2114 placement.placement = &requested; 2115 2116 /* 2117 * Stolen needs to be handled like below VRAM handling if we ever need 2118 * to support it. 2119 */ 2120 drm_WARN_ON(&xe->drm, mem_type == XE_PL_STOLEN); 2121 2122 if (mem_type_is_vram(mem_type)) { 2123 u32 c = 0; 2124 2125 add_vram(xe, bo, &requested, bo->flags, mem_type, &c); 2126 } 2127 2128 return ttm_bo_validate(&bo->ttm, &placement, &ctx); 2129 } 2130 2131 /** 2132 * xe_bo_evict - Evict an object to evict placement 2133 * @bo: The buffer object to migrate. 2134 * @force_alloc: Set force_alloc in ttm_operation_ctx 2135 * 2136 * On successful completion, the object memory will be moved to evict 2137 * placement. Ths function blocks until the object has been fully moved. 2138 * 2139 * Return: 0 on success. Negative error code on failure. 2140 */ 2141 int xe_bo_evict(struct xe_bo *bo, bool force_alloc) 2142 { 2143 struct ttm_operation_ctx ctx = { 2144 .interruptible = false, 2145 .no_wait_gpu = false, 2146 .force_alloc = force_alloc, 2147 }; 2148 struct ttm_placement placement; 2149 int ret; 2150 2151 xe_evict_flags(&bo->ttm, &placement); 2152 ret = ttm_bo_validate(&bo->ttm, &placement, &ctx); 2153 if (ret) 2154 return ret; 2155 2156 dma_resv_wait_timeout(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL, 2157 false, MAX_SCHEDULE_TIMEOUT); 2158 2159 return 0; 2160 } 2161 2162 /** 2163 * xe_bo_needs_ccs_pages - Whether a bo needs to back up CCS pages when 2164 * placed in system memory. 2165 * @bo: The xe_bo 2166 * 2167 * Return: true if extra pages need to be allocated, false otherwise. 2168 */ 2169 bool xe_bo_needs_ccs_pages(struct xe_bo *bo) 2170 { 2171 struct xe_device *xe = xe_bo_device(bo); 2172 2173 if (!xe_device_has_flat_ccs(xe) || bo->ttm.type != ttm_bo_type_device) 2174 return false; 2175 2176 /* On discrete GPUs, if the GPU can access this buffer from 2177 * system memory (i.e., it allows XE_PL_TT placement), FlatCCS 2178 * can't be used since there's no CCS storage associated with 2179 * non-VRAM addresses. 2180 */ 2181 if (IS_DGFX(xe) && (bo->flags & XE_BO_CREATE_SYSTEM_BIT)) 2182 return false; 2183 2184 return true; 2185 } 2186 2187 /** 2188 * __xe_bo_release_dummy() - Dummy kref release function 2189 * @kref: The embedded struct kref. 2190 * 2191 * Dummy release function for xe_bo_put_deferred(). Keep off. 2192 */ 2193 void __xe_bo_release_dummy(struct kref *kref) 2194 { 2195 } 2196 2197 /** 2198 * xe_bo_put_commit() - Put bos whose put was deferred by xe_bo_put_deferred(). 2199 * @deferred: The lockless list used for the call to xe_bo_put_deferred(). 2200 * 2201 * Puts all bos whose put was deferred by xe_bo_put_deferred(). 2202 * The @deferred list can be either an onstack local list or a global 2203 * shared list used by a workqueue. 2204 */ 2205 void xe_bo_put_commit(struct llist_head *deferred) 2206 { 2207 struct llist_node *freed; 2208 struct xe_bo *bo, *next; 2209 2210 if (!deferred) 2211 return; 2212 2213 freed = llist_del_all(deferred); 2214 if (!freed) 2215 return; 2216 2217 llist_for_each_entry_safe(bo, next, freed, freed) 2218 drm_gem_object_free(&bo->ttm.base.refcount); 2219 } 2220 2221 /** 2222 * xe_bo_dumb_create - Create a dumb bo as backing for a fb 2223 * @file_priv: ... 2224 * @dev: ... 2225 * @args: ... 2226 * 2227 * See dumb_create() hook in include/drm/drm_drv.h 2228 * 2229 * Return: ... 2230 */ 2231 int xe_bo_dumb_create(struct drm_file *file_priv, 2232 struct drm_device *dev, 2233 struct drm_mode_create_dumb *args) 2234 { 2235 struct xe_device *xe = to_xe_device(dev); 2236 struct xe_bo *bo; 2237 uint32_t handle; 2238 int cpp = DIV_ROUND_UP(args->bpp, 8); 2239 int err; 2240 u32 page_size = max_t(u32, PAGE_SIZE, 2241 xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : SZ_4K); 2242 2243 args->pitch = ALIGN(args->width * cpp, 64); 2244 args->size = ALIGN(mul_u32_u32(args->pitch, args->height), 2245 page_size); 2246 2247 bo = xe_bo_create_user(xe, NULL, NULL, args->size, 2248 DRM_XE_GEM_CPU_CACHING_WC, 2249 ttm_bo_type_device, 2250 XE_BO_CREATE_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) | 2251 XE_BO_CREATE_USER_BIT | XE_BO_SCANOUT_BIT | 2252 XE_BO_NEEDS_CPU_ACCESS); 2253 if (IS_ERR(bo)) 2254 return PTR_ERR(bo); 2255 2256 err = drm_gem_handle_create(file_priv, &bo->ttm.base, &handle); 2257 /* drop reference from allocate - handle holds it now */ 2258 drm_gem_object_put(&bo->ttm.base); 2259 if (!err) 2260 args->handle = handle; 2261 return err; 2262 } 2263 2264 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 2265 #include "tests/xe_bo.c" 2266 #endif 2267