1 /* 2 * Copyright 2008 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 22 * DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: 25 * Jerome Glisse <glisse@freedesktop.org> 26 */ 27 #include <linux/pagemap.h> 28 #include <linux/sync_file.h> 29 #include <drm/drmP.h> 30 #include <drm/amdgpu_drm.h> 31 #include <drm/drm_syncobj.h> 32 #include "amdgpu.h" 33 #include "amdgpu_trace.h" 34 #include "amdgpu_gmc.h" 35 36 static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p, 37 struct drm_amdgpu_cs_chunk_fence *data, 38 uint32_t *offset) 39 { 40 struct drm_gem_object *gobj; 41 unsigned long size; 42 int r; 43 44 gobj = drm_gem_object_lookup(p->filp, data->handle); 45 if (gobj == NULL) 46 return -EINVAL; 47 48 p->uf_entry.robj = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj)); 49 p->uf_entry.priority = 0; 50 p->uf_entry.tv.bo = &p->uf_entry.robj->tbo; 51 p->uf_entry.tv.shared = true; 52 p->uf_entry.user_pages = NULL; 53 54 drm_gem_object_put_unlocked(gobj); 55 56 size = amdgpu_bo_size(p->uf_entry.robj); 57 if (size != PAGE_SIZE || (data->offset + 8) > size) { 58 r = -EINVAL; 59 goto error_unref; 60 } 61 62 if (amdgpu_ttm_tt_get_usermm(p->uf_entry.robj->tbo.ttm)) { 63 r = -EINVAL; 64 goto error_unref; 65 } 66 67 *offset = data->offset; 68 69 return 0; 70 71 error_unref: 72 amdgpu_bo_unref(&p->uf_entry.robj); 73 return r; 74 } 75 76 static int amdgpu_cs_bo_handles_chunk(struct amdgpu_cs_parser *p, 77 struct drm_amdgpu_bo_list_in *data) 78 { 79 int r; 80 struct drm_amdgpu_bo_list_entry *info = NULL; 81 82 r = amdgpu_bo_create_list_entry_array(data, &info); 83 if (r) 84 return r; 85 86 r = amdgpu_bo_list_create(p->adev, p->filp, info, data->bo_number, 87 &p->bo_list); 88 if (r) 89 goto error_free; 90 91 kvfree(info); 92 return 0; 93 94 error_free: 95 if (info) 96 kvfree(info); 97 98 return r; 99 } 100 101 static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, union drm_amdgpu_cs *cs) 102 { 103 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 104 struct amdgpu_vm *vm = &fpriv->vm; 105 uint64_t *chunk_array_user; 106 uint64_t *chunk_array; 107 unsigned size, num_ibs = 0; 108 uint32_t uf_offset = 0; 109 int i; 110 int ret; 111 112 if (cs->in.num_chunks == 0) 113 return 0; 114 115 chunk_array = kmalloc_array(cs->in.num_chunks, sizeof(uint64_t), GFP_KERNEL); 116 if (!chunk_array) 117 return -ENOMEM; 118 119 p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id); 120 if (!p->ctx) { 121 ret = -EINVAL; 122 goto free_chunk; 123 } 124 125 mutex_lock(&p->ctx->lock); 126 127 /* skip guilty context job */ 128 if (atomic_read(&p->ctx->guilty) == 1) { 129 ret = -ECANCELED; 130 goto free_chunk; 131 } 132 133 /* get chunks */ 134 chunk_array_user = u64_to_user_ptr(cs->in.chunks); 135 if (copy_from_user(chunk_array, chunk_array_user, 136 sizeof(uint64_t)*cs->in.num_chunks)) { 137 ret = -EFAULT; 138 goto free_chunk; 139 } 140 141 p->nchunks = cs->in.num_chunks; 142 p->chunks = kmalloc_array(p->nchunks, sizeof(struct amdgpu_cs_chunk), 143 GFP_KERNEL); 144 if (!p->chunks) { 145 ret = -ENOMEM; 146 goto free_chunk; 147 } 148 149 for (i = 0; i < p->nchunks; i++) { 150 struct drm_amdgpu_cs_chunk __user **chunk_ptr = NULL; 151 struct drm_amdgpu_cs_chunk user_chunk; 152 uint32_t __user *cdata; 153 154 chunk_ptr = u64_to_user_ptr(chunk_array[i]); 155 if (copy_from_user(&user_chunk, chunk_ptr, 156 sizeof(struct drm_amdgpu_cs_chunk))) { 157 ret = -EFAULT; 158 i--; 159 goto free_partial_kdata; 160 } 161 p->chunks[i].chunk_id = user_chunk.chunk_id; 162 p->chunks[i].length_dw = user_chunk.length_dw; 163 164 size = p->chunks[i].length_dw; 165 cdata = u64_to_user_ptr(user_chunk.chunk_data); 166 167 p->chunks[i].kdata = kvmalloc_array(size, sizeof(uint32_t), GFP_KERNEL); 168 if (p->chunks[i].kdata == NULL) { 169 ret = -ENOMEM; 170 i--; 171 goto free_partial_kdata; 172 } 173 size *= sizeof(uint32_t); 174 if (copy_from_user(p->chunks[i].kdata, cdata, size)) { 175 ret = -EFAULT; 176 goto free_partial_kdata; 177 } 178 179 switch (p->chunks[i].chunk_id) { 180 case AMDGPU_CHUNK_ID_IB: 181 ++num_ibs; 182 break; 183 184 case AMDGPU_CHUNK_ID_FENCE: 185 size = sizeof(struct drm_amdgpu_cs_chunk_fence); 186 if (p->chunks[i].length_dw * sizeof(uint32_t) < size) { 187 ret = -EINVAL; 188 goto free_partial_kdata; 189 } 190 191 ret = amdgpu_cs_user_fence_chunk(p, p->chunks[i].kdata, 192 &uf_offset); 193 if (ret) 194 goto free_partial_kdata; 195 196 break; 197 198 case AMDGPU_CHUNK_ID_BO_HANDLES: 199 size = sizeof(struct drm_amdgpu_bo_list_in); 200 if (p->chunks[i].length_dw * sizeof(uint32_t) < size) { 201 ret = -EINVAL; 202 goto free_partial_kdata; 203 } 204 205 ret = amdgpu_cs_bo_handles_chunk(p, p->chunks[i].kdata); 206 if (ret) 207 goto free_partial_kdata; 208 209 break; 210 211 case AMDGPU_CHUNK_ID_DEPENDENCIES: 212 case AMDGPU_CHUNK_ID_SYNCOBJ_IN: 213 case AMDGPU_CHUNK_ID_SYNCOBJ_OUT: 214 break; 215 216 default: 217 ret = -EINVAL; 218 goto free_partial_kdata; 219 } 220 } 221 222 ret = amdgpu_job_alloc(p->adev, num_ibs, &p->job, vm); 223 if (ret) 224 goto free_all_kdata; 225 226 if (p->ctx->vram_lost_counter != p->job->vram_lost_counter) { 227 ret = -ECANCELED; 228 goto free_all_kdata; 229 } 230 231 if (p->uf_entry.robj) 232 p->job->uf_addr = uf_offset; 233 kfree(chunk_array); 234 235 /* Use this opportunity to fill in task info for the vm */ 236 amdgpu_vm_set_task_info(vm); 237 238 return 0; 239 240 free_all_kdata: 241 i = p->nchunks - 1; 242 free_partial_kdata: 243 for (; i >= 0; i--) 244 kvfree(p->chunks[i].kdata); 245 kfree(p->chunks); 246 p->chunks = NULL; 247 p->nchunks = 0; 248 free_chunk: 249 kfree(chunk_array); 250 251 return ret; 252 } 253 254 /* Convert microseconds to bytes. */ 255 static u64 us_to_bytes(struct amdgpu_device *adev, s64 us) 256 { 257 if (us <= 0 || !adev->mm_stats.log2_max_MBps) 258 return 0; 259 260 /* Since accum_us is incremented by a million per second, just 261 * multiply it by the number of MB/s to get the number of bytes. 262 */ 263 return us << adev->mm_stats.log2_max_MBps; 264 } 265 266 static s64 bytes_to_us(struct amdgpu_device *adev, u64 bytes) 267 { 268 if (!adev->mm_stats.log2_max_MBps) 269 return 0; 270 271 return bytes >> adev->mm_stats.log2_max_MBps; 272 } 273 274 /* Returns how many bytes TTM can move right now. If no bytes can be moved, 275 * it returns 0. If it returns non-zero, it's OK to move at least one buffer, 276 * which means it can go over the threshold once. If that happens, the driver 277 * will be in debt and no other buffer migrations can be done until that debt 278 * is repaid. 279 * 280 * This approach allows moving a buffer of any size (it's important to allow 281 * that). 282 * 283 * The currency is simply time in microseconds and it increases as the clock 284 * ticks. The accumulated microseconds (us) are converted to bytes and 285 * returned. 286 */ 287 static void amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev, 288 u64 *max_bytes, 289 u64 *max_vis_bytes) 290 { 291 s64 time_us, increment_us; 292 u64 free_vram, total_vram, used_vram; 293 294 /* Allow a maximum of 200 accumulated ms. This is basically per-IB 295 * throttling. 296 * 297 * It means that in order to get full max MBps, at least 5 IBs per 298 * second must be submitted and not more than 200ms apart from each 299 * other. 300 */ 301 const s64 us_upper_bound = 200000; 302 303 if (!adev->mm_stats.log2_max_MBps) { 304 *max_bytes = 0; 305 *max_vis_bytes = 0; 306 return; 307 } 308 309 total_vram = adev->gmc.real_vram_size - atomic64_read(&adev->vram_pin_size); 310 used_vram = amdgpu_vram_mgr_usage(&adev->mman.bdev.man[TTM_PL_VRAM]); 311 free_vram = used_vram >= total_vram ? 0 : total_vram - used_vram; 312 313 spin_lock(&adev->mm_stats.lock); 314 315 /* Increase the amount of accumulated us. */ 316 time_us = ktime_to_us(ktime_get()); 317 increment_us = time_us - adev->mm_stats.last_update_us; 318 adev->mm_stats.last_update_us = time_us; 319 adev->mm_stats.accum_us = min(adev->mm_stats.accum_us + increment_us, 320 us_upper_bound); 321 322 /* This prevents the short period of low performance when the VRAM 323 * usage is low and the driver is in debt or doesn't have enough 324 * accumulated us to fill VRAM quickly. 325 * 326 * The situation can occur in these cases: 327 * - a lot of VRAM is freed by userspace 328 * - the presence of a big buffer causes a lot of evictions 329 * (solution: split buffers into smaller ones) 330 * 331 * If 128 MB or 1/8th of VRAM is free, start filling it now by setting 332 * accum_us to a positive number. 333 */ 334 if (free_vram >= 128 * 1024 * 1024 || free_vram >= total_vram / 8) { 335 s64 min_us; 336 337 /* Be more aggresive on dGPUs. Try to fill a portion of free 338 * VRAM now. 339 */ 340 if (!(adev->flags & AMD_IS_APU)) 341 min_us = bytes_to_us(adev, free_vram / 4); 342 else 343 min_us = 0; /* Reset accum_us on APUs. */ 344 345 adev->mm_stats.accum_us = max(min_us, adev->mm_stats.accum_us); 346 } 347 348 /* This is set to 0 if the driver is in debt to disallow (optional) 349 * buffer moves. 350 */ 351 *max_bytes = us_to_bytes(adev, adev->mm_stats.accum_us); 352 353 /* Do the same for visible VRAM if half of it is free */ 354 if (!amdgpu_gmc_vram_full_visible(&adev->gmc)) { 355 u64 total_vis_vram = adev->gmc.visible_vram_size; 356 u64 used_vis_vram = 357 amdgpu_vram_mgr_vis_usage(&adev->mman.bdev.man[TTM_PL_VRAM]); 358 359 if (used_vis_vram < total_vis_vram) { 360 u64 free_vis_vram = total_vis_vram - used_vis_vram; 361 adev->mm_stats.accum_us_vis = min(adev->mm_stats.accum_us_vis + 362 increment_us, us_upper_bound); 363 364 if (free_vis_vram >= total_vis_vram / 2) 365 adev->mm_stats.accum_us_vis = 366 max(bytes_to_us(adev, free_vis_vram / 2), 367 adev->mm_stats.accum_us_vis); 368 } 369 370 *max_vis_bytes = us_to_bytes(adev, adev->mm_stats.accum_us_vis); 371 } else { 372 *max_vis_bytes = 0; 373 } 374 375 spin_unlock(&adev->mm_stats.lock); 376 } 377 378 /* Report how many bytes have really been moved for the last command 379 * submission. This can result in a debt that can stop buffer migrations 380 * temporarily. 381 */ 382 void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes, 383 u64 num_vis_bytes) 384 { 385 spin_lock(&adev->mm_stats.lock); 386 adev->mm_stats.accum_us -= bytes_to_us(adev, num_bytes); 387 adev->mm_stats.accum_us_vis -= bytes_to_us(adev, num_vis_bytes); 388 spin_unlock(&adev->mm_stats.lock); 389 } 390 391 static int amdgpu_cs_bo_validate(struct amdgpu_cs_parser *p, 392 struct amdgpu_bo *bo) 393 { 394 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 395 struct ttm_operation_ctx ctx = { 396 .interruptible = true, 397 .no_wait_gpu = false, 398 .resv = bo->tbo.resv, 399 .flags = 0 400 }; 401 uint32_t domain; 402 int r; 403 404 if (bo->pin_count) 405 return 0; 406 407 /* Don't move this buffer if we have depleted our allowance 408 * to move it. Don't move anything if the threshold is zero. 409 */ 410 if (p->bytes_moved < p->bytes_moved_threshold) { 411 if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && 412 (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)) { 413 /* And don't move a CPU_ACCESS_REQUIRED BO to limited 414 * visible VRAM if we've depleted our allowance to do 415 * that. 416 */ 417 if (p->bytes_moved_vis < p->bytes_moved_vis_threshold) 418 domain = bo->preferred_domains; 419 else 420 domain = bo->allowed_domains; 421 } else { 422 domain = bo->preferred_domains; 423 } 424 } else { 425 domain = bo->allowed_domains; 426 } 427 428 retry: 429 amdgpu_bo_placement_from_domain(bo, domain); 430 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 431 432 p->bytes_moved += ctx.bytes_moved; 433 if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && 434 amdgpu_bo_in_cpu_visible_vram(bo)) 435 p->bytes_moved_vis += ctx.bytes_moved; 436 437 if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) { 438 domain = bo->allowed_domains; 439 goto retry; 440 } 441 442 return r; 443 } 444 445 /* Last resort, try to evict something from the current working set */ 446 static bool amdgpu_cs_try_evict(struct amdgpu_cs_parser *p, 447 struct amdgpu_bo *validated) 448 { 449 uint32_t domain = validated->allowed_domains; 450 struct ttm_operation_ctx ctx = { true, false }; 451 int r; 452 453 if (!p->evictable) 454 return false; 455 456 for (;&p->evictable->tv.head != &p->validated; 457 p->evictable = list_prev_entry(p->evictable, tv.head)) { 458 459 struct amdgpu_bo_list_entry *candidate = p->evictable; 460 struct amdgpu_bo *bo = candidate->robj; 461 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 462 bool update_bytes_moved_vis; 463 uint32_t other; 464 465 /* If we reached our current BO we can forget it */ 466 if (candidate->robj == validated) 467 break; 468 469 /* We can't move pinned BOs here */ 470 if (bo->pin_count) 471 continue; 472 473 other = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type); 474 475 /* Check if this BO is in one of the domains we need space for */ 476 if (!(other & domain)) 477 continue; 478 479 /* Check if we can move this BO somewhere else */ 480 other = bo->allowed_domains & ~domain; 481 if (!other) 482 continue; 483 484 /* Good we can try to move this BO somewhere else */ 485 update_bytes_moved_vis = 486 !amdgpu_gmc_vram_full_visible(&adev->gmc) && 487 amdgpu_bo_in_cpu_visible_vram(bo); 488 amdgpu_bo_placement_from_domain(bo, other); 489 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 490 p->bytes_moved += ctx.bytes_moved; 491 if (update_bytes_moved_vis) 492 p->bytes_moved_vis += ctx.bytes_moved; 493 494 if (unlikely(r)) 495 break; 496 497 p->evictable = list_prev_entry(p->evictable, tv.head); 498 list_move(&candidate->tv.head, &p->validated); 499 500 return true; 501 } 502 503 return false; 504 } 505 506 static int amdgpu_cs_validate(void *param, struct amdgpu_bo *bo) 507 { 508 struct amdgpu_cs_parser *p = param; 509 int r; 510 511 do { 512 r = amdgpu_cs_bo_validate(p, bo); 513 } while (r == -ENOMEM && amdgpu_cs_try_evict(p, bo)); 514 if (r) 515 return r; 516 517 if (bo->shadow) 518 r = amdgpu_cs_bo_validate(p, bo->shadow); 519 520 return r; 521 } 522 523 static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p, 524 struct list_head *validated) 525 { 526 struct ttm_operation_ctx ctx = { true, false }; 527 struct amdgpu_bo_list_entry *lobj; 528 int r; 529 530 list_for_each_entry(lobj, validated, tv.head) { 531 struct amdgpu_bo *bo = lobj->robj; 532 bool binding_userptr = false; 533 struct mm_struct *usermm; 534 535 usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm); 536 #if 0 537 if (usermm && usermm != current->mm) 538 return -EPERM; 539 #endif 540 541 /* Check if we have user pages and nobody bound the BO already */ 542 if (amdgpu_ttm_tt_userptr_needs_pages(bo->tbo.ttm) && 543 lobj->user_pages) { 544 amdgpu_bo_placement_from_domain(bo, 545 AMDGPU_GEM_DOMAIN_CPU); 546 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 547 if (r) 548 return r; 549 amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, 550 lobj->user_pages); 551 binding_userptr = true; 552 } 553 554 if (p->evictable == lobj) 555 p->evictable = NULL; 556 557 r = amdgpu_cs_validate(p, bo); 558 if (r) 559 return r; 560 561 if (binding_userptr) { 562 kvfree(lobj->user_pages); 563 lobj->user_pages = NULL; 564 } 565 } 566 return 0; 567 } 568 569 static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, 570 union drm_amdgpu_cs *cs) 571 { 572 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 573 struct amdgpu_vm *vm = &fpriv->vm; 574 struct amdgpu_bo_list_entry *e; 575 struct list_head duplicates; 576 struct amdgpu_bo *gds; 577 struct amdgpu_bo *gws; 578 struct amdgpu_bo *oa; 579 unsigned tries = 10; 580 int r; 581 582 INIT_LIST_HEAD(&p->validated); 583 584 /* p->bo_list could already be assigned if AMDGPU_CHUNK_ID_BO_HANDLES is present */ 585 if (cs->in.bo_list_handle) { 586 if (p->bo_list) 587 return -EINVAL; 588 589 r = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle, 590 &p->bo_list); 591 if (r) 592 return r; 593 } else if (!p->bo_list) { 594 /* Create a empty bo_list when no handle is provided */ 595 r = amdgpu_bo_list_create(p->adev, p->filp, NULL, 0, 596 &p->bo_list); 597 if (r) 598 return r; 599 } 600 601 amdgpu_bo_list_get_list(p->bo_list, &p->validated); 602 if (p->bo_list->first_userptr != p->bo_list->num_entries) 603 p->mn = amdgpu_mn_get(p->adev, AMDGPU_MN_TYPE_GFX); 604 605 INIT_LIST_HEAD(&duplicates); 606 amdgpu_vm_get_pd_bo(&fpriv->vm, &p->validated, &p->vm_pd); 607 608 if (p->uf_entry.robj && !p->uf_entry.robj->parent) 609 list_add(&p->uf_entry.tv.head, &p->validated); 610 611 while (1) { 612 struct list_head need_pages; 613 614 r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true, 615 &duplicates); 616 if (unlikely(r != 0)) { 617 if (r != -ERESTARTSYS) 618 DRM_ERROR("ttm_eu_reserve_buffers failed.\n"); 619 goto error_free_pages; 620 } 621 622 INIT_LIST_HEAD(&need_pages); 623 amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) { 624 struct amdgpu_bo *bo = e->robj; 625 626 if (amdgpu_ttm_tt_userptr_invalidated(bo->tbo.ttm, 627 &e->user_invalidated) && e->user_pages) { 628 629 /* We acquired a page array, but somebody 630 * invalidated it. Free it and try again 631 */ 632 release_pages(e->user_pages, 633 bo->tbo.ttm->num_pages); 634 kvfree(e->user_pages); 635 e->user_pages = NULL; 636 } 637 638 if (amdgpu_ttm_tt_userptr_needs_pages(bo->tbo.ttm) && 639 !e->user_pages) { 640 list_del(&e->tv.head); 641 list_add(&e->tv.head, &need_pages); 642 643 amdgpu_bo_unreserve(e->robj); 644 } 645 } 646 647 if (list_empty(&need_pages)) 648 break; 649 650 /* Unreserve everything again. */ 651 ttm_eu_backoff_reservation(&p->ticket, &p->validated); 652 653 /* We tried too many times, just abort */ 654 if (!--tries) { 655 r = -EDEADLK; 656 DRM_ERROR("deadlock in %s\n", __func__); 657 goto error_free_pages; 658 } 659 660 /* Fill the page arrays for all userptrs. */ 661 list_for_each_entry(e, &need_pages, tv.head) { 662 struct ttm_tt *ttm = e->robj->tbo.ttm; 663 664 e->user_pages = kvmalloc_array(ttm->num_pages, 665 sizeof(struct page*), 666 GFP_KERNEL | __GFP_ZERO); 667 if (!e->user_pages) { 668 r = -ENOMEM; 669 DRM_ERROR("calloc failure in %s\n", __func__); 670 goto error_free_pages; 671 } 672 673 r = amdgpu_ttm_tt_get_user_pages(ttm, e->user_pages); 674 if (r) { 675 DRM_ERROR("amdgpu_ttm_tt_get_user_pages failed.\n"); 676 kvfree(e->user_pages); 677 e->user_pages = NULL; 678 goto error_free_pages; 679 } 680 } 681 682 /* And try again. */ 683 list_splice(&need_pages, &p->validated); 684 } 685 686 amdgpu_cs_get_threshold_for_moves(p->adev, (u64 *)&p->bytes_moved_threshold, 687 (u64 *)&p->bytes_moved_vis_threshold); 688 p->bytes_moved = 0; 689 p->bytes_moved_vis = 0; 690 p->evictable = list_last_entry(&p->validated, 691 struct amdgpu_bo_list_entry, 692 tv.head); 693 694 r = amdgpu_vm_validate_pt_bos(p->adev, &fpriv->vm, 695 amdgpu_cs_validate, p); 696 if (r) { 697 DRM_ERROR("amdgpu_vm_validate_pt_bos() failed.\n"); 698 goto error_validate; 699 } 700 701 r = amdgpu_cs_list_validate(p, &duplicates); 702 if (r) { 703 DRM_ERROR("amdgpu_cs_list_validate(duplicates) failed.\n"); 704 goto error_validate; 705 } 706 707 r = amdgpu_cs_list_validate(p, &p->validated); 708 if (r) { 709 DRM_ERROR("amdgpu_cs_list_validate(validated) failed.\n"); 710 goto error_validate; 711 } 712 713 amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved, 714 p->bytes_moved_vis); 715 716 gds = p->bo_list->gds_obj; 717 gws = p->bo_list->gws_obj; 718 oa = p->bo_list->oa_obj; 719 720 amdgpu_bo_list_for_each_entry(e, p->bo_list) 721 e->bo_va = amdgpu_vm_bo_find(vm, e->robj); 722 723 if (gds) { 724 p->job->gds_base = amdgpu_bo_gpu_offset(gds); 725 p->job->gds_size = amdgpu_bo_size(gds); 726 } 727 if (gws) { 728 p->job->gws_base = amdgpu_bo_gpu_offset(gws); 729 p->job->gws_size = amdgpu_bo_size(gws); 730 } 731 if (oa) { 732 p->job->oa_base = amdgpu_bo_gpu_offset(oa); 733 p->job->oa_size = amdgpu_bo_size(oa); 734 } 735 736 if (!r && p->uf_entry.robj) { 737 struct amdgpu_bo *uf = p->uf_entry.robj; 738 739 r = amdgpu_ttm_alloc_gart(&uf->tbo); 740 p->job->uf_addr += amdgpu_bo_gpu_offset(uf); 741 } 742 743 error_validate: 744 if (r) 745 ttm_eu_backoff_reservation(&p->ticket, &p->validated); 746 747 error_free_pages: 748 749 amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) { 750 if (!e->user_pages) 751 continue; 752 753 release_pages(e->user_pages, 754 e->robj->tbo.ttm->num_pages); 755 kvfree(e->user_pages); 756 } 757 758 return r; 759 } 760 761 static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p) 762 { 763 struct amdgpu_bo_list_entry *e; 764 int r; 765 766 list_for_each_entry(e, &p->validated, tv.head) { 767 struct reservation_object *resv = e->robj->tbo.resv; 768 r = amdgpu_sync_resv(p->adev, &p->job->sync, resv, p->filp, 769 amdgpu_bo_explicit_sync(e->robj)); 770 771 if (r) 772 return r; 773 } 774 return 0; 775 } 776 777 /** 778 * cs_parser_fini() - clean parser states 779 * @parser: parser structure holding parsing context. 780 * @error: error number 781 * 782 * If error is set than unvalidate buffer, otherwise just free memory 783 * used by parsing context. 784 **/ 785 static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, 786 bool backoff) 787 { 788 unsigned i; 789 790 if (error && backoff) 791 ttm_eu_backoff_reservation(&parser->ticket, 792 &parser->validated); 793 794 for (i = 0; i < parser->num_post_dep_syncobjs; i++) 795 drm_syncobj_put(parser->post_dep_syncobjs[i]); 796 kfree(parser->post_dep_syncobjs); 797 798 dma_fence_put(parser->fence); 799 800 if (parser->ctx) { 801 mutex_unlock(&parser->ctx->lock); 802 amdgpu_ctx_put(parser->ctx); 803 } 804 if (parser->bo_list) 805 amdgpu_bo_list_put(parser->bo_list); 806 807 for (i = 0; i < parser->nchunks; i++) 808 kvfree(parser->chunks[i].kdata); 809 kfree(parser->chunks); 810 if (parser->job) 811 amdgpu_job_free(parser->job); 812 amdgpu_bo_unref(&parser->uf_entry.robj); 813 } 814 815 static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p) 816 { 817 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 818 struct amdgpu_device *adev = p->adev; 819 struct amdgpu_vm *vm = &fpriv->vm; 820 struct amdgpu_bo_list_entry *e; 821 struct amdgpu_bo_va *bo_va; 822 struct amdgpu_bo *bo; 823 int r; 824 825 r = amdgpu_vm_clear_freed(adev, vm, NULL); 826 if (r) 827 return r; 828 829 r = amdgpu_vm_bo_update(adev, fpriv->prt_va, false); 830 if (r) 831 return r; 832 833 r = amdgpu_sync_fence(adev, &p->job->sync, 834 fpriv->prt_va->last_pt_update, false); 835 if (r) 836 return r; 837 838 if (amdgpu_sriov_vf(adev)) { 839 struct dma_fence *f; 840 841 bo_va = fpriv->csa_va; 842 BUG_ON(!bo_va); 843 r = amdgpu_vm_bo_update(adev, bo_va, false); 844 if (r) 845 return r; 846 847 f = bo_va->last_pt_update; 848 r = amdgpu_sync_fence(adev, &p->job->sync, f, false); 849 if (r) 850 return r; 851 } 852 853 amdgpu_bo_list_for_each_entry(e, p->bo_list) { 854 struct dma_fence *f; 855 856 /* ignore duplicates */ 857 bo = e->robj; 858 if (!bo) 859 continue; 860 861 bo_va = e->bo_va; 862 if (bo_va == NULL) 863 continue; 864 865 r = amdgpu_vm_bo_update(adev, bo_va, false); 866 if (r) 867 return r; 868 869 f = bo_va->last_pt_update; 870 r = amdgpu_sync_fence(adev, &p->job->sync, f, false); 871 if (r) 872 return r; 873 } 874 875 r = amdgpu_vm_handle_moved(adev, vm); 876 if (r) 877 return r; 878 879 r = amdgpu_vm_update_directories(adev, vm); 880 if (r) 881 return r; 882 883 r = amdgpu_sync_fence(adev, &p->job->sync, vm->last_update, false); 884 if (r) 885 return r; 886 887 if (amdgpu_vm_debug) { 888 /* Invalidate all BOs to test for userspace bugs */ 889 amdgpu_bo_list_for_each_entry(e, p->bo_list) { 890 /* ignore duplicates */ 891 if (!e->robj) 892 continue; 893 894 amdgpu_vm_bo_invalidate(adev, e->robj, false); 895 } 896 } 897 898 return r; 899 } 900 901 static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev, 902 struct amdgpu_cs_parser *p) 903 { 904 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 905 struct amdgpu_vm *vm = &fpriv->vm; 906 struct amdgpu_ring *ring = p->ring; 907 int r; 908 909 /* Only for UVD/VCE VM emulation */ 910 if (p->ring->funcs->parse_cs || p->ring->funcs->patch_cs_in_place) { 911 unsigned i, j; 912 913 for (i = 0, j = 0; i < p->nchunks && j < p->job->num_ibs; i++) { 914 struct drm_amdgpu_cs_chunk_ib *chunk_ib; 915 struct amdgpu_bo_va_mapping *m; 916 struct amdgpu_bo *aobj = NULL; 917 struct amdgpu_cs_chunk *chunk; 918 uint64_t offset, va_start; 919 struct amdgpu_ib *ib; 920 uint8_t *kptr; 921 922 chunk = &p->chunks[i]; 923 ib = &p->job->ibs[j]; 924 chunk_ib = chunk->kdata; 925 926 if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB) 927 continue; 928 929 va_start = chunk_ib->va_start & AMDGPU_VA_HOLE_MASK; 930 r = amdgpu_cs_find_mapping(p, va_start, &aobj, &m); 931 if (r) { 932 DRM_ERROR("IB va_start is invalid\n"); 933 return r; 934 } 935 936 if ((va_start + chunk_ib->ib_bytes) > 937 (m->last + 1) * AMDGPU_GPU_PAGE_SIZE) { 938 DRM_ERROR("IB va_start+ib_bytes is invalid\n"); 939 return -EINVAL; 940 } 941 942 /* the IB should be reserved at this point */ 943 r = amdgpu_bo_kmap(aobj, (void **)&kptr); 944 if (r) { 945 return r; 946 } 947 948 offset = m->start * AMDGPU_GPU_PAGE_SIZE; 949 kptr += va_start - offset; 950 951 if (p->ring->funcs->parse_cs) { 952 memcpy(ib->ptr, kptr, chunk_ib->ib_bytes); 953 amdgpu_bo_kunmap(aobj); 954 955 r = amdgpu_ring_parse_cs(ring, p, j); 956 if (r) 957 return r; 958 } else { 959 ib->ptr = (uint32_t *)kptr; 960 r = amdgpu_ring_patch_cs_in_place(ring, p, j); 961 amdgpu_bo_kunmap(aobj); 962 if (r) 963 return r; 964 } 965 966 j++; 967 } 968 } 969 970 if (p->job->vm) { 971 p->job->vm_pd_addr = amdgpu_bo_gpu_offset(vm->root.base.bo); 972 973 r = amdgpu_bo_vm_update_pte(p); 974 if (r) 975 return r; 976 977 r = reservation_object_reserve_shared(vm->root.base.bo->tbo.resv); 978 if (r) 979 return r; 980 } 981 982 return amdgpu_cs_sync_rings(p); 983 } 984 985 static int amdgpu_cs_ib_fill(struct amdgpu_device *adev, 986 struct amdgpu_cs_parser *parser) 987 { 988 struct amdgpu_fpriv *fpriv = parser->filp->driver_priv; 989 struct amdgpu_vm *vm = &fpriv->vm; 990 int i, j; 991 int r, ce_preempt = 0, de_preempt = 0; 992 993 for (i = 0, j = 0; i < parser->nchunks && j < parser->job->num_ibs; i++) { 994 struct amdgpu_cs_chunk *chunk; 995 struct amdgpu_ib *ib; 996 struct drm_amdgpu_cs_chunk_ib *chunk_ib; 997 struct amdgpu_ring *ring; 998 999 chunk = &parser->chunks[i]; 1000 ib = &parser->job->ibs[j]; 1001 chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata; 1002 1003 if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB) 1004 continue; 1005 1006 if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX && amdgpu_sriov_vf(adev)) { 1007 if (chunk_ib->flags & AMDGPU_IB_FLAG_PREEMPT) { 1008 if (chunk_ib->flags & AMDGPU_IB_FLAG_CE) 1009 ce_preempt++; 1010 else 1011 de_preempt++; 1012 } 1013 1014 /* each GFX command submit allows 0 or 1 IB preemptible for CE & DE */ 1015 if (ce_preempt > 1 || de_preempt > 1) 1016 return -EINVAL; 1017 } 1018 1019 r = amdgpu_queue_mgr_map(adev, &parser->ctx->queue_mgr, chunk_ib->ip_type, 1020 chunk_ib->ip_instance, chunk_ib->ring, &ring); 1021 if (r) 1022 return r; 1023 1024 if (chunk_ib->flags & AMDGPU_IB_FLAG_PREAMBLE) 1025 parser->job->preamble_status |= 1026 AMDGPU_PREAMBLE_IB_PRESENT; 1027 1028 if (parser->ring && parser->ring != ring) 1029 return -EINVAL; 1030 1031 parser->ring = ring; 1032 1033 r = amdgpu_ib_get(adev, vm, 1034 ring->funcs->parse_cs ? chunk_ib->ib_bytes : 0, 1035 ib); 1036 if (r) { 1037 DRM_ERROR("Failed to get ib !\n"); 1038 return r; 1039 } 1040 1041 ib->gpu_addr = chunk_ib->va_start; 1042 ib->length_dw = chunk_ib->ib_bytes / 4; 1043 ib->flags = chunk_ib->flags; 1044 1045 j++; 1046 } 1047 1048 /* UVD & VCE fw doesn't support user fences */ 1049 if (parser->job->uf_addr && ( 1050 parser->ring->funcs->type == AMDGPU_RING_TYPE_UVD || 1051 parser->ring->funcs->type == AMDGPU_RING_TYPE_VCE)) 1052 return -EINVAL; 1053 1054 return amdgpu_ctx_wait_prev_fence(parser->ctx, parser->ring->idx); 1055 } 1056 1057 static int amdgpu_cs_process_fence_dep(struct amdgpu_cs_parser *p, 1058 struct amdgpu_cs_chunk *chunk) 1059 { 1060 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 1061 unsigned num_deps; 1062 int i, r; 1063 struct drm_amdgpu_cs_chunk_dep *deps; 1064 1065 deps = (struct drm_amdgpu_cs_chunk_dep *)chunk->kdata; 1066 num_deps = chunk->length_dw * 4 / 1067 sizeof(struct drm_amdgpu_cs_chunk_dep); 1068 1069 for (i = 0; i < num_deps; ++i) { 1070 struct amdgpu_ring *ring; 1071 struct amdgpu_ctx *ctx; 1072 struct dma_fence *fence; 1073 1074 ctx = amdgpu_ctx_get(fpriv, deps[i].ctx_id); 1075 if (ctx == NULL) 1076 return -EINVAL; 1077 1078 r = amdgpu_queue_mgr_map(p->adev, &ctx->queue_mgr, 1079 deps[i].ip_type, 1080 deps[i].ip_instance, 1081 deps[i].ring, &ring); 1082 if (r) { 1083 amdgpu_ctx_put(ctx); 1084 return r; 1085 } 1086 1087 fence = amdgpu_ctx_get_fence(ctx, ring, 1088 deps[i].handle); 1089 if (IS_ERR(fence)) { 1090 r = PTR_ERR(fence); 1091 amdgpu_ctx_put(ctx); 1092 return r; 1093 } else if (fence) { 1094 r = amdgpu_sync_fence(p->adev, &p->job->sync, fence, 1095 true); 1096 dma_fence_put(fence); 1097 amdgpu_ctx_put(ctx); 1098 if (r) 1099 return r; 1100 } 1101 } 1102 return 0; 1103 } 1104 1105 static int amdgpu_syncobj_lookup_and_add_to_sync(struct amdgpu_cs_parser *p, 1106 uint32_t handle) 1107 { 1108 int r; 1109 struct dma_fence *fence; 1110 r = drm_syncobj_find_fence(p->filp, handle, &fence); 1111 if (r) 1112 return r; 1113 1114 r = amdgpu_sync_fence(p->adev, &p->job->sync, fence, true); 1115 dma_fence_put(fence); 1116 1117 return r; 1118 } 1119 1120 static int amdgpu_cs_process_syncobj_in_dep(struct amdgpu_cs_parser *p, 1121 struct amdgpu_cs_chunk *chunk) 1122 { 1123 unsigned num_deps; 1124 int i, r; 1125 struct drm_amdgpu_cs_chunk_sem *deps; 1126 1127 deps = (struct drm_amdgpu_cs_chunk_sem *)chunk->kdata; 1128 num_deps = chunk->length_dw * 4 / 1129 sizeof(struct drm_amdgpu_cs_chunk_sem); 1130 1131 for (i = 0; i < num_deps; ++i) { 1132 r = amdgpu_syncobj_lookup_and_add_to_sync(p, deps[i].handle); 1133 if (r) 1134 return r; 1135 } 1136 return 0; 1137 } 1138 1139 static int amdgpu_cs_process_syncobj_out_dep(struct amdgpu_cs_parser *p, 1140 struct amdgpu_cs_chunk *chunk) 1141 { 1142 unsigned num_deps; 1143 int i; 1144 struct drm_amdgpu_cs_chunk_sem *deps; 1145 deps = (struct drm_amdgpu_cs_chunk_sem *)chunk->kdata; 1146 num_deps = chunk->length_dw * 4 / 1147 sizeof(struct drm_amdgpu_cs_chunk_sem); 1148 1149 p->post_dep_syncobjs = kmalloc_array(num_deps, 1150 sizeof(struct drm_syncobj *), 1151 GFP_KERNEL); 1152 p->num_post_dep_syncobjs = 0; 1153 1154 if (!p->post_dep_syncobjs) 1155 return -ENOMEM; 1156 1157 for (i = 0; i < num_deps; ++i) { 1158 p->post_dep_syncobjs[i] = drm_syncobj_find(p->filp, deps[i].handle); 1159 if (!p->post_dep_syncobjs[i]) 1160 return -EINVAL; 1161 p->num_post_dep_syncobjs++; 1162 } 1163 return 0; 1164 } 1165 1166 static int amdgpu_cs_dependencies(struct amdgpu_device *adev, 1167 struct amdgpu_cs_parser *p) 1168 { 1169 int i, r; 1170 1171 for (i = 0; i < p->nchunks; ++i) { 1172 struct amdgpu_cs_chunk *chunk; 1173 1174 chunk = &p->chunks[i]; 1175 1176 if (chunk->chunk_id == AMDGPU_CHUNK_ID_DEPENDENCIES) { 1177 r = amdgpu_cs_process_fence_dep(p, chunk); 1178 if (r) 1179 return r; 1180 } else if (chunk->chunk_id == AMDGPU_CHUNK_ID_SYNCOBJ_IN) { 1181 r = amdgpu_cs_process_syncobj_in_dep(p, chunk); 1182 if (r) 1183 return r; 1184 } else if (chunk->chunk_id == AMDGPU_CHUNK_ID_SYNCOBJ_OUT) { 1185 r = amdgpu_cs_process_syncobj_out_dep(p, chunk); 1186 if (r) 1187 return r; 1188 } 1189 } 1190 1191 return 0; 1192 } 1193 1194 static void amdgpu_cs_post_dependencies(struct amdgpu_cs_parser *p) 1195 { 1196 int i; 1197 1198 for (i = 0; i < p->num_post_dep_syncobjs; ++i) 1199 drm_syncobj_replace_fence(p->post_dep_syncobjs[i], p->fence); 1200 } 1201 1202 static int amdgpu_cs_submit(struct amdgpu_cs_parser *p, 1203 union drm_amdgpu_cs *cs) 1204 { 1205 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 1206 struct amdgpu_ring *ring = p->ring; 1207 struct drm_sched_entity *entity = &p->ctx->rings[ring->idx].entity; 1208 enum drm_sched_priority priority; 1209 struct amdgpu_bo_list_entry *e; 1210 struct amdgpu_job *job; 1211 uint64_t seq; 1212 1213 int r; 1214 1215 job = p->job; 1216 p->job = NULL; 1217 1218 r = drm_sched_job_init(&job->base, entity, p->filp); 1219 if (r) 1220 goto error_unlock; 1221 1222 /* No memory allocation is allowed while holding the mn lock */ 1223 amdgpu_mn_lock(p->mn); 1224 amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) { 1225 struct amdgpu_bo *bo = e->robj; 1226 1227 if (amdgpu_ttm_tt_userptr_needs_pages(bo->tbo.ttm)) { 1228 r = -ERESTARTSYS; 1229 goto error_abort; 1230 } 1231 } 1232 1233 job->owner = p->filp; 1234 p->fence = dma_fence_get(&job->base.s_fence->finished); 1235 1236 r = amdgpu_ctx_add_fence(p->ctx, ring, p->fence, &seq); 1237 if (r) { 1238 dma_fence_put(p->fence); 1239 dma_fence_put(&job->base.s_fence->finished); 1240 amdgpu_job_free(job); 1241 amdgpu_mn_unlock(p->mn); 1242 return r; 1243 } 1244 1245 amdgpu_cs_post_dependencies(p); 1246 1247 if ((job->preamble_status & AMDGPU_PREAMBLE_IB_PRESENT) && 1248 !p->ctx->preamble_presented) { 1249 job->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT_FIRST; 1250 p->ctx->preamble_presented = true; 1251 } 1252 1253 cs->out.handle = seq; 1254 job->uf_sequence = seq; 1255 1256 amdgpu_job_free_resources(job); 1257 1258 #if 0 1259 trace_amdgpu_cs_ioctl(job); 1260 #endif 1261 amdgpu_vm_bo_trace_cs(&fpriv->vm, &p->ticket); 1262 priority = job->base.s_priority; 1263 drm_sched_entity_push_job(&job->base, entity); 1264 1265 ring = to_amdgpu_ring(entity->rq->sched); 1266 amdgpu_ring_priority_get(ring, priority); 1267 1268 ttm_eu_fence_buffer_objects(&p->ticket, &p->validated, p->fence); 1269 amdgpu_mn_unlock(p->mn); 1270 1271 return 0; 1272 1273 error_abort: 1274 dma_fence_put(&job->base.s_fence->finished); 1275 job->base.s_fence = NULL; 1276 amdgpu_mn_unlock(p->mn); 1277 1278 error_unlock: 1279 amdgpu_job_free(job); 1280 return r; 1281 } 1282 1283 int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 1284 { 1285 struct amdgpu_device *adev = dev->dev_private; 1286 union drm_amdgpu_cs *cs = data; 1287 struct amdgpu_cs_parser parser = {}; 1288 bool reserved_buffers = false; 1289 int r; 1290 1291 if (!adev->accel_working) 1292 return -EBUSY; 1293 1294 parser.adev = adev; 1295 parser.filp = filp; 1296 1297 r = amdgpu_cs_parser_init(&parser, data); 1298 if (r) { 1299 DRM_ERROR("Failed to initialize parser !\n"); 1300 goto out; 1301 } 1302 1303 r = amdgpu_cs_ib_fill(adev, &parser); 1304 if (r) 1305 goto out; 1306 1307 r = amdgpu_cs_parser_bos(&parser, data); 1308 if (r) { 1309 if (r == -ENOMEM) 1310 DRM_ERROR("Not enough memory for command submission!\n"); 1311 else if (r != -ERESTARTSYS) 1312 DRM_ERROR("Failed to process the buffer list %d!\n", r); 1313 goto out; 1314 } 1315 1316 reserved_buffers = true; 1317 1318 r = amdgpu_cs_dependencies(adev, &parser); 1319 if (r) { 1320 DRM_ERROR("Failed in the dependencies handling %d!\n", r); 1321 goto out; 1322 } 1323 1324 #if 0 1325 for (i = 0; i < parser.job->num_ibs; i++) 1326 trace_amdgpu_cs(&parser, i); 1327 #endif 1328 1329 r = amdgpu_cs_ib_vm_chunk(adev, &parser); 1330 if (r) 1331 goto out; 1332 1333 r = amdgpu_cs_submit(&parser, cs); 1334 1335 out: 1336 amdgpu_cs_parser_fini(&parser, r, reserved_buffers); 1337 return r; 1338 } 1339 1340 /** 1341 * amdgpu_cs_wait_ioctl - wait for a command submission to finish 1342 * 1343 * @dev: drm device 1344 * @data: data from userspace 1345 * @filp: file private 1346 * 1347 * Wait for the command submission identified by handle to finish. 1348 */ 1349 int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data, 1350 struct drm_file *filp) 1351 { 1352 union drm_amdgpu_wait_cs *wait = data; 1353 struct amdgpu_device *adev = dev->dev_private; 1354 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout); 1355 struct amdgpu_ring *ring = NULL; 1356 struct amdgpu_ctx *ctx; 1357 struct dma_fence *fence; 1358 long r; 1359 1360 ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id); 1361 if (ctx == NULL) 1362 return -EINVAL; 1363 1364 r = amdgpu_queue_mgr_map(adev, &ctx->queue_mgr, 1365 wait->in.ip_type, wait->in.ip_instance, 1366 wait->in.ring, &ring); 1367 if (r) { 1368 amdgpu_ctx_put(ctx); 1369 return r; 1370 } 1371 1372 fence = amdgpu_ctx_get_fence(ctx, ring, wait->in.handle); 1373 if (IS_ERR(fence)) 1374 r = PTR_ERR(fence); 1375 else if (fence) { 1376 r = dma_fence_wait_timeout(fence, true, timeout); 1377 if (r > 0 && fence->error) 1378 r = fence->error; 1379 dma_fence_put(fence); 1380 } else 1381 r = 1; 1382 1383 amdgpu_ctx_put(ctx); 1384 if (r < 0) 1385 return r; 1386 1387 memset(wait, 0, sizeof(*wait)); 1388 wait->out.status = (r == 0); 1389 1390 return 0; 1391 } 1392 1393 /** 1394 * amdgpu_cs_get_fence - helper to get fence from drm_amdgpu_fence 1395 * 1396 * @adev: amdgpu device 1397 * @filp: file private 1398 * @user: drm_amdgpu_fence copied from user space 1399 */ 1400 static struct dma_fence *amdgpu_cs_get_fence(struct amdgpu_device *adev, 1401 struct drm_file *filp, 1402 struct drm_amdgpu_fence *user) 1403 { 1404 struct amdgpu_ring *ring; 1405 struct amdgpu_ctx *ctx; 1406 struct dma_fence *fence; 1407 int r; 1408 1409 ctx = amdgpu_ctx_get(filp->driver_priv, user->ctx_id); 1410 if (ctx == NULL) 1411 return ERR_PTR(-EINVAL); 1412 1413 r = amdgpu_queue_mgr_map(adev, &ctx->queue_mgr, user->ip_type, 1414 user->ip_instance, user->ring, &ring); 1415 if (r) { 1416 amdgpu_ctx_put(ctx); 1417 return ERR_PTR(r); 1418 } 1419 1420 fence = amdgpu_ctx_get_fence(ctx, ring, user->seq_no); 1421 amdgpu_ctx_put(ctx); 1422 1423 return fence; 1424 } 1425 1426 int amdgpu_cs_fence_to_handle_ioctl(struct drm_device *dev, void *data, 1427 struct drm_file *filp) 1428 { 1429 struct amdgpu_device *adev = dev->dev_private; 1430 union drm_amdgpu_fence_to_handle *info = data; 1431 struct dma_fence *fence; 1432 struct drm_syncobj *syncobj; 1433 struct sync_file *sync_file; 1434 int fd, r; 1435 1436 fence = amdgpu_cs_get_fence(adev, filp, &info->in.fence); 1437 if (IS_ERR(fence)) 1438 return PTR_ERR(fence); 1439 1440 switch (info->in.what) { 1441 case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ: 1442 r = drm_syncobj_create(&syncobj, 0, fence); 1443 dma_fence_put(fence); 1444 if (r) 1445 return r; 1446 r = drm_syncobj_get_handle(filp, syncobj, &info->out.handle); 1447 drm_syncobj_put(syncobj); 1448 return r; 1449 1450 case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ_FD: 1451 r = drm_syncobj_create(&syncobj, 0, fence); 1452 dma_fence_put(fence); 1453 if (r) 1454 return r; 1455 r = drm_syncobj_get_fd(syncobj, (int*)&info->out.handle); 1456 drm_syncobj_put(syncobj); 1457 return r; 1458 1459 case AMDGPU_FENCE_TO_HANDLE_GET_SYNC_FILE_FD: 1460 fd = get_unused_fd_flags(O_CLOEXEC); 1461 if (fd < 0) { 1462 dma_fence_put(fence); 1463 return fd; 1464 } 1465 1466 sync_file = sync_file_create(fence); 1467 dma_fence_put(fence); 1468 if (!sync_file) { 1469 put_unused_fd(fd); 1470 return -ENOMEM; 1471 } 1472 1473 fd_install(fd, sync_file->file); 1474 info->out.handle = fd; 1475 return 0; 1476 1477 default: 1478 return -EINVAL; 1479 } 1480 } 1481 1482 /** 1483 * amdgpu_cs_wait_all_fence - wait on all fences to signal 1484 * 1485 * @adev: amdgpu device 1486 * @filp: file private 1487 * @wait: wait parameters 1488 * @fences: array of drm_amdgpu_fence 1489 */ 1490 static int amdgpu_cs_wait_all_fences(struct amdgpu_device *adev, 1491 struct drm_file *filp, 1492 union drm_amdgpu_wait_fences *wait, 1493 struct drm_amdgpu_fence *fences) 1494 { 1495 uint32_t fence_count = wait->in.fence_count; 1496 unsigned int i; 1497 long r = 1; 1498 1499 for (i = 0; i < fence_count; i++) { 1500 struct dma_fence *fence; 1501 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns); 1502 1503 fence = amdgpu_cs_get_fence(adev, filp, &fences[i]); 1504 if (IS_ERR(fence)) 1505 return PTR_ERR(fence); 1506 else if (!fence) 1507 continue; 1508 1509 r = dma_fence_wait_timeout(fence, true, timeout); 1510 dma_fence_put(fence); 1511 if (r < 0) 1512 return r; 1513 1514 if (r == 0) 1515 break; 1516 1517 if (fence->error) 1518 return fence->error; 1519 } 1520 1521 memset(wait, 0, sizeof(*wait)); 1522 wait->out.status = (r > 0); 1523 1524 return 0; 1525 } 1526 1527 /** 1528 * amdgpu_cs_wait_any_fence - wait on any fence to signal 1529 * 1530 * @adev: amdgpu device 1531 * @filp: file private 1532 * @wait: wait parameters 1533 * @fences: array of drm_amdgpu_fence 1534 */ 1535 static int amdgpu_cs_wait_any_fence(struct amdgpu_device *adev, 1536 struct drm_file *filp, 1537 union drm_amdgpu_wait_fences *wait, 1538 struct drm_amdgpu_fence *fences) 1539 { 1540 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns); 1541 uint32_t fence_count = wait->in.fence_count; 1542 uint32_t first = ~0; 1543 struct dma_fence **array; 1544 unsigned int i; 1545 long r; 1546 1547 /* Prepare the fence array */ 1548 array = kcalloc(fence_count, sizeof(struct dma_fence *), GFP_KERNEL); 1549 1550 if (array == NULL) 1551 return -ENOMEM; 1552 1553 for (i = 0; i < fence_count; i++) { 1554 struct dma_fence *fence; 1555 1556 fence = amdgpu_cs_get_fence(adev, filp, &fences[i]); 1557 if (IS_ERR(fence)) { 1558 r = PTR_ERR(fence); 1559 goto err_free_fence_array; 1560 } else if (fence) { 1561 array[i] = fence; 1562 } else { /* NULL, the fence has been already signaled */ 1563 r = 1; 1564 first = i; 1565 goto out; 1566 } 1567 } 1568 1569 r = dma_fence_wait_any_timeout(array, fence_count, true, timeout, 1570 &first); 1571 if (r < 0) 1572 goto err_free_fence_array; 1573 1574 out: 1575 memset(wait, 0, sizeof(*wait)); 1576 wait->out.status = (r > 0); 1577 wait->out.first_signaled = first; 1578 1579 if (first < fence_count && array[first]) 1580 r = array[first]->error; 1581 else 1582 r = 0; 1583 1584 err_free_fence_array: 1585 for (i = 0; i < fence_count; i++) 1586 dma_fence_put(array[i]); 1587 kfree(array); 1588 1589 return r; 1590 } 1591 1592 /** 1593 * amdgpu_cs_wait_fences_ioctl - wait for multiple command submissions to finish 1594 * 1595 * @dev: drm device 1596 * @data: data from userspace 1597 * @filp: file private 1598 */ 1599 int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data, 1600 struct drm_file *filp) 1601 { 1602 struct amdgpu_device *adev = dev->dev_private; 1603 union drm_amdgpu_wait_fences *wait = data; 1604 uint32_t fence_count = wait->in.fence_count; 1605 struct drm_amdgpu_fence *fences_user; 1606 struct drm_amdgpu_fence *fences; 1607 int r; 1608 1609 /* Get the fences from userspace */ 1610 fences = kmalloc_array(fence_count, sizeof(struct drm_amdgpu_fence), 1611 GFP_KERNEL); 1612 if (fences == NULL) 1613 return -ENOMEM; 1614 1615 fences_user = u64_to_user_ptr(wait->in.fences); 1616 if (copy_from_user(fences, fences_user, 1617 sizeof(struct drm_amdgpu_fence) * fence_count)) { 1618 r = -EFAULT; 1619 goto err_free_fences; 1620 } 1621 1622 if (wait->in.wait_all) 1623 r = amdgpu_cs_wait_all_fences(adev, filp, wait, fences); 1624 else 1625 r = amdgpu_cs_wait_any_fence(adev, filp, wait, fences); 1626 1627 err_free_fences: 1628 kfree(fences); 1629 1630 return r; 1631 } 1632 1633 /** 1634 * amdgpu_cs_find_bo_va - find bo_va for VM address 1635 * 1636 * @parser: command submission parser context 1637 * @addr: VM address 1638 * @bo: resulting BO of the mapping found 1639 * 1640 * Search the buffer objects in the command submission context for a certain 1641 * virtual memory address. Returns allocation structure when found, NULL 1642 * otherwise. 1643 */ 1644 int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser, 1645 uint64_t addr, struct amdgpu_bo **bo, 1646 struct amdgpu_bo_va_mapping **map) 1647 { 1648 struct amdgpu_fpriv *fpriv = parser->filp->driver_priv; 1649 struct ttm_operation_ctx ctx = { false, false }; 1650 struct amdgpu_vm *vm = &fpriv->vm; 1651 struct amdgpu_bo_va_mapping *mapping; 1652 int r; 1653 1654 addr /= AMDGPU_GPU_PAGE_SIZE; 1655 1656 mapping = amdgpu_vm_bo_lookup_mapping(vm, addr); 1657 if (!mapping || !mapping->bo_va || !mapping->bo_va->base.bo) 1658 return -EINVAL; 1659 1660 *bo = mapping->bo_va->base.bo; 1661 *map = mapping; 1662 1663 /* Double check that the BO is reserved by this CS */ 1664 if (READ_ONCE((*bo)->tbo.resv->lock.ctx) != &parser->ticket) 1665 return -EINVAL; 1666 1667 if (!((*bo)->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)) { 1668 (*bo)->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; 1669 amdgpu_bo_placement_from_domain(*bo, (*bo)->allowed_domains); 1670 r = ttm_bo_validate(&(*bo)->tbo, &(*bo)->placement, &ctx); 1671 if (r) 1672 return r; 1673 } 1674 1675 return amdgpu_ttm_alloc_gart(&(*bo)->tbo); 1676 } 1677