1 /* 2 * Copyright (C) 2008 Ben Skeggs. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sublicense, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial 15 * portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26 27 #include "nouveau_drv.h" 28 #include "nouveau_dma.h" 29 #include "nouveau_fence.h" 30 #include "nouveau_abi16.h" 31 32 #include "nouveau_ttm.h" 33 #include "nouveau_gem.h" 34 #include "nouveau_mem.h" 35 #include "nouveau_vmm.h" 36 37 #include <nvif/class.h> 38 39 void 40 nouveau_gem_object_del(struct drm_gem_object *gem) 41 { 42 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 43 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 44 struct device *dev = drm->dev->dev; 45 int ret; 46 47 ret = pm_runtime_get_sync(dev); 48 if (WARN_ON(ret < 0 && ret != -EACCES)) 49 return; 50 51 if (gem->import_attach) 52 drm_prime_gem_destroy(gem, nvbo->bo.sg); 53 54 ttm_bo_put(&nvbo->bo); 55 56 pm_runtime_mark_last_busy(dev); 57 pm_runtime_put_autosuspend(dev); 58 } 59 60 int 61 nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) 62 { 63 struct nouveau_cli *cli = nouveau_cli(file_priv); 64 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 65 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 66 struct device *dev = drm->dev->dev; 67 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm; 68 struct nouveau_vma *vma; 69 int ret; 70 71 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) 72 return 0; 73 74 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 75 if (ret) 76 return ret; 77 78 ret = pm_runtime_get_sync(dev); 79 if (ret < 0 && ret != -EACCES) 80 goto out; 81 82 ret = nouveau_vma_new(nvbo, vmm, &vma); 83 pm_runtime_mark_last_busy(dev); 84 pm_runtime_put_autosuspend(dev); 85 out: 86 ttm_bo_unreserve(&nvbo->bo); 87 return ret; 88 } 89 90 struct nouveau_gem_object_unmap { 91 struct nouveau_cli_work work; 92 struct nouveau_vma *vma; 93 }; 94 95 static void 96 nouveau_gem_object_delete(struct nouveau_vma *vma) 97 { 98 nouveau_fence_unref(&vma->fence); 99 nouveau_vma_del(&vma); 100 } 101 102 static void 103 nouveau_gem_object_delete_work(struct nouveau_cli_work *w) 104 { 105 struct nouveau_gem_object_unmap *work = 106 container_of(w, typeof(*work), work); 107 nouveau_gem_object_delete(work->vma); 108 kfree(work); 109 } 110 111 static void 112 nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) 113 { 114 struct dma_fence *fence = vma->fence ? &vma->fence->base : NULL; 115 struct nouveau_gem_object_unmap *work; 116 117 list_del_init(&vma->head); 118 119 if (!fence) { 120 nouveau_gem_object_delete(vma); 121 return; 122 } 123 124 if (!(work = kmalloc(sizeof(*work), GFP_KERNEL))) { 125 WARN_ON(dma_fence_wait_timeout(fence, false, 2 * HZ) <= 0); 126 nouveau_gem_object_delete(vma); 127 return; 128 } 129 130 work->work.func = nouveau_gem_object_delete_work; 131 work->vma = vma; 132 nouveau_cli_work_queue(vma->vmm->cli, fence, &work->work); 133 } 134 135 void 136 nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) 137 { 138 struct nouveau_cli *cli = nouveau_cli(file_priv); 139 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 140 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 141 struct device *dev = drm->dev->dev; 142 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : & cli->vmm; 143 struct nouveau_vma *vma; 144 int ret; 145 146 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) 147 return; 148 149 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 150 if (ret) 151 return; 152 153 vma = nouveau_vma_find(nvbo, vmm); 154 if (vma) { 155 if (--vma->refs == 0) { 156 ret = pm_runtime_get_sync(dev); 157 if (!WARN_ON(ret < 0 && ret != -EACCES)) { 158 nouveau_gem_object_unmap(nvbo, vma); 159 pm_runtime_mark_last_busy(dev); 160 pm_runtime_put_autosuspend(dev); 161 } 162 } 163 } 164 ttm_bo_unreserve(&nvbo->bo); 165 } 166 167 int 168 nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain, 169 uint32_t tile_mode, uint32_t tile_flags, 170 struct nouveau_bo **pnvbo) 171 { 172 struct nouveau_drm *drm = cli->drm; 173 struct nouveau_bo *nvbo; 174 u32 flags = 0; 175 int ret; 176 177 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) 178 flags |= TTM_PL_FLAG_VRAM; 179 if (domain & NOUVEAU_GEM_DOMAIN_GART) 180 flags |= TTM_PL_FLAG_TT; 181 if (!flags || domain & NOUVEAU_GEM_DOMAIN_CPU) 182 flags |= TTM_PL_FLAG_SYSTEM; 183 184 if (domain & NOUVEAU_GEM_DOMAIN_COHERENT) 185 flags |= TTM_PL_FLAG_UNCACHED; 186 187 nvbo = nouveau_bo_alloc(cli, &size, &align, flags, tile_mode, 188 tile_flags); 189 if (IS_ERR(nvbo)) 190 return PTR_ERR(nvbo); 191 192 /* Initialize the embedded gem-object. We return a single gem-reference 193 * to the caller, instead of a normal nouveau_bo ttm reference. */ 194 ret = drm_gem_object_init(drm->dev, &nvbo->bo.base, size); 195 if (ret) { 196 nouveau_bo_ref(NULL, &nvbo); 197 return ret; 198 } 199 200 ret = nouveau_bo_init(nvbo, size, align, flags, NULL, NULL); 201 if (ret) { 202 nouveau_bo_ref(NULL, &nvbo); 203 return ret; 204 } 205 206 /* we restrict allowed domains on nv50+ to only the types 207 * that were requested at creation time. not possibly on 208 * earlier chips without busting the ABI. 209 */ 210 nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM | 211 NOUVEAU_GEM_DOMAIN_GART; 212 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) 213 nvbo->valid_domains &= domain; 214 215 nvbo->bo.persistent_swap_storage = nvbo->bo.base.filp; 216 *pnvbo = nvbo; 217 return 0; 218 } 219 220 static int 221 nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, 222 struct drm_nouveau_gem_info *rep) 223 { 224 struct nouveau_cli *cli = nouveau_cli(file_priv); 225 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 226 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm; 227 struct nouveau_vma *vma; 228 229 if (is_power_of_2(nvbo->valid_domains)) 230 rep->domain = nvbo->valid_domains; 231 else if (nvbo->bo.mem.mem_type == TTM_PL_TT) 232 rep->domain = NOUVEAU_GEM_DOMAIN_GART; 233 else 234 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; 235 rep->offset = nvbo->bo.offset; 236 if (vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 237 vma = nouveau_vma_find(nvbo, vmm); 238 if (!vma) 239 return -EINVAL; 240 241 rep->offset = vma->addr; 242 } 243 244 rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT; 245 rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.base.vma_node); 246 rep->tile_mode = nvbo->mode; 247 rep->tile_flags = nvbo->contig ? 0 : NOUVEAU_GEM_TILE_NONCONTIG; 248 if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) 249 rep->tile_flags |= nvbo->kind << 8; 250 else 251 if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) 252 rep->tile_flags |= nvbo->kind << 8 | nvbo->comp << 16; 253 else 254 rep->tile_flags |= nvbo->zeta; 255 return 0; 256 } 257 258 int 259 nouveau_gem_ioctl_new(struct drm_device *dev, void *data, 260 struct drm_file *file_priv) 261 { 262 struct nouveau_cli *cli = nouveau_cli(file_priv); 263 struct drm_nouveau_gem_new *req = data; 264 struct nouveau_bo *nvbo = NULL; 265 int ret = 0; 266 267 ret = nouveau_gem_new(cli, req->info.size, req->align, 268 req->info.domain, req->info.tile_mode, 269 req->info.tile_flags, &nvbo); 270 if (ret) 271 return ret; 272 273 ret = drm_gem_handle_create(file_priv, &nvbo->bo.base, 274 &req->info.handle); 275 if (ret == 0) { 276 ret = nouveau_gem_info(file_priv, &nvbo->bo.base, &req->info); 277 if (ret) 278 drm_gem_handle_delete(file_priv, req->info.handle); 279 } 280 281 /* drop reference from allocate - handle holds it now */ 282 drm_gem_object_put_unlocked(&nvbo->bo.base); 283 return ret; 284 } 285 286 static int 287 nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains, 288 uint32_t write_domains, uint32_t valid_domains) 289 { 290 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 291 struct ttm_buffer_object *bo = &nvbo->bo; 292 uint32_t domains = valid_domains & nvbo->valid_domains & 293 (write_domains ? write_domains : read_domains); 294 uint32_t pref_flags = 0, valid_flags = 0; 295 296 if (!domains) 297 return -EINVAL; 298 299 if (valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 300 valid_flags |= TTM_PL_FLAG_VRAM; 301 302 if (valid_domains & NOUVEAU_GEM_DOMAIN_GART) 303 valid_flags |= TTM_PL_FLAG_TT; 304 305 if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) && 306 bo->mem.mem_type == TTM_PL_VRAM) 307 pref_flags |= TTM_PL_FLAG_VRAM; 308 309 else if ((domains & NOUVEAU_GEM_DOMAIN_GART) && 310 bo->mem.mem_type == TTM_PL_TT) 311 pref_flags |= TTM_PL_FLAG_TT; 312 313 else if (domains & NOUVEAU_GEM_DOMAIN_VRAM) 314 pref_flags |= TTM_PL_FLAG_VRAM; 315 316 else 317 pref_flags |= TTM_PL_FLAG_TT; 318 319 nouveau_bo_placement_set(nvbo, pref_flags, valid_flags); 320 321 return 0; 322 } 323 324 struct validate_op { 325 struct list_head list; 326 struct ww_acquire_ctx ticket; 327 }; 328 329 static void 330 validate_fini_no_ticket(struct validate_op *op, struct nouveau_channel *chan, 331 struct nouveau_fence *fence, 332 struct drm_nouveau_gem_pushbuf_bo *pbbo) 333 { 334 struct nouveau_bo *nvbo; 335 struct drm_nouveau_gem_pushbuf_bo *b; 336 337 while (!list_empty(&op->list)) { 338 nvbo = list_entry(op->list.next, struct nouveau_bo, entry); 339 b = &pbbo[nvbo->pbbo_index]; 340 341 if (likely(fence)) { 342 nouveau_bo_fence(nvbo, fence, !!b->write_domains); 343 344 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 345 struct nouveau_vma *vma = 346 (void *)(unsigned long)b->user_priv; 347 nouveau_fence_unref(&vma->fence); 348 dma_fence_get(&fence->base); 349 vma->fence = fence; 350 } 351 } 352 353 if (unlikely(nvbo->validate_mapped)) { 354 ttm_bo_kunmap(&nvbo->kmap); 355 nvbo->validate_mapped = false; 356 } 357 358 list_del(&nvbo->entry); 359 nvbo->reserved_by = NULL; 360 ttm_bo_unreserve(&nvbo->bo); 361 drm_gem_object_put_unlocked(&nvbo->bo.base); 362 } 363 } 364 365 static void 366 validate_fini(struct validate_op *op, struct nouveau_channel *chan, 367 struct nouveau_fence *fence, 368 struct drm_nouveau_gem_pushbuf_bo *pbbo) 369 { 370 validate_fini_no_ticket(op, chan, fence, pbbo); 371 ww_acquire_fini(&op->ticket); 372 } 373 374 static int 375 validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, 376 struct drm_nouveau_gem_pushbuf_bo *pbbo, 377 int nr_buffers, struct validate_op *op) 378 { 379 struct nouveau_cli *cli = nouveau_cli(file_priv); 380 int trycnt = 0; 381 int ret = -EINVAL, i; 382 struct nouveau_bo *res_bo = NULL; 383 LIST_HEAD(gart_list); 384 LIST_HEAD(vram_list); 385 LIST_HEAD(both_list); 386 387 ww_acquire_init(&op->ticket, &reservation_ww_class); 388 retry: 389 if (++trycnt > 100000) { 390 NV_PRINTK(err, cli, "%s failed and gave up.\n", __func__); 391 return -EINVAL; 392 } 393 394 for (i = 0; i < nr_buffers; i++) { 395 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i]; 396 struct drm_gem_object *gem; 397 struct nouveau_bo *nvbo; 398 399 gem = drm_gem_object_lookup(file_priv, b->handle); 400 if (!gem) { 401 NV_PRINTK(err, cli, "Unknown handle 0x%08x\n", b->handle); 402 ret = -ENOENT; 403 break; 404 } 405 nvbo = nouveau_gem_object(gem); 406 if (nvbo == res_bo) { 407 res_bo = NULL; 408 drm_gem_object_put_unlocked(gem); 409 continue; 410 } 411 412 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) { 413 NV_PRINTK(err, cli, "multiple instances of buffer %d on " 414 "validation list\n", b->handle); 415 drm_gem_object_put_unlocked(gem); 416 ret = -EINVAL; 417 break; 418 } 419 420 ret = ttm_bo_reserve(&nvbo->bo, true, false, &op->ticket); 421 if (ret) { 422 list_splice_tail_init(&vram_list, &op->list); 423 list_splice_tail_init(&gart_list, &op->list); 424 list_splice_tail_init(&both_list, &op->list); 425 validate_fini_no_ticket(op, chan, NULL, NULL); 426 if (unlikely(ret == -EDEADLK)) { 427 ret = ttm_bo_reserve_slowpath(&nvbo->bo, true, 428 &op->ticket); 429 if (!ret) 430 res_bo = nvbo; 431 } 432 if (unlikely(ret)) { 433 if (ret != -ERESTARTSYS) 434 NV_PRINTK(err, cli, "fail reserve\n"); 435 break; 436 } 437 } 438 439 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 440 struct nouveau_vmm *vmm = chan->vmm; 441 struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm); 442 if (!vma) { 443 NV_PRINTK(err, cli, "vma not found!\n"); 444 ret = -EINVAL; 445 break; 446 } 447 448 b->user_priv = (uint64_t)(unsigned long)vma; 449 } else { 450 b->user_priv = (uint64_t)(unsigned long)nvbo; 451 } 452 453 nvbo->reserved_by = file_priv; 454 nvbo->pbbo_index = i; 455 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 456 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) 457 list_add_tail(&nvbo->entry, &both_list); 458 else 459 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 460 list_add_tail(&nvbo->entry, &vram_list); 461 else 462 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART) 463 list_add_tail(&nvbo->entry, &gart_list); 464 else { 465 NV_PRINTK(err, cli, "invalid valid domains: 0x%08x\n", 466 b->valid_domains); 467 list_add_tail(&nvbo->entry, &both_list); 468 ret = -EINVAL; 469 break; 470 } 471 if (nvbo == res_bo) 472 goto retry; 473 } 474 475 ww_acquire_done(&op->ticket); 476 list_splice_tail(&vram_list, &op->list); 477 list_splice_tail(&gart_list, &op->list); 478 list_splice_tail(&both_list, &op->list); 479 if (ret) 480 validate_fini(op, chan, NULL, NULL); 481 return ret; 482 483 } 484 485 static int 486 validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli, 487 struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo) 488 { 489 struct nouveau_drm *drm = chan->drm; 490 struct nouveau_bo *nvbo; 491 int ret, relocs = 0; 492 493 list_for_each_entry(nvbo, list, entry) { 494 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index]; 495 496 ret = nouveau_gem_set_domain(&nvbo->bo.base, b->read_domains, 497 b->write_domains, 498 b->valid_domains); 499 if (unlikely(ret)) { 500 NV_PRINTK(err, cli, "fail set_domain\n"); 501 return ret; 502 } 503 504 ret = nouveau_bo_validate(nvbo, true, false); 505 if (unlikely(ret)) { 506 if (ret != -ERESTARTSYS) 507 NV_PRINTK(err, cli, "fail ttm_validate\n"); 508 return ret; 509 } 510 511 ret = nouveau_fence_sync(nvbo, chan, !!b->write_domains, true); 512 if (unlikely(ret)) { 513 if (ret != -ERESTARTSYS) 514 NV_PRINTK(err, cli, "fail post-validate sync\n"); 515 return ret; 516 } 517 518 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { 519 if (nvbo->bo.offset == b->presumed.offset && 520 ((nvbo->bo.mem.mem_type == TTM_PL_VRAM && 521 b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) || 522 (nvbo->bo.mem.mem_type == TTM_PL_TT && 523 b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART))) 524 continue; 525 526 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 527 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; 528 else 529 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; 530 b->presumed.offset = nvbo->bo.offset; 531 b->presumed.valid = 0; 532 relocs++; 533 } 534 } 535 536 return relocs; 537 } 538 539 static int 540 nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, 541 struct drm_file *file_priv, 542 struct drm_nouveau_gem_pushbuf_bo *pbbo, 543 int nr_buffers, 544 struct validate_op *op, bool *apply_relocs) 545 { 546 struct nouveau_cli *cli = nouveau_cli(file_priv); 547 int ret; 548 549 INIT_LIST_HEAD(&op->list); 550 551 if (nr_buffers == 0) 552 return 0; 553 554 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); 555 if (unlikely(ret)) { 556 if (ret != -ERESTARTSYS) 557 NV_PRINTK(err, cli, "validate_init\n"); 558 return ret; 559 } 560 561 ret = validate_list(chan, cli, &op->list, pbbo); 562 if (unlikely(ret < 0)) { 563 if (ret != -ERESTARTSYS) 564 NV_PRINTK(err, cli, "validating bo list\n"); 565 validate_fini(op, chan, NULL, NULL); 566 return ret; 567 } 568 *apply_relocs = ret; 569 return 0; 570 } 571 572 static inline void 573 u_free(void *addr) 574 { 575 kvfree(addr); 576 } 577 578 static inline void * 579 u_memcpya(uint64_t user, unsigned nmemb, unsigned size) 580 { 581 void *mem; 582 void __user *userptr = (void __force __user *)(uintptr_t)user; 583 584 size *= nmemb; 585 586 mem = kvmalloc(size, GFP_KERNEL); 587 if (!mem) 588 return ERR_PTR(-ENOMEM); 589 590 if (copy_from_user(mem, userptr, size)) { 591 u_free(mem); 592 return ERR_PTR(-EFAULT); 593 } 594 595 return mem; 596 } 597 598 static int 599 nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli, 600 struct drm_nouveau_gem_pushbuf *req, 601 struct drm_nouveau_gem_pushbuf_reloc *reloc, 602 struct drm_nouveau_gem_pushbuf_bo *bo) 603 { 604 int ret = 0; 605 unsigned i; 606 607 for (i = 0; i < req->nr_relocs; i++) { 608 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i]; 609 struct drm_nouveau_gem_pushbuf_bo *b; 610 struct nouveau_bo *nvbo; 611 uint32_t data; 612 613 if (unlikely(r->bo_index >= req->nr_buffers)) { 614 NV_PRINTK(err, cli, "reloc bo index invalid\n"); 615 ret = -EINVAL; 616 break; 617 } 618 619 b = &bo[r->bo_index]; 620 if (b->presumed.valid) 621 continue; 622 623 if (unlikely(r->reloc_bo_index >= req->nr_buffers)) { 624 NV_PRINTK(err, cli, "reloc container bo index invalid\n"); 625 ret = -EINVAL; 626 break; 627 } 628 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv; 629 630 if (unlikely(r->reloc_bo_offset + 4 > 631 nvbo->bo.mem.num_pages << PAGE_SHIFT)) { 632 NV_PRINTK(err, cli, "reloc outside of bo\n"); 633 ret = -EINVAL; 634 break; 635 } 636 637 if (!nvbo->kmap.virtual) { 638 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, 639 &nvbo->kmap); 640 if (ret) { 641 NV_PRINTK(err, cli, "failed kmap for reloc\n"); 642 break; 643 } 644 nvbo->validate_mapped = true; 645 } 646 647 if (r->flags & NOUVEAU_GEM_RELOC_LOW) 648 data = b->presumed.offset + r->data; 649 else 650 if (r->flags & NOUVEAU_GEM_RELOC_HIGH) 651 data = (b->presumed.offset + r->data) >> 32; 652 else 653 data = r->data; 654 655 if (r->flags & NOUVEAU_GEM_RELOC_OR) { 656 if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART) 657 data |= r->tor; 658 else 659 data |= r->vor; 660 } 661 662 ret = ttm_bo_wait(&nvbo->bo, false, false); 663 if (ret) { 664 NV_PRINTK(err, cli, "reloc wait_idle failed: %d\n", ret); 665 break; 666 } 667 668 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data); 669 } 670 671 u_free(reloc); 672 return ret; 673 } 674 675 int 676 nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, 677 struct drm_file *file_priv) 678 { 679 struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv); 680 struct nouveau_cli *cli = nouveau_cli(file_priv); 681 struct nouveau_abi16_chan *temp; 682 struct nouveau_drm *drm = nouveau_drm(dev); 683 struct drm_nouveau_gem_pushbuf *req = data; 684 struct drm_nouveau_gem_pushbuf_push *push; 685 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL; 686 struct drm_nouveau_gem_pushbuf_bo *bo; 687 struct nouveau_channel *chan = NULL; 688 struct validate_op op; 689 struct nouveau_fence *fence = NULL; 690 int i, j, ret = 0; 691 bool do_reloc = false, sync = false; 692 693 if (unlikely(!abi16)) 694 return -ENOMEM; 695 696 list_for_each_entry(temp, &abi16->channels, head) { 697 if (temp->chan->chid == req->channel) { 698 chan = temp->chan; 699 break; 700 } 701 } 702 703 if (!chan) 704 return nouveau_abi16_put(abi16, -ENOENT); 705 if (unlikely(atomic_read(&chan->killed))) 706 return nouveau_abi16_put(abi16, -ENODEV); 707 708 sync = req->vram_available & NOUVEAU_GEM_PUSHBUF_SYNC; 709 710 req->vram_available = drm->gem.vram_available; 711 req->gart_available = drm->gem.gart_available; 712 if (unlikely(req->nr_push == 0)) 713 goto out_next; 714 715 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) { 716 NV_PRINTK(err, cli, "pushbuf push count exceeds limit: %d max %d\n", 717 req->nr_push, NOUVEAU_GEM_MAX_PUSH); 718 return nouveau_abi16_put(abi16, -EINVAL); 719 } 720 721 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) { 722 NV_PRINTK(err, cli, "pushbuf bo count exceeds limit: %d max %d\n", 723 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS); 724 return nouveau_abi16_put(abi16, -EINVAL); 725 } 726 727 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) { 728 NV_PRINTK(err, cli, "pushbuf reloc count exceeds limit: %d max %d\n", 729 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS); 730 return nouveau_abi16_put(abi16, -EINVAL); 731 } 732 733 push = u_memcpya(req->push, req->nr_push, sizeof(*push)); 734 if (IS_ERR(push)) 735 return nouveau_abi16_put(abi16, PTR_ERR(push)); 736 737 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo)); 738 if (IS_ERR(bo)) { 739 u_free(push); 740 return nouveau_abi16_put(abi16, PTR_ERR(bo)); 741 } 742 743 /* Ensure all push buffers are on validate list */ 744 for (i = 0; i < req->nr_push; i++) { 745 if (push[i].bo_index >= req->nr_buffers) { 746 NV_PRINTK(err, cli, "push %d buffer not in list\n", i); 747 ret = -EINVAL; 748 goto out_prevalid; 749 } 750 } 751 752 /* Validate buffer list */ 753 revalidate: 754 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, 755 req->nr_buffers, &op, &do_reloc); 756 if (ret) { 757 if (ret != -ERESTARTSYS) 758 NV_PRINTK(err, cli, "validate: %d\n", ret); 759 goto out_prevalid; 760 } 761 762 /* Apply any relocations that are required */ 763 if (do_reloc) { 764 if (!reloc) { 765 validate_fini(&op, chan, NULL, bo); 766 reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc)); 767 if (IS_ERR(reloc)) { 768 ret = PTR_ERR(reloc); 769 goto out_prevalid; 770 } 771 772 goto revalidate; 773 } 774 775 ret = nouveau_gem_pushbuf_reloc_apply(cli, req, reloc, bo); 776 if (ret) { 777 NV_PRINTK(err, cli, "reloc apply: %d\n", ret); 778 goto out; 779 } 780 } 781 782 if (chan->dma.ib_max) { 783 ret = nouveau_dma_wait(chan, req->nr_push + 1, 16); 784 if (ret) { 785 NV_PRINTK(err, cli, "nv50cal_space: %d\n", ret); 786 goto out; 787 } 788 789 for (i = 0; i < req->nr_push; i++) { 790 struct nouveau_vma *vma = (void *)(unsigned long) 791 bo[push[i].bo_index].user_priv; 792 793 nv50_dma_push(chan, vma->addr + push[i].offset, 794 push[i].length); 795 } 796 } else 797 if (drm->client.device.info.chipset >= 0x25) { 798 ret = RING_SPACE(chan, req->nr_push * 2); 799 if (ret) { 800 NV_PRINTK(err, cli, "cal_space: %d\n", ret); 801 goto out; 802 } 803 804 for (i = 0; i < req->nr_push; i++) { 805 struct nouveau_bo *nvbo = (void *)(unsigned long) 806 bo[push[i].bo_index].user_priv; 807 808 OUT_RING(chan, (nvbo->bo.offset + push[i].offset) | 2); 809 OUT_RING(chan, 0); 810 } 811 } else { 812 ret = RING_SPACE(chan, req->nr_push * (2 + NOUVEAU_DMA_SKIPS)); 813 if (ret) { 814 NV_PRINTK(err, cli, "jmp_space: %d\n", ret); 815 goto out; 816 } 817 818 for (i = 0; i < req->nr_push; i++) { 819 struct nouveau_bo *nvbo = (void *)(unsigned long) 820 bo[push[i].bo_index].user_priv; 821 uint32_t cmd; 822 823 cmd = chan->push.addr + ((chan->dma.cur + 2) << 2); 824 cmd |= 0x20000000; 825 if (unlikely(cmd != req->suffix0)) { 826 if (!nvbo->kmap.virtual) { 827 ret = ttm_bo_kmap(&nvbo->bo, 0, 828 nvbo->bo.mem. 829 num_pages, 830 &nvbo->kmap); 831 if (ret) { 832 WIND_RING(chan); 833 goto out; 834 } 835 nvbo->validate_mapped = true; 836 } 837 838 nouveau_bo_wr32(nvbo, (push[i].offset + 839 push[i].length - 8) / 4, cmd); 840 } 841 842 OUT_RING(chan, 0x20000000 | 843 (nvbo->bo.offset + push[i].offset)); 844 OUT_RING(chan, 0); 845 for (j = 0; j < NOUVEAU_DMA_SKIPS; j++) 846 OUT_RING(chan, 0); 847 } 848 } 849 850 ret = nouveau_fence_new(chan, false, &fence); 851 if (ret) { 852 NV_PRINTK(err, cli, "error fencing pushbuf: %d\n", ret); 853 WIND_RING(chan); 854 goto out; 855 } 856 857 if (sync) { 858 if (!(ret = nouveau_fence_wait(fence, false, false))) { 859 if ((ret = dma_fence_get_status(&fence->base)) == 1) 860 ret = 0; 861 } 862 } 863 864 out: 865 validate_fini(&op, chan, fence, bo); 866 nouveau_fence_unref(&fence); 867 868 if (do_reloc) { 869 struct drm_nouveau_gem_pushbuf_bo __user *upbbo = 870 u64_to_user_ptr(req->buffers); 871 872 for (i = 0; i < req->nr_buffers; i++) { 873 if (bo[i].presumed.valid) 874 continue; 875 876 if (copy_to_user(&upbbo[i].presumed, &bo[i].presumed, 877 sizeof(bo[i].presumed))) { 878 ret = -EFAULT; 879 break; 880 } 881 } 882 u_free(reloc); 883 } 884 out_prevalid: 885 u_free(bo); 886 u_free(push); 887 888 out_next: 889 if (chan->dma.ib_max) { 890 req->suffix0 = 0x00000000; 891 req->suffix1 = 0x00000000; 892 } else 893 if (drm->client.device.info.chipset >= 0x25) { 894 req->suffix0 = 0x00020000; 895 req->suffix1 = 0x00000000; 896 } else { 897 req->suffix0 = 0x20000000 | 898 (chan->push.addr + ((chan->dma.cur + 2) << 2)); 899 req->suffix1 = 0x00000000; 900 } 901 902 return nouveau_abi16_put(abi16, ret); 903 } 904 905 int 906 nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, 907 struct drm_file *file_priv) 908 { 909 struct drm_nouveau_gem_cpu_prep *req = data; 910 struct drm_gem_object *gem; 911 struct nouveau_bo *nvbo; 912 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); 913 bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE); 914 long lret; 915 int ret; 916 917 gem = drm_gem_object_lookup(file_priv, req->handle); 918 if (!gem) 919 return -ENOENT; 920 nvbo = nouveau_gem_object(gem); 921 922 lret = dma_resv_wait_timeout_rcu(nvbo->bo.base.resv, write, true, 923 no_wait ? 0 : 30 * HZ); 924 if (!lret) 925 ret = -EBUSY; 926 else if (lret > 0) 927 ret = 0; 928 else 929 ret = lret; 930 931 nouveau_bo_sync_for_cpu(nvbo); 932 drm_gem_object_put_unlocked(gem); 933 934 return ret; 935 } 936 937 int 938 nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, 939 struct drm_file *file_priv) 940 { 941 struct drm_nouveau_gem_cpu_fini *req = data; 942 struct drm_gem_object *gem; 943 struct nouveau_bo *nvbo; 944 945 gem = drm_gem_object_lookup(file_priv, req->handle); 946 if (!gem) 947 return -ENOENT; 948 nvbo = nouveau_gem_object(gem); 949 950 nouveau_bo_sync_for_device(nvbo); 951 drm_gem_object_put_unlocked(gem); 952 return 0; 953 } 954 955 int 956 nouveau_gem_ioctl_info(struct drm_device *dev, void *data, 957 struct drm_file *file_priv) 958 { 959 struct drm_nouveau_gem_info *req = data; 960 struct drm_gem_object *gem; 961 int ret; 962 963 gem = drm_gem_object_lookup(file_priv, req->handle); 964 if (!gem) 965 return -ENOENT; 966 967 ret = nouveau_gem_info(file_priv, gem, req); 968 drm_gem_object_put_unlocked(gem); 969 return ret; 970 } 971 972