1 /* 2 * Copyright 2009 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 /* 27 * Authors: 28 * Jerome Glisse <glisse@freedesktop.org> 29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> 30 * Dave Airlie 31 * 32 * $FreeBSD: head/sys/dev/drm2/radeon/radeon_ttm.c 254885 2013-08-25 19:37:15Z dumbbell $ 33 */ 34 35 #include <drm/ttm/ttm_bo_api.h> 36 #include <drm/ttm/ttm_bo_driver.h> 37 #include <drm/ttm/ttm_placement.h> 38 #include <drm/ttm/ttm_module.h> 39 #include <drm/ttm/ttm_page_alloc.h> 40 #include <drm/drmP.h> 41 #include <uapi_drm/radeon_drm.h> 42 #include <linux/seq_file.h> 43 #include <linux/slab.h> 44 #include "radeon_reg.h" 45 #include "radeon.h" 46 47 #define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT) 48 49 static int radeon_ttm_debugfs_init(struct radeon_device *rdev); 50 static void radeon_ttm_debugfs_fini(struct radeon_device *rdev); 51 52 static struct radeon_device *radeon_get_rdev(struct ttm_bo_device *bdev) 53 { 54 struct radeon_mman *mman; 55 struct radeon_device *rdev; 56 57 mman = container_of(bdev, struct radeon_mman, bdev); 58 rdev = container_of(mman, struct radeon_device, mman); 59 return rdev; 60 } 61 62 63 /* 64 * Global memory. 65 */ 66 static int radeon_ttm_mem_global_init(struct drm_global_reference *ref) 67 { 68 return ttm_mem_global_init(ref->object); 69 } 70 71 static void radeon_ttm_mem_global_release(struct drm_global_reference *ref) 72 { 73 ttm_mem_global_release(ref->object); 74 } 75 76 static int radeon_ttm_global_init(struct radeon_device *rdev) 77 { 78 struct drm_global_reference *global_ref; 79 int r; 80 81 rdev->mman.mem_global_referenced = false; 82 global_ref = &rdev->mman.mem_global_ref; 83 global_ref->global_type = DRM_GLOBAL_TTM_MEM; 84 global_ref->size = sizeof(struct ttm_mem_global); 85 global_ref->init = &radeon_ttm_mem_global_init; 86 global_ref->release = &radeon_ttm_mem_global_release; 87 r = drm_global_item_ref(global_ref); 88 if (r != 0) { 89 DRM_ERROR("Failed setting up TTM memory accounting " 90 "subsystem.\n"); 91 return r; 92 } 93 94 rdev->mman.bo_global_ref.mem_glob = 95 rdev->mman.mem_global_ref.object; 96 global_ref = &rdev->mman.bo_global_ref.ref; 97 global_ref->global_type = DRM_GLOBAL_TTM_BO; 98 global_ref->size = sizeof(struct ttm_bo_global); 99 global_ref->init = &ttm_bo_global_init; 100 global_ref->release = &ttm_bo_global_release; 101 r = drm_global_item_ref(global_ref); 102 if (r != 0) { 103 DRM_ERROR("Failed setting up TTM BO subsystem.\n"); 104 drm_global_item_unref(&rdev->mman.mem_global_ref); 105 return r; 106 } 107 108 rdev->mman.mem_global_referenced = true; 109 return 0; 110 } 111 112 static void radeon_ttm_global_fini(struct radeon_device *rdev) 113 { 114 if (rdev->mman.mem_global_referenced) { 115 drm_global_item_unref(&rdev->mman.bo_global_ref.ref); 116 drm_global_item_unref(&rdev->mman.mem_global_ref); 117 rdev->mman.mem_global_referenced = false; 118 } 119 } 120 121 static int radeon_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags) 122 { 123 return 0; 124 } 125 126 static int radeon_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, 127 struct ttm_mem_type_manager *man) 128 { 129 struct radeon_device *rdev; 130 131 rdev = radeon_get_rdev(bdev); 132 133 switch (type) { 134 case TTM_PL_SYSTEM: 135 /* System memory */ 136 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; 137 man->available_caching = TTM_PL_MASK_CACHING; 138 man->default_caching = TTM_PL_FLAG_CACHED; 139 break; 140 case TTM_PL_TT: 141 man->func = &ttm_bo_manager_func; 142 man->gpu_offset = rdev->mc.gtt_start; 143 man->available_caching = TTM_PL_MASK_CACHING; 144 man->default_caching = TTM_PL_FLAG_CACHED; 145 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA; 146 #if __OS_HAS_AGP 147 if (rdev->flags & RADEON_IS_AGP) { 148 if (!(drm_core_has_AGP(rdev->ddev) && rdev->ddev->agp)) { 149 DRM_ERROR("AGP is not enabled for memory type %u\n", 150 (unsigned)type); 151 return -EINVAL; 152 } 153 if (!rdev->ddev->agp->cant_use_aperture) 154 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; 155 man->available_caching = TTM_PL_FLAG_UNCACHED | 156 TTM_PL_FLAG_WC; 157 man->default_caching = TTM_PL_FLAG_WC; 158 } 159 #endif 160 break; 161 case TTM_PL_VRAM: 162 /* "On-card" video ram */ 163 man->func = &ttm_bo_manager_func; 164 man->gpu_offset = rdev->mc.vram_start; 165 man->flags = TTM_MEMTYPE_FLAG_FIXED | 166 TTM_MEMTYPE_FLAG_MAPPABLE; 167 man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC; 168 man->default_caching = TTM_PL_FLAG_WC; 169 break; 170 default: 171 DRM_ERROR("Unsupported memory type %u\n", (unsigned)type); 172 return -EINVAL; 173 } 174 return 0; 175 } 176 177 static void radeon_evict_flags(struct ttm_buffer_object *bo, 178 struct ttm_placement *placement) 179 { 180 struct radeon_bo *rbo; 181 static u32 placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM; 182 183 if (!radeon_ttm_bo_is_radeon_bo(bo)) { 184 placement->fpfn = 0; 185 placement->lpfn = 0; 186 placement->placement = &placements; 187 placement->busy_placement = &placements; 188 placement->num_placement = 1; 189 placement->num_busy_placement = 1; 190 return; 191 } 192 rbo = container_of(bo, struct radeon_bo, tbo); 193 switch (bo->mem.mem_type) { 194 case TTM_PL_VRAM: 195 if (rbo->rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready == false) 196 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU); 197 else 198 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_GTT); 199 break; 200 case TTM_PL_TT: 201 default: 202 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU); 203 } 204 *placement = rbo->placement; 205 } 206 207 static int radeon_verify_access(struct ttm_buffer_object *bo) 208 { 209 return 0; 210 } 211 212 static void radeon_move_null(struct ttm_buffer_object *bo, 213 struct ttm_mem_reg *new_mem) 214 { 215 struct ttm_mem_reg *old_mem = &bo->mem; 216 217 BUG_ON(old_mem->mm_node != NULL); 218 *old_mem = *new_mem; 219 new_mem->mm_node = NULL; 220 } 221 222 static int radeon_move_blit(struct ttm_buffer_object *bo, 223 bool evict, bool no_wait_gpu, 224 struct ttm_mem_reg *new_mem, 225 struct ttm_mem_reg *old_mem) 226 { 227 struct radeon_device *rdev; 228 uint64_t old_start, new_start; 229 struct radeon_fence *fence; 230 int r, ridx; 231 232 rdev = radeon_get_rdev(bo->bdev); 233 ridx = radeon_copy_ring_index(rdev); 234 old_start = old_mem->start << PAGE_SHIFT; 235 new_start = new_mem->start << PAGE_SHIFT; 236 237 switch (old_mem->mem_type) { 238 case TTM_PL_VRAM: 239 old_start += rdev->mc.vram_start; 240 break; 241 case TTM_PL_TT: 242 old_start += rdev->mc.gtt_start; 243 break; 244 default: 245 DRM_ERROR("Unknown placement %d\n", old_mem->mem_type); 246 return -EINVAL; 247 } 248 switch (new_mem->mem_type) { 249 case TTM_PL_VRAM: 250 new_start += rdev->mc.vram_start; 251 break; 252 case TTM_PL_TT: 253 new_start += rdev->mc.gtt_start; 254 break; 255 default: 256 DRM_ERROR("Unknown placement %d\n", old_mem->mem_type); 257 return -EINVAL; 258 } 259 if (!rdev->ring[ridx].ready) { 260 DRM_ERROR("Trying to move memory with ring turned off.\n"); 261 return -EINVAL; 262 } 263 264 BUILD_BUG_ON((PAGE_SIZE % RADEON_GPU_PAGE_SIZE) != 0); 265 266 /* sync other rings */ 267 fence = bo->sync_obj; 268 r = radeon_copy(rdev, old_start, new_start, 269 new_mem->num_pages * (PAGE_SIZE / RADEON_GPU_PAGE_SIZE), /* GPU pages */ 270 &fence); 271 /* FIXME: handle copy error */ 272 r = ttm_bo_move_accel_cleanup(bo, (void *)fence, 273 evict, no_wait_gpu, new_mem); 274 radeon_fence_unref(&fence); 275 return r; 276 } 277 278 static int radeon_move_vram_ram(struct ttm_buffer_object *bo, 279 bool evict, bool interruptible, 280 bool no_wait_gpu, 281 struct ttm_mem_reg *new_mem) 282 { 283 struct radeon_device *rdev; 284 struct ttm_mem_reg *old_mem = &bo->mem; 285 struct ttm_mem_reg tmp_mem; 286 u32 placements; 287 struct ttm_placement placement; 288 int r; 289 290 rdev = radeon_get_rdev(bo->bdev); 291 tmp_mem = *new_mem; 292 tmp_mem.mm_node = NULL; 293 placement.fpfn = 0; 294 placement.lpfn = 0; 295 placement.num_placement = 1; 296 placement.placement = &placements; 297 placement.num_busy_placement = 1; 298 placement.busy_placement = &placements; 299 placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; 300 r = ttm_bo_mem_space(bo, &placement, &tmp_mem, 301 interruptible, no_wait_gpu); 302 if (unlikely(r)) { 303 return r; 304 } 305 306 r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement); 307 if (unlikely(r)) { 308 goto out_cleanup; 309 } 310 311 r = ttm_tt_bind(bo->ttm, &tmp_mem); 312 if (unlikely(r)) { 313 goto out_cleanup; 314 } 315 r = radeon_move_blit(bo, true, no_wait_gpu, &tmp_mem, old_mem); 316 if (unlikely(r)) { 317 goto out_cleanup; 318 } 319 r = ttm_bo_move_ttm(bo, true, no_wait_gpu, new_mem); 320 out_cleanup: 321 ttm_bo_mem_put(bo, &tmp_mem); 322 return r; 323 } 324 325 static int radeon_move_ram_vram(struct ttm_buffer_object *bo, 326 bool evict, bool interruptible, 327 bool no_wait_gpu, 328 struct ttm_mem_reg *new_mem) 329 { 330 struct radeon_device *rdev; 331 struct ttm_mem_reg *old_mem = &bo->mem; 332 struct ttm_mem_reg tmp_mem; 333 struct ttm_placement placement; 334 u32 placements; 335 int r; 336 337 rdev = radeon_get_rdev(bo->bdev); 338 tmp_mem = *new_mem; 339 tmp_mem.mm_node = NULL; 340 placement.fpfn = 0; 341 placement.lpfn = 0; 342 placement.num_placement = 1; 343 placement.placement = &placements; 344 placement.num_busy_placement = 1; 345 placement.busy_placement = &placements; 346 placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; 347 r = ttm_bo_mem_space(bo, &placement, &tmp_mem, 348 interruptible, no_wait_gpu); 349 if (unlikely(r)) { 350 return r; 351 } 352 r = ttm_bo_move_ttm(bo, true, no_wait_gpu, &tmp_mem); 353 if (unlikely(r)) { 354 goto out_cleanup; 355 } 356 r = radeon_move_blit(bo, true, no_wait_gpu, new_mem, old_mem); 357 if (unlikely(r)) { 358 goto out_cleanup; 359 } 360 out_cleanup: 361 ttm_bo_mem_put(bo, &tmp_mem); 362 return r; 363 } 364 365 static int radeon_bo_move(struct ttm_buffer_object *bo, 366 bool evict, bool interruptible, 367 bool no_wait_gpu, 368 struct ttm_mem_reg *new_mem) 369 { 370 struct radeon_device *rdev; 371 struct ttm_mem_reg *old_mem = &bo->mem; 372 int r; 373 374 rdev = radeon_get_rdev(bo->bdev); 375 if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { 376 radeon_move_null(bo, new_mem); 377 return 0; 378 } 379 if ((old_mem->mem_type == TTM_PL_TT && 380 new_mem->mem_type == TTM_PL_SYSTEM) || 381 (old_mem->mem_type == TTM_PL_SYSTEM && 382 new_mem->mem_type == TTM_PL_TT)) { 383 /* bind is enough */ 384 radeon_move_null(bo, new_mem); 385 return 0; 386 } 387 if (!rdev->ring[radeon_copy_ring_index(rdev)].ready || 388 rdev->asic->copy.copy == NULL) { 389 /* use memcpy */ 390 goto memcpy; 391 } 392 393 if (old_mem->mem_type == TTM_PL_VRAM && 394 new_mem->mem_type == TTM_PL_SYSTEM) { 395 r = radeon_move_vram_ram(bo, evict, interruptible, 396 no_wait_gpu, new_mem); 397 } else if (old_mem->mem_type == TTM_PL_SYSTEM && 398 new_mem->mem_type == TTM_PL_VRAM) { 399 r = radeon_move_ram_vram(bo, evict, interruptible, 400 no_wait_gpu, new_mem); 401 } else { 402 r = radeon_move_blit(bo, evict, no_wait_gpu, new_mem, old_mem); 403 } 404 405 if (r) { 406 memcpy: 407 r = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem); 408 if (r) { 409 return r; 410 } 411 } 412 413 /* update statistics */ 414 atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &rdev->num_bytes_moved); 415 return 0; 416 } 417 418 static int radeon_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 419 { 420 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 421 struct radeon_device *rdev = radeon_get_rdev(bdev); 422 423 mem->bus.addr = NULL; 424 mem->bus.offset = 0; 425 mem->bus.size = mem->num_pages << PAGE_SHIFT; 426 mem->bus.base = 0; 427 mem->bus.is_iomem = false; 428 if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE)) 429 return -EINVAL; 430 switch (mem->mem_type) { 431 case TTM_PL_SYSTEM: 432 /* system memory */ 433 return 0; 434 case TTM_PL_TT: 435 #if __OS_HAS_AGP 436 if (rdev->flags & RADEON_IS_AGP) { 437 /* RADEON_IS_AGP is set only if AGP is active */ 438 mem->bus.offset = mem->start << PAGE_SHIFT; 439 mem->bus.base = rdev->mc.agp_base; 440 mem->bus.is_iomem = !rdev->ddev->agp->cant_use_aperture; 441 } 442 #endif 443 break; 444 case TTM_PL_VRAM: 445 mem->bus.offset = mem->start << PAGE_SHIFT; 446 /* check if it's visible */ 447 if ((mem->bus.offset + mem->bus.size) > rdev->mc.visible_vram_size) 448 return -EINVAL; 449 mem->bus.base = rdev->mc.aper_base; 450 mem->bus.is_iomem = true; 451 #ifdef __alpha__ 452 /* 453 * Alpha: use bus.addr to hold the ioremap() return, 454 * so we can modify bus.base below. 455 */ 456 if (mem->placement & TTM_PL_FLAG_WC) 457 mem->bus.addr = 458 ioremap_wc(mem->bus.base + mem->bus.offset, 459 mem->bus.size); 460 else 461 mem->bus.addr = 462 ioremap_nocache(mem->bus.base + mem->bus.offset, 463 mem->bus.size); 464 465 /* 466 * Alpha: Use just the bus offset plus 467 * the hose/domain memory base for bus.base. 468 * It then can be used to build PTEs for VRAM 469 * access, as done in ttm_bo_vm_fault(). 470 */ 471 mem->bus.base = (mem->bus.base & 0x0ffffffffUL) + 472 rdev->ddev->hose->dense_mem_base; 473 #endif 474 break; 475 default: 476 return -EINVAL; 477 } 478 return 0; 479 } 480 481 static void radeon_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 482 { 483 } 484 485 static int radeon_sync_obj_wait(void *sync_obj, bool lazy, bool interruptible) 486 { 487 return radeon_fence_wait((struct radeon_fence *)sync_obj, interruptible); 488 } 489 490 static int radeon_sync_obj_flush(void *sync_obj) 491 { 492 return 0; 493 } 494 495 static void radeon_sync_obj_unref(void **sync_obj) 496 { 497 radeon_fence_unref((struct radeon_fence **)sync_obj); 498 } 499 500 static void *radeon_sync_obj_ref(void *sync_obj) 501 { 502 return radeon_fence_ref((struct radeon_fence *)sync_obj); 503 } 504 505 static bool radeon_sync_obj_signaled(void *sync_obj) 506 { 507 return radeon_fence_signaled((struct radeon_fence *)sync_obj); 508 } 509 510 /* 511 * TTM backend functions. 512 */ 513 struct radeon_ttm_tt { 514 struct ttm_dma_tt ttm; 515 struct radeon_device *rdev; 516 u64 offset; 517 }; 518 519 static int radeon_ttm_backend_bind(struct ttm_tt *ttm, 520 struct ttm_mem_reg *bo_mem) 521 { 522 struct radeon_ttm_tt *gtt = (void*)ttm; 523 uint32_t flags = RADEON_GART_PAGE_VALID | RADEON_GART_PAGE_READ | 524 RADEON_GART_PAGE_WRITE; 525 int r; 526 527 gtt->offset = (unsigned long)(bo_mem->start << PAGE_SHIFT); 528 if (!ttm->num_pages) { 529 WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n", 530 ttm->num_pages, bo_mem, ttm); 531 } 532 if (ttm->caching_state == tt_cached) 533 flags |= RADEON_GART_PAGE_SNOOP; 534 r = radeon_gart_bind(gtt->rdev, gtt->offset, ttm->num_pages, 535 ttm->pages, gtt->ttm.dma_address, flags); 536 if (r) { 537 DRM_ERROR("failed to bind %lu pages at 0x%08X\n", 538 ttm->num_pages, (unsigned)gtt->offset); 539 return r; 540 } 541 return 0; 542 } 543 544 static int radeon_ttm_backend_unbind(struct ttm_tt *ttm) 545 { 546 struct radeon_ttm_tt *gtt = (void *)ttm; 547 548 radeon_gart_unbind(gtt->rdev, gtt->offset, ttm->num_pages); 549 return 0; 550 } 551 552 static void radeon_ttm_backend_destroy(struct ttm_tt *ttm) 553 { 554 struct radeon_ttm_tt *gtt = (void *)ttm; 555 556 ttm_dma_tt_fini(>t->ttm); 557 kfree(gtt); 558 } 559 560 static struct ttm_backend_func radeon_backend_func = { 561 .bind = &radeon_ttm_backend_bind, 562 .unbind = &radeon_ttm_backend_unbind, 563 .destroy = &radeon_ttm_backend_destroy, 564 }; 565 566 static struct ttm_tt *radeon_ttm_tt_create(struct ttm_bo_device *bdev, 567 unsigned long size, uint32_t page_flags, 568 vm_page_t dummy_read_page) 569 { 570 struct radeon_device *rdev; 571 struct radeon_ttm_tt *gtt; 572 573 rdev = radeon_get_rdev(bdev); 574 #if __OS_HAS_AGP 575 #ifdef DUMBBELL_WIP 576 if (rdev->flags & RADEON_IS_AGP) { 577 return ttm_agp_tt_create(bdev, rdev->ddev->agp->agpdev, 578 size, page_flags, dummy_read_page); 579 } 580 #endif /* DUMBBELL_WIP */ 581 #endif 582 583 gtt = kzalloc(sizeof(struct radeon_ttm_tt), GFP_KERNEL); 584 if (gtt == NULL) { 585 return NULL; 586 } 587 gtt->ttm.ttm.func = &radeon_backend_func; 588 gtt->rdev = rdev; 589 if (ttm_dma_tt_init(>t->ttm, bdev, size, page_flags, dummy_read_page)) { 590 kfree(gtt); 591 return NULL; 592 } 593 return >t->ttm.ttm; 594 } 595 596 static int radeon_ttm_tt_populate(struct ttm_tt *ttm) 597 { 598 struct radeon_device *rdev; 599 struct radeon_ttm_tt *gtt = (void *)ttm; 600 unsigned i; 601 int r; 602 #ifdef DUMBBELL_WIP 603 bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); 604 #endif /* DUMBBELL_WIP */ 605 606 if (ttm->state != tt_unpopulated) 607 return 0; 608 609 #ifdef DUMBBELL_WIP 610 /* 611 * Maybe unneeded on FreeBSD. 612 * -- dumbbell@ 613 */ 614 if (slave && ttm->sg) { 615 drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages, 616 gtt->ttm.dma_address, ttm->num_pages); 617 ttm->state = tt_unbound; 618 return 0; 619 } 620 #endif /* DUMBBELL_WIP */ 621 622 rdev = radeon_get_rdev(ttm->bdev); 623 #if __OS_HAS_AGP 624 #ifdef DUMBBELL_WIP 625 if (rdev->flags & RADEON_IS_AGP) { 626 return ttm_agp_tt_populate(ttm); 627 } 628 #endif /* DUMBBELL_WIP */ 629 #endif 630 631 #ifdef CONFIG_SWIOTLB 632 if (swiotlb_nr_tbl()) { 633 return ttm_dma_populate(>t->ttm, rdev->dev); 634 } 635 #endif 636 637 r = ttm_pool_populate(ttm); 638 if (r) { 639 return r; 640 } 641 642 for (i = 0; i < ttm->num_pages; i++) { 643 gtt->ttm.dma_address[i] = VM_PAGE_TO_PHYS(ttm->pages[i]); 644 #ifdef DUMBBELL_WIP 645 gtt->ttm.dma_address[i] = pci_map_page(rdev->pdev, ttm->pages[i], 646 0, PAGE_SIZE, 647 PCI_DMA_BIDIRECTIONAL); 648 if (pci_dma_mapping_error(rdev->pdev, gtt->ttm.dma_address[i])) { 649 while (--i) { 650 pci_unmap_page(rdev->pdev, gtt->ttm.dma_address[i], 651 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 652 gtt->ttm.dma_address[i] = 0; 653 } 654 ttm_pool_unpopulate(ttm); 655 return -EFAULT; 656 } 657 #endif /* DUMBBELL_WIP */ 658 } 659 return 0; 660 } 661 662 static void radeon_ttm_tt_unpopulate(struct ttm_tt *ttm) 663 { 664 struct radeon_device *rdev; 665 struct radeon_ttm_tt *gtt = (void *)ttm; 666 unsigned i; 667 bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); 668 669 if (slave) 670 return; 671 672 rdev = radeon_get_rdev(ttm->bdev); 673 #if __OS_HAS_AGP 674 #ifdef DUMBBELL_WIP 675 if (rdev->flags & RADEON_IS_AGP) { 676 ttm_agp_tt_unpopulate(ttm); 677 return; 678 } 679 #endif /* DUMBBELL_WIP */ 680 #endif 681 682 #ifdef CONFIG_SWIOTLB 683 if (swiotlb_nr_tbl()) { 684 ttm_dma_unpopulate(>t->ttm, rdev->dev); 685 return; 686 } 687 #endif 688 689 for (i = 0; i < ttm->num_pages; i++) { 690 if (gtt->ttm.dma_address[i]) { 691 gtt->ttm.dma_address[i] = 0; 692 #ifdef DUMBBELL_WIP 693 pci_unmap_page(rdev->pdev, gtt->ttm.dma_address[i], 694 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 695 #endif /* DUMBBELL_WIP */ 696 } 697 } 698 699 ttm_pool_unpopulate(ttm); 700 } 701 702 static struct ttm_bo_driver radeon_bo_driver = { 703 .ttm_tt_create = &radeon_ttm_tt_create, 704 .ttm_tt_populate = &radeon_ttm_tt_populate, 705 .ttm_tt_unpopulate = &radeon_ttm_tt_unpopulate, 706 .invalidate_caches = &radeon_invalidate_caches, 707 .init_mem_type = &radeon_init_mem_type, 708 .evict_flags = &radeon_evict_flags, 709 .move = &radeon_bo_move, 710 .verify_access = &radeon_verify_access, 711 .sync_obj_signaled = &radeon_sync_obj_signaled, 712 .sync_obj_wait = &radeon_sync_obj_wait, 713 .sync_obj_flush = &radeon_sync_obj_flush, 714 .sync_obj_unref = &radeon_sync_obj_unref, 715 .sync_obj_ref = &radeon_sync_obj_ref, 716 .move_notify = &radeon_bo_move_notify, 717 .fault_reserve_notify = &radeon_bo_fault_reserve_notify, 718 .io_mem_reserve = &radeon_ttm_io_mem_reserve, 719 .io_mem_free = &radeon_ttm_io_mem_free, 720 }; 721 722 int radeon_ttm_init(struct radeon_device *rdev) 723 { 724 int r, r2; 725 726 r = radeon_ttm_global_init(rdev); 727 if (r) { 728 return r; 729 } 730 /* No others user of address space so set it to 0 */ 731 r = ttm_bo_device_init(&rdev->mman.bdev, 732 rdev->mman.bo_global_ref.ref.object, 733 &radeon_bo_driver, DRM_FILE_PAGE_OFFSET, 734 rdev->need_dma32); 735 if (r) { 736 DRM_ERROR("failed initializing buffer object driver(%d).\n", r); 737 return r; 738 } 739 rdev->mman.initialized = true; 740 rdev->ddev->drm_ttm_bdev = &rdev->mman.bdev; 741 r = ttm_bo_init_mm(&rdev->mman.bdev, TTM_PL_VRAM, 742 rdev->mc.real_vram_size >> PAGE_SHIFT); 743 if (r) { 744 DRM_ERROR("Failed initializing VRAM heap.\n"); 745 return r; 746 } 747 /* Change the size here instead of the init above so only lpfn is affected */ 748 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); 749 750 r = radeon_bo_create(rdev, 256 * 1024, PAGE_SIZE, true, 751 RADEON_GEM_DOMAIN_VRAM, 0, 752 NULL, &rdev->stollen_vga_memory); 753 if (r) { 754 return r; 755 } 756 r = radeon_bo_reserve(rdev->stollen_vga_memory, false); 757 if (r) { 758 radeon_bo_unref(&rdev->stollen_vga_memory); 759 return r; 760 } 761 r = radeon_bo_pin(rdev->stollen_vga_memory, RADEON_GEM_DOMAIN_VRAM, NULL); 762 radeon_bo_unreserve(rdev->stollen_vga_memory); 763 if (r) { 764 radeon_bo_unref(&rdev->stollen_vga_memory); 765 return r; 766 } 767 DRM_INFO("radeon: %uM of VRAM memory ready\n", 768 (unsigned) (rdev->mc.real_vram_size / (1024 * 1024))); 769 r = ttm_bo_init_mm(&rdev->mman.bdev, TTM_PL_TT, 770 rdev->mc.gtt_size >> PAGE_SHIFT); 771 if (r) { 772 DRM_ERROR("Failed initializing GTT heap.\n"); 773 r2 = radeon_bo_reserve(rdev->stollen_vga_memory, false); 774 if (likely(r2 == 0)) { 775 radeon_bo_unpin(rdev->stollen_vga_memory); 776 radeon_bo_unreserve(rdev->stollen_vga_memory); 777 } 778 radeon_bo_unref(&rdev->stollen_vga_memory); 779 return r; 780 } 781 DRM_INFO("radeon: %uM of GTT memory ready.\n", 782 (unsigned)(rdev->mc.gtt_size / (1024 * 1024))); 783 784 r = radeon_ttm_debugfs_init(rdev); 785 if (r) { 786 DRM_ERROR("Failed to init debugfs\n"); 787 r2 = radeon_bo_reserve(rdev->stollen_vga_memory, false); 788 if (likely(r2 == 0)) { 789 radeon_bo_unpin(rdev->stollen_vga_memory); 790 radeon_bo_unreserve(rdev->stollen_vga_memory); 791 } 792 radeon_bo_unref(&rdev->stollen_vga_memory); 793 return r; 794 } 795 return 0; 796 } 797 798 void radeon_ttm_fini(struct radeon_device *rdev) 799 { 800 int r; 801 802 if (!rdev->mman.initialized) 803 return; 804 radeon_ttm_debugfs_fini(rdev); 805 if (rdev->stollen_vga_memory) { 806 r = radeon_bo_reserve(rdev->stollen_vga_memory, false); 807 if (r == 0) { 808 radeon_bo_unpin(rdev->stollen_vga_memory); 809 radeon_bo_unreserve(rdev->stollen_vga_memory); 810 } 811 radeon_bo_unref(&rdev->stollen_vga_memory); 812 } 813 ttm_bo_clean_mm(&rdev->mman.bdev, TTM_PL_VRAM); 814 ttm_bo_clean_mm(&rdev->mman.bdev, TTM_PL_TT); 815 ttm_bo_device_release(&rdev->mman.bdev); 816 radeon_gart_fini(rdev); 817 radeon_ttm_global_fini(rdev); 818 rdev->mman.initialized = false; 819 DRM_INFO("radeon: ttm finalized\n"); 820 } 821 822 /* this should only be called at bootup or when userspace 823 * isn't running */ 824 void radeon_ttm_set_active_vram_size(struct radeon_device *rdev, u64 size) 825 { 826 struct ttm_mem_type_manager *man; 827 828 if (!rdev->mman.initialized) 829 return; 830 831 man = &rdev->mman.bdev.man[TTM_PL_VRAM]; 832 /* this just adjusts TTM size idea, which sets lpfn to the correct value */ 833 man->size = size >> PAGE_SHIFT; 834 } 835 836 #ifdef DUMBBELL_WIP 837 static struct vm_operations_struct radeon_ttm_vm_ops; 838 static const struct vm_operations_struct *ttm_vm_ops = NULL; 839 840 static int radeon_ttm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 841 { 842 struct ttm_buffer_object *bo; 843 struct radeon_device *rdev; 844 int r; 845 846 bo = (struct ttm_buffer_object *)vma->vm_private_data; 847 if (bo == NULL) { 848 return VM_FAULT_NOPAGE; 849 } 850 rdev = radeon_get_rdev(bo->bdev); 851 lockmgr(&rdev->pm.mclk_lock, LK_SHARED); 852 r = ttm_vm_ops->fault(vma, vmf); 853 lockmgr(&rdev->pm.mclk_lock, LK_RELEASE); 854 return r; 855 } 856 857 int radeon_mmap(struct file *filp, struct vm_area_struct *vma) 858 { 859 struct drm_file *file_priv; 860 struct radeon_device *rdev; 861 int r; 862 863 if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET)) { 864 return drm_mmap(filp, vma); 865 } 866 867 file_priv = filp->private_data; 868 rdev = file_priv->minor->dev->dev_private; 869 if (rdev == NULL) { 870 return -EINVAL; 871 } 872 r = ttm_bo_mmap(filp, vma, &rdev->mman.bdev); 873 if (unlikely(r != 0)) { 874 return r; 875 } 876 if (unlikely(ttm_vm_ops == NULL)) { 877 ttm_vm_ops = vma->vm_ops; 878 radeon_ttm_vm_ops = *ttm_vm_ops; 879 radeon_ttm_vm_ops.fault = &radeon_ttm_fault; 880 } 881 vma->vm_ops = &radeon_ttm_vm_ops; 882 return 0; 883 } 884 #endif /* DUMBBELL_WIP */ 885 886 #if defined(CONFIG_DEBUG_FS) 887 888 static int radeon_mm_dump_table(struct seq_file *m, void *data) 889 { 890 struct drm_info_node *node = (struct drm_info_node *)m->private; 891 unsigned ttm_pl = *(int *)node->info_ent->data; 892 struct drm_device *dev = node->minor->dev; 893 struct radeon_device *rdev = dev->dev_private; 894 struct drm_mm *mm = (struct drm_mm *)rdev->mman.bdev.man[ttm_pl].priv; 895 int ret; 896 struct ttm_bo_global *glob = rdev->mman.bdev.glob; 897 898 spin_lock(&glob->lru_lock); 899 ret = drm_mm_dump_table(m, mm); 900 spin_unlock(&glob->lru_lock); 901 return ret; 902 } 903 904 static int ttm_pl_vram = TTM_PL_VRAM; 905 static int ttm_pl_tt = TTM_PL_TT; 906 907 static struct drm_info_list radeon_ttm_debugfs_list[] = { 908 {"radeon_vram_mm", radeon_mm_dump_table, 0, &ttm_pl_vram}, 909 {"radeon_gtt_mm", radeon_mm_dump_table, 0, &ttm_pl_tt}, 910 {"ttm_page_pool", ttm_page_alloc_debugfs, 0, NULL}, 911 #ifdef CONFIG_SWIOTLB 912 {"ttm_dma_page_pool", ttm_dma_page_alloc_debugfs, 0, NULL} 913 #endif 914 }; 915 916 static int radeon_ttm_vram_open(struct inode *inode, struct file *filep) 917 { 918 struct radeon_device *rdev = inode->i_private; 919 i_size_write(inode, rdev->mc.mc_vram_size); 920 filep->private_data = inode->i_private; 921 return 0; 922 } 923 924 static ssize_t radeon_ttm_vram_read(struct file *f, char __user *buf, 925 size_t size, loff_t *pos) 926 { 927 struct radeon_device *rdev = f->private_data; 928 ssize_t result = 0; 929 int r; 930 931 if (size & 0x3 || *pos & 0x3) 932 return -EINVAL; 933 934 while (size) { 935 unsigned long flags; 936 uint32_t value; 937 938 if (*pos >= rdev->mc.mc_vram_size) 939 return result; 940 941 spin_lock_irqsave(&rdev->mmio_idx_lock, flags); 942 WREG32(RADEON_MM_INDEX, ((uint32_t)*pos) | 0x80000000); 943 if (rdev->family >= CHIP_CEDAR) 944 WREG32(EVERGREEN_MM_INDEX_HI, *pos >> 31); 945 value = RREG32(RADEON_MM_DATA); 946 spin_unlock_irqrestore(&rdev->mmio_idx_lock, flags); 947 948 r = put_user(value, (uint32_t *)buf); 949 if (r) 950 return r; 951 952 result += 4; 953 buf += 4; 954 *pos += 4; 955 size -= 4; 956 } 957 958 return result; 959 } 960 961 static const struct file_operations radeon_ttm_vram_fops = { 962 .owner = THIS_MODULE, 963 .open = radeon_ttm_vram_open, 964 .read = radeon_ttm_vram_read, 965 .llseek = default_llseek 966 }; 967 968 static int radeon_ttm_gtt_open(struct inode *inode, struct file *filep) 969 { 970 struct radeon_device *rdev = inode->i_private; 971 i_size_write(inode, rdev->mc.gtt_size); 972 filep->private_data = inode->i_private; 973 return 0; 974 } 975 976 static ssize_t radeon_ttm_gtt_read(struct file *f, char __user *buf, 977 size_t size, loff_t *pos) 978 { 979 struct radeon_device *rdev = f->private_data; 980 ssize_t result = 0; 981 int r; 982 983 while (size) { 984 loff_t p = *pos / PAGE_SIZE; 985 unsigned off = *pos & ~PAGE_MASK; 986 size_t cur_size = min_t(size_t, size, PAGE_SIZE - off); 987 struct page *page; 988 void *ptr; 989 990 if (p >= rdev->gart.num_cpu_pages) 991 return result; 992 993 page = rdev->gart.pages[p]; 994 if (page) { 995 ptr = kmap(page); 996 ptr += off; 997 998 r = copy_to_user(buf, ptr, cur_size); 999 kunmap(rdev->gart.pages[p]); 1000 } else 1001 r = clear_user(buf, cur_size); 1002 1003 if (r) 1004 return -EFAULT; 1005 1006 result += cur_size; 1007 buf += cur_size; 1008 *pos += cur_size; 1009 size -= cur_size; 1010 } 1011 1012 return result; 1013 } 1014 1015 static const struct file_operations radeon_ttm_gtt_fops = { 1016 .owner = THIS_MODULE, 1017 .open = radeon_ttm_gtt_open, 1018 .read = radeon_ttm_gtt_read, 1019 .llseek = default_llseek 1020 }; 1021 1022 #endif 1023 1024 static int radeon_ttm_debugfs_init(struct radeon_device *rdev) 1025 { 1026 #if defined(CONFIG_DEBUG_FS) 1027 unsigned count; 1028 1029 struct drm_minor *minor = rdev->ddev->primary; 1030 struct dentry *ent, *root = minor->debugfs_root; 1031 1032 ent = debugfs_create_file("radeon_vram", S_IFREG | S_IRUGO, root, 1033 rdev, &radeon_ttm_vram_fops); 1034 if (IS_ERR(ent)) 1035 return PTR_ERR(ent); 1036 rdev->mman.vram = ent; 1037 1038 ent = debugfs_create_file("radeon_gtt", S_IFREG | S_IRUGO, root, 1039 rdev, &radeon_ttm_gtt_fops); 1040 if (IS_ERR(ent)) 1041 return PTR_ERR(ent); 1042 rdev->mman.gtt = ent; 1043 1044 count = ARRAY_SIZE(radeon_ttm_debugfs_list); 1045 1046 #ifdef CONFIG_SWIOTLB 1047 if (!swiotlb_nr_tbl()) 1048 --count; 1049 #endif 1050 1051 return radeon_debugfs_add_files(rdev, radeon_ttm_debugfs_list, count); 1052 #else 1053 1054 return 0; 1055 #endif 1056 } 1057 1058 static void radeon_ttm_debugfs_fini(struct radeon_device *rdev) 1059 { 1060 #if defined(CONFIG_DEBUG_FS) 1061 1062 debugfs_remove(rdev->mman.vram); 1063 rdev->mman.vram = NULL; 1064 1065 debugfs_remove(rdev->mman.gtt); 1066 rdev->mman.gtt = NULL; 1067 #endif 1068 } 1069