1 /* 2 * Copyright 2008 Advanced Micro Devices, Inc. 3 * Copyright 2008 Red Hat Inc. 4 * Copyright 2009 Jerome Glisse. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: Dave Airlie 25 * Alex Deucher 26 * Jerome Glisse 27 * 28 * $FreeBSD: head/sys/dev/drm2/radeon/radeon_gart.c 254885 2013-08-25 19:37:15Z dumbbell $ 29 */ 30 31 #include <drm/drmP.h> 32 #include <uapi_drm/radeon_drm.h> 33 #include "radeon.h" 34 #include "radeon_reg.h" 35 36 /* 37 * GART 38 * The GART (Graphics Aperture Remapping Table) is an aperture 39 * in the GPU's address space. System pages can be mapped into 40 * the aperture and look like contiguous pages from the GPU's 41 * perspective. A page table maps the pages in the aperture 42 * to the actual backing pages in system memory. 43 * 44 * Radeon GPUs support both an internal GART, as described above, 45 * and AGP. AGP works similarly, but the GART table is configured 46 * and maintained by the northbridge rather than the driver. 47 * Radeon hw has a separate AGP aperture that is programmed to 48 * point to the AGP aperture provided by the northbridge and the 49 * requests are passed through to the northbridge aperture. 50 * Both AGP and internal GART can be used at the same time, however 51 * that is not currently supported by the driver. 52 * 53 * This file handles the common internal GART management. 54 */ 55 56 /* 57 * Common GART table functions. 58 */ 59 /** 60 * radeon_gart_table_ram_alloc - allocate system ram for gart page table 61 * 62 * @rdev: radeon_device pointer 63 * 64 * Allocate system memory for GART page table 65 * (r1xx-r3xx, non-pcie r4xx, rs400). These asics require the 66 * gart table to be in system memory. 67 * Returns 0 for success, -ENOMEM for failure. 68 */ 69 int radeon_gart_table_ram_alloc(struct radeon_device *rdev) 70 { 71 drm_dma_handle_t *dmah; 72 73 dmah = drm_pci_alloc(rdev->ddev, rdev->gart.table_size, 74 PAGE_SIZE); 75 if (dmah == NULL) { 76 return -ENOMEM; 77 } 78 rdev->gart.dmah = dmah; 79 rdev->gart.ptr = dmah->vaddr; 80 #if defined(__i386) || defined(__amd64) 81 if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 || 82 rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) { 83 pmap_change_attr((vm_offset_t)rdev->gart.ptr, 84 rdev->gart.table_size >> PAGE_SHIFT, PAT_UNCACHED); 85 } 86 #endif 87 rdev->gart.table_addr = dmah->busaddr; 88 memset((void *)rdev->gart.ptr, 0, rdev->gart.table_size); 89 return 0; 90 } 91 92 /** 93 * radeon_gart_table_ram_free - free system ram for gart page table 94 * 95 * @rdev: radeon_device pointer 96 * 97 * Free system memory for GART page table 98 * (r1xx-r3xx, non-pcie r4xx, rs400). These asics require the 99 * gart table to be in system memory. 100 */ 101 void radeon_gart_table_ram_free(struct radeon_device *rdev) 102 { 103 if (rdev->gart.ptr == NULL) { 104 return; 105 } 106 #if defined(__i386) || defined(__amd64) 107 if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 || 108 rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) { 109 pmap_change_attr((vm_offset_t)rdev->gart.ptr, 110 rdev->gart.table_size >> PAGE_SHIFT, PAT_WRITE_COMBINING); 111 } 112 #endif 113 drm_pci_free(rdev->ddev, rdev->gart.dmah); 114 rdev->gart.dmah = NULL; 115 rdev->gart.ptr = NULL; 116 rdev->gart.table_addr = 0; 117 } 118 119 /** 120 * radeon_gart_table_vram_alloc - allocate vram for gart page table 121 * 122 * @rdev: radeon_device pointer 123 * 124 * Allocate video memory for GART page table 125 * (pcie r4xx, r5xx+). These asics require the 126 * gart table to be in video memory. 127 * Returns 0 for success, error for failure. 128 */ 129 int radeon_gart_table_vram_alloc(struct radeon_device *rdev) 130 { 131 int r; 132 133 if (rdev->gart.robj == NULL) { 134 r = radeon_bo_create(rdev, rdev->gart.table_size, 135 PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM, 136 NULL, &rdev->gart.robj); 137 if (r) { 138 return r; 139 } 140 } 141 return 0; 142 } 143 144 /** 145 * radeon_gart_table_vram_pin - pin gart page table in vram 146 * 147 * @rdev: radeon_device pointer 148 * 149 * Pin the GART page table in vram so it will not be moved 150 * by the memory manager (pcie r4xx, r5xx+). These asics require the 151 * gart table to be in video memory. 152 * Returns 0 for success, error for failure. 153 */ 154 int radeon_gart_table_vram_pin(struct radeon_device *rdev) 155 { 156 uint64_t gpu_addr; 157 int r; 158 159 r = radeon_bo_reserve(rdev->gart.robj, false); 160 if (unlikely(r != 0)) 161 return r; 162 r = radeon_bo_pin(rdev->gart.robj, 163 RADEON_GEM_DOMAIN_VRAM, &gpu_addr); 164 if (r) { 165 radeon_bo_unreserve(rdev->gart.robj); 166 return r; 167 } 168 r = radeon_bo_kmap(rdev->gart.robj, &rdev->gart.ptr); 169 if (r) 170 radeon_bo_unpin(rdev->gart.robj); 171 radeon_bo_unreserve(rdev->gart.robj); 172 rdev->gart.table_addr = gpu_addr; 173 return r; 174 } 175 176 /** 177 * radeon_gart_table_vram_unpin - unpin gart page table in vram 178 * 179 * @rdev: radeon_device pointer 180 * 181 * Unpin the GART page table in vram (pcie r4xx, r5xx+). 182 * These asics require the gart table to be in video memory. 183 */ 184 void radeon_gart_table_vram_unpin(struct radeon_device *rdev) 185 { 186 int r; 187 188 if (rdev->gart.robj == NULL) { 189 return; 190 } 191 r = radeon_bo_reserve(rdev->gart.robj, false); 192 if (likely(r == 0)) { 193 radeon_bo_kunmap(rdev->gart.robj); 194 radeon_bo_unpin(rdev->gart.robj); 195 radeon_bo_unreserve(rdev->gart.robj); 196 rdev->gart.ptr = NULL; 197 } 198 } 199 200 /** 201 * radeon_gart_table_vram_free - free gart page table vram 202 * 203 * @rdev: radeon_device pointer 204 * 205 * Free the video memory used for the GART page table 206 * (pcie r4xx, r5xx+). These asics require the gart table to 207 * be in video memory. 208 */ 209 void radeon_gart_table_vram_free(struct radeon_device *rdev) 210 { 211 if (rdev->gart.robj == NULL) { 212 return; 213 } 214 radeon_bo_unref(&rdev->gart.robj); 215 } 216 217 /* 218 * Common gart functions. 219 */ 220 /** 221 * radeon_gart_unbind - unbind pages from the gart page table 222 * 223 * @rdev: radeon_device pointer 224 * @offset: offset into the GPU's gart aperture 225 * @pages: number of pages to unbind 226 * 227 * Unbinds the requested pages from the gart page table and 228 * replaces them with the dummy page (all asics). 229 */ 230 void radeon_gart_unbind(struct radeon_device *rdev, unsigned offset, 231 int pages) 232 { 233 unsigned t; 234 unsigned p; 235 int i, j; 236 u64 page_base; 237 238 if (!rdev->gart.ready) { 239 WARN(1, "trying to unbind memory from uninitialized GART !\n"); 240 return; 241 } 242 t = offset / RADEON_GPU_PAGE_SIZE; 243 p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); 244 for (i = 0; i < pages; i++, p++) { 245 if (rdev->gart.pages[p]) { 246 rdev->gart.pages[p] = NULL; 247 rdev->gart.pages_addr[p] = rdev->dummy_page.addr; 248 page_base = rdev->gart.pages_addr[p]; 249 for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) { 250 if (rdev->gart.ptr) { 251 radeon_gart_set_page(rdev, t, page_base); 252 } 253 page_base += RADEON_GPU_PAGE_SIZE; 254 } 255 } 256 } 257 mb(); 258 radeon_gart_tlb_flush(rdev); 259 } 260 261 /** 262 * radeon_gart_bind - bind pages into the gart page table 263 * 264 * @rdev: radeon_device pointer 265 * @offset: offset into the GPU's gart aperture 266 * @pages: number of pages to bind 267 * @pagelist: pages to bind 268 * @dma_addr: DMA addresses of pages 269 * 270 * Binds the requested pages to the gart page table 271 * (all asics). 272 * Returns 0 for success, -EINVAL for failure. 273 */ 274 int radeon_gart_bind(struct radeon_device *rdev, unsigned offset, 275 int pages, vm_page_t *pagelist, dma_addr_t *dma_addr) 276 { 277 unsigned t; 278 unsigned p; 279 uint64_t page_base; 280 int i, j; 281 282 if (!rdev->gart.ready) { 283 WARN(1, "trying to bind memory to uninitialized GART !\n"); 284 return -EINVAL; 285 } 286 t = offset / RADEON_GPU_PAGE_SIZE; 287 p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); 288 289 for (i = 0; i < pages; i++, p++) { 290 rdev->gart.pages_addr[p] = dma_addr[i]; 291 rdev->gart.pages[p] = pagelist[i]; 292 if (rdev->gart.ptr) { 293 page_base = rdev->gart.pages_addr[p]; 294 for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) { 295 radeon_gart_set_page(rdev, t, page_base); 296 page_base += RADEON_GPU_PAGE_SIZE; 297 } 298 } 299 } 300 mb(); 301 radeon_gart_tlb_flush(rdev); 302 return 0; 303 } 304 305 /** 306 * radeon_gart_restore - bind all pages in the gart page table 307 * 308 * @rdev: radeon_device pointer 309 * 310 * Binds all pages in the gart page table (all asics). 311 * Used to rebuild the gart table on device startup or resume. 312 */ 313 void radeon_gart_restore(struct radeon_device *rdev) 314 { 315 int i, j, t; 316 u64 page_base; 317 318 if (!rdev->gart.ptr) { 319 return; 320 } 321 for (i = 0, t = 0; i < rdev->gart.num_cpu_pages; i++) { 322 page_base = rdev->gart.pages_addr[i]; 323 for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) { 324 radeon_gart_set_page(rdev, t, page_base); 325 page_base += RADEON_GPU_PAGE_SIZE; 326 } 327 } 328 mb(); 329 radeon_gart_tlb_flush(rdev); 330 } 331 332 /** 333 * radeon_gart_init - init the driver info for managing the gart 334 * 335 * @rdev: radeon_device pointer 336 * 337 * Allocate the dummy page and init the gart driver info (all asics). 338 * Returns 0 for success, error for failure. 339 */ 340 int radeon_gart_init(struct radeon_device *rdev) 341 { 342 int r, i; 343 344 if (rdev->gart.pages) { 345 return 0; 346 } 347 /* We need PAGE_SIZE >= RADEON_GPU_PAGE_SIZE */ 348 if (PAGE_SIZE < RADEON_GPU_PAGE_SIZE) { 349 DRM_ERROR("Page size is smaller than GPU page size!\n"); 350 return -EINVAL; 351 } 352 r = radeon_dummy_page_init(rdev); 353 if (r) 354 return r; 355 /* Compute table size */ 356 rdev->gart.num_cpu_pages = rdev->mc.gtt_size / PAGE_SIZE; 357 rdev->gart.num_gpu_pages = rdev->mc.gtt_size / RADEON_GPU_PAGE_SIZE; 358 DRM_INFO("GART: num cpu pages %u, num gpu pages %u\n", 359 rdev->gart.num_cpu_pages, rdev->gart.num_gpu_pages); 360 /* Allocate pages table */ 361 rdev->gart.pages = kmalloc(sizeof(void *) * rdev->gart.num_cpu_pages, 362 M_DRM, M_ZERO | M_WAITOK); 363 if (rdev->gart.pages == NULL) { 364 radeon_gart_fini(rdev); 365 return -ENOMEM; 366 } 367 rdev->gart.pages_addr = kmalloc(sizeof(dma_addr_t) * rdev->gart.num_cpu_pages, 368 M_DRM, M_ZERO | M_WAITOK); 369 if (rdev->gart.pages_addr == NULL) { 370 radeon_gart_fini(rdev); 371 return -ENOMEM; 372 } 373 /* set GART entry to point to the dummy page by default */ 374 for (i = 0; i < rdev->gart.num_cpu_pages; i++) { 375 rdev->gart.pages_addr[i] = rdev->dummy_page.addr; 376 } 377 return 0; 378 } 379 380 /** 381 * radeon_gart_fini - tear down the driver info for managing the gart 382 * 383 * @rdev: radeon_device pointer 384 * 385 * Tear down the gart driver info and free the dummy page (all asics). 386 */ 387 void radeon_gart_fini(struct radeon_device *rdev) 388 { 389 if (rdev->gart.pages && rdev->gart.pages_addr && rdev->gart.ready) { 390 /* unbind pages */ 391 radeon_gart_unbind(rdev, 0, rdev->gart.num_cpu_pages); 392 } 393 rdev->gart.ready = false; 394 drm_free(rdev->gart.pages, M_DRM); 395 drm_free(rdev->gart.pages_addr, M_DRM); 396 rdev->gart.pages = NULL; 397 rdev->gart.pages_addr = NULL; 398 399 radeon_dummy_page_fini(rdev); 400 } 401 402 /* 403 * GPUVM 404 * GPUVM is similar to the legacy gart on older asics, however 405 * rather than there being a single global gart table 406 * for the entire GPU, there are multiple VM page tables active 407 * at any given time. The VM page tables can contain a mix 408 * vram pages and system memory pages and system memory pages 409 * can be mapped as snooped (cached system pages) or unsnooped 410 * (uncached system pages). 411 * Each VM has an ID associated with it and there is a page table 412 * associated with each VMID. When execting a command buffer, 413 * the kernel tells the the ring what VMID to use for that command 414 * buffer. VMIDs are allocated dynamically as commands are submitted. 415 * The userspace drivers maintain their own address space and the kernel 416 * sets up their pages tables accordingly when they submit their 417 * command buffers and a VMID is assigned. 418 * Cayman/Trinity support up to 8 active VMs at any given time; 419 * SI supports 16. 420 */ 421 422 /* 423 * vm helpers 424 * 425 * TODO bind a default page at vm initialization for default address 426 */ 427 428 /** 429 * radeon_vm_num_pde - return the number of page directory entries 430 * 431 * @rdev: radeon_device pointer 432 * 433 * Calculate the number of page directory entries (cayman+). 434 */ 435 static unsigned radeon_vm_num_pdes(struct radeon_device *rdev) 436 { 437 return rdev->vm_manager.max_pfn >> RADEON_VM_BLOCK_SIZE; 438 } 439 440 /** 441 * radeon_vm_directory_size - returns the size of the page directory in bytes 442 * 443 * @rdev: radeon_device pointer 444 * 445 * Calculate the size of the page directory in bytes (cayman+). 446 */ 447 static unsigned radeon_vm_directory_size(struct radeon_device *rdev) 448 { 449 return RADEON_GPU_PAGE_ALIGN(radeon_vm_num_pdes(rdev) * 8); 450 } 451 452 /** 453 * radeon_vm_manager_init - init the vm manager 454 * 455 * @rdev: radeon_device pointer 456 * 457 * Init the vm manager (cayman+). 458 * Returns 0 for success, error for failure. 459 */ 460 int radeon_vm_manager_init(struct radeon_device *rdev) 461 { 462 struct radeon_vm *vm; 463 struct radeon_bo_va *bo_va; 464 int r; 465 unsigned size; 466 467 if (!rdev->vm_manager.enabled) { 468 /* allocate enough for 2 full VM pts */ 469 size = radeon_vm_directory_size(rdev); 470 size += rdev->vm_manager.max_pfn * 8; 471 size *= 2; 472 r = radeon_sa_bo_manager_init(rdev, &rdev->vm_manager.sa_manager, 473 RADEON_GPU_PAGE_ALIGN(size), 474 RADEON_VM_PTB_ALIGN_SIZE, 475 RADEON_GEM_DOMAIN_VRAM); 476 if (r) { 477 dev_err(rdev->dev, "failed to allocate vm bo (%dKB)\n", 478 (rdev->vm_manager.max_pfn * 8) >> 10); 479 return r; 480 } 481 482 r = radeon_asic_vm_init(rdev); 483 if (r) 484 return r; 485 486 rdev->vm_manager.enabled = true; 487 488 r = radeon_sa_bo_manager_start(rdev, &rdev->vm_manager.sa_manager); 489 if (r) 490 return r; 491 } 492 493 /* restore page table */ 494 list_for_each_entry(vm, &rdev->vm_manager.lru_vm, list) { 495 if (vm->page_directory == NULL) 496 continue; 497 498 list_for_each_entry(bo_va, &vm->va, vm_list) { 499 bo_va->valid = false; 500 } 501 } 502 return 0; 503 } 504 505 /** 506 * radeon_vm_free_pt - free the page table for a specific vm 507 * 508 * @rdev: radeon_device pointer 509 * @vm: vm to unbind 510 * 511 * Free the page table of a specific vm (cayman+). 512 * 513 * Global and local mutex must be lock! 514 */ 515 static void radeon_vm_free_pt(struct radeon_device *rdev, 516 struct radeon_vm *vm) 517 { 518 struct radeon_bo_va *bo_va; 519 int i; 520 521 if (!vm->page_directory) 522 return; 523 524 list_del_init(&vm->list); 525 radeon_sa_bo_free(rdev, &vm->page_directory, vm->fence); 526 527 list_for_each_entry(bo_va, &vm->va, vm_list) { 528 bo_va->valid = false; 529 } 530 531 if (vm->page_tables == NULL) 532 return; 533 534 for (i = 0; i < radeon_vm_num_pdes(rdev); i++) 535 radeon_sa_bo_free(rdev, &vm->page_tables[i], vm->fence); 536 537 kfree(vm->page_tables); 538 } 539 540 /** 541 * radeon_vm_manager_fini - tear down the vm manager 542 * 543 * @rdev: radeon_device pointer 544 * 545 * Tear down the VM manager (cayman+). 546 */ 547 void radeon_vm_manager_fini(struct radeon_device *rdev) 548 { 549 struct radeon_vm *vm, *tmp; 550 int i; 551 552 if (!rdev->vm_manager.enabled) 553 return; 554 555 lockmgr(&rdev->vm_manager.lock, LK_EXCLUSIVE); 556 /* free all allocated page tables */ 557 list_for_each_entry_safe(vm, tmp, &rdev->vm_manager.lru_vm, list) { 558 lockmgr(&vm->mutex, LK_EXCLUSIVE); 559 radeon_vm_free_pt(rdev, vm); 560 lockmgr(&vm->mutex, LK_RELEASE); 561 } 562 for (i = 0; i < RADEON_NUM_VM; ++i) { 563 radeon_fence_unref(&rdev->vm_manager.active[i]); 564 } 565 radeon_asic_vm_fini(rdev); 566 lockmgr(&rdev->vm_manager.lock, LK_RELEASE); 567 568 radeon_sa_bo_manager_suspend(rdev, &rdev->vm_manager.sa_manager); 569 radeon_sa_bo_manager_fini(rdev, &rdev->vm_manager.sa_manager); 570 rdev->vm_manager.enabled = false; 571 } 572 573 /** 574 * radeon_vm_evict - evict page table to make room for new one 575 * 576 * @rdev: radeon_device pointer 577 * @vm: VM we want to allocate something for 578 * 579 * Evict a VM from the lru, making sure that it isn't @vm. (cayman+). 580 * Returns 0 for success, -ENOMEM for failure. 581 * 582 * Global and local mutex must be locked! 583 */ 584 static int radeon_vm_evict(struct radeon_device *rdev, struct radeon_vm *vm) 585 { 586 struct radeon_vm *vm_evict; 587 588 if (list_empty(&rdev->vm_manager.lru_vm)) 589 return -ENOMEM; 590 591 vm_evict = list_first_entry(&rdev->vm_manager.lru_vm, 592 struct radeon_vm, list); 593 if (vm_evict == vm) 594 return -ENOMEM; 595 596 lockmgr(&vm_evict->mutex, LK_EXCLUSIVE); 597 radeon_vm_free_pt(rdev, vm_evict); 598 lockmgr(&vm_evict->mutex, LK_RELEASE); 599 return 0; 600 } 601 602 /** 603 * radeon_vm_alloc_pt - allocates a page table for a VM 604 * 605 * @rdev: radeon_device pointer 606 * @vm: vm to bind 607 * 608 * Allocate a page table for the requested vm (cayman+). 609 * Returns 0 for success, error for failure. 610 * 611 * Global and local mutex must be locked! 612 */ 613 int radeon_vm_alloc_pt(struct radeon_device *rdev, struct radeon_vm *vm) 614 { 615 unsigned pd_size, pts_size; 616 u64 *pd_addr; 617 int r; 618 619 if (vm == NULL) { 620 return -EINVAL; 621 } 622 623 if (vm->page_directory != NULL) { 624 return 0; 625 } 626 627 retry: 628 pd_size = radeon_vm_directory_size(rdev); 629 r = radeon_sa_bo_new(rdev, &rdev->vm_manager.sa_manager, 630 &vm->page_directory, pd_size, 631 RADEON_VM_PTB_ALIGN_SIZE, false); 632 if (r == -ENOMEM) { 633 r = radeon_vm_evict(rdev, vm); 634 if (r) 635 return r; 636 goto retry; 637 638 } else if (r) { 639 return r; 640 } 641 642 vm->pd_gpu_addr = radeon_sa_bo_gpu_addr(vm->page_directory); 643 644 /* Initially clear the page directory */ 645 pd_addr = radeon_sa_bo_cpu_addr(vm->page_directory); 646 memset(pd_addr, 0, pd_size); 647 648 pts_size = radeon_vm_num_pdes(rdev) * sizeof(struct radeon_sa_bo *); 649 vm->page_tables = kzalloc(pts_size, GFP_KERNEL); 650 651 if (vm->page_tables == NULL) { 652 DRM_ERROR("Cannot allocate memory for page table array\n"); 653 radeon_sa_bo_free(rdev, &vm->page_directory, vm->fence); 654 return -ENOMEM; 655 } 656 657 return 0; 658 } 659 660 /** 661 * radeon_vm_add_to_lru - add VMs page table to LRU list 662 * 663 * @rdev: radeon_device pointer 664 * @vm: vm to add to LRU 665 * 666 * Add the allocated page table to the LRU list (cayman+). 667 * 668 * Global mutex must be locked! 669 */ 670 void radeon_vm_add_to_lru(struct radeon_device *rdev, struct radeon_vm *vm) 671 { 672 list_del_init(&vm->list); 673 list_add_tail(&vm->list, &rdev->vm_manager.lru_vm); 674 } 675 676 /** 677 * radeon_vm_grab_id - allocate the next free VMID 678 * 679 * @rdev: radeon_device pointer 680 * @vm: vm to allocate id for 681 * @ring: ring we want to submit job to 682 * 683 * Allocate an id for the vm (cayman+). 684 * Returns the fence we need to sync to (if any). 685 * 686 * Global and local mutex must be locked! 687 */ 688 struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev, 689 struct radeon_vm *vm, int ring) 690 { 691 struct radeon_fence *best[RADEON_NUM_RINGS] = {}; 692 unsigned choices[2] = {}; 693 unsigned i; 694 695 /* check if the id is still valid */ 696 if (vm->fence && vm->fence == rdev->vm_manager.active[vm->id]) 697 return NULL; 698 699 /* we definately need to flush */ 700 radeon_fence_unref(&vm->last_flush); 701 702 /* skip over VMID 0, since it is the system VM */ 703 for (i = 1; i < rdev->vm_manager.nvm; ++i) { 704 struct radeon_fence *fence = rdev->vm_manager.active[i]; 705 706 if (fence == NULL) { 707 /* found a free one */ 708 vm->id = i; 709 return NULL; 710 } 711 712 if (radeon_fence_is_earlier(fence, best[fence->ring])) { 713 best[fence->ring] = fence; 714 choices[fence->ring == ring ? 0 : 1] = i; 715 } 716 } 717 718 for (i = 0; i < 2; ++i) { 719 if (choices[i]) { 720 vm->id = choices[i]; 721 return rdev->vm_manager.active[choices[i]]; 722 } 723 } 724 725 /* should never happen */ 726 BUG(); 727 return NULL; 728 } 729 730 /** 731 * radeon_vm_fence - remember fence for vm 732 * 733 * @rdev: radeon_device pointer 734 * @vm: vm we want to fence 735 * @fence: fence to remember 736 * 737 * Fence the vm (cayman+). 738 * Set the fence used to protect page table and id. 739 * 740 * Global and local mutex must be locked! 741 */ 742 void radeon_vm_fence(struct radeon_device *rdev, 743 struct radeon_vm *vm, 744 struct radeon_fence *fence) 745 { 746 radeon_fence_unref(&rdev->vm_manager.active[vm->id]); 747 rdev->vm_manager.active[vm->id] = radeon_fence_ref(fence); 748 749 radeon_fence_unref(&vm->fence); 750 vm->fence = radeon_fence_ref(fence); 751 } 752 753 /** 754 * radeon_vm_bo_find - find the bo_va for a specific vm & bo 755 * 756 * @vm: requested vm 757 * @bo: requested buffer object 758 * 759 * Find @bo inside the requested vm (cayman+). 760 * Search inside the @bos vm list for the requested vm 761 * Returns the found bo_va or NULL if none is found 762 * 763 * Object has to be reserved! 764 */ 765 struct radeon_bo_va *radeon_vm_bo_find(struct radeon_vm *vm, 766 struct radeon_bo *bo) 767 { 768 struct radeon_bo_va *bo_va; 769 770 list_for_each_entry(bo_va, &bo->va, bo_list) { 771 if (bo_va->vm == vm) { 772 return bo_va; 773 } 774 } 775 return NULL; 776 } 777 778 /** 779 * radeon_vm_bo_add - add a bo to a specific vm 780 * 781 * @rdev: radeon_device pointer 782 * @vm: requested vm 783 * @bo: radeon buffer object 784 * 785 * Add @bo into the requested vm (cayman+). 786 * Add @bo to the list of bos associated with the vm 787 * Returns newly added bo_va or NULL for failure 788 * 789 * Object has to be reserved! 790 */ 791 struct radeon_bo_va *radeon_vm_bo_add(struct radeon_device *rdev, 792 struct radeon_vm *vm, 793 struct radeon_bo *bo) 794 { 795 struct radeon_bo_va *bo_va; 796 797 bo_va = kzalloc(sizeof(struct radeon_bo_va), GFP_KERNEL); 798 if (bo_va == NULL) { 799 return NULL; 800 } 801 bo_va->vm = vm; 802 bo_va->bo = bo; 803 bo_va->soffset = 0; 804 bo_va->eoffset = 0; 805 bo_va->flags = 0; 806 bo_va->valid = false; 807 bo_va->ref_count = 1; 808 INIT_LIST_HEAD(&bo_va->bo_list); 809 INIT_LIST_HEAD(&bo_va->vm_list); 810 811 lockmgr(&vm->mutex, LK_EXCLUSIVE); 812 list_add(&bo_va->vm_list, &vm->va); 813 list_add_tail(&bo_va->bo_list, &bo->va); 814 lockmgr(&vm->mutex, LK_RELEASE); 815 816 return bo_va; 817 } 818 819 /** 820 * radeon_vm_bo_set_addr - set bos virtual address inside a vm 821 * 822 * @rdev: radeon_device pointer 823 * @bo_va: bo_va to store the address 824 * @soffset: requested offset of the buffer in the VM address space 825 * @flags: attributes of pages (read/write/valid/etc.) 826 * 827 * Set offset of @bo_va (cayman+). 828 * Validate and set the offset requested within the vm address space. 829 * Returns 0 for success, error for failure. 830 * 831 * Object has to be reserved! 832 */ 833 int radeon_vm_bo_set_addr(struct radeon_device *rdev, 834 struct radeon_bo_va *bo_va, 835 uint64_t soffset, 836 uint32_t flags) 837 { 838 uint64_t size = radeon_bo_size(bo_va->bo); 839 uint64_t eoffset, last_offset = 0; 840 struct radeon_vm *vm = bo_va->vm; 841 struct radeon_bo_va *tmp; 842 struct list_head *head; 843 unsigned last_pfn; 844 845 if (soffset) { 846 /* make sure object fit at this offset */ 847 eoffset = soffset + size; 848 if (soffset >= eoffset) { 849 return -EINVAL; 850 } 851 852 last_pfn = eoffset / RADEON_GPU_PAGE_SIZE; 853 if (last_pfn > rdev->vm_manager.max_pfn) { 854 dev_err(rdev->dev, "va above limit (0x%08X > 0x%08X)\n", 855 last_pfn, rdev->vm_manager.max_pfn); 856 return -EINVAL; 857 } 858 859 } else { 860 eoffset = last_pfn = 0; 861 } 862 863 lockmgr(&vm->mutex, LK_EXCLUSIVE); 864 head = &vm->va; 865 last_offset = 0; 866 list_for_each_entry(tmp, &vm->va, vm_list) { 867 if (bo_va == tmp) { 868 /* skip over currently modified bo */ 869 continue; 870 } 871 872 if (soffset >= last_offset && eoffset <= tmp->soffset) { 873 /* bo can be added before this one */ 874 break; 875 } 876 if (eoffset > tmp->soffset && soffset < tmp->eoffset) { 877 /* bo and tmp overlap, invalid offset */ 878 dev_err(rdev->dev, "bo %p va 0x%08X conflict with (bo %p 0x%08X 0x%08X)\n", 879 bo_va->bo, (unsigned)bo_va->soffset, tmp->bo, 880 (unsigned)tmp->soffset, (unsigned)tmp->eoffset); 881 lockmgr(&vm->mutex, LK_RELEASE); 882 return -EINVAL; 883 } 884 last_offset = tmp->eoffset; 885 head = &tmp->vm_list; 886 } 887 888 bo_va->soffset = soffset; 889 bo_va->eoffset = eoffset; 890 bo_va->flags = flags; 891 bo_va->valid = false; 892 list_move(&bo_va->vm_list, head); 893 894 lockmgr(&vm->mutex, LK_RELEASE); 895 return 0; 896 } 897 898 /** 899 * radeon_vm_map_gart - get the physical address of a gart page 900 * 901 * @rdev: radeon_device pointer 902 * @addr: the unmapped addr 903 * 904 * Look up the physical address of the page that the pte resolves 905 * to (cayman+). 906 * Returns the physical address of the page. 907 */ 908 uint64_t radeon_vm_map_gart(struct radeon_device *rdev, uint64_t addr) 909 { 910 uint64_t result; 911 912 /* page table offset */ 913 result = rdev->gart.pages_addr[addr >> PAGE_SHIFT]; 914 915 /* in case cpu page size != gpu page size*/ 916 result |= addr & (PAGE_MASK); 917 918 return result; 919 } 920 921 /** 922 * radeon_vm_update_pdes - make sure that page directory is valid 923 * 924 * @rdev: radeon_device pointer 925 * @vm: requested vm 926 * @start: start of GPU address range 927 * @end: end of GPU address range 928 * 929 * Allocates new page tables if necessary 930 * and updates the page directory (cayman+). 931 * Returns 0 for success, error for failure. 932 * 933 * Global and local mutex must be locked! 934 */ 935 static int radeon_vm_update_pdes(struct radeon_device *rdev, 936 struct radeon_vm *vm, 937 struct radeon_ib *ib, 938 uint64_t start, uint64_t end) 939 { 940 static const uint32_t incr = RADEON_VM_PTE_COUNT * 8; 941 942 uint64_t last_pde = ~0, last_pt = ~0; 943 unsigned count = 0; 944 uint64_t pt_idx; 945 int r; 946 947 start = (start / RADEON_GPU_PAGE_SIZE) >> RADEON_VM_BLOCK_SIZE; 948 end = (end / RADEON_GPU_PAGE_SIZE) >> RADEON_VM_BLOCK_SIZE; 949 950 /* walk over the address space and update the page directory */ 951 for (pt_idx = start; pt_idx <= end; ++pt_idx) { 952 uint64_t pde, pt; 953 954 if (vm->page_tables[pt_idx]) 955 continue; 956 957 retry: 958 r = radeon_sa_bo_new(rdev, &rdev->vm_manager.sa_manager, 959 &vm->page_tables[pt_idx], 960 RADEON_VM_PTE_COUNT * 8, 961 RADEON_GPU_PAGE_SIZE, false); 962 963 if (r == -ENOMEM) { 964 r = radeon_vm_evict(rdev, vm); 965 if (r) 966 return r; 967 goto retry; 968 } else if (r) { 969 return r; 970 } 971 972 pde = vm->pd_gpu_addr + pt_idx * 8; 973 974 pt = radeon_sa_bo_gpu_addr(vm->page_tables[pt_idx]); 975 976 if (((last_pde + 8 * count) != pde) || 977 ((last_pt + incr * count) != pt)) { 978 979 if (count) { 980 radeon_asic_vm_set_page(rdev, ib, last_pde, 981 last_pt, count, incr, 982 RADEON_VM_PAGE_VALID); 983 } 984 985 count = 1; 986 last_pde = pde; 987 last_pt = pt; 988 } else { 989 ++count; 990 } 991 } 992 993 if (count) { 994 radeon_asic_vm_set_page(rdev, ib, last_pde, last_pt, count, 995 incr, RADEON_VM_PAGE_VALID); 996 997 } 998 999 return 0; 1000 } 1001 1002 /** 1003 * radeon_vm_update_ptes - make sure that page tables are valid 1004 * 1005 * @rdev: radeon_device pointer 1006 * @vm: requested vm 1007 * @start: start of GPU address range 1008 * @end: end of GPU address range 1009 * @dst: destination address to map to 1010 * @flags: mapping flags 1011 * 1012 * Update the page tables in the range @start - @end (cayman+). 1013 * 1014 * Global and local mutex must be locked! 1015 */ 1016 static void radeon_vm_update_ptes(struct radeon_device *rdev, 1017 struct radeon_vm *vm, 1018 struct radeon_ib *ib, 1019 uint64_t start, uint64_t end, 1020 uint64_t dst, uint32_t flags) 1021 { 1022 static const uint64_t mask = RADEON_VM_PTE_COUNT - 1; 1023 1024 uint64_t last_pte = ~0, last_dst = ~0; 1025 unsigned count = 0; 1026 uint64_t addr; 1027 1028 start = start / RADEON_GPU_PAGE_SIZE; 1029 end = end / RADEON_GPU_PAGE_SIZE; 1030 1031 /* walk over the address space and update the page tables */ 1032 for (addr = start; addr < end; ) { 1033 uint64_t pt_idx = addr >> RADEON_VM_BLOCK_SIZE; 1034 unsigned nptes; 1035 uint64_t pte; 1036 1037 if ((addr & ~mask) == (end & ~mask)) 1038 nptes = end - addr; 1039 else 1040 nptes = RADEON_VM_PTE_COUNT - (addr & mask); 1041 1042 pte = radeon_sa_bo_gpu_addr(vm->page_tables[pt_idx]); 1043 pte += (addr & mask) * 8; 1044 1045 if ((last_pte + 8 * count) != pte) { 1046 1047 if (count) { 1048 radeon_asic_vm_set_page(rdev, ib, last_pte, 1049 last_dst, count, 1050 RADEON_GPU_PAGE_SIZE, 1051 flags); 1052 } 1053 1054 count = nptes; 1055 last_pte = pte; 1056 last_dst = dst; 1057 } else { 1058 count += nptes; 1059 } 1060 1061 addr += nptes; 1062 dst += nptes * RADEON_GPU_PAGE_SIZE; 1063 } 1064 1065 if (count) { 1066 radeon_asic_vm_set_page(rdev, ib, last_pte, 1067 last_dst, count, 1068 RADEON_GPU_PAGE_SIZE, flags); 1069 } 1070 } 1071 1072 /** 1073 * radeon_vm_bo_update_pte - map a bo into the vm page table 1074 * 1075 * @rdev: radeon_device pointer 1076 * @vm: requested vm 1077 * @bo: radeon buffer object 1078 * @mem: ttm mem 1079 * 1080 * Fill in the page table entries for @bo (cayman+). 1081 * Returns 0 for success, -EINVAL for failure. 1082 * 1083 * Object have to be reserved & global and local mutex must be locked! 1084 */ 1085 int radeon_vm_bo_update_pte(struct radeon_device *rdev, 1086 struct radeon_vm *vm, 1087 struct radeon_bo *bo, 1088 struct ttm_mem_reg *mem) 1089 { 1090 unsigned ridx = rdev->asic->vm.pt_ring_index; 1091 struct radeon_ib ib; 1092 struct radeon_bo_va *bo_va; 1093 unsigned nptes, npdes, ndw; 1094 uint64_t addr; 1095 int r; 1096 1097 /* nothing to do if vm isn't bound */ 1098 if (vm->page_directory == NULL) 1099 return 0; 1100 1101 bo_va = radeon_vm_bo_find(vm, bo); 1102 if (bo_va == NULL) { 1103 dev_err(rdev->dev, "bo %p not in vm %p\n", bo, vm); 1104 return -EINVAL; 1105 } 1106 1107 if (!bo_va->soffset) { 1108 dev_err(rdev->dev, "bo %p don't has a mapping in vm %p\n", 1109 bo, vm); 1110 return -EINVAL; 1111 } 1112 1113 if ((bo_va->valid && mem) || (!bo_va->valid && mem == NULL)) 1114 return 0; 1115 1116 bo_va->flags &= ~RADEON_VM_PAGE_VALID; 1117 bo_va->flags &= ~RADEON_VM_PAGE_SYSTEM; 1118 if (mem) { 1119 addr = mem->start << PAGE_SHIFT; 1120 if (mem->mem_type != TTM_PL_SYSTEM) { 1121 bo_va->flags |= RADEON_VM_PAGE_VALID; 1122 bo_va->valid = true; 1123 } 1124 if (mem->mem_type == TTM_PL_TT) { 1125 bo_va->flags |= RADEON_VM_PAGE_SYSTEM; 1126 } else { 1127 addr += rdev->vm_manager.vram_base_offset; 1128 } 1129 } else { 1130 addr = 0; 1131 bo_va->valid = false; 1132 } 1133 1134 nptes = radeon_bo_ngpu_pages(bo); 1135 1136 /* assume two extra pdes in case the mapping overlaps the borders */ 1137 npdes = (nptes >> RADEON_VM_BLOCK_SIZE) + 2; 1138 1139 /* padding, etc. */ 1140 ndw = 64; 1141 1142 if (RADEON_VM_BLOCK_SIZE > 11) 1143 /* reserve space for one header for every 2k dwords */ 1144 ndw += (nptes >> 11) * 4; 1145 else 1146 /* reserve space for one header for 1147 every (1 << BLOCK_SIZE) entries */ 1148 ndw += (nptes >> RADEON_VM_BLOCK_SIZE) * 4; 1149 1150 /* reserve space for pte addresses */ 1151 ndw += nptes * 2; 1152 1153 /* reserve space for one header for every 2k dwords */ 1154 ndw += (npdes >> 11) * 4; 1155 1156 /* reserve space for pde addresses */ 1157 ndw += npdes * 2; 1158 1159 /* update too big for an IB */ 1160 if (ndw > 0xfffff) 1161 return -ENOMEM; 1162 1163 r = radeon_ib_get(rdev, ridx, &ib, NULL, ndw * 4); 1164 ib.length_dw = 0; 1165 1166 r = radeon_vm_update_pdes(rdev, vm, &ib, bo_va->soffset, bo_va->eoffset); 1167 if (r) { 1168 radeon_ib_free(rdev, &ib); 1169 return r; 1170 } 1171 1172 radeon_vm_update_ptes(rdev, vm, &ib, bo_va->soffset, bo_va->eoffset, 1173 addr, bo_va->flags); 1174 1175 radeon_ib_sync_to(&ib, vm->fence); 1176 r = radeon_ib_schedule(rdev, &ib, NULL); 1177 if (r) { 1178 radeon_ib_free(rdev, &ib); 1179 return r; 1180 } 1181 radeon_fence_unref(&vm->fence); 1182 vm->fence = radeon_fence_ref(ib.fence); 1183 radeon_ib_free(rdev, &ib); 1184 radeon_fence_unref(&vm->last_flush); 1185 1186 return 0; 1187 } 1188 1189 /** 1190 * radeon_vm_bo_rmv - remove a bo to a specific vm 1191 * 1192 * @rdev: radeon_device pointer 1193 * @bo_va: requested bo_va 1194 * 1195 * Remove @bo_va->bo from the requested vm (cayman+). 1196 * Remove @bo_va->bo from the list of bos associated with the bo_va->vm and 1197 * remove the ptes for @bo_va in the page table. 1198 * Returns 0 for success. 1199 * 1200 * Object have to be reserved! 1201 */ 1202 int radeon_vm_bo_rmv(struct radeon_device *rdev, 1203 struct radeon_bo_va *bo_va) 1204 { 1205 int r = 0; 1206 1207 lockmgr(&rdev->vm_manager.lock, LK_EXCLUSIVE); 1208 lockmgr(&bo_va->vm->mutex, LK_EXCLUSIVE); 1209 if (bo_va->soffset) { 1210 r = radeon_vm_bo_update_pte(rdev, bo_va->vm, bo_va->bo, NULL); 1211 } 1212 lockmgr(&rdev->vm_manager.lock, LK_RELEASE); 1213 list_del(&bo_va->vm_list); 1214 lockmgr(&bo_va->vm->mutex, LK_RELEASE); 1215 list_del(&bo_va->bo_list); 1216 1217 kfree(bo_va); 1218 return r; 1219 } 1220 1221 /** 1222 * radeon_vm_bo_invalidate - mark the bo as invalid 1223 * 1224 * @rdev: radeon_device pointer 1225 * @vm: requested vm 1226 * @bo: radeon buffer object 1227 * 1228 * Mark @bo as invalid (cayman+). 1229 */ 1230 void radeon_vm_bo_invalidate(struct radeon_device *rdev, 1231 struct radeon_bo *bo) 1232 { 1233 struct radeon_bo_va *bo_va; 1234 1235 list_for_each_entry(bo_va, &bo->va, bo_list) { 1236 bo_va->valid = false; 1237 } 1238 } 1239 1240 /** 1241 * radeon_vm_init - initialize a vm instance 1242 * 1243 * @rdev: radeon_device pointer 1244 * @vm: requested vm 1245 * 1246 * Init @vm fields (cayman+). 1247 */ 1248 void radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm) 1249 { 1250 vm->id = 0; 1251 vm->fence = NULL; 1252 lockinit(&vm->mutex, "rvmmtx", 0, LK_CANRECURSE); 1253 INIT_LIST_HEAD(&vm->list); 1254 INIT_LIST_HEAD(&vm->va); 1255 } 1256 1257 /** 1258 * radeon_vm_fini - tear down a vm instance 1259 * 1260 * @rdev: radeon_device pointer 1261 * @vm: requested vm 1262 * 1263 * Tear down @vm (cayman+). 1264 * Unbind the VM and remove all bos from the vm bo list 1265 */ 1266 void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm) 1267 { 1268 struct radeon_bo_va *bo_va, *tmp; 1269 int r; 1270 1271 lockmgr(&rdev->vm_manager.lock, LK_EXCLUSIVE); 1272 lockmgr(&vm->mutex, LK_EXCLUSIVE); 1273 radeon_vm_free_pt(rdev, vm); 1274 lockmgr(&rdev->vm_manager.lock, LK_RELEASE); 1275 1276 if (!list_empty(&vm->va)) { 1277 dev_err(rdev->dev, "still active bo inside vm\n"); 1278 } 1279 list_for_each_entry_safe(bo_va, tmp, &vm->va, vm_list) { 1280 list_del_init(&bo_va->vm_list); 1281 r = radeon_bo_reserve(bo_va->bo, false); 1282 if (!r) { 1283 list_del_init(&bo_va->bo_list); 1284 radeon_bo_unreserve(bo_va->bo); 1285 kfree(bo_va); 1286 } 1287 } 1288 radeon_fence_unref(&vm->fence); 1289 radeon_fence_unref(&vm->last_flush); 1290 lockmgr(&vm->mutex, LK_RELEASE); 1291 } 1292