1 /* 2 * Copyright 2008 Advanced Micro Devices, Inc. 3 * Copyright 2008 Red Hat Inc. 4 * Copyright 2009 Jerome Glisse. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: Dave Airlie 25 * Alex Deucher 26 * Jerome Glisse 27 * 28 * $FreeBSD: head/sys/dev/drm2/radeon/radeon_gart.c 254885 2013-08-25 19:37:15Z dumbbell $ 29 */ 30 31 #include <drm/drmP.h> 32 #include <uapi_drm/radeon_drm.h> 33 #include "radeon.h" 34 #include "radeon_reg.h" 35 36 /* 37 * GART 38 * The GART (Graphics Aperture Remapping Table) is an aperture 39 * in the GPU's address space. System pages can be mapped into 40 * the aperture and look like contiguous pages from the GPU's 41 * perspective. A page table maps the pages in the aperture 42 * to the actual backing pages in system memory. 43 * 44 * Radeon GPUs support both an internal GART, as described above, 45 * and AGP. AGP works similarly, but the GART table is configured 46 * and maintained by the northbridge rather than the driver. 47 * Radeon hw has a separate AGP aperture that is programmed to 48 * point to the AGP aperture provided by the northbridge and the 49 * requests are passed through to the northbridge aperture. 50 * Both AGP and internal GART can be used at the same time, however 51 * that is not currently supported by the driver. 52 * 53 * This file handles the common internal GART management. 54 */ 55 56 /* 57 * Common GART table functions. 58 */ 59 /** 60 * radeon_gart_table_ram_alloc - allocate system ram for gart page table 61 * 62 * @rdev: radeon_device pointer 63 * 64 * Allocate system memory for GART page table 65 * (r1xx-r3xx, non-pcie r4xx, rs400). These asics require the 66 * gart table to be in system memory. 67 * Returns 0 for success, -ENOMEM for failure. 68 */ 69 int radeon_gart_table_ram_alloc(struct radeon_device *rdev) 70 { 71 drm_dma_handle_t *dmah; 72 73 dmah = drm_pci_alloc(rdev->ddev, rdev->gart.table_size, 74 PAGE_SIZE, 0xFFFFFFFFUL); 75 if (dmah == NULL) { 76 return -ENOMEM; 77 } 78 rdev->gart.dmah = dmah; 79 rdev->gart.ptr = dmah->vaddr; 80 #if defined(__i386) || defined(__amd64) 81 if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 || 82 rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) { 83 pmap_change_attr((vm_offset_t)rdev->gart.ptr, 84 rdev->gart.table_size >> PAGE_SHIFT, PAT_UNCACHED); 85 } 86 #endif 87 rdev->gart.table_addr = dmah->busaddr; 88 memset((void *)rdev->gart.ptr, 0, rdev->gart.table_size); 89 return 0; 90 } 91 92 /** 93 * radeon_gart_table_ram_free - free system ram for gart page table 94 * 95 * @rdev: radeon_device pointer 96 * 97 * Free system memory for GART page table 98 * (r1xx-r3xx, non-pcie r4xx, rs400). These asics require the 99 * gart table to be in system memory. 100 */ 101 void radeon_gart_table_ram_free(struct radeon_device *rdev) 102 { 103 if (rdev->gart.ptr == NULL) { 104 return; 105 } 106 #if defined(__i386) || defined(__amd64) 107 if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 || 108 rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) { 109 pmap_change_attr((vm_offset_t)rdev->gart.ptr, 110 rdev->gart.table_size >> PAGE_SHIFT, PAT_WRITE_COMBINING); 111 } 112 #endif 113 drm_pci_free(rdev->ddev, rdev->gart.dmah); 114 rdev->gart.dmah = NULL; 115 rdev->gart.ptr = NULL; 116 rdev->gart.table_addr = 0; 117 } 118 119 /** 120 * radeon_gart_table_vram_alloc - allocate vram for gart page table 121 * 122 * @rdev: radeon_device pointer 123 * 124 * Allocate video memory for GART page table 125 * (pcie r4xx, r5xx+). These asics require the 126 * gart table to be in video memory. 127 * Returns 0 for success, error for failure. 128 */ 129 int radeon_gart_table_vram_alloc(struct radeon_device *rdev) 130 { 131 int r; 132 133 if (rdev->gart.robj == NULL) { 134 r = radeon_bo_create(rdev, rdev->gart.table_size, 135 PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM, 136 NULL, &rdev->gart.robj); 137 if (r) { 138 return r; 139 } 140 } 141 return 0; 142 } 143 144 /** 145 * radeon_gart_table_vram_pin - pin gart page table in vram 146 * 147 * @rdev: radeon_device pointer 148 * 149 * Pin the GART page table in vram so it will not be moved 150 * by the memory manager (pcie r4xx, r5xx+). These asics require the 151 * gart table to be in video memory. 152 * Returns 0 for success, error for failure. 153 */ 154 int radeon_gart_table_vram_pin(struct radeon_device *rdev) 155 { 156 uint64_t gpu_addr; 157 int r; 158 159 r = radeon_bo_reserve(rdev->gart.robj, false); 160 if (unlikely(r != 0)) 161 return r; 162 r = radeon_bo_pin(rdev->gart.robj, 163 RADEON_GEM_DOMAIN_VRAM, &gpu_addr); 164 if (r) { 165 radeon_bo_unreserve(rdev->gart.robj); 166 return r; 167 } 168 r = radeon_bo_kmap(rdev->gart.robj, &rdev->gart.ptr); 169 if (r) 170 radeon_bo_unpin(rdev->gart.robj); 171 radeon_bo_unreserve(rdev->gart.robj); 172 rdev->gart.table_addr = gpu_addr; 173 return r; 174 } 175 176 /** 177 * radeon_gart_table_vram_unpin - unpin gart page table in vram 178 * 179 * @rdev: radeon_device pointer 180 * 181 * Unpin the GART page table in vram (pcie r4xx, r5xx+). 182 * These asics require the gart table to be in video memory. 183 */ 184 void radeon_gart_table_vram_unpin(struct radeon_device *rdev) 185 { 186 int r; 187 188 if (rdev->gart.robj == NULL) { 189 return; 190 } 191 r = radeon_bo_reserve(rdev->gart.robj, false); 192 if (likely(r == 0)) { 193 radeon_bo_kunmap(rdev->gart.robj); 194 radeon_bo_unpin(rdev->gart.robj); 195 radeon_bo_unreserve(rdev->gart.robj); 196 rdev->gart.ptr = NULL; 197 } 198 } 199 200 /** 201 * radeon_gart_table_vram_free - free gart page table vram 202 * 203 * @rdev: radeon_device pointer 204 * 205 * Free the video memory used for the GART page table 206 * (pcie r4xx, r5xx+). These asics require the gart table to 207 * be in video memory. 208 */ 209 void radeon_gart_table_vram_free(struct radeon_device *rdev) 210 { 211 if (rdev->gart.robj == NULL) { 212 return; 213 } 214 radeon_gart_table_vram_unpin(rdev); 215 radeon_bo_unref(&rdev->gart.robj); 216 } 217 218 /* 219 * Common gart functions. 220 */ 221 /** 222 * radeon_gart_unbind - unbind pages from the gart page table 223 * 224 * @rdev: radeon_device pointer 225 * @offset: offset into the GPU's gart aperture 226 * @pages: number of pages to unbind 227 * 228 * Unbinds the requested pages from the gart page table and 229 * replaces them with the dummy page (all asics). 230 */ 231 void radeon_gart_unbind(struct radeon_device *rdev, unsigned offset, 232 int pages) 233 { 234 unsigned t; 235 unsigned p; 236 int i, j; 237 u64 page_base; 238 239 if (!rdev->gart.ready) { 240 DRM_ERROR("trying to unbind memory from uninitialized GART !\n"); 241 return; 242 } 243 t = offset / RADEON_GPU_PAGE_SIZE; 244 p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); 245 for (i = 0; i < pages; i++, p++) { 246 if (rdev->gart.pages[p]) { 247 rdev->gart.pages[p] = NULL; 248 rdev->gart.pages_addr[p] = rdev->dummy_page.addr; 249 page_base = rdev->gart.pages_addr[p]; 250 for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) { 251 if (rdev->gart.ptr) { 252 radeon_gart_set_page(rdev, t, page_base); 253 } 254 page_base += RADEON_GPU_PAGE_SIZE; 255 } 256 } 257 } 258 cpu_mfence(); 259 radeon_gart_tlb_flush(rdev); 260 } 261 262 /** 263 * radeon_gart_bind - bind pages into the gart page table 264 * 265 * @rdev: radeon_device pointer 266 * @offset: offset into the GPU's gart aperture 267 * @pages: number of pages to bind 268 * @pagelist: pages to bind 269 * @dma_addr: DMA addresses of pages 270 * 271 * Binds the requested pages to the gart page table 272 * (all asics). 273 * Returns 0 for success, -EINVAL for failure. 274 */ 275 int radeon_gart_bind(struct radeon_device *rdev, unsigned offset, 276 int pages, vm_page_t *pagelist, dma_addr_t *dma_addr) 277 { 278 unsigned t; 279 unsigned p; 280 uint64_t page_base; 281 int i, j; 282 283 if (!rdev->gart.ready) { 284 DRM_ERROR("trying to bind memory to uninitialized GART !\n"); 285 return -EINVAL; 286 } 287 t = offset / RADEON_GPU_PAGE_SIZE; 288 p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); 289 290 for (i = 0; i < pages; i++, p++) { 291 rdev->gart.pages_addr[p] = dma_addr[i]; 292 rdev->gart.pages[p] = pagelist[i]; 293 if (rdev->gart.ptr) { 294 page_base = rdev->gart.pages_addr[p]; 295 for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) { 296 radeon_gart_set_page(rdev, t, page_base); 297 page_base += RADEON_GPU_PAGE_SIZE; 298 } 299 } 300 } 301 cpu_mfence(); 302 radeon_gart_tlb_flush(rdev); 303 return 0; 304 } 305 306 /** 307 * radeon_gart_restore - bind all pages in the gart page table 308 * 309 * @rdev: radeon_device pointer 310 * 311 * Binds all pages in the gart page table (all asics). 312 * Used to rebuild the gart table on device startup or resume. 313 */ 314 void radeon_gart_restore(struct radeon_device *rdev) 315 { 316 int i, j, t; 317 u64 page_base; 318 319 if (!rdev->gart.ptr) { 320 return; 321 } 322 for (i = 0, t = 0; i < rdev->gart.num_cpu_pages; i++) { 323 page_base = rdev->gart.pages_addr[i]; 324 for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) { 325 radeon_gart_set_page(rdev, t, page_base); 326 page_base += RADEON_GPU_PAGE_SIZE; 327 } 328 } 329 cpu_mfence(); 330 radeon_gart_tlb_flush(rdev); 331 } 332 333 /** 334 * radeon_gart_init - init the driver info for managing the gart 335 * 336 * @rdev: radeon_device pointer 337 * 338 * Allocate the dummy page and init the gart driver info (all asics). 339 * Returns 0 for success, error for failure. 340 */ 341 int radeon_gart_init(struct radeon_device *rdev) 342 { 343 int r, i; 344 345 if (rdev->gart.pages) { 346 return 0; 347 } 348 /* We need PAGE_SIZE >= RADEON_GPU_PAGE_SIZE */ 349 if (PAGE_SIZE < RADEON_GPU_PAGE_SIZE) { 350 DRM_ERROR("Page size is smaller than GPU page size!\n"); 351 return -EINVAL; 352 } 353 r = radeon_dummy_page_init(rdev); 354 if (r) 355 return r; 356 /* Compute table size */ 357 rdev->gart.num_cpu_pages = rdev->mc.gtt_size / PAGE_SIZE; 358 rdev->gart.num_gpu_pages = rdev->mc.gtt_size / RADEON_GPU_PAGE_SIZE; 359 DRM_INFO("GART: num cpu pages %u, num gpu pages %u\n", 360 rdev->gart.num_cpu_pages, rdev->gart.num_gpu_pages); 361 /* Allocate pages table */ 362 rdev->gart.pages = kmalloc(sizeof(void *) * rdev->gart.num_cpu_pages, 363 DRM_MEM_DRIVER, M_ZERO | M_WAITOK); 364 if (rdev->gart.pages == NULL) { 365 radeon_gart_fini(rdev); 366 return -ENOMEM; 367 } 368 rdev->gart.pages_addr = kmalloc(sizeof(dma_addr_t) * rdev->gart.num_cpu_pages, 369 DRM_MEM_DRIVER, M_ZERO | M_WAITOK); 370 if (rdev->gart.pages_addr == NULL) { 371 radeon_gart_fini(rdev); 372 return -ENOMEM; 373 } 374 /* set GART entry to point to the dummy page by default */ 375 for (i = 0; i < rdev->gart.num_cpu_pages; i++) { 376 rdev->gart.pages_addr[i] = rdev->dummy_page.addr; 377 } 378 return 0; 379 } 380 381 /** 382 * radeon_gart_fini - tear down the driver info for managing the gart 383 * 384 * @rdev: radeon_device pointer 385 * 386 * Tear down the gart driver info and free the dummy page (all asics). 387 */ 388 void radeon_gart_fini(struct radeon_device *rdev) 389 { 390 if (rdev->gart.pages && rdev->gart.pages_addr && rdev->gart.ready) { 391 /* unbind pages */ 392 radeon_gart_unbind(rdev, 0, rdev->gart.num_cpu_pages); 393 } 394 rdev->gart.ready = false; 395 drm_free(rdev->gart.pages, DRM_MEM_DRIVER); 396 drm_free(rdev->gart.pages_addr, DRM_MEM_DRIVER); 397 rdev->gart.pages = NULL; 398 rdev->gart.pages_addr = NULL; 399 400 radeon_dummy_page_fini(rdev); 401 } 402 403 /* 404 * GPUVM 405 * GPUVM is similar to the legacy gart on older asics, however 406 * rather than there being a single global gart table 407 * for the entire GPU, there are multiple VM page tables active 408 * at any given time. The VM page tables can contain a mix 409 * vram pages and system memory pages and system memory pages 410 * can be mapped as snooped (cached system pages) or unsnooped 411 * (uncached system pages). 412 * Each VM has an ID associated with it and there is a page table 413 * associated with each VMID. When execting a command buffer, 414 * the kernel tells the the ring what VMID to use for that command 415 * buffer. VMIDs are allocated dynamically as commands are submitted. 416 * The userspace drivers maintain their own address space and the kernel 417 * sets up their pages tables accordingly when they submit their 418 * command buffers and a VMID is assigned. 419 * Cayman/Trinity support up to 8 active VMs at any given time; 420 * SI supports 16. 421 */ 422 423 /* 424 * vm helpers 425 * 426 * TODO bind a default page at vm initialization for default address 427 */ 428 429 /** 430 * radeon_vm_num_pde - return the number of page directory entries 431 * 432 * @rdev: radeon_device pointer 433 * 434 * Calculate the number of page directory entries (cayman+). 435 */ 436 static unsigned radeon_vm_num_pdes(struct radeon_device *rdev) 437 { 438 return rdev->vm_manager.max_pfn >> RADEON_VM_BLOCK_SIZE; 439 } 440 441 /** 442 * radeon_vm_directory_size - returns the size of the page directory in bytes 443 * 444 * @rdev: radeon_device pointer 445 * 446 * Calculate the size of the page directory in bytes (cayman+). 447 */ 448 static unsigned radeon_vm_directory_size(struct radeon_device *rdev) 449 { 450 return RADEON_GPU_PAGE_ALIGN(radeon_vm_num_pdes(rdev) * 8); 451 } 452 453 /** 454 * radeon_vm_manager_init - init the vm manager 455 * 456 * @rdev: radeon_device pointer 457 * 458 * Init the vm manager (cayman+). 459 * Returns 0 for success, error for failure. 460 */ 461 int radeon_vm_manager_init(struct radeon_device *rdev) 462 { 463 struct radeon_vm *vm; 464 struct radeon_bo_va *bo_va; 465 int r; 466 unsigned size; 467 468 if (!rdev->vm_manager.enabled) { 469 /* allocate enough for 2 full VM pts */ 470 size = radeon_vm_directory_size(rdev); 471 size += rdev->vm_manager.max_pfn * 8; 472 size *= 2; 473 r = radeon_sa_bo_manager_init(rdev, &rdev->vm_manager.sa_manager, 474 RADEON_GPU_PAGE_ALIGN(size), 475 RADEON_GEM_DOMAIN_VRAM); 476 if (r) { 477 dev_err(rdev->dev, "failed to allocate vm bo (%dKB)\n", 478 (rdev->vm_manager.max_pfn * 8) >> 10); 479 return r; 480 } 481 482 r = radeon_asic_vm_init(rdev); 483 if (r) 484 return r; 485 486 rdev->vm_manager.enabled = true; 487 488 r = radeon_sa_bo_manager_start(rdev, &rdev->vm_manager.sa_manager); 489 if (r) 490 return r; 491 } 492 493 /* restore page table */ 494 list_for_each_entry(vm, &rdev->vm_manager.lru_vm, list) { 495 if (vm->page_directory == NULL) 496 continue; 497 498 list_for_each_entry(bo_va, &vm->va, vm_list) { 499 bo_va->valid = false; 500 } 501 } 502 return 0; 503 } 504 505 /** 506 * radeon_vm_free_pt - free the page table for a specific vm 507 * 508 * @rdev: radeon_device pointer 509 * @vm: vm to unbind 510 * 511 * Free the page table of a specific vm (cayman+). 512 * 513 * Global and local mutex must be lock! 514 */ 515 static void radeon_vm_free_pt(struct radeon_device *rdev, 516 struct radeon_vm *vm) 517 { 518 struct radeon_bo_va *bo_va; 519 int i; 520 521 if (!vm->page_directory) 522 return; 523 524 list_del_init(&vm->list); 525 radeon_sa_bo_free(rdev, &vm->page_directory, vm->fence); 526 527 list_for_each_entry(bo_va, &vm->va, vm_list) { 528 bo_va->valid = false; 529 } 530 531 if (vm->page_tables == NULL) 532 return; 533 534 for (i = 0; i < radeon_vm_num_pdes(rdev); i++) 535 radeon_sa_bo_free(rdev, &vm->page_tables[i], vm->fence); 536 537 drm_free(vm->page_tables, DRM_MEM_DRIVER); 538 } 539 540 /** 541 * radeon_vm_manager_fini - tear down the vm manager 542 * 543 * @rdev: radeon_device pointer 544 * 545 * Tear down the VM manager (cayman+). 546 */ 547 void radeon_vm_manager_fini(struct radeon_device *rdev) 548 { 549 struct radeon_vm *vm, *tmp; 550 int i; 551 552 if (!rdev->vm_manager.enabled) 553 return; 554 555 lockmgr(&rdev->vm_manager.lock, LK_EXCLUSIVE); 556 /* free all allocated page tables */ 557 list_for_each_entry_safe(vm, tmp, &rdev->vm_manager.lru_vm, list) { 558 lockmgr(&vm->mutex, LK_EXCLUSIVE); 559 radeon_vm_free_pt(rdev, vm); 560 lockmgr(&vm->mutex, LK_RELEASE); 561 } 562 for (i = 0; i < RADEON_NUM_VM; ++i) { 563 radeon_fence_unref(&rdev->vm_manager.active[i]); 564 } 565 radeon_asic_vm_fini(rdev); 566 lockmgr(&rdev->vm_manager.lock, LK_RELEASE); 567 568 radeon_sa_bo_manager_suspend(rdev, &rdev->vm_manager.sa_manager); 569 radeon_sa_bo_manager_fini(rdev, &rdev->vm_manager.sa_manager); 570 rdev->vm_manager.enabled = false; 571 } 572 573 /** 574 * radeon_vm_evict - evict page table to make room for new one 575 * 576 * @rdev: radeon_device pointer 577 * @vm: VM we want to allocate something for 578 * 579 * Evict a VM from the lru, making sure that it isn't @vm. (cayman+). 580 * Returns 0 for success, -ENOMEM for failure. 581 * 582 * Global and local mutex must be locked! 583 */ 584 static int radeon_vm_evict(struct radeon_device *rdev, struct radeon_vm *vm) 585 { 586 struct radeon_vm *vm_evict; 587 588 if (list_empty(&rdev->vm_manager.lru_vm)) 589 return -ENOMEM; 590 591 vm_evict = list_first_entry(&rdev->vm_manager.lru_vm, 592 struct radeon_vm, list); 593 if (vm_evict == vm) 594 return -ENOMEM; 595 596 lockmgr(&vm_evict->mutex, LK_EXCLUSIVE); 597 radeon_vm_free_pt(rdev, vm_evict); 598 lockmgr(&vm_evict->mutex, LK_RELEASE); 599 return 0; 600 } 601 602 /** 603 * radeon_vm_alloc_pt - allocates a page table for a VM 604 * 605 * @rdev: radeon_device pointer 606 * @vm: vm to bind 607 * 608 * Allocate a page table for the requested vm (cayman+). 609 * Returns 0 for success, error for failure. 610 * 611 * Global and local mutex must be locked! 612 */ 613 int radeon_vm_alloc_pt(struct radeon_device *rdev, struct radeon_vm *vm) 614 { 615 unsigned pd_size, pts_size; 616 u64 *pd_addr; 617 int r; 618 619 if (vm == NULL) { 620 return -EINVAL; 621 } 622 623 if (vm->page_directory != NULL) { 624 return 0; 625 } 626 627 retry: 628 pd_size = RADEON_GPU_PAGE_ALIGN(radeon_vm_directory_size(rdev)); 629 r = radeon_sa_bo_new(rdev, &rdev->vm_manager.sa_manager, 630 &vm->page_directory, pd_size, 631 RADEON_GPU_PAGE_SIZE, false); 632 if (r == -ENOMEM) { 633 r = radeon_vm_evict(rdev, vm); 634 if (r) 635 return r; 636 goto retry; 637 638 } else if (r) { 639 return r; 640 } 641 642 vm->pd_gpu_addr = radeon_sa_bo_gpu_addr(vm->page_directory); 643 644 /* Initially clear the page directory */ 645 pd_addr = radeon_sa_bo_cpu_addr(vm->page_directory); 646 memset(pd_addr, 0, pd_size); 647 648 pts_size = radeon_vm_num_pdes(rdev) * sizeof(struct radeon_sa_bo *); 649 vm->page_tables = kmalloc(pts_size, DRM_MEM_DRIVER, M_ZERO | M_WAITOK); 650 651 if (vm->page_tables == NULL) { 652 DRM_ERROR("Cannot allocate memory for page table array\n"); 653 radeon_sa_bo_free(rdev, &vm->page_directory, vm->fence); 654 return -ENOMEM; 655 } 656 657 return 0; 658 } 659 660 /** 661 * radeon_vm_add_to_lru - add VMs page table to LRU list 662 * 663 * @rdev: radeon_device pointer 664 * @vm: vm to add to LRU 665 * 666 * Add the allocated page table to the LRU list (cayman+). 667 * 668 * Global mutex must be locked! 669 */ 670 void radeon_vm_add_to_lru(struct radeon_device *rdev, struct radeon_vm *vm) 671 { 672 list_del_init(&vm->list); 673 list_add_tail(&vm->list, &rdev->vm_manager.lru_vm); 674 } 675 676 /** 677 * radeon_vm_grab_id - allocate the next free VMID 678 * 679 * @rdev: radeon_device pointer 680 * @vm: vm to allocate id for 681 * @ring: ring we want to submit job to 682 * 683 * Allocate an id for the vm (cayman+). 684 * Returns the fence we need to sync to (if any). 685 * 686 * Global and local mutex must be locked! 687 */ 688 struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev, 689 struct radeon_vm *vm, int ring) 690 { 691 struct radeon_fence *best[RADEON_NUM_RINGS] = {}; 692 unsigned choices[2] = {}; 693 unsigned i; 694 695 /* check if the id is still valid */ 696 if (vm->fence && vm->fence == rdev->vm_manager.active[vm->id]) 697 return NULL; 698 699 /* we definately need to flush */ 700 radeon_fence_unref(&vm->last_flush); 701 702 /* skip over VMID 0, since it is the system VM */ 703 for (i = 1; i < rdev->vm_manager.nvm; ++i) { 704 struct radeon_fence *fence = rdev->vm_manager.active[i]; 705 706 if (fence == NULL) { 707 /* found a free one */ 708 vm->id = i; 709 return NULL; 710 } 711 712 if (radeon_fence_is_earlier(fence, best[fence->ring])) { 713 best[fence->ring] = fence; 714 choices[fence->ring == ring ? 0 : 1] = i; 715 } 716 } 717 718 for (i = 0; i < 2; ++i) { 719 if (choices[i]) { 720 vm->id = choices[i]; 721 return rdev->vm_manager.active[choices[i]]; 722 } 723 } 724 725 /* should never happen */ 726 panic("%s: failed to allocate next VMID", __func__); 727 return NULL; 728 } 729 730 /** 731 * radeon_vm_fence - remember fence for vm 732 * 733 * @rdev: radeon_device pointer 734 * @vm: vm we want to fence 735 * @fence: fence to remember 736 * 737 * Fence the vm (cayman+). 738 * Set the fence used to protect page table and id. 739 * 740 * Global and local mutex must be locked! 741 */ 742 void radeon_vm_fence(struct radeon_device *rdev, 743 struct radeon_vm *vm, 744 struct radeon_fence *fence) 745 { 746 radeon_fence_unref(&rdev->vm_manager.active[vm->id]); 747 rdev->vm_manager.active[vm->id] = radeon_fence_ref(fence); 748 749 radeon_fence_unref(&vm->fence); 750 vm->fence = radeon_fence_ref(fence); 751 } 752 753 /** 754 * radeon_vm_bo_find - find the bo_va for a specific vm & bo 755 * 756 * @vm: requested vm 757 * @bo: requested buffer object 758 * 759 * Find @bo inside the requested vm (cayman+). 760 * Search inside the @bos vm list for the requested vm 761 * Returns the found bo_va or NULL if none is found 762 * 763 * Object has to be reserved! 764 */ 765 struct radeon_bo_va *radeon_vm_bo_find(struct radeon_vm *vm, 766 struct radeon_bo *bo) 767 { 768 struct radeon_bo_va *bo_va; 769 770 list_for_each_entry(bo_va, &bo->va, bo_list) { 771 if (bo_va->vm == vm) { 772 return bo_va; 773 } 774 } 775 return NULL; 776 } 777 778 /** 779 * radeon_vm_bo_add - add a bo to a specific vm 780 * 781 * @rdev: radeon_device pointer 782 * @vm: requested vm 783 * @bo: radeon buffer object 784 * 785 * Add @bo into the requested vm (cayman+). 786 * Add @bo to the list of bos associated with the vm 787 * Returns newly added bo_va or NULL for failure 788 * 789 * Object has to be reserved! 790 */ 791 struct radeon_bo_va *radeon_vm_bo_add(struct radeon_device *rdev, 792 struct radeon_vm *vm, 793 struct radeon_bo *bo) 794 { 795 struct radeon_bo_va *bo_va; 796 797 bo_va = kmalloc(sizeof(struct radeon_bo_va), DRM_MEM_DRIVER, 798 M_ZERO | M_WAITOK); 799 if (bo_va == NULL) { 800 return NULL; 801 } 802 bo_va->vm = vm; 803 bo_va->bo = bo; 804 bo_va->soffset = 0; 805 bo_va->eoffset = 0; 806 bo_va->flags = 0; 807 bo_va->valid = false; 808 bo_va->ref_count = 1; 809 INIT_LIST_HEAD(&bo_va->bo_list); 810 INIT_LIST_HEAD(&bo_va->vm_list); 811 812 lockmgr(&vm->mutex, LK_EXCLUSIVE); 813 list_add(&bo_va->vm_list, &vm->va); 814 list_add_tail(&bo_va->bo_list, &bo->va); 815 lockmgr(&vm->mutex, LK_RELEASE); 816 817 return bo_va; 818 } 819 820 /** 821 * radeon_vm_bo_set_addr - set bos virtual address inside a vm 822 * 823 * @rdev: radeon_device pointer 824 * @bo_va: bo_va to store the address 825 * @soffset: requested offset of the buffer in the VM address space 826 * @flags: attributes of pages (read/write/valid/etc.) 827 * 828 * Set offset of @bo_va (cayman+). 829 * Validate and set the offset requested within the vm address space. 830 * Returns 0 for success, error for failure. 831 * 832 * Object has to be reserved! 833 */ 834 int radeon_vm_bo_set_addr(struct radeon_device *rdev, 835 struct radeon_bo_va *bo_va, 836 uint64_t soffset, 837 uint32_t flags) 838 { 839 uint64_t size = radeon_bo_size(bo_va->bo); 840 uint64_t eoffset, last_offset = 0; 841 struct radeon_vm *vm = bo_va->vm; 842 struct radeon_bo_va *tmp; 843 struct list_head *head; 844 unsigned last_pfn; 845 846 if (soffset) { 847 /* make sure object fit at this offset */ 848 eoffset = soffset + size; 849 if (soffset >= eoffset) { 850 return -EINVAL; 851 } 852 853 last_pfn = eoffset / RADEON_GPU_PAGE_SIZE; 854 if (last_pfn > rdev->vm_manager.max_pfn) { 855 dev_err(rdev->dev, "va above limit (0x%08X > 0x%08X)\n", 856 last_pfn, rdev->vm_manager.max_pfn); 857 return -EINVAL; 858 } 859 860 } else { 861 eoffset = last_pfn = 0; 862 } 863 864 lockmgr(&vm->mutex, LK_EXCLUSIVE); 865 head = &vm->va; 866 last_offset = 0; 867 list_for_each_entry(tmp, &vm->va, vm_list) { 868 if (bo_va == tmp) { 869 /* skip over currently modified bo */ 870 continue; 871 } 872 873 if (soffset >= last_offset && eoffset <= tmp->soffset) { 874 /* bo can be added before this one */ 875 break; 876 } 877 if (eoffset > tmp->soffset && soffset < tmp->eoffset) { 878 /* bo and tmp overlap, invalid offset */ 879 dev_err(rdev->dev, "bo %p va 0x%08X conflict with (bo %p 0x%08X 0x%08X)\n", 880 bo_va->bo, (unsigned)bo_va->soffset, tmp->bo, 881 (unsigned)tmp->soffset, (unsigned)tmp->eoffset); 882 lockmgr(&vm->mutex, LK_RELEASE); 883 return -EINVAL; 884 } 885 last_offset = tmp->eoffset; 886 head = &tmp->vm_list; 887 } 888 889 bo_va->soffset = soffset; 890 bo_va->eoffset = eoffset; 891 bo_va->flags = flags; 892 bo_va->valid = false; 893 list_move(&bo_va->vm_list, head); 894 895 lockmgr(&vm->mutex, LK_RELEASE); 896 return 0; 897 } 898 899 /** 900 * radeon_vm_map_gart - get the physical address of a gart page 901 * 902 * @rdev: radeon_device pointer 903 * @addr: the unmapped addr 904 * 905 * Look up the physical address of the page that the pte resolves 906 * to (cayman+). 907 * Returns the physical address of the page. 908 */ 909 uint64_t radeon_vm_map_gart(struct radeon_device *rdev, uint64_t addr) 910 { 911 uint64_t result; 912 913 /* page table offset */ 914 result = rdev->gart.pages_addr[addr >> PAGE_SHIFT]; 915 916 /* in case cpu page size != gpu page size*/ 917 result |= addr & (~PAGE_MASK); 918 919 return result; 920 } 921 922 /** 923 * radeon_vm_update_pdes - make sure that page directory is valid 924 * 925 * @rdev: radeon_device pointer 926 * @vm: requested vm 927 * @start: start of GPU address range 928 * @end: end of GPU address range 929 * 930 * Allocates new page tables if necessary 931 * and updates the page directory (cayman+). 932 * Returns 0 for success, error for failure. 933 * 934 * Global and local mutex must be locked! 935 */ 936 static int radeon_vm_update_pdes(struct radeon_device *rdev, 937 struct radeon_vm *vm, 938 uint64_t start, uint64_t end) 939 { 940 static const uint32_t incr = RADEON_VM_PTE_COUNT * 8; 941 942 uint64_t last_pde = ~0, last_pt = ~0; 943 unsigned count = 0; 944 uint64_t pt_idx; 945 int r; 946 947 start = (start / RADEON_GPU_PAGE_SIZE) >> RADEON_VM_BLOCK_SIZE; 948 end = (end / RADEON_GPU_PAGE_SIZE) >> RADEON_VM_BLOCK_SIZE; 949 950 /* walk over the address space and update the page directory */ 951 for (pt_idx = start; pt_idx <= end; ++pt_idx) { 952 uint64_t pde, pt; 953 954 if (vm->page_tables[pt_idx]) 955 continue; 956 957 retry: 958 r = radeon_sa_bo_new(rdev, &rdev->vm_manager.sa_manager, 959 &vm->page_tables[pt_idx], 960 RADEON_VM_PTE_COUNT * 8, 961 RADEON_GPU_PAGE_SIZE, false); 962 963 if (r == -ENOMEM) { 964 r = radeon_vm_evict(rdev, vm); 965 if (r) 966 return r; 967 goto retry; 968 } else if (r) { 969 return r; 970 } 971 972 pde = vm->pd_gpu_addr + pt_idx * 8; 973 974 pt = radeon_sa_bo_gpu_addr(vm->page_tables[pt_idx]); 975 976 if (((last_pde + 8 * count) != pde) || 977 ((last_pt + incr * count) != pt)) { 978 979 if (count) { 980 radeon_asic_vm_set_page(rdev, last_pde, 981 last_pt, count, incr, 982 RADEON_VM_PAGE_VALID); 983 } 984 985 count = 1; 986 last_pde = pde; 987 last_pt = pt; 988 } else { 989 ++count; 990 } 991 } 992 993 if (count) { 994 radeon_asic_vm_set_page(rdev, last_pde, last_pt, count, 995 incr, RADEON_VM_PAGE_VALID); 996 997 } 998 999 return 0; 1000 } 1001 1002 /** 1003 * radeon_vm_update_ptes - make sure that page tables are valid 1004 * 1005 * @rdev: radeon_device pointer 1006 * @vm: requested vm 1007 * @start: start of GPU address range 1008 * @end: end of GPU address range 1009 * @dst: destination address to map to 1010 * @flags: mapping flags 1011 * 1012 * Update the page tables in the range @start - @end (cayman+). 1013 * 1014 * Global and local mutex must be locked! 1015 */ 1016 static void radeon_vm_update_ptes(struct radeon_device *rdev, 1017 struct radeon_vm *vm, 1018 uint64_t start, uint64_t end, 1019 uint64_t dst, uint32_t flags) 1020 { 1021 static const uint64_t mask = RADEON_VM_PTE_COUNT - 1; 1022 1023 uint64_t last_pte = ~0, last_dst = ~0; 1024 unsigned count = 0; 1025 uint64_t addr; 1026 1027 start = start / RADEON_GPU_PAGE_SIZE; 1028 end = end / RADEON_GPU_PAGE_SIZE; 1029 1030 /* walk over the address space and update the page tables */ 1031 for (addr = start; addr < end; ) { 1032 uint64_t pt_idx = addr >> RADEON_VM_BLOCK_SIZE; 1033 unsigned nptes; 1034 uint64_t pte; 1035 1036 if ((addr & ~mask) == (end & ~mask)) 1037 nptes = end - addr; 1038 else 1039 nptes = RADEON_VM_PTE_COUNT - (addr & mask); 1040 1041 pte = radeon_sa_bo_gpu_addr(vm->page_tables[pt_idx]); 1042 pte += (addr & mask) * 8; 1043 1044 if ((last_pte + 8 * count) != pte) { 1045 1046 if (count) { 1047 radeon_asic_vm_set_page(rdev, last_pte, 1048 last_dst, count, 1049 RADEON_GPU_PAGE_SIZE, 1050 flags); 1051 } 1052 1053 count = nptes; 1054 last_pte = pte; 1055 last_dst = dst; 1056 } else { 1057 count += nptes; 1058 } 1059 1060 addr += nptes; 1061 dst += nptes * RADEON_GPU_PAGE_SIZE; 1062 } 1063 1064 if (count) { 1065 radeon_asic_vm_set_page(rdev, last_pte, last_dst, count, 1066 RADEON_GPU_PAGE_SIZE, flags); 1067 } 1068 } 1069 1070 /** 1071 * radeon_vm_bo_update_pte - map a bo into the vm page table 1072 * 1073 * @rdev: radeon_device pointer 1074 * @vm: requested vm 1075 * @bo: radeon buffer object 1076 * @mem: ttm mem 1077 * 1078 * Fill in the page table entries for @bo (cayman+). 1079 * Returns 0 for success, -EINVAL for failure. 1080 * 1081 * Object have to be reserved & global and local mutex must be locked! 1082 */ 1083 int radeon_vm_bo_update_pte(struct radeon_device *rdev, 1084 struct radeon_vm *vm, 1085 struct radeon_bo *bo, 1086 struct ttm_mem_reg *mem) 1087 { 1088 unsigned ridx = rdev->asic->vm.pt_ring_index; 1089 struct radeon_ring *ring = &rdev->ring[ridx]; 1090 struct radeon_semaphore *sem = NULL; 1091 struct radeon_bo_va *bo_va; 1092 unsigned nptes, npdes, ndw; 1093 uint64_t addr; 1094 int r; 1095 1096 /* nothing to do if vm isn't bound */ 1097 if (vm->page_directory == NULL) 1098 return 0; 1099 1100 bo_va = radeon_vm_bo_find(vm, bo); 1101 if (bo_va == NULL) { 1102 dev_err(rdev->dev, "bo %p not in vm %p\n", bo, vm); 1103 return -EINVAL; 1104 } 1105 1106 if (!bo_va->soffset) { 1107 dev_err(rdev->dev, "bo %p don't has a mapping in vm %p\n", 1108 bo, vm); 1109 return -EINVAL; 1110 } 1111 1112 if ((bo_va->valid && mem) || (!bo_va->valid && mem == NULL)) 1113 return 0; 1114 1115 bo_va->flags &= ~RADEON_VM_PAGE_VALID; 1116 bo_va->flags &= ~RADEON_VM_PAGE_SYSTEM; 1117 if (mem) { 1118 addr = mem->start << PAGE_SHIFT; 1119 if (mem->mem_type != TTM_PL_SYSTEM) { 1120 bo_va->flags |= RADEON_VM_PAGE_VALID; 1121 bo_va->valid = true; 1122 } 1123 if (mem->mem_type == TTM_PL_TT) { 1124 bo_va->flags |= RADEON_VM_PAGE_SYSTEM; 1125 } else { 1126 addr += rdev->vm_manager.vram_base_offset; 1127 } 1128 } else { 1129 addr = 0; 1130 bo_va->valid = false; 1131 } 1132 1133 if (vm->fence && radeon_fence_signaled(vm->fence)) { 1134 radeon_fence_unref(&vm->fence); 1135 } 1136 1137 if (vm->fence && vm->fence->ring != ridx) { 1138 r = radeon_semaphore_create(rdev, &sem); 1139 if (r) { 1140 return r; 1141 } 1142 } 1143 1144 nptes = radeon_bo_ngpu_pages(bo); 1145 1146 /* assume two extra pdes in case the mapping overlaps the borders */ 1147 npdes = (nptes >> RADEON_VM_BLOCK_SIZE) + 2; 1148 1149 /* estimate number of dw needed */ 1150 /* semaphore, fence and padding */ 1151 ndw = 32; 1152 1153 if (RADEON_VM_BLOCK_SIZE > 11) 1154 /* reserve space for one header for every 2k dwords */ 1155 ndw += (nptes >> 11) * 4; 1156 else 1157 /* reserve space for one header for 1158 every (1 << BLOCK_SIZE) entries */ 1159 ndw += (nptes >> RADEON_VM_BLOCK_SIZE) * 4; 1160 1161 /* reserve space for pte addresses */ 1162 ndw += nptes * 2; 1163 1164 /* reserve space for one header for every 2k dwords */ 1165 ndw += (npdes >> 11) * 4; 1166 1167 /* reserve space for pde addresses */ 1168 ndw += npdes * 2; 1169 1170 r = radeon_ring_lock(rdev, ring, ndw); 1171 if (r) { 1172 return r; 1173 } 1174 1175 if (sem && radeon_fence_need_sync(vm->fence, ridx)) { 1176 radeon_semaphore_sync_rings(rdev, sem, vm->fence->ring, ridx); 1177 radeon_fence_note_sync(vm->fence, ridx); 1178 } 1179 1180 r = radeon_vm_update_pdes(rdev, vm, bo_va->soffset, bo_va->eoffset); 1181 if (r) { 1182 radeon_ring_unlock_undo(rdev, ring); 1183 return r; 1184 } 1185 1186 radeon_vm_update_ptes(rdev, vm, bo_va->soffset, bo_va->eoffset, 1187 addr, bo_va->flags); 1188 1189 radeon_fence_unref(&vm->fence); 1190 r = radeon_fence_emit(rdev, &vm->fence, ridx); 1191 if (r) { 1192 radeon_ring_unlock_undo(rdev, ring); 1193 return r; 1194 } 1195 radeon_ring_unlock_commit(rdev, ring); 1196 radeon_semaphore_free(rdev, &sem, vm->fence); 1197 radeon_fence_unref(&vm->last_flush); 1198 1199 return 0; 1200 } 1201 1202 /** 1203 * radeon_vm_bo_rmv - remove a bo to a specific vm 1204 * 1205 * @rdev: radeon_device pointer 1206 * @bo_va: requested bo_va 1207 * 1208 * Remove @bo_va->bo from the requested vm (cayman+). 1209 * Remove @bo_va->bo from the list of bos associated with the bo_va->vm and 1210 * remove the ptes for @bo_va in the page table. 1211 * Returns 0 for success. 1212 * 1213 * Object have to be reserved! 1214 */ 1215 int radeon_vm_bo_rmv(struct radeon_device *rdev, 1216 struct radeon_bo_va *bo_va) 1217 { 1218 int r; 1219 1220 lockmgr(&rdev->vm_manager.lock, LK_EXCLUSIVE); 1221 lockmgr(&bo_va->vm->mutex, LK_EXCLUSIVE); 1222 r = radeon_vm_bo_update_pte(rdev, bo_va->vm, bo_va->bo, NULL); 1223 lockmgr(&rdev->vm_manager.lock, LK_RELEASE); 1224 list_del(&bo_va->vm_list); 1225 lockmgr(&bo_va->vm->mutex, LK_RELEASE); 1226 list_del(&bo_va->bo_list); 1227 1228 drm_free(bo_va, DRM_MEM_DRIVER); 1229 return r; 1230 } 1231 1232 /** 1233 * radeon_vm_bo_invalidate - mark the bo as invalid 1234 * 1235 * @rdev: radeon_device pointer 1236 * @vm: requested vm 1237 * @bo: radeon buffer object 1238 * 1239 * Mark @bo as invalid (cayman+). 1240 */ 1241 void radeon_vm_bo_invalidate(struct radeon_device *rdev, 1242 struct radeon_bo *bo) 1243 { 1244 struct radeon_bo_va *bo_va; 1245 1246 list_for_each_entry(bo_va, &bo->va, bo_list) { 1247 bo_va->valid = false; 1248 } 1249 } 1250 1251 /** 1252 * radeon_vm_init - initialize a vm instance 1253 * 1254 * @rdev: radeon_device pointer 1255 * @vm: requested vm 1256 * 1257 * Init @vm fields (cayman+). 1258 */ 1259 void radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm) 1260 { 1261 vm->id = 0; 1262 vm->fence = NULL; 1263 lockinit(&vm->mutex, "rvmmtx", 0, LK_CANRECURSE); 1264 INIT_LIST_HEAD(&vm->list); 1265 INIT_LIST_HEAD(&vm->va); 1266 } 1267 1268 /** 1269 * radeon_vm_fini - tear down a vm instance 1270 * 1271 * @rdev: radeon_device pointer 1272 * @vm: requested vm 1273 * 1274 * Tear down @vm (cayman+). 1275 * Unbind the VM and remove all bos from the vm bo list 1276 */ 1277 void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm) 1278 { 1279 struct radeon_bo_va *bo_va, *tmp; 1280 int r; 1281 1282 lockmgr(&rdev->vm_manager.lock, LK_EXCLUSIVE); 1283 lockmgr(&vm->mutex, LK_EXCLUSIVE); 1284 radeon_vm_free_pt(rdev, vm); 1285 lockmgr(&rdev->vm_manager.lock, LK_RELEASE); 1286 1287 if (!list_empty(&vm->va)) { 1288 dev_err(rdev->dev, "still active bo inside vm\n"); 1289 } 1290 list_for_each_entry_safe(bo_va, tmp, &vm->va, vm_list) { 1291 list_del_init(&bo_va->vm_list); 1292 r = radeon_bo_reserve(bo_va->bo, false); 1293 if (!r) { 1294 list_del_init(&bo_va->bo_list); 1295 radeon_bo_unreserve(bo_va->bo); 1296 drm_free(bo_va, DRM_MEM_DRIVER); 1297 } 1298 } 1299 radeon_fence_unref(&vm->fence); 1300 radeon_fence_unref(&vm->last_flush); 1301 lockmgr(&vm->mutex, LK_RELEASE); 1302 } 1303