1 /* 2 * Copyright © 2010 Daniel Vetter 3 * Copyright © 2011-2014 Intel Corporation 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 22 * IN THE SOFTWARE. 23 * 24 */ 25 26 #include <linux/seq_file.h> 27 #include <drm/drmP.h> 28 #include <drm/i915_drm.h> 29 #include "i915_drv.h" 30 #include "i915_vgpu.h" 31 #include "i915_trace.h" 32 #include "intel_drv.h" 33 34 #include <linux/bitmap.h> 35 #include <linux/highmem.h> 36 37 /** 38 * DOC: Global GTT views 39 * 40 * Background and previous state 41 * 42 * Historically objects could exists (be bound) in global GTT space only as 43 * singular instances with a view representing all of the object's backing pages 44 * in a linear fashion. This view will be called a normal view. 45 * 46 * To support multiple views of the same object, where the number of mapped 47 * pages is not equal to the backing store, or where the layout of the pages 48 * is not linear, concept of a GGTT view was added. 49 * 50 * One example of an alternative view is a stereo display driven by a single 51 * image. In this case we would have a framebuffer looking like this 52 * (2x2 pages): 53 * 54 * 12 55 * 34 56 * 57 * Above would represent a normal GGTT view as normally mapped for GPU or CPU 58 * rendering. In contrast, fed to the display engine would be an alternative 59 * view which could look something like this: 60 * 61 * 1212 62 * 3434 63 * 64 * In this example both the size and layout of pages in the alternative view is 65 * different from the normal view. 66 * 67 * Implementation and usage 68 * 69 * GGTT views are implemented using VMAs and are distinguished via enum 70 * i915_ggtt_view_type and struct i915_ggtt_view. 71 * 72 * A new flavour of core GEM functions which work with GGTT bound objects were 73 * added with the _ggtt_ infix, and sometimes with _view postfix to avoid 74 * renaming in large amounts of code. They take the struct i915_ggtt_view 75 * parameter encapsulating all metadata required to implement a view. 76 * 77 * As a helper for callers which are only interested in the normal view, 78 * globally const i915_ggtt_view_normal singleton instance exists. All old core 79 * GEM API functions, the ones not taking the view parameter, are operating on, 80 * or with the normal GGTT view. 81 * 82 * Code wanting to add or use a new GGTT view needs to: 83 * 84 * 1. Add a new enum with a suitable name. 85 * 2. Extend the metadata in the i915_ggtt_view structure if required. 86 * 3. Add support to i915_get_vma_pages(). 87 * 88 * New views are required to build a scatter-gather table from within the 89 * i915_get_vma_pages function. This table is stored in the vma.ggtt_view and 90 * exists for the lifetime of an VMA. 91 * 92 * Core API is designed to have copy semantics which means that passed in 93 * struct i915_ggtt_view does not need to be persistent (left around after 94 * calling the core API functions). 95 * 96 */ 97 98 static int 99 i915_get_ggtt_vma_pages(struct i915_vma *vma); 100 101 const struct i915_ggtt_view i915_ggtt_view_normal; 102 const struct i915_ggtt_view i915_ggtt_view_rotated = { 103 .type = I915_GGTT_VIEW_ROTATED 104 }; 105 106 static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt) 107 { 108 bool has_aliasing_ppgtt; 109 bool has_full_ppgtt; 110 111 has_aliasing_ppgtt = INTEL_INFO(dev)->gen >= 6; 112 has_full_ppgtt = INTEL_INFO(dev)->gen >= 7; 113 114 if (intel_vgpu_active(dev)) 115 has_full_ppgtt = false; /* emulation is too hard */ 116 117 /* 118 * We don't allow disabling PPGTT for gen9+ as it's a requirement for 119 * execlists, the sole mechanism available to submit work. 120 */ 121 if (INTEL_INFO(dev)->gen < 9 && 122 (enable_ppgtt == 0 || !has_aliasing_ppgtt)) 123 return 0; 124 125 if (enable_ppgtt == 1) 126 return 1; 127 128 if (enable_ppgtt == 2 && has_full_ppgtt) 129 return 2; 130 131 #ifdef CONFIG_INTEL_IOMMU 132 /* Disable ppgtt on SNB if VT-d is on. */ 133 if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) { 134 DRM_INFO("Disabling PPGTT because VT-d is on\n"); 135 return 0; 136 } 137 #endif 138 139 /* Early VLV doesn't have this */ 140 if (IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) && 141 dev->pdev->revision < 0xb) { 142 DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n"); 143 return 0; 144 } 145 146 if (INTEL_INFO(dev)->gen >= 8 && i915.enable_execlists) 147 return 2; 148 else 149 return has_aliasing_ppgtt ? 1 : 0; 150 } 151 152 static int ppgtt_bind_vma(struct i915_vma *vma, 153 enum i915_cache_level cache_level, 154 u32 unused) 155 { 156 u32 pte_flags = 0; 157 158 /* Currently applicable only to VLV */ 159 if (vma->obj->gt_ro) 160 pte_flags |= PTE_READ_ONLY; 161 162 vma->vm->insert_entries(vma->vm, vma->obj->pages, vma->node.start, 163 cache_level, pte_flags); 164 165 return 0; 166 } 167 168 static void ppgtt_unbind_vma(struct i915_vma *vma) 169 { 170 vma->vm->clear_range(vma->vm, 171 vma->node.start, 172 vma->obj->base.size, 173 true); 174 } 175 176 static gen8_pte_t gen8_pte_encode(dma_addr_t addr, 177 enum i915_cache_level level, 178 bool valid) 179 { 180 gen8_pte_t pte = valid ? _PAGE_PRESENT | _PAGE_RW : 0; 181 pte |= addr; 182 183 switch (level) { 184 case I915_CACHE_NONE: 185 pte |= PPAT_UNCACHED_INDEX; 186 break; 187 case I915_CACHE_WT: 188 pte |= PPAT_DISPLAY_ELLC_INDEX; 189 break; 190 default: 191 pte |= PPAT_CACHED_INDEX; 192 break; 193 } 194 195 return pte; 196 } 197 198 static gen8_pde_t gen8_pde_encode(const dma_addr_t addr, 199 const enum i915_cache_level level) 200 { 201 gen8_pde_t pde = _PAGE_PRESENT | _PAGE_RW; 202 pde |= addr; 203 if (level != I915_CACHE_NONE) 204 pde |= PPAT_CACHED_PDE_INDEX; 205 else 206 pde |= PPAT_UNCACHED_INDEX; 207 return pde; 208 } 209 210 static gen6_pte_t snb_pte_encode(dma_addr_t addr, 211 enum i915_cache_level level, 212 bool valid, u32 unused) 213 { 214 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; 215 pte |= GEN6_PTE_ADDR_ENCODE(addr); 216 217 switch (level) { 218 case I915_CACHE_L3_LLC: 219 case I915_CACHE_LLC: 220 pte |= GEN6_PTE_CACHE_LLC; 221 break; 222 case I915_CACHE_NONE: 223 pte |= GEN6_PTE_UNCACHED; 224 break; 225 default: 226 MISSING_CASE(level); 227 } 228 229 return pte; 230 } 231 232 static gen6_pte_t ivb_pte_encode(dma_addr_t addr, 233 enum i915_cache_level level, 234 bool valid, u32 unused) 235 { 236 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; 237 pte |= GEN6_PTE_ADDR_ENCODE(addr); 238 239 switch (level) { 240 case I915_CACHE_L3_LLC: 241 pte |= GEN7_PTE_CACHE_L3_LLC; 242 break; 243 case I915_CACHE_LLC: 244 pte |= GEN6_PTE_CACHE_LLC; 245 break; 246 case I915_CACHE_NONE: 247 pte |= GEN6_PTE_UNCACHED; 248 break; 249 default: 250 MISSING_CASE(level); 251 } 252 253 return pte; 254 } 255 256 static gen6_pte_t byt_pte_encode(dma_addr_t addr, 257 enum i915_cache_level level, 258 bool valid, u32 flags) 259 { 260 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; 261 pte |= GEN6_PTE_ADDR_ENCODE(addr); 262 263 if (!(flags & PTE_READ_ONLY)) 264 pte |= BYT_PTE_WRITEABLE; 265 266 if (level != I915_CACHE_NONE) 267 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES; 268 269 return pte; 270 } 271 272 static gen6_pte_t hsw_pte_encode(dma_addr_t addr, 273 enum i915_cache_level level, 274 bool valid, u32 unused) 275 { 276 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; 277 pte |= HSW_PTE_ADDR_ENCODE(addr); 278 279 if (level != I915_CACHE_NONE) 280 pte |= HSW_WB_LLC_AGE3; 281 282 return pte; 283 } 284 285 static gen6_pte_t iris_pte_encode(dma_addr_t addr, 286 enum i915_cache_level level, 287 bool valid, u32 unused) 288 { 289 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; 290 pte |= HSW_PTE_ADDR_ENCODE(addr); 291 292 switch (level) { 293 case I915_CACHE_NONE: 294 break; 295 case I915_CACHE_WT: 296 pte |= HSW_WT_ELLC_LLC_AGE3; 297 break; 298 default: 299 pte |= HSW_WB_ELLC_LLC_AGE3; 300 break; 301 } 302 303 return pte; 304 } 305 306 static int __setup_page_dma(struct drm_device *dev, 307 struct i915_page_dma *p, gfp_t flags) 308 { 309 struct device *device = dev->pdev->dev; 310 311 p->page = alloc_page(flags); 312 if (!p->page) 313 return -ENOMEM; 314 315 p->daddr = dma_map_page(device, 316 p->page, 0, 4096, PCI_DMA_BIDIRECTIONAL); 317 318 if (dma_mapping_error(device, p->daddr)) { 319 __free_page(p->page); 320 return -EINVAL; 321 } 322 323 return 0; 324 } 325 326 static int setup_page_dma(struct drm_device *dev, struct i915_page_dma *p) 327 { 328 return __setup_page_dma(dev, p, GFP_KERNEL); 329 } 330 331 static void cleanup_page_dma(struct drm_device *dev, struct i915_page_dma *p) 332 { 333 if (WARN_ON(!p->page)) 334 return; 335 336 dma_unmap_page(dev->pdev->dev, p->daddr, 4096, PCI_DMA_BIDIRECTIONAL); 337 __free_page(p->page); 338 memset(p, 0, sizeof(*p)); 339 } 340 341 static void *kmap_page_dma(struct i915_page_dma *p) 342 { 343 return kmap_atomic(p->page); 344 } 345 346 /* We use the flushing unmap only with ppgtt structures: 347 * page directories, page tables and scratch pages. 348 */ 349 static void kunmap_page_dma(struct drm_device *dev, void *vaddr) 350 { 351 /* There are only few exceptions for gen >=6. chv and bxt. 352 * And we are not sure about the latter so play safe for now. 353 */ 354 if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev)) 355 drm_clflush_virt_range(vaddr, PAGE_SIZE); 356 357 kunmap_atomic(vaddr); 358 } 359 360 #define kmap_px(px) kmap_page_dma(px_base(px)) 361 #define kunmap_px(ppgtt, vaddr) kunmap_page_dma((ppgtt)->base.dev, (vaddr)) 362 363 #define setup_px(dev, px) setup_page_dma((dev), px_base(px)) 364 #define cleanup_px(dev, px) cleanup_page_dma((dev), px_base(px)) 365 #define fill_px(dev, px, v) fill_page_dma((dev), px_base(px), (v)) 366 #define fill32_px(dev, px, v) fill_page_dma_32((dev), px_base(px), (v)) 367 368 static void fill_page_dma(struct drm_device *dev, struct i915_page_dma *p, 369 const uint64_t val) 370 { 371 int i; 372 uint64_t * const vaddr = kmap_page_dma(p); 373 374 for (i = 0; i < 512; i++) 375 vaddr[i] = val; 376 377 kunmap_page_dma(dev, vaddr); 378 } 379 380 static void fill_page_dma_32(struct drm_device *dev, struct i915_page_dma *p, 381 const uint32_t val32) 382 { 383 uint64_t v = val32; 384 385 v = v << 32 | val32; 386 387 fill_page_dma(dev, p, v); 388 } 389 390 static struct i915_page_scratch *alloc_scratch_page(struct drm_device *dev) 391 { 392 struct i915_page_scratch *sp; 393 int ret; 394 395 sp = kzalloc(sizeof(*sp), GFP_KERNEL); 396 if (sp == NULL) 397 return ERR_PTR(-ENOMEM); 398 399 ret = __setup_page_dma(dev, px_base(sp), GFP_DMA32 | __GFP_ZERO); 400 if (ret) { 401 kfree(sp); 402 return ERR_PTR(ret); 403 } 404 405 set_pages_uc(px_page(sp), 1); 406 407 return sp; 408 } 409 410 static void free_scratch_page(struct drm_device *dev, 411 struct i915_page_scratch *sp) 412 { 413 set_pages_wb(px_page(sp), 1); 414 415 cleanup_px(dev, sp); 416 kfree(sp); 417 } 418 419 static struct i915_page_table *alloc_pt(struct drm_device *dev) 420 { 421 struct i915_page_table *pt; 422 const size_t count = INTEL_INFO(dev)->gen >= 8 ? 423 GEN8_PTES : GEN6_PTES; 424 int ret = -ENOMEM; 425 426 pt = kzalloc(sizeof(*pt), GFP_KERNEL); 427 if (!pt) 428 return ERR_PTR(-ENOMEM); 429 430 pt->used_ptes = kcalloc(BITS_TO_LONGS(count), sizeof(*pt->used_ptes), 431 GFP_KERNEL); 432 433 if (!pt->used_ptes) 434 goto fail_bitmap; 435 436 ret = setup_px(dev, pt); 437 if (ret) 438 goto fail_page_m; 439 440 return pt; 441 442 fail_page_m: 443 kfree(pt->used_ptes); 444 fail_bitmap: 445 kfree(pt); 446 447 return ERR_PTR(ret); 448 } 449 450 static void free_pt(struct drm_device *dev, struct i915_page_table *pt) 451 { 452 cleanup_px(dev, pt); 453 kfree(pt->used_ptes); 454 kfree(pt); 455 } 456 457 /** 458 * alloc_pt_range() - Allocate a multiple page tables 459 * @pd: The page directory which will have at least @count entries 460 * available to point to the allocated page tables. 461 * @pde: First page directory entry for which we are allocating. 462 * @count: Number of pages to allocate. 463 * @dev: DRM device. 464 * 465 * Allocates multiple page table pages and sets the appropriate entries in the 466 * page table structure within the page directory. Function cleans up after 467 * itself on any failures. 468 * 469 * Return: 0 if allocation succeeded. 470 */ 471 static int alloc_pt_range(struct i915_page_directory *pd, uint16_t pde, size_t count, 472 struct drm_device *dev) 473 { 474 int i, ret; 475 476 /* 512 is the max page tables per page_directory on any platform. */ 477 if (WARN_ON(pde + count > I915_PDES)) 478 return -EINVAL; 479 480 for (i = pde; i < pde + count; i++) { 481 struct i915_page_table *pt = alloc_pt(dev); 482 483 if (IS_ERR(pt)) { 484 ret = PTR_ERR(pt); 485 goto err_out; 486 } 487 WARN(pd->page_table[i], 488 "Leaking page directory entry %d (%p)\n", 489 i, pd->page_table[i]); 490 pd->page_table[i] = pt; 491 } 492 493 return 0; 494 495 err_out: 496 while (i-- > pde) 497 free_pt(dev, pd->page_table[i]); 498 return ret; 499 } 500 501 static void gen8_initialize_pt(struct i915_address_space *vm, 502 struct i915_page_table *pt) 503 { 504 gen8_pte_t scratch_pte; 505 506 scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), 507 I915_CACHE_LLC, true); 508 509 fill_px(vm->dev, pt, scratch_pte); 510 } 511 512 static void gen6_initialize_pt(struct i915_address_space *vm, 513 struct i915_page_table *pt) 514 { 515 gen6_pte_t scratch_pte; 516 517 WARN_ON(px_dma(vm->scratch_page) == 0); 518 519 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), 520 I915_CACHE_LLC, true, 0); 521 522 fill32_px(vm->dev, pt, scratch_pte); 523 } 524 525 static struct i915_page_directory *alloc_pd(struct drm_device *dev) 526 { 527 struct i915_page_directory *pd; 528 int ret = -ENOMEM; 529 530 pd = kzalloc(sizeof(*pd), GFP_KERNEL); 531 if (!pd) 532 return ERR_PTR(-ENOMEM); 533 534 pd->used_pdes = kcalloc(BITS_TO_LONGS(I915_PDES), 535 sizeof(*pd->used_pdes), GFP_KERNEL); 536 if (!pd->used_pdes) 537 goto fail_bitmap; 538 539 ret = setup_px(dev, pd); 540 if (ret) 541 goto fail_page_m; 542 543 return pd; 544 545 fail_page_m: 546 kfree(pd->used_pdes); 547 fail_bitmap: 548 kfree(pd); 549 550 return ERR_PTR(ret); 551 } 552 553 static void free_pd(struct drm_device *dev, struct i915_page_directory *pd) 554 { 555 if (px_page(pd)) { 556 cleanup_px(dev, pd); 557 kfree(pd->used_pdes); 558 kfree(pd); 559 } 560 } 561 562 static void gen8_initialize_pd(struct i915_address_space *vm, 563 struct i915_page_directory *pd) 564 { 565 gen8_pde_t scratch_pde; 566 567 scratch_pde = gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC); 568 569 fill_px(vm->dev, pd, scratch_pde); 570 } 571 572 /* Broadwell Page Directory Pointer Descriptors */ 573 static int gen8_write_pdp(struct drm_i915_gem_request *req, 574 unsigned entry, 575 dma_addr_t addr) 576 { 577 struct intel_engine_cs *ring = req->ring; 578 int ret; 579 580 BUG_ON(entry >= 4); 581 582 ret = intel_ring_begin(req, 6); 583 if (ret) 584 return ret; 585 586 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); 587 intel_ring_emit(ring, GEN8_RING_PDP_UDW(ring, entry)); 588 intel_ring_emit(ring, upper_32_bits(addr)); 589 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); 590 intel_ring_emit(ring, GEN8_RING_PDP_LDW(ring, entry)); 591 intel_ring_emit(ring, lower_32_bits(addr)); 592 intel_ring_advance(ring); 593 594 return 0; 595 } 596 597 static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt, 598 struct drm_i915_gem_request *req) 599 { 600 int i, ret; 601 602 for (i = GEN8_LEGACY_PDPES - 1; i >= 0; i--) { 603 const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i); 604 605 ret = gen8_write_pdp(req, i, pd_daddr); 606 if (ret) 607 return ret; 608 } 609 610 return 0; 611 } 612 613 static void gen8_ppgtt_clear_range(struct i915_address_space *vm, 614 uint64_t start, 615 uint64_t length, 616 bool use_scratch) 617 { 618 struct i915_hw_ppgtt *ppgtt = 619 container_of(vm, struct i915_hw_ppgtt, base); 620 gen8_pte_t *pt_vaddr, scratch_pte; 621 unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK; 622 unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK; 623 unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK; 624 unsigned num_entries = length >> PAGE_SHIFT; 625 unsigned last_pte, i; 626 627 scratch_pte = gen8_pte_encode(px_dma(ppgtt->base.scratch_page), 628 I915_CACHE_LLC, use_scratch); 629 630 while (num_entries) { 631 struct i915_page_directory *pd; 632 struct i915_page_table *pt; 633 634 if (WARN_ON(!ppgtt->pdp.page_directory[pdpe])) 635 continue; 636 637 pd = ppgtt->pdp.page_directory[pdpe]; 638 639 if (WARN_ON(!pd->page_table[pde])) 640 continue; 641 642 pt = pd->page_table[pde]; 643 644 if (WARN_ON(!px_page(pt))) 645 continue; 646 647 last_pte = pte + num_entries; 648 if (last_pte > GEN8_PTES) 649 last_pte = GEN8_PTES; 650 651 pt_vaddr = kmap_px(pt); 652 653 for (i = pte; i < last_pte; i++) { 654 pt_vaddr[i] = scratch_pte; 655 num_entries--; 656 } 657 658 kunmap_px(ppgtt, pt); 659 660 pte = 0; 661 if (++pde == I915_PDES) { 662 pdpe++; 663 pde = 0; 664 } 665 } 666 } 667 668 static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, 669 struct sg_table *pages, 670 uint64_t start, 671 enum i915_cache_level cache_level, u32 unused) 672 { 673 struct i915_hw_ppgtt *ppgtt = 674 container_of(vm, struct i915_hw_ppgtt, base); 675 gen8_pte_t *pt_vaddr; 676 unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK; 677 unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK; 678 unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK; 679 struct sg_page_iter sg_iter; 680 681 pt_vaddr = NULL; 682 683 for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) { 684 if (WARN_ON(pdpe >= GEN8_LEGACY_PDPES)) 685 break; 686 687 if (pt_vaddr == NULL) { 688 struct i915_page_directory *pd = ppgtt->pdp.page_directory[pdpe]; 689 struct i915_page_table *pt = pd->page_table[pde]; 690 691 pt_vaddr = kmap_px(pt); 692 } 693 694 pt_vaddr[pte] = 695 gen8_pte_encode(sg_page_iter_dma_address(&sg_iter), 696 cache_level, true); 697 if (++pte == GEN8_PTES) { 698 kunmap_px(ppgtt, pt_vaddr); 699 pt_vaddr = NULL; 700 if (++pde == I915_PDES) { 701 pdpe++; 702 pde = 0; 703 } 704 pte = 0; 705 } 706 } 707 708 if (pt_vaddr) 709 kunmap_px(ppgtt, pt_vaddr); 710 } 711 712 static void gen8_free_page_tables(struct drm_device *dev, 713 struct i915_page_directory *pd) 714 { 715 int i; 716 717 if (!px_page(pd)) 718 return; 719 720 for_each_set_bit(i, pd->used_pdes, I915_PDES) { 721 if (WARN_ON(!pd->page_table[i])) 722 continue; 723 724 free_pt(dev, pd->page_table[i]); 725 pd->page_table[i] = NULL; 726 } 727 } 728 729 static int gen8_init_scratch(struct i915_address_space *vm) 730 { 731 struct drm_device *dev = vm->dev; 732 733 vm->scratch_page = alloc_scratch_page(dev); 734 if (IS_ERR(vm->scratch_page)) 735 return PTR_ERR(vm->scratch_page); 736 737 vm->scratch_pt = alloc_pt(dev); 738 if (IS_ERR(vm->scratch_pt)) { 739 free_scratch_page(dev, vm->scratch_page); 740 return PTR_ERR(vm->scratch_pt); 741 } 742 743 vm->scratch_pd = alloc_pd(dev); 744 if (IS_ERR(vm->scratch_pd)) { 745 free_pt(dev, vm->scratch_pt); 746 free_scratch_page(dev, vm->scratch_page); 747 return PTR_ERR(vm->scratch_pd); 748 } 749 750 gen8_initialize_pt(vm, vm->scratch_pt); 751 gen8_initialize_pd(vm, vm->scratch_pd); 752 753 return 0; 754 } 755 756 static void gen8_free_scratch(struct i915_address_space *vm) 757 { 758 struct drm_device *dev = vm->dev; 759 760 free_pd(dev, vm->scratch_pd); 761 free_pt(dev, vm->scratch_pt); 762 free_scratch_page(dev, vm->scratch_page); 763 } 764 765 static void gen8_ppgtt_cleanup(struct i915_address_space *vm) 766 { 767 struct i915_hw_ppgtt *ppgtt = 768 container_of(vm, struct i915_hw_ppgtt, base); 769 int i; 770 771 for_each_set_bit(i, ppgtt->pdp.used_pdpes, GEN8_LEGACY_PDPES) { 772 if (WARN_ON(!ppgtt->pdp.page_directory[i])) 773 continue; 774 775 gen8_free_page_tables(ppgtt->base.dev, 776 ppgtt->pdp.page_directory[i]); 777 free_pd(ppgtt->base.dev, ppgtt->pdp.page_directory[i]); 778 } 779 780 gen8_free_scratch(vm); 781 } 782 783 /** 784 * gen8_ppgtt_alloc_pagetabs() - Allocate page tables for VA range. 785 * @ppgtt: Master ppgtt structure. 786 * @pd: Page directory for this address range. 787 * @start: Starting virtual address to begin allocations. 788 * @length Size of the allocations. 789 * @new_pts: Bitmap set by function with new allocations. Likely used by the 790 * caller to free on error. 791 * 792 * Allocate the required number of page tables. Extremely similar to 793 * gen8_ppgtt_alloc_page_directories(). The main difference is here we are limited by 794 * the page directory boundary (instead of the page directory pointer). That 795 * boundary is 1GB virtual. Therefore, unlike gen8_ppgtt_alloc_page_directories(), it is 796 * possible, and likely that the caller will need to use multiple calls of this 797 * function to achieve the appropriate allocation. 798 * 799 * Return: 0 if success; negative error code otherwise. 800 */ 801 static int gen8_ppgtt_alloc_pagetabs(struct i915_hw_ppgtt *ppgtt, 802 struct i915_page_directory *pd, 803 uint64_t start, 804 uint64_t length, 805 unsigned long *new_pts) 806 { 807 struct drm_device *dev = ppgtt->base.dev; 808 struct i915_page_table *pt; 809 uint64_t temp; 810 uint32_t pde; 811 812 gen8_for_each_pde(pt, pd, start, length, temp, pde) { 813 /* Don't reallocate page tables */ 814 if (pt) { 815 /* Scratch is never allocated this way */ 816 WARN_ON(pt == ppgtt->base.scratch_pt); 817 continue; 818 } 819 820 pt = alloc_pt(dev); 821 if (IS_ERR(pt)) 822 goto unwind_out; 823 824 gen8_initialize_pt(&ppgtt->base, pt); 825 pd->page_table[pde] = pt; 826 __set_bit(pde, new_pts); 827 } 828 829 return 0; 830 831 unwind_out: 832 for_each_set_bit(pde, new_pts, I915_PDES) 833 free_pt(dev, pd->page_table[pde]); 834 835 return -ENOMEM; 836 } 837 838 /** 839 * gen8_ppgtt_alloc_page_directories() - Allocate page directories for VA range. 840 * @ppgtt: Master ppgtt structure. 841 * @pdp: Page directory pointer for this address range. 842 * @start: Starting virtual address to begin allocations. 843 * @length Size of the allocations. 844 * @new_pds Bitmap set by function with new allocations. Likely used by the 845 * caller to free on error. 846 * 847 * Allocate the required number of page directories starting at the pde index of 848 * @start, and ending at the pde index @start + @length. This function will skip 849 * over already allocated page directories within the range, and only allocate 850 * new ones, setting the appropriate pointer within the pdp as well as the 851 * correct position in the bitmap @new_pds. 852 * 853 * The function will only allocate the pages within the range for a give page 854 * directory pointer. In other words, if @start + @length straddles a virtually 855 * addressed PDP boundary (512GB for 4k pages), there will be more allocations 856 * required by the caller, This is not currently possible, and the BUG in the 857 * code will prevent it. 858 * 859 * Return: 0 if success; negative error code otherwise. 860 */ 861 static int gen8_ppgtt_alloc_page_directories(struct i915_hw_ppgtt *ppgtt, 862 struct i915_page_directory_pointer *pdp, 863 uint64_t start, 864 uint64_t length, 865 unsigned long *new_pds) 866 { 867 struct drm_device *dev = ppgtt->base.dev; 868 struct i915_page_directory *pd; 869 uint64_t temp; 870 uint32_t pdpe; 871 872 WARN_ON(!bitmap_empty(new_pds, GEN8_LEGACY_PDPES)); 873 874 gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) { 875 if (pd) 876 continue; 877 878 pd = alloc_pd(dev); 879 if (IS_ERR(pd)) 880 goto unwind_out; 881 882 gen8_initialize_pd(&ppgtt->base, pd); 883 pdp->page_directory[pdpe] = pd; 884 __set_bit(pdpe, new_pds); 885 } 886 887 return 0; 888 889 unwind_out: 890 for_each_set_bit(pdpe, new_pds, GEN8_LEGACY_PDPES) 891 free_pd(dev, pdp->page_directory[pdpe]); 892 893 return -ENOMEM; 894 } 895 896 static void 897 free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long **new_pts) 898 { 899 int i; 900 901 for (i = 0; i < GEN8_LEGACY_PDPES; i++) 902 kfree(new_pts[i]); 903 kfree(new_pts); 904 kfree(new_pds); 905 } 906 907 /* Fills in the page directory bitmap, and the array of page tables bitmap. Both 908 * of these are based on the number of PDPEs in the system. 909 */ 910 static 911 int __must_check alloc_gen8_temp_bitmaps(unsigned long **new_pds, 912 unsigned long ***new_pts) 913 { 914 int i; 915 unsigned long *pds; 916 unsigned long **pts; 917 918 pds = kcalloc(BITS_TO_LONGS(GEN8_LEGACY_PDPES), sizeof(unsigned long), GFP_KERNEL); 919 if (!pds) 920 return -ENOMEM; 921 922 pts = kcalloc(GEN8_LEGACY_PDPES, sizeof(unsigned long *), GFP_KERNEL); 923 if (!pts) { 924 kfree(pds); 925 return -ENOMEM; 926 } 927 928 for (i = 0; i < GEN8_LEGACY_PDPES; i++) { 929 pts[i] = kcalloc(BITS_TO_LONGS(I915_PDES), 930 sizeof(unsigned long), GFP_KERNEL); 931 if (!pts[i]) 932 goto err_out; 933 } 934 935 *new_pds = pds; 936 *new_pts = pts; 937 938 return 0; 939 940 err_out: 941 free_gen8_temp_bitmaps(pds, pts); 942 return -ENOMEM; 943 } 944 945 /* PDE TLBs are a pain to invalidate on GEN8+. When we modify 946 * the page table structures, we mark them dirty so that 947 * context switching/execlist queuing code takes extra steps 948 * to ensure that tlbs are flushed. 949 */ 950 static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt) 951 { 952 ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.dev)->ring_mask; 953 } 954 955 static int gen8_alloc_va_range(struct i915_address_space *vm, 956 uint64_t start, 957 uint64_t length) 958 { 959 struct i915_hw_ppgtt *ppgtt = 960 container_of(vm, struct i915_hw_ppgtt, base); 961 unsigned long *new_page_dirs, **new_page_tables; 962 struct i915_page_directory *pd; 963 const uint64_t orig_start = start; 964 const uint64_t orig_length = length; 965 uint64_t temp; 966 uint32_t pdpe; 967 int ret; 968 969 /* Wrap is never okay since we can only represent 48b, and we don't 970 * actually use the other side of the canonical address space. 971 */ 972 if (WARN_ON(start + length < start)) 973 return -ENODEV; 974 975 if (WARN_ON(start + length > ppgtt->base.total)) 976 return -ENODEV; 977 978 ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables); 979 if (ret) 980 return ret; 981 982 /* Do the allocations first so we can easily bail out */ 983 ret = gen8_ppgtt_alloc_page_directories(ppgtt, &ppgtt->pdp, start, length, 984 new_page_dirs); 985 if (ret) { 986 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); 987 return ret; 988 } 989 990 /* For every page directory referenced, allocate page tables */ 991 gen8_for_each_pdpe(pd, &ppgtt->pdp, start, length, temp, pdpe) { 992 ret = gen8_ppgtt_alloc_pagetabs(ppgtt, pd, start, length, 993 new_page_tables[pdpe]); 994 if (ret) 995 goto err_out; 996 } 997 998 start = orig_start; 999 length = orig_length; 1000 1001 /* Allocations have completed successfully, so set the bitmaps, and do 1002 * the mappings. */ 1003 gen8_for_each_pdpe(pd, &ppgtt->pdp, start, length, temp, pdpe) { 1004 gen8_pde_t *const page_directory = kmap_px(pd); 1005 struct i915_page_table *pt; 1006 uint64_t pd_len = gen8_clamp_pd(start, length); 1007 uint64_t pd_start = start; 1008 uint32_t pde; 1009 1010 /* Every pd should be allocated, we just did that above. */ 1011 WARN_ON(!pd); 1012 1013 gen8_for_each_pde(pt, pd, pd_start, pd_len, temp, pde) { 1014 /* Same reasoning as pd */ 1015 WARN_ON(!pt); 1016 WARN_ON(!pd_len); 1017 WARN_ON(!gen8_pte_count(pd_start, pd_len)); 1018 1019 /* Set our used ptes within the page table */ 1020 bitmap_set(pt->used_ptes, 1021 gen8_pte_index(pd_start), 1022 gen8_pte_count(pd_start, pd_len)); 1023 1024 /* Our pde is now pointing to the pagetable, pt */ 1025 __set_bit(pde, pd->used_pdes); 1026 1027 /* Map the PDE to the page table */ 1028 page_directory[pde] = gen8_pde_encode(px_dma(pt), 1029 I915_CACHE_LLC); 1030 1031 /* NB: We haven't yet mapped ptes to pages. At this 1032 * point we're still relying on insert_entries() */ 1033 } 1034 1035 kunmap_px(ppgtt, page_directory); 1036 1037 __set_bit(pdpe, ppgtt->pdp.used_pdpes); 1038 } 1039 1040 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); 1041 mark_tlbs_dirty(ppgtt); 1042 return 0; 1043 1044 err_out: 1045 while (pdpe--) { 1046 for_each_set_bit(temp, new_page_tables[pdpe], I915_PDES) 1047 free_pt(vm->dev, ppgtt->pdp.page_directory[pdpe]->page_table[temp]); 1048 } 1049 1050 for_each_set_bit(pdpe, new_page_dirs, GEN8_LEGACY_PDPES) 1051 free_pd(vm->dev, ppgtt->pdp.page_directory[pdpe]); 1052 1053 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); 1054 mark_tlbs_dirty(ppgtt); 1055 return ret; 1056 } 1057 1058 /* 1059 * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers 1060 * with a net effect resembling a 2-level page table in normal x86 terms. Each 1061 * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address 1062 * space. 1063 * 1064 */ 1065 static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt) 1066 { 1067 int ret; 1068 1069 ret = gen8_init_scratch(&ppgtt->base); 1070 if (ret) 1071 return ret; 1072 1073 ppgtt->base.start = 0; 1074 #ifndef __DragonFly__ 1075 ppgtt->base.total = 1ULL << 32; 1076 if (IS_ENABLED(CONFIG_X86_32)) 1077 /* While we have a proliferation of size_t variables 1078 * we cannot represent the full ppgtt size on 32bit, 1079 * so limit it to the same size as the GGTT (currently 1080 * 2GiB). 1081 */ 1082 #endif 1083 ppgtt->base.total = to_i915(ppgtt->base.dev)->gtt.base.total; 1084 ppgtt->base.cleanup = gen8_ppgtt_cleanup; 1085 ppgtt->base.allocate_va_range = gen8_alloc_va_range; 1086 ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; 1087 ppgtt->base.clear_range = gen8_ppgtt_clear_range; 1088 ppgtt->base.unbind_vma = ppgtt_unbind_vma; 1089 ppgtt->base.bind_vma = ppgtt_bind_vma; 1090 1091 ppgtt->switch_mm = gen8_mm_switch; 1092 1093 return 0; 1094 } 1095 1096 static int gen8_aliasing_ppgtt_init(struct i915_hw_ppgtt *ppgtt) 1097 { 1098 struct drm_device *dev = ppgtt->base.dev; 1099 struct drm_i915_private *dev_priv = dev->dev_private; 1100 uint64_t start = 0, size = dev_priv->gtt.base.total; 1101 int ret; 1102 1103 ret = gen8_ppgtt_init(ppgtt); 1104 if (ret) 1105 return ret; 1106 1107 /* Aliasing PPGTT has to always work and be mapped because of the way we 1108 * use RESTORE_INHIBIT in the context switch. This will be fixed 1109 * eventually. */ 1110 ret = gen8_alloc_va_range(&ppgtt->base, start, size); 1111 if (ret) { 1112 free_pd(ppgtt->base.dev, ppgtt->base.scratch_pd); 1113 free_pt(ppgtt->base.dev, ppgtt->base.scratch_pt); 1114 return ret; 1115 } 1116 1117 ppgtt->base.allocate_va_range = NULL; 1118 ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true); 1119 1120 return 0; 1121 } 1122 1123 static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) 1124 { 1125 struct i915_address_space *vm = &ppgtt->base; 1126 struct i915_page_table *unused; 1127 gen6_pte_t scratch_pte; 1128 uint32_t pd_entry; 1129 uint32_t pte, pde, temp; 1130 uint32_t start = ppgtt->base.start, length = ppgtt->base.total; 1131 1132 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), 1133 I915_CACHE_LLC, true, 0); 1134 1135 gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde) { 1136 u32 expected; 1137 gen6_pte_t *pt_vaddr; 1138 const dma_addr_t pt_addr = px_dma(ppgtt->pd.page_table[pde]); 1139 pd_entry = readl(ppgtt->pd_addr + pde); 1140 expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID); 1141 1142 if (pd_entry != expected) 1143 seq_printf(m, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n", 1144 pde, 1145 pd_entry, 1146 expected); 1147 seq_printf(m, "\tPDE: %x\n", pd_entry); 1148 1149 pt_vaddr = kmap_px(ppgtt->pd.page_table[pde]); 1150 1151 for (pte = 0; pte < GEN6_PTES; pte+=4) { 1152 unsigned long va = 1153 (pde * PAGE_SIZE * GEN6_PTES) + 1154 (pte * PAGE_SIZE); 1155 int i; 1156 bool found = false; 1157 for (i = 0; i < 4; i++) 1158 if (pt_vaddr[pte + i] != scratch_pte) 1159 found = true; 1160 if (!found) 1161 continue; 1162 1163 seq_printf(m, "\t\t0x%lx [%03d,%04d]: =", va, pde, pte); 1164 for (i = 0; i < 4; i++) { 1165 if (pt_vaddr[pte + i] != scratch_pte) 1166 seq_printf(m, " %08x", pt_vaddr[pte + i]); 1167 else 1168 seq_puts(m, " SCRATCH "); 1169 } 1170 seq_puts(m, "\n"); 1171 } 1172 kunmap_px(ppgtt, pt_vaddr); 1173 } 1174 } 1175 1176 /* Write pde (index) from the page directory @pd to the page table @pt */ 1177 static void gen6_write_pde(struct i915_page_directory *pd, 1178 const int pde, struct i915_page_table *pt) 1179 { 1180 /* Caller needs to make sure the write completes if necessary */ 1181 struct i915_hw_ppgtt *ppgtt = 1182 container_of(pd, struct i915_hw_ppgtt, pd); 1183 u32 pd_entry; 1184 1185 pd_entry = GEN6_PDE_ADDR_ENCODE(px_dma(pt)); 1186 pd_entry |= GEN6_PDE_VALID; 1187 1188 writel(pd_entry, ppgtt->pd_addr + pde); 1189 } 1190 1191 /* Write all the page tables found in the ppgtt structure to incrementing page 1192 * directories. */ 1193 static void gen6_write_page_range(struct drm_i915_private *dev_priv, 1194 struct i915_page_directory *pd, 1195 uint32_t start, uint32_t length) 1196 { 1197 struct i915_page_table *pt; 1198 uint32_t pde, temp; 1199 1200 gen6_for_each_pde(pt, pd, start, length, temp, pde) 1201 gen6_write_pde(pd, pde, pt); 1202 1203 /* Make sure write is complete before other code can use this page 1204 * table. Also require for WC mapped PTEs */ 1205 readl(dev_priv->gtt.gsm); 1206 } 1207 1208 static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt) 1209 { 1210 BUG_ON(ppgtt->pd.base.ggtt_offset & 0x3f); 1211 1212 return (ppgtt->pd.base.ggtt_offset / 64) << 16; 1213 } 1214 1215 static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt, 1216 struct drm_i915_gem_request *req) 1217 { 1218 struct intel_engine_cs *ring = req->ring; 1219 int ret; 1220 1221 /* NB: TLBs must be flushed and invalidated before a switch */ 1222 ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); 1223 if (ret) 1224 return ret; 1225 1226 ret = intel_ring_begin(req, 6); 1227 if (ret) 1228 return ret; 1229 1230 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2)); 1231 intel_ring_emit(ring, RING_PP_DIR_DCLV(ring)); 1232 intel_ring_emit(ring, PP_DIR_DCLV_2G); 1233 intel_ring_emit(ring, RING_PP_DIR_BASE(ring)); 1234 intel_ring_emit(ring, get_pd_offset(ppgtt)); 1235 intel_ring_emit(ring, MI_NOOP); 1236 intel_ring_advance(ring); 1237 1238 return 0; 1239 } 1240 1241 static int vgpu_mm_switch(struct i915_hw_ppgtt *ppgtt, 1242 struct drm_i915_gem_request *req) 1243 { 1244 struct intel_engine_cs *ring = req->ring; 1245 struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev); 1246 1247 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); 1248 I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt)); 1249 return 0; 1250 } 1251 1252 static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt, 1253 struct drm_i915_gem_request *req) 1254 { 1255 struct intel_engine_cs *ring = req->ring; 1256 int ret; 1257 1258 /* NB: TLBs must be flushed and invalidated before a switch */ 1259 ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); 1260 if (ret) 1261 return ret; 1262 1263 ret = intel_ring_begin(req, 6); 1264 if (ret) 1265 return ret; 1266 1267 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2)); 1268 intel_ring_emit(ring, RING_PP_DIR_DCLV(ring)); 1269 intel_ring_emit(ring, PP_DIR_DCLV_2G); 1270 intel_ring_emit(ring, RING_PP_DIR_BASE(ring)); 1271 intel_ring_emit(ring, get_pd_offset(ppgtt)); 1272 intel_ring_emit(ring, MI_NOOP); 1273 intel_ring_advance(ring); 1274 1275 /* XXX: RCS is the only one to auto invalidate the TLBs? */ 1276 if (ring->id != RCS) { 1277 ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); 1278 if (ret) 1279 return ret; 1280 } 1281 1282 return 0; 1283 } 1284 1285 static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt, 1286 struct drm_i915_gem_request *req) 1287 { 1288 struct intel_engine_cs *ring = req->ring; 1289 struct drm_device *dev = ppgtt->base.dev; 1290 struct drm_i915_private *dev_priv = dev->dev_private; 1291 1292 1293 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); 1294 I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt)); 1295 1296 POSTING_READ(RING_PP_DIR_DCLV(ring)); 1297 1298 return 0; 1299 } 1300 1301 static void gen8_ppgtt_enable(struct drm_device *dev) 1302 { 1303 struct drm_i915_private *dev_priv = dev->dev_private; 1304 struct intel_engine_cs *ring; 1305 int j; 1306 1307 for_each_ring(ring, dev_priv, j) { 1308 I915_WRITE(RING_MODE_GEN7(ring), 1309 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); 1310 } 1311 } 1312 1313 static void gen7_ppgtt_enable(struct drm_device *dev) 1314 { 1315 struct drm_i915_private *dev_priv = dev->dev_private; 1316 struct intel_engine_cs *ring; 1317 uint32_t ecochk, ecobits; 1318 int i; 1319 1320 ecobits = I915_READ(GAC_ECO_BITS); 1321 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B); 1322 1323 ecochk = I915_READ(GAM_ECOCHK); 1324 if (IS_HASWELL(dev)) { 1325 ecochk |= ECOCHK_PPGTT_WB_HSW; 1326 } else { 1327 ecochk |= ECOCHK_PPGTT_LLC_IVB; 1328 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB; 1329 } 1330 I915_WRITE(GAM_ECOCHK, ecochk); 1331 1332 for_each_ring(ring, dev_priv, i) { 1333 /* GFX_MODE is per-ring on gen7+ */ 1334 I915_WRITE(RING_MODE_GEN7(ring), 1335 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); 1336 } 1337 } 1338 1339 static void gen6_ppgtt_enable(struct drm_device *dev) 1340 { 1341 struct drm_i915_private *dev_priv = dev->dev_private; 1342 uint32_t ecochk, gab_ctl, ecobits; 1343 1344 ecobits = I915_READ(GAC_ECO_BITS); 1345 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT | 1346 ECOBITS_PPGTT_CACHE64B); 1347 1348 gab_ctl = I915_READ(GAB_CTL); 1349 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT); 1350 1351 ecochk = I915_READ(GAM_ECOCHK); 1352 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B); 1353 1354 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); 1355 } 1356 1357 /* PPGTT support for Sandybdrige/Gen6 and later */ 1358 static void gen6_ppgtt_clear_range(struct i915_address_space *vm, 1359 uint64_t start, 1360 uint64_t length, 1361 bool use_scratch) 1362 { 1363 struct i915_hw_ppgtt *ppgtt = 1364 container_of(vm, struct i915_hw_ppgtt, base); 1365 gen6_pte_t *pt_vaddr, scratch_pte; 1366 unsigned first_entry = start >> PAGE_SHIFT; 1367 unsigned num_entries = length >> PAGE_SHIFT; 1368 unsigned act_pt = first_entry / GEN6_PTES; 1369 unsigned first_pte = first_entry % GEN6_PTES; 1370 unsigned last_pte, i; 1371 1372 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), 1373 I915_CACHE_LLC, true, 0); 1374 1375 while (num_entries) { 1376 last_pte = first_pte + num_entries; 1377 if (last_pte > GEN6_PTES) 1378 last_pte = GEN6_PTES; 1379 1380 pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]); 1381 1382 for (i = first_pte; i < last_pte; i++) 1383 pt_vaddr[i] = scratch_pte; 1384 1385 kunmap_px(ppgtt, pt_vaddr); 1386 1387 num_entries -= last_pte - first_pte; 1388 first_pte = 0; 1389 act_pt++; 1390 } 1391 } 1392 1393 static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, 1394 struct sg_table *pages, 1395 uint64_t start, 1396 enum i915_cache_level cache_level, u32 flags) 1397 { 1398 struct i915_hw_ppgtt *ppgtt = 1399 container_of(vm, struct i915_hw_ppgtt, base); 1400 gen6_pte_t *pt_vaddr; 1401 unsigned first_entry = start >> PAGE_SHIFT; 1402 unsigned act_pt = first_entry / GEN6_PTES; 1403 unsigned act_pte = first_entry % GEN6_PTES; 1404 struct sg_page_iter sg_iter; 1405 1406 pt_vaddr = NULL; 1407 for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) { 1408 if (pt_vaddr == NULL) 1409 pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]); 1410 1411 pt_vaddr[act_pte] = 1412 vm->pte_encode(sg_page_iter_dma_address(&sg_iter), 1413 cache_level, true, flags); 1414 1415 if (++act_pte == GEN6_PTES) { 1416 kunmap_px(ppgtt, pt_vaddr); 1417 pt_vaddr = NULL; 1418 act_pt++; 1419 act_pte = 0; 1420 } 1421 } 1422 if (pt_vaddr) 1423 kunmap_px(ppgtt, pt_vaddr); 1424 } 1425 1426 static int gen6_alloc_va_range(struct i915_address_space *vm, 1427 uint64_t start_in, uint64_t length_in) 1428 { 1429 DECLARE_BITMAP(new_page_tables, I915_PDES); 1430 struct drm_device *dev = vm->dev; 1431 struct drm_i915_private *dev_priv = dev->dev_private; 1432 struct i915_hw_ppgtt *ppgtt = 1433 container_of(vm, struct i915_hw_ppgtt, base); 1434 struct i915_page_table *pt; 1435 uint32_t start, length, start_save, length_save; 1436 uint32_t pde, temp; 1437 int ret; 1438 1439 if (WARN_ON(start_in + length_in > ppgtt->base.total)) 1440 return -ENODEV; 1441 1442 start = start_save = start_in; 1443 length = length_save = length_in; 1444 1445 bitmap_zero(new_page_tables, I915_PDES); 1446 1447 /* The allocation is done in two stages so that we can bail out with 1448 * minimal amount of pain. The first stage finds new page tables that 1449 * need allocation. The second stage marks use ptes within the page 1450 * tables. 1451 */ 1452 gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) { 1453 if (pt != vm->scratch_pt) { 1454 // WARN_ON(bitmap_empty(pt->used_ptes, GEN6_PTES)); 1455 continue; 1456 } 1457 1458 /* We've already allocated a page table */ 1459 WARN_ON(!bitmap_empty(pt->used_ptes, GEN6_PTES)); 1460 1461 pt = alloc_pt(dev); 1462 if (IS_ERR(pt)) { 1463 ret = PTR_ERR(pt); 1464 goto unwind_out; 1465 } 1466 1467 gen6_initialize_pt(vm, pt); 1468 1469 ppgtt->pd.page_table[pde] = pt; 1470 __set_bit(pde, new_page_tables); 1471 trace_i915_page_table_entry_alloc(vm, pde, start, GEN6_PDE_SHIFT); 1472 } 1473 1474 start = start_save; 1475 length = length_save; 1476 1477 gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) { 1478 DECLARE_BITMAP(tmp_bitmap, GEN6_PTES); 1479 1480 bitmap_zero(tmp_bitmap, GEN6_PTES); 1481 bitmap_set(tmp_bitmap, gen6_pte_index(start), 1482 gen6_pte_count(start, length)); 1483 1484 if (__test_and_clear_bit(pde, new_page_tables)) 1485 gen6_write_pde(&ppgtt->pd, pde, pt); 1486 1487 trace_i915_page_table_entry_map(vm, pde, pt, 1488 gen6_pte_index(start), 1489 gen6_pte_count(start, length), 1490 GEN6_PTES); 1491 bitmap_or(pt->used_ptes, tmp_bitmap, pt->used_ptes, 1492 GEN6_PTES); 1493 } 1494 1495 WARN_ON(!bitmap_empty(new_page_tables, I915_PDES)); 1496 1497 /* Make sure write is complete before other code can use this page 1498 * table. Also require for WC mapped PTEs */ 1499 readl(dev_priv->gtt.gsm); 1500 1501 mark_tlbs_dirty(ppgtt); 1502 return 0; 1503 1504 unwind_out: 1505 for_each_set_bit(pde, new_page_tables, I915_PDES) { 1506 struct i915_page_table *pt = ppgtt->pd.page_table[pde]; 1507 1508 ppgtt->pd.page_table[pde] = vm->scratch_pt; 1509 free_pt(vm->dev, pt); 1510 } 1511 1512 mark_tlbs_dirty(ppgtt); 1513 return ret; 1514 } 1515 1516 static int gen6_init_scratch(struct i915_address_space *vm) 1517 { 1518 struct drm_device *dev = vm->dev; 1519 1520 vm->scratch_page = alloc_scratch_page(dev); 1521 if (IS_ERR(vm->scratch_page)) 1522 return PTR_ERR(vm->scratch_page); 1523 1524 vm->scratch_pt = alloc_pt(dev); 1525 if (IS_ERR(vm->scratch_pt)) { 1526 free_scratch_page(dev, vm->scratch_page); 1527 return PTR_ERR(vm->scratch_pt); 1528 } 1529 1530 gen6_initialize_pt(vm, vm->scratch_pt); 1531 1532 return 0; 1533 } 1534 1535 static void gen6_free_scratch(struct i915_address_space *vm) 1536 { 1537 struct drm_device *dev = vm->dev; 1538 1539 free_pt(dev, vm->scratch_pt); 1540 free_scratch_page(dev, vm->scratch_page); 1541 } 1542 1543 static void gen6_ppgtt_cleanup(struct i915_address_space *vm) 1544 { 1545 struct i915_hw_ppgtt *ppgtt = 1546 container_of(vm, struct i915_hw_ppgtt, base); 1547 struct i915_page_table *pt; 1548 uint32_t pde; 1549 1550 drm_mm_remove_node(&ppgtt->node); 1551 1552 gen6_for_all_pdes(pt, ppgtt, pde) { 1553 if (pt != vm->scratch_pt) 1554 free_pt(ppgtt->base.dev, pt); 1555 } 1556 1557 gen6_free_scratch(vm); 1558 } 1559 1560 static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt) 1561 { 1562 struct i915_address_space *vm = &ppgtt->base; 1563 struct drm_device *dev = ppgtt->base.dev; 1564 struct drm_i915_private *dev_priv = dev->dev_private; 1565 bool retried = false; 1566 int ret; 1567 1568 /* PPGTT PDEs reside in the GGTT and consists of 512 entries. The 1569 * allocator works in address space sizes, so it's multiplied by page 1570 * size. We allocate at the top of the GTT to avoid fragmentation. 1571 */ 1572 BUG_ON(!drm_mm_initialized(&dev_priv->gtt.base.mm)); 1573 1574 ret = gen6_init_scratch(vm); 1575 if (ret) 1576 return ret; 1577 1578 alloc: 1579 ret = drm_mm_insert_node_in_range_generic(&dev_priv->gtt.base.mm, 1580 &ppgtt->node, GEN6_PD_SIZE, 1581 GEN6_PD_ALIGN, 0, 1582 0, dev_priv->gtt.base.total, 1583 DRM_MM_TOPDOWN); 1584 if (ret == -ENOSPC && !retried) { 1585 ret = i915_gem_evict_something(dev, &dev_priv->gtt.base, 1586 GEN6_PD_SIZE, GEN6_PD_ALIGN, 1587 I915_CACHE_NONE, 1588 0, dev_priv->gtt.base.total, 1589 0); 1590 if (ret) 1591 goto err_out; 1592 1593 retried = true; 1594 goto alloc; 1595 } 1596 1597 if (ret) 1598 goto err_out; 1599 1600 1601 if (ppgtt->node.start < dev_priv->gtt.mappable_end) 1602 DRM_DEBUG("Forced to use aperture for PDEs\n"); 1603 1604 return 0; 1605 1606 err_out: 1607 gen6_free_scratch(vm); 1608 return ret; 1609 } 1610 1611 static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt) 1612 { 1613 return gen6_ppgtt_allocate_page_directories(ppgtt); 1614 } 1615 1616 static void gen6_scratch_va_range(struct i915_hw_ppgtt *ppgtt, 1617 uint64_t start, uint64_t length) 1618 { 1619 struct i915_page_table *unused; 1620 uint32_t pde, temp; 1621 1622 gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde) 1623 ppgtt->pd.page_table[pde] = ppgtt->base.scratch_pt; 1624 } 1625 1626 static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt, bool aliasing) 1627 { 1628 struct drm_device *dev = ppgtt->base.dev; 1629 struct drm_i915_private *dev_priv = dev->dev_private; 1630 int ret; 1631 1632 ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode; 1633 if (IS_GEN6(dev)) { 1634 ppgtt->switch_mm = gen6_mm_switch; 1635 } else if (IS_HASWELL(dev)) { 1636 ppgtt->switch_mm = hsw_mm_switch; 1637 } else if (IS_GEN7(dev)) { 1638 ppgtt->switch_mm = gen7_mm_switch; 1639 } else 1640 BUG(); 1641 1642 if (intel_vgpu_active(dev)) 1643 ppgtt->switch_mm = vgpu_mm_switch; 1644 1645 ret = gen6_ppgtt_alloc(ppgtt); 1646 if (ret) 1647 return ret; 1648 1649 if (aliasing) { 1650 /* preallocate all pts */ 1651 ret = alloc_pt_range(&ppgtt->pd, 0, I915_PDES, 1652 ppgtt->base.dev); 1653 1654 if (ret) { 1655 gen6_ppgtt_cleanup(&ppgtt->base); 1656 return ret; 1657 } 1658 } 1659 1660 ppgtt->base.allocate_va_range = gen6_alloc_va_range; 1661 ppgtt->base.clear_range = gen6_ppgtt_clear_range; 1662 ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; 1663 ppgtt->base.unbind_vma = ppgtt_unbind_vma; 1664 ppgtt->base.bind_vma = ppgtt_bind_vma; 1665 ppgtt->base.cleanup = gen6_ppgtt_cleanup; 1666 ppgtt->base.start = 0; 1667 ppgtt->base.total = I915_PDES * GEN6_PTES * PAGE_SIZE; 1668 ppgtt->debug_dump = gen6_dump_ppgtt; 1669 1670 ppgtt->pd.base.ggtt_offset = 1671 ppgtt->node.start / PAGE_SIZE * sizeof(gen6_pte_t); 1672 1673 ppgtt->pd_addr = (gen6_pte_t __iomem *)dev_priv->gtt.gsm + 1674 ppgtt->pd.base.ggtt_offset / sizeof(gen6_pte_t); 1675 1676 if (aliasing) 1677 ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true); 1678 else 1679 gen6_scratch_va_range(ppgtt, 0, ppgtt->base.total); 1680 1681 gen6_write_page_range(dev_priv, &ppgtt->pd, 0, ppgtt->base.total); 1682 1683 DRM_DEBUG_DRIVER("Allocated pde space (%ldM) at GTT entry: %lx\n", 1684 ppgtt->node.size >> 20, 1685 ppgtt->node.start / PAGE_SIZE); 1686 1687 DRM_DEBUG("Adding PPGTT at offset %x\n", 1688 ppgtt->pd.base.ggtt_offset << 10); 1689 1690 return 0; 1691 } 1692 1693 static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt, 1694 bool aliasing) 1695 { 1696 ppgtt->base.dev = dev; 1697 1698 if (INTEL_INFO(dev)->gen < 8) 1699 return gen6_ppgtt_init(ppgtt, aliasing); 1700 else if (aliasing) 1701 return gen8_aliasing_ppgtt_init(ppgtt); 1702 else 1703 return gen8_ppgtt_init(ppgtt); 1704 } 1705 1706 int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt) 1707 { 1708 struct drm_i915_private *dev_priv = dev->dev_private; 1709 int ret = 0; 1710 1711 ret = __hw_ppgtt_init(dev, ppgtt, false); 1712 if (ret == 0) { 1713 kref_init(&ppgtt->ref); 1714 drm_mm_init(&ppgtt->base.mm, ppgtt->base.start, 1715 ppgtt->base.total); 1716 i915_init_vm(dev_priv, &ppgtt->base); 1717 } 1718 1719 return ret; 1720 } 1721 1722 int i915_ppgtt_init_hw(struct drm_device *dev) 1723 { 1724 /* In the case of execlists, PPGTT is enabled by the context descriptor 1725 * and the PDPs are contained within the context itself. We don't 1726 * need to do anything here. */ 1727 if (i915.enable_execlists) 1728 return 0; 1729 1730 if (!USES_PPGTT(dev)) 1731 return 0; 1732 1733 if (IS_GEN6(dev)) 1734 gen6_ppgtt_enable(dev); 1735 else if (IS_GEN7(dev)) 1736 gen7_ppgtt_enable(dev); 1737 else if (INTEL_INFO(dev)->gen >= 8) 1738 gen8_ppgtt_enable(dev); 1739 else 1740 MISSING_CASE(INTEL_INFO(dev)->gen); 1741 1742 return 0; 1743 } 1744 1745 int i915_ppgtt_init_ring(struct drm_i915_gem_request *req) 1746 { 1747 struct drm_i915_private *dev_priv = req->ring->dev->dev_private; 1748 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; 1749 1750 if (i915.enable_execlists) 1751 return 0; 1752 1753 if (!ppgtt) 1754 return 0; 1755 1756 return ppgtt->switch_mm(ppgtt, req); 1757 } 1758 1759 struct i915_hw_ppgtt * 1760 i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv) 1761 { 1762 struct i915_hw_ppgtt *ppgtt; 1763 int ret; 1764 1765 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); 1766 if (!ppgtt) 1767 return ERR_PTR(-ENOMEM); 1768 1769 ret = i915_ppgtt_init(dev, ppgtt); 1770 if (ret) { 1771 kfree(ppgtt); 1772 return ERR_PTR(ret); 1773 } 1774 1775 ppgtt->file_priv = fpriv; 1776 1777 trace_i915_ppgtt_create(&ppgtt->base); 1778 1779 return ppgtt; 1780 } 1781 1782 void i915_ppgtt_release(struct kref *kref) 1783 { 1784 struct i915_hw_ppgtt *ppgtt = 1785 container_of(kref, struct i915_hw_ppgtt, ref); 1786 1787 trace_i915_ppgtt_release(&ppgtt->base); 1788 1789 /* vmas should already be unbound */ 1790 WARN_ON(!list_empty(&ppgtt->base.active_list)); 1791 WARN_ON(!list_empty(&ppgtt->base.inactive_list)); 1792 1793 list_del(&ppgtt->base.global_link); 1794 drm_mm_takedown(&ppgtt->base.mm); 1795 1796 ppgtt->base.cleanup(&ppgtt->base); 1797 kfree(ppgtt); 1798 } 1799 1800 extern int intel_iommu_gfx_mapped; 1801 /* Certain Gen5 chipsets require require idling the GPU before 1802 * unmapping anything from the GTT when VT-d is enabled. 1803 */ 1804 static bool needs_idle_maps(struct drm_device *dev) 1805 { 1806 #ifdef CONFIG_INTEL_IOMMU 1807 /* Query intel_iommu to see if we need the workaround. Presumably that 1808 * was loaded first. 1809 */ 1810 if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped) 1811 return true; 1812 #endif 1813 return false; 1814 } 1815 1816 static bool do_idling(struct drm_i915_private *dev_priv) 1817 { 1818 bool ret = dev_priv->mm.interruptible; 1819 1820 if (unlikely(dev_priv->gtt.do_idle_maps)) { 1821 dev_priv->mm.interruptible = false; 1822 if (i915_gpu_idle(dev_priv->dev)) { 1823 DRM_ERROR("Couldn't idle GPU\n"); 1824 /* Wait a bit, in hopes it avoids the hang */ 1825 udelay(10); 1826 } 1827 } 1828 1829 return ret; 1830 } 1831 1832 static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible) 1833 { 1834 if (unlikely(dev_priv->gtt.do_idle_maps)) 1835 dev_priv->mm.interruptible = interruptible; 1836 } 1837 1838 void i915_check_and_clear_faults(struct drm_device *dev) 1839 { 1840 struct drm_i915_private *dev_priv = dev->dev_private; 1841 struct intel_engine_cs *ring; 1842 int i; 1843 1844 if (INTEL_INFO(dev)->gen < 6) 1845 return; 1846 1847 for_each_ring(ring, dev_priv, i) { 1848 u32 fault_reg; 1849 fault_reg = I915_READ(RING_FAULT_REG(ring)); 1850 if (fault_reg & RING_FAULT_VALID) { 1851 #if 0 1852 DRM_DEBUG_DRIVER("Unexpected fault\n" 1853 "\tAddr: 0x%08lx\n" 1854 "\tAddress space: %s\n" 1855 "\tSource ID: %d\n" 1856 "\tType: %d\n", 1857 fault_reg & PAGE_MASK, 1858 fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT", 1859 RING_FAULT_SRCID(fault_reg), 1860 RING_FAULT_FAULT_TYPE(fault_reg)); 1861 #endif 1862 I915_WRITE(RING_FAULT_REG(ring), 1863 fault_reg & ~RING_FAULT_VALID); 1864 } 1865 } 1866 POSTING_READ(RING_FAULT_REG(&dev_priv->ring[RCS])); 1867 } 1868 1869 static void i915_ggtt_flush(struct drm_i915_private *dev_priv) 1870 { 1871 if (INTEL_INFO(dev_priv->dev)->gen < 6) { 1872 intel_gtt_chipset_flush(); 1873 } else { 1874 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 1875 POSTING_READ(GFX_FLSH_CNTL_GEN6); 1876 } 1877 } 1878 1879 void i915_gem_suspend_gtt_mappings(struct drm_device *dev) 1880 { 1881 struct drm_i915_private *dev_priv = dev->dev_private; 1882 1883 /* Don't bother messing with faults pre GEN6 as we have little 1884 * documentation supporting that it's a good idea. 1885 */ 1886 if (INTEL_INFO(dev)->gen < 6) 1887 return; 1888 1889 i915_check_and_clear_faults(dev); 1890 1891 dev_priv->gtt.base.clear_range(&dev_priv->gtt.base, 1892 dev_priv->gtt.base.start, 1893 dev_priv->gtt.base.total, 1894 true); 1895 1896 i915_ggtt_flush(dev_priv); 1897 } 1898 1899 int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj) 1900 { 1901 if (!dma_map_sg(obj->base.dev->pdev->dev, 1902 obj->pages->sgl, obj->pages->nents, 1903 PCI_DMA_BIDIRECTIONAL)) 1904 return -ENOSPC; 1905 1906 return 0; 1907 } 1908 1909 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) 1910 { 1911 #if 0 1912 writeq(pte, addr); 1913 #else 1914 iowrite32((u32)pte, addr); 1915 iowrite32(pte >> 32, addr + 4); 1916 #endif 1917 } 1918 1919 static void gen8_ggtt_insert_entries(struct i915_address_space *vm, 1920 struct sg_table *st, 1921 uint64_t start, 1922 enum i915_cache_level level, u32 unused) 1923 { 1924 struct drm_i915_private *dev_priv = vm->dev->dev_private; 1925 unsigned first_entry = start >> PAGE_SHIFT; 1926 gen8_pte_t __iomem *gtt_entries = 1927 (gen8_pte_t __iomem *)dev_priv->gtt.gsm + first_entry; 1928 int i = 0; 1929 struct sg_page_iter sg_iter; 1930 dma_addr_t addr = 0; /* shut up gcc */ 1931 1932 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) { 1933 addr = sg_dma_address(sg_iter.sg) + 1934 (sg_iter.sg_pgoffset << PAGE_SHIFT); 1935 gen8_set_pte(>t_entries[i], 1936 gen8_pte_encode(addr, level, true)); 1937 i++; 1938 } 1939 1940 /* 1941 * XXX: This serves as a posting read to make sure that the PTE has 1942 * actually been updated. There is some concern that even though 1943 * registers and PTEs are within the same BAR that they are potentially 1944 * of NUMA access patterns. Therefore, even with the way we assume 1945 * hardware should work, we must keep this posting read for paranoia. 1946 */ 1947 if (i != 0) 1948 WARN_ON(readq(>t_entries[i-1]) 1949 != gen8_pte_encode(addr, level, true)); 1950 1951 /* This next bit makes the above posting read even more important. We 1952 * want to flush the TLBs only after we're certain all the PTE updates 1953 * have finished. 1954 */ 1955 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 1956 POSTING_READ(GFX_FLSH_CNTL_GEN6); 1957 } 1958 1959 /* 1960 * Binds an object into the global gtt with the specified cache level. The object 1961 * will be accessible to the GPU via commands whose operands reference offsets 1962 * within the global GTT as well as accessible by the GPU through the GMADR 1963 * mapped BAR (dev_priv->mm.gtt->gtt). 1964 */ 1965 static void gen6_ggtt_insert_entries(struct i915_address_space *vm, 1966 struct sg_table *st, 1967 uint64_t start, 1968 enum i915_cache_level level, u32 flags) 1969 { 1970 struct drm_i915_private *dev_priv = vm->dev->dev_private; 1971 unsigned first_entry = start >> PAGE_SHIFT; 1972 gen6_pte_t __iomem *gtt_entries = 1973 (gen6_pte_t __iomem *)dev_priv->gtt.gsm + first_entry; 1974 int i = 0; 1975 struct sg_page_iter sg_iter; 1976 dma_addr_t addr = 0; 1977 1978 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) { 1979 addr = sg_page_iter_dma_address(&sg_iter); 1980 iowrite32(vm->pte_encode(addr, level, true, flags), >t_entries[i]); 1981 i++; 1982 } 1983 1984 /* XXX: This serves as a posting read to make sure that the PTE has 1985 * actually been updated. There is some concern that even though 1986 * registers and PTEs are within the same BAR that they are potentially 1987 * of NUMA access patterns. Therefore, even with the way we assume 1988 * hardware should work, we must keep this posting read for paranoia. 1989 */ 1990 if (i != 0) { 1991 unsigned long gtt = readl(>t_entries[i-1]); 1992 WARN_ON(gtt != vm->pte_encode(addr, level, true, flags)); 1993 } 1994 1995 /* This next bit makes the above posting read even more important. We 1996 * want to flush the TLBs only after we're certain all the PTE updates 1997 * have finished. 1998 */ 1999 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 2000 POSTING_READ(GFX_FLSH_CNTL_GEN6); 2001 } 2002 2003 static void gen8_ggtt_clear_range(struct i915_address_space *vm, 2004 uint64_t start, 2005 uint64_t length, 2006 bool use_scratch) 2007 { 2008 struct drm_i915_private *dev_priv = vm->dev->dev_private; 2009 unsigned first_entry = start >> PAGE_SHIFT; 2010 unsigned num_entries = length >> PAGE_SHIFT; 2011 gen8_pte_t scratch_pte, __iomem *gtt_base = 2012 (gen8_pte_t __iomem *) dev_priv->gtt.gsm + first_entry; 2013 const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry; 2014 int i; 2015 2016 if (WARN(num_entries > max_entries, 2017 "First entry = %d; Num entries = %d (max=%d)\n", 2018 first_entry, num_entries, max_entries)) 2019 num_entries = max_entries; 2020 2021 scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), 2022 I915_CACHE_LLC, 2023 use_scratch); 2024 for (i = 0; i < num_entries; i++) 2025 gen8_set_pte(>t_base[i], scratch_pte); 2026 readl(gtt_base); 2027 } 2028 2029 static void gen6_ggtt_clear_range(struct i915_address_space *vm, 2030 uint64_t start, 2031 uint64_t length, 2032 bool use_scratch) 2033 { 2034 struct drm_i915_private *dev_priv = vm->dev->dev_private; 2035 unsigned first_entry = start >> PAGE_SHIFT; 2036 unsigned num_entries = length >> PAGE_SHIFT; 2037 gen6_pte_t scratch_pte, __iomem *gtt_base = 2038 (gen6_pte_t __iomem *) dev_priv->gtt.gsm + first_entry; 2039 const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry; 2040 int i; 2041 2042 if (WARN(num_entries > max_entries, 2043 "First entry = %d; Num entries = %d (max=%d)\n", 2044 first_entry, num_entries, max_entries)) 2045 num_entries = max_entries; 2046 2047 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), 2048 I915_CACHE_LLC, use_scratch, 0); 2049 2050 for (i = 0; i < num_entries; i++) 2051 iowrite32(scratch_pte, >t_base[i]); 2052 readl(gtt_base); 2053 } 2054 2055 static void i915_ggtt_insert_entries(struct i915_address_space *vm, 2056 struct sg_table *pages, 2057 uint64_t start, 2058 enum i915_cache_level cache_level, u32 unused) 2059 { 2060 unsigned int flags = (cache_level == I915_CACHE_NONE) ? 2061 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; 2062 2063 intel_gtt_insert_sg_entries(pages, start >> PAGE_SHIFT, flags); 2064 } 2065 2066 static void i915_ggtt_clear_range(struct i915_address_space *vm, 2067 uint64_t start, 2068 uint64_t length, 2069 bool unused) 2070 { 2071 unsigned first_entry = start >> PAGE_SHIFT; 2072 unsigned num_entries = length >> PAGE_SHIFT; 2073 intel_gtt_clear_range(first_entry, num_entries); 2074 } 2075 2076 static int ggtt_bind_vma(struct i915_vma *vma, 2077 enum i915_cache_level cache_level, 2078 u32 flags) 2079 { 2080 struct drm_device *dev = vma->vm->dev; 2081 struct drm_i915_private *dev_priv = dev->dev_private; 2082 struct drm_i915_gem_object *obj = vma->obj; 2083 struct sg_table *pages = obj->pages; 2084 u32 pte_flags = 0; 2085 int ret; 2086 2087 ret = i915_get_ggtt_vma_pages(vma); 2088 if (ret) 2089 return ret; 2090 pages = vma->ggtt_view.pages; 2091 2092 /* Currently applicable only to VLV */ 2093 if (obj->gt_ro) 2094 pte_flags |= PTE_READ_ONLY; 2095 2096 2097 if (!dev_priv->mm.aliasing_ppgtt || flags & GLOBAL_BIND) { 2098 vma->vm->insert_entries(vma->vm, pages, 2099 vma->node.start, 2100 cache_level, pte_flags); 2101 } 2102 2103 if (dev_priv->mm.aliasing_ppgtt && flags & LOCAL_BIND) { 2104 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt; 2105 appgtt->base.insert_entries(&appgtt->base, pages, 2106 vma->node.start, 2107 cache_level, pte_flags); 2108 } 2109 2110 return 0; 2111 } 2112 2113 static void ggtt_unbind_vma(struct i915_vma *vma) 2114 { 2115 struct drm_device *dev = vma->vm->dev; 2116 struct drm_i915_private *dev_priv = dev->dev_private; 2117 struct drm_i915_gem_object *obj = vma->obj; 2118 const uint64_t size = min_t(uint64_t, 2119 obj->base.size, 2120 vma->node.size); 2121 2122 if (vma->bound & GLOBAL_BIND) { 2123 vma->vm->clear_range(vma->vm, 2124 vma->node.start, 2125 size, 2126 true); 2127 } 2128 2129 if (dev_priv->mm.aliasing_ppgtt && vma->bound & LOCAL_BIND) { 2130 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt; 2131 2132 appgtt->base.clear_range(&appgtt->base, 2133 vma->node.start, 2134 size, 2135 true); 2136 } 2137 } 2138 2139 void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) 2140 { 2141 struct drm_device *dev = obj->base.dev; 2142 struct drm_i915_private *dev_priv = dev->dev_private; 2143 bool interruptible; 2144 2145 interruptible = do_idling(dev_priv); 2146 2147 dma_unmap_sg(dev->pdev->dev, obj->pages->sgl, obj->pages->nents, 2148 PCI_DMA_BIDIRECTIONAL); 2149 2150 undo_idling(dev_priv, interruptible); 2151 } 2152 2153 static void i915_gtt_color_adjust(struct drm_mm_node *node, 2154 unsigned long color, 2155 u64 *start, 2156 u64 *end) 2157 { 2158 if (node->color != color) 2159 *start += 4096; 2160 2161 if (!list_empty(&node->node_list)) { 2162 node = list_entry(node->node_list.next, 2163 struct drm_mm_node, 2164 node_list); 2165 if (node->allocated && node->color != color) 2166 *end -= 4096; 2167 } 2168 } 2169 2170 static int i915_gem_setup_global_gtt(struct drm_device *dev, 2171 unsigned long start, 2172 unsigned long mappable_end, 2173 unsigned long end) 2174 { 2175 /* Let GEM Manage all of the aperture. 2176 * 2177 * However, leave one page at the end still bound to the scratch page. 2178 * There are a number of places where the hardware apparently prefetches 2179 * past the end of the object, and we've seen multiple hangs with the 2180 * GPU head pointer stuck in a batchbuffer bound at the last page of the 2181 * aperture. One page should be enough to keep any prefetching inside 2182 * of the aperture. 2183 */ 2184 struct drm_i915_private *dev_priv = dev->dev_private; 2185 struct i915_address_space *ggtt_vm = &dev_priv->gtt.base; 2186 unsigned long mappable; 2187 int error; 2188 struct drm_mm_node *entry; 2189 struct drm_i915_gem_object *obj; 2190 unsigned long hole_start, hole_end; 2191 int ret; 2192 2193 mappable = min(end, mappable_end) - start; 2194 BUG_ON(mappable_end > end); 2195 2196 /* Subtract the guard page ... */ 2197 drm_mm_init(&ggtt_vm->mm, start, end - start - PAGE_SIZE); 2198 2199 dev_priv->gtt.base.start = start; 2200 dev_priv->gtt.base.total = end - start; 2201 2202 if (intel_vgpu_active(dev)) { 2203 ret = intel_vgt_balloon(dev); 2204 if (ret) 2205 return ret; 2206 } 2207 2208 if (!HAS_LLC(dev)) 2209 dev_priv->gtt.base.mm.color_adjust = i915_gtt_color_adjust; 2210 2211 /* Mark any preallocated objects as occupied */ 2212 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { 2213 struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm); 2214 2215 DRM_DEBUG_KMS("reserving preallocated space: %lx + %zx\n", 2216 i915_gem_obj_ggtt_offset(obj), obj->base.size); 2217 2218 WARN_ON(i915_gem_obj_ggtt_bound(obj)); 2219 ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node); 2220 if (ret) { 2221 DRM_DEBUG_KMS("Reservation failed: %i\n", ret); 2222 return ret; 2223 } 2224 vma->bound |= GLOBAL_BIND; 2225 } 2226 2227 /* Clear any non-preallocated blocks */ 2228 drm_mm_for_each_hole(entry, &ggtt_vm->mm, hole_start, hole_end) { 2229 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n", 2230 hole_start, hole_end); 2231 ggtt_vm->clear_range(ggtt_vm, hole_start, 2232 hole_end - hole_start, true); 2233 } 2234 2235 #ifdef __DragonFly__ 2236 device_printf(dev->dev, 2237 "taking over the fictitious range 0x%lx-0x%lx\n", 2238 dev_priv->gtt.mappable_base + start, dev_priv->gtt.mappable_base + start + mappable); 2239 error = -vm_phys_fictitious_reg_range(dev_priv->gtt.mappable_base + start, 2240 dev_priv->gtt.mappable_base + start + mappable, VM_MEMATTR_WRITE_COMBINING); 2241 #endif 2242 2243 /* And finally clear the reserved guard page */ 2244 ggtt_vm->clear_range(ggtt_vm, end - PAGE_SIZE, PAGE_SIZE, true); 2245 2246 if (USES_PPGTT(dev) && !USES_FULL_PPGTT(dev)) { 2247 struct i915_hw_ppgtt *ppgtt; 2248 2249 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); 2250 if (!ppgtt) 2251 return -ENOMEM; 2252 2253 ret = __hw_ppgtt_init(dev, ppgtt, true); 2254 if (ret) { 2255 ppgtt->base.cleanup(&ppgtt->base); 2256 kfree(ppgtt); 2257 return ret; 2258 } 2259 2260 if (ppgtt->base.allocate_va_range) 2261 ret = ppgtt->base.allocate_va_range(&ppgtt->base, 0, 2262 ppgtt->base.total); 2263 if (ret) { 2264 ppgtt->base.cleanup(&ppgtt->base); 2265 kfree(ppgtt); 2266 return ret; 2267 } 2268 2269 ppgtt->base.clear_range(&ppgtt->base, 2270 ppgtt->base.start, 2271 ppgtt->base.total, 2272 true); 2273 2274 dev_priv->mm.aliasing_ppgtt = ppgtt; 2275 } 2276 2277 return 0; 2278 } 2279 2280 void i915_gem_init_global_gtt(struct drm_device *dev) 2281 { 2282 struct drm_i915_private *dev_priv = dev->dev_private; 2283 u64 gtt_size, mappable_size; 2284 2285 gtt_size = dev_priv->gtt.base.total; 2286 mappable_size = dev_priv->gtt.mappable_end; 2287 2288 i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size); 2289 } 2290 2291 void i915_global_gtt_cleanup(struct drm_device *dev) 2292 { 2293 struct drm_i915_private *dev_priv = dev->dev_private; 2294 struct i915_address_space *vm = &dev_priv->gtt.base; 2295 2296 if (dev_priv->mm.aliasing_ppgtt) { 2297 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; 2298 2299 ppgtt->base.cleanup(&ppgtt->base); 2300 } 2301 2302 if (drm_mm_initialized(&vm->mm)) { 2303 if (intel_vgpu_active(dev)) 2304 intel_vgt_deballoon(); 2305 2306 drm_mm_takedown(&vm->mm); 2307 list_del(&vm->global_link); 2308 } 2309 2310 vm->cleanup(vm); 2311 } 2312 2313 static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) 2314 { 2315 snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT; 2316 snb_gmch_ctl &= SNB_GMCH_GGMS_MASK; 2317 return snb_gmch_ctl << 20; 2318 } 2319 2320 static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl) 2321 { 2322 bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT; 2323 bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK; 2324 if (bdw_gmch_ctl) 2325 bdw_gmch_ctl = 1 << bdw_gmch_ctl; 2326 2327 #ifdef CONFIG_X86_32 2328 /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * PAGE_SIZE */ 2329 if (bdw_gmch_ctl > 4) 2330 bdw_gmch_ctl = 4; 2331 #endif 2332 2333 return bdw_gmch_ctl << 20; 2334 } 2335 2336 static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl) 2337 { 2338 gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT; 2339 gmch_ctrl &= SNB_GMCH_GGMS_MASK; 2340 2341 if (gmch_ctrl) 2342 return 1 << (20 + gmch_ctrl); 2343 2344 return 0; 2345 } 2346 2347 static size_t gen6_get_stolen_size(u16 snb_gmch_ctl) 2348 { 2349 snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT; 2350 snb_gmch_ctl &= SNB_GMCH_GMS_MASK; 2351 return snb_gmch_ctl << 25; /* 32 MB units */ 2352 } 2353 2354 static size_t gen8_get_stolen_size(u16 bdw_gmch_ctl) 2355 { 2356 bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT; 2357 bdw_gmch_ctl &= BDW_GMCH_GMS_MASK; 2358 return bdw_gmch_ctl << 25; /* 32 MB units */ 2359 } 2360 2361 static size_t chv_get_stolen_size(u16 gmch_ctrl) 2362 { 2363 gmch_ctrl >>= SNB_GMCH_GMS_SHIFT; 2364 gmch_ctrl &= SNB_GMCH_GMS_MASK; 2365 2366 /* 2367 * 0x0 to 0x10: 32MB increments starting at 0MB 2368 * 0x11 to 0x16: 4MB increments starting at 8MB 2369 * 0x17 to 0x1d: 4MB increments start at 36MB 2370 */ 2371 if (gmch_ctrl < 0x11) 2372 return gmch_ctrl << 25; 2373 else if (gmch_ctrl < 0x17) 2374 return (gmch_ctrl - 0x11 + 2) << 22; 2375 else 2376 return (gmch_ctrl - 0x17 + 9) << 22; 2377 } 2378 2379 static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl) 2380 { 2381 gen9_gmch_ctl >>= BDW_GMCH_GMS_SHIFT; 2382 gen9_gmch_ctl &= BDW_GMCH_GMS_MASK; 2383 2384 if (gen9_gmch_ctl < 0xf0) 2385 return gen9_gmch_ctl << 25; /* 32 MB units */ 2386 else 2387 /* 4MB increments starting at 0xf0 for 4MB */ 2388 return (gen9_gmch_ctl - 0xf0 + 1) << 22; 2389 } 2390 2391 static int ggtt_probe_common(struct drm_device *dev, 2392 size_t gtt_size) 2393 { 2394 struct drm_i915_private *dev_priv = dev->dev_private; 2395 struct i915_page_scratch *scratch_page; 2396 phys_addr_t gtt_phys_addr; 2397 2398 /* For Modern GENs the PTEs and register space are split in the BAR */ 2399 gtt_phys_addr = pci_resource_start(dev->pdev, 0) + 2400 (pci_resource_len(dev->pdev, 0) / 2); 2401 2402 /* 2403 * On BXT writes larger than 64 bit to the GTT pagetable range will be 2404 * dropped. For WC mappings in general we have 64 byte burst writes 2405 * when the WC buffer is flushed, so we can't use it, but have to 2406 * resort to an uncached mapping. The WC issue is easily caught by the 2407 * readback check when writing GTT PTE entries. 2408 */ 2409 if (IS_BROXTON(dev)) 2410 dev_priv->gtt.gsm = ioremap_nocache(gtt_phys_addr, gtt_size); 2411 else 2412 dev_priv->gtt.gsm = ioremap_wc(gtt_phys_addr, gtt_size); 2413 if (!dev_priv->gtt.gsm) { 2414 DRM_ERROR("Failed to map the gtt page table\n"); 2415 return -ENOMEM; 2416 } 2417 2418 scratch_page = alloc_scratch_page(dev); 2419 if (IS_ERR(scratch_page)) { 2420 DRM_ERROR("Scratch setup failed\n"); 2421 /* iounmap will also get called at remove, but meh */ 2422 iounmap(dev_priv->gtt.gsm); 2423 return PTR_ERR(scratch_page); 2424 } 2425 2426 dev_priv->gtt.base.scratch_page = scratch_page; 2427 2428 return 0; 2429 } 2430 2431 /* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability 2432 * bits. When using advanced contexts each context stores its own PAT, but 2433 * writing this data shouldn't be harmful even in those cases. */ 2434 static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv) 2435 { 2436 uint64_t pat; 2437 2438 pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC) | /* for normal objects, no eLLC */ 2439 GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */ 2440 GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */ 2441 GEN8_PPAT(3, GEN8_PPAT_UC) | /* Uncached objects, mostly for scanout */ 2442 GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) | 2443 GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) | 2444 GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) | 2445 GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3)); 2446 2447 if (!USES_PPGTT(dev_priv->dev)) 2448 /* Spec: "For GGTT, there is NO pat_sel[2:0] from the entry, 2449 * so RTL will always use the value corresponding to 2450 * pat_sel = 000". 2451 * So let's disable cache for GGTT to avoid screen corruptions. 2452 * MOCS still can be used though. 2453 * - System agent ggtt writes (i.e. cpu gtt mmaps) already work 2454 * before this patch, i.e. the same uncached + snooping access 2455 * like on gen6/7 seems to be in effect. 2456 * - So this just fixes blitter/render access. Again it looks 2457 * like it's not just uncached access, but uncached + snooping. 2458 * So we can still hold onto all our assumptions wrt cpu 2459 * clflushing on LLC machines. 2460 */ 2461 pat = GEN8_PPAT(0, GEN8_PPAT_UC); 2462 2463 /* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b 2464 * write would work. */ 2465 I915_WRITE(GEN8_PRIVATE_PAT, pat); 2466 I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32); 2467 } 2468 2469 static void chv_setup_private_ppat(struct drm_i915_private *dev_priv) 2470 { 2471 uint64_t pat; 2472 2473 /* 2474 * Map WB on BDW to snooped on CHV. 2475 * 2476 * Only the snoop bit has meaning for CHV, the rest is 2477 * ignored. 2478 * 2479 * The hardware will never snoop for certain types of accesses: 2480 * - CPU GTT (GMADR->GGTT->no snoop->memory) 2481 * - PPGTT page tables 2482 * - some other special cycles 2483 * 2484 * As with BDW, we also need to consider the following for GT accesses: 2485 * "For GGTT, there is NO pat_sel[2:0] from the entry, 2486 * so RTL will always use the value corresponding to 2487 * pat_sel = 000". 2488 * Which means we must set the snoop bit in PAT entry 0 2489 * in order to keep the global status page working. 2490 */ 2491 pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) | 2492 GEN8_PPAT(1, 0) | 2493 GEN8_PPAT(2, 0) | 2494 GEN8_PPAT(3, 0) | 2495 GEN8_PPAT(4, CHV_PPAT_SNOOP) | 2496 GEN8_PPAT(5, CHV_PPAT_SNOOP) | 2497 GEN8_PPAT(6, CHV_PPAT_SNOOP) | 2498 GEN8_PPAT(7, CHV_PPAT_SNOOP); 2499 2500 I915_WRITE(GEN8_PRIVATE_PAT, pat); 2501 I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32); 2502 } 2503 2504 static int gen8_gmch_probe(struct drm_device *dev, 2505 u64 *gtt_total, 2506 size_t *stolen, 2507 phys_addr_t *mappable_base, 2508 u64 *mappable_end) 2509 { 2510 struct drm_i915_private *dev_priv = dev->dev_private; 2511 u64 gtt_size; 2512 u16 snb_gmch_ctl; 2513 int ret; 2514 2515 /* TODO: We're not aware of mappable constraints on gen8 yet */ 2516 *mappable_base = pci_resource_start(dev->pdev, 2); 2517 *mappable_end = pci_resource_len(dev->pdev, 2); 2518 2519 #if 0 2520 if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(39))) 2521 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(39)); 2522 #endif 2523 2524 pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); 2525 2526 if (INTEL_INFO(dev)->gen >= 9) { 2527 *stolen = gen9_get_stolen_size(snb_gmch_ctl); 2528 gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl); 2529 } else if (IS_CHERRYVIEW(dev)) { 2530 *stolen = chv_get_stolen_size(snb_gmch_ctl); 2531 gtt_size = chv_get_total_gtt_size(snb_gmch_ctl); 2532 } else { 2533 *stolen = gen8_get_stolen_size(snb_gmch_ctl); 2534 gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl); 2535 } 2536 2537 *gtt_total = (gtt_size / sizeof(gen8_pte_t)) << PAGE_SHIFT; 2538 2539 if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev)) 2540 chv_setup_private_ppat(dev_priv); 2541 else 2542 bdw_setup_private_ppat(dev_priv); 2543 2544 ret = ggtt_probe_common(dev, gtt_size); 2545 2546 dev_priv->gtt.base.clear_range = gen8_ggtt_clear_range; 2547 dev_priv->gtt.base.insert_entries = gen8_ggtt_insert_entries; 2548 dev_priv->gtt.base.bind_vma = ggtt_bind_vma; 2549 dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma; 2550 2551 return ret; 2552 } 2553 2554 static int gen6_gmch_probe(struct drm_device *dev, 2555 u64 *gtt_total, 2556 size_t *stolen, 2557 phys_addr_t *mappable_base, 2558 u64 *mappable_end) 2559 { 2560 struct drm_i915_private *dev_priv = dev->dev_private; 2561 unsigned int gtt_size; 2562 u16 snb_gmch_ctl; 2563 int ret; 2564 2565 *mappable_base = pci_resource_start(dev->pdev, 2); 2566 *mappable_end = pci_resource_len(dev->pdev, 2); 2567 2568 /* 64/512MB is the current min/max we actually know of, but this is just 2569 * a coarse sanity check. 2570 */ 2571 if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) { 2572 DRM_ERROR("Unknown GMADR size (%lx)\n", 2573 dev_priv->gtt.mappable_end); 2574 return -ENXIO; 2575 } 2576 2577 #if 0 2578 if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40))) 2579 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40)); 2580 #endif 2581 pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); 2582 2583 *stolen = gen6_get_stolen_size(snb_gmch_ctl); 2584 2585 gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl); 2586 *gtt_total = (gtt_size / sizeof(gen6_pte_t)) << PAGE_SHIFT; 2587 2588 ret = ggtt_probe_common(dev, gtt_size); 2589 2590 dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range; 2591 dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries; 2592 dev_priv->gtt.base.bind_vma = ggtt_bind_vma; 2593 dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma; 2594 2595 return ret; 2596 } 2597 2598 static void gen6_gmch_remove(struct i915_address_space *vm) 2599 { 2600 struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base); 2601 2602 iounmap(gtt->gsm); 2603 free_scratch_page(vm->dev, vm->scratch_page); 2604 } 2605 2606 static int i915_gmch_probe(struct drm_device *dev, 2607 u64 *gtt_total, 2608 size_t *stolen, 2609 phys_addr_t *mappable_base, 2610 u64 *mappable_end) 2611 { 2612 struct drm_i915_private *dev_priv = dev->dev_private; 2613 #if 0 2614 int ret; 2615 2616 ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL); 2617 if (!ret) { 2618 DRM_ERROR("failed to set up gmch\n"); 2619 return -EIO; 2620 } 2621 #endif 2622 2623 intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end); 2624 2625 dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev); 2626 dev_priv->gtt.base.insert_entries = i915_ggtt_insert_entries; 2627 dev_priv->gtt.base.clear_range = i915_ggtt_clear_range; 2628 dev_priv->gtt.base.bind_vma = ggtt_bind_vma; 2629 dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma; 2630 2631 if (unlikely(dev_priv->gtt.do_idle_maps)) 2632 DRM_INFO("applying Ironlake quirks for intel_iommu\n"); 2633 2634 return 0; 2635 } 2636 2637 static void i915_gmch_remove(struct i915_address_space *vm) 2638 { 2639 intel_gmch_remove(); 2640 } 2641 2642 int i915_gem_gtt_init(struct drm_device *dev) 2643 { 2644 struct drm_i915_private *dev_priv = dev->dev_private; 2645 struct i915_gtt *gtt = &dev_priv->gtt; 2646 int ret; 2647 2648 if (INTEL_INFO(dev)->gen <= 5) { 2649 gtt->gtt_probe = i915_gmch_probe; 2650 gtt->base.cleanup = i915_gmch_remove; 2651 } else if (INTEL_INFO(dev)->gen < 8) { 2652 gtt->gtt_probe = gen6_gmch_probe; 2653 gtt->base.cleanup = gen6_gmch_remove; 2654 if (IS_HASWELL(dev) && dev_priv->ellc_size) 2655 gtt->base.pte_encode = iris_pte_encode; 2656 else if (IS_HASWELL(dev)) 2657 gtt->base.pte_encode = hsw_pte_encode; 2658 else if (IS_VALLEYVIEW(dev)) 2659 gtt->base.pte_encode = byt_pte_encode; 2660 else if (INTEL_INFO(dev)->gen >= 7) 2661 gtt->base.pte_encode = ivb_pte_encode; 2662 else 2663 gtt->base.pte_encode = snb_pte_encode; 2664 } else { 2665 dev_priv->gtt.gtt_probe = gen8_gmch_probe; 2666 dev_priv->gtt.base.cleanup = gen6_gmch_remove; 2667 } 2668 2669 gtt->base.dev = dev; 2670 2671 ret = gtt->gtt_probe(dev, >t->base.total, >t->stolen_size, 2672 >t->mappable_base, >t->mappable_end); 2673 if (ret) 2674 return ret; 2675 2676 /* GMADR is the PCI mmio aperture into the global GTT. */ 2677 DRM_INFO("Memory usable by graphics device = %luM\n", 2678 gtt->base.total >> 20); 2679 DRM_DEBUG_DRIVER("GMADR size = %ldM\n", gtt->mappable_end >> 20); 2680 DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20); 2681 #ifdef CONFIG_INTEL_IOMMU 2682 if (intel_iommu_gfx_mapped) 2683 DRM_INFO("VT-d active for gfx access\n"); 2684 #endif 2685 /* 2686 * i915.enable_ppgtt is read-only, so do an early pass to validate the 2687 * user's requested state against the hardware/driver capabilities. We 2688 * do this now so that we can print out any log messages once rather 2689 * than every time we check intel_enable_ppgtt(). 2690 */ 2691 i915.enable_ppgtt = sanitize_enable_ppgtt(dev, i915.enable_ppgtt); 2692 DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt); 2693 2694 return 0; 2695 } 2696 2697 void i915_gem_restore_gtt_mappings(struct drm_device *dev) 2698 { 2699 struct drm_i915_private *dev_priv = dev->dev_private; 2700 struct drm_i915_gem_object *obj; 2701 struct i915_address_space *vm; 2702 2703 i915_check_and_clear_faults(dev); 2704 2705 /* First fill our portion of the GTT with scratch pages */ 2706 dev_priv->gtt.base.clear_range(&dev_priv->gtt.base, 2707 dev_priv->gtt.base.start, 2708 dev_priv->gtt.base.total, 2709 true); 2710 2711 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { 2712 struct i915_vma *vma = i915_gem_obj_to_vma(obj, 2713 &dev_priv->gtt.base); 2714 if (!vma) 2715 continue; 2716 2717 i915_gem_clflush_object(obj, obj->pin_display); 2718 WARN_ON(i915_vma_bind(vma, obj->cache_level, PIN_UPDATE)); 2719 } 2720 2721 2722 if (INTEL_INFO(dev)->gen >= 8) { 2723 if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev)) 2724 chv_setup_private_ppat(dev_priv); 2725 else 2726 bdw_setup_private_ppat(dev_priv); 2727 2728 return; 2729 } 2730 2731 if (USES_PPGTT(dev)) { 2732 list_for_each_entry(vm, &dev_priv->vm_list, global_link) { 2733 /* TODO: Perhaps it shouldn't be gen6 specific */ 2734 2735 struct i915_hw_ppgtt *ppgtt = 2736 container_of(vm, struct i915_hw_ppgtt, 2737 base); 2738 2739 if (i915_is_ggtt(vm)) 2740 ppgtt = dev_priv->mm.aliasing_ppgtt; 2741 2742 gen6_write_page_range(dev_priv, &ppgtt->pd, 2743 0, ppgtt->base.total); 2744 } 2745 } 2746 2747 i915_ggtt_flush(dev_priv); 2748 } 2749 2750 static struct i915_vma * 2751 __i915_gem_vma_create(struct drm_i915_gem_object *obj, 2752 struct i915_address_space *vm, 2753 const struct i915_ggtt_view *ggtt_view) 2754 { 2755 struct i915_vma *vma; 2756 2757 if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view)) 2758 return ERR_PTR(-EINVAL); 2759 2760 vma = kzalloc(sizeof(*vma), GFP_KERNEL); 2761 if (vma == NULL) 2762 return ERR_PTR(-ENOMEM); 2763 2764 INIT_LIST_HEAD(&vma->vma_link); 2765 INIT_LIST_HEAD(&vma->mm_list); 2766 INIT_LIST_HEAD(&vma->exec_list); 2767 vma->vm = vm; 2768 vma->obj = obj; 2769 2770 if (i915_is_ggtt(vm)) 2771 vma->ggtt_view = *ggtt_view; 2772 2773 list_add_tail(&vma->vma_link, &obj->vma_list); 2774 if (!i915_is_ggtt(vm)) 2775 i915_ppgtt_get(i915_vm_to_ppgtt(vm)); 2776 2777 return vma; 2778 } 2779 2780 struct i915_vma * 2781 i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj, 2782 struct i915_address_space *vm) 2783 { 2784 struct i915_vma *vma; 2785 2786 vma = i915_gem_obj_to_vma(obj, vm); 2787 if (!vma) 2788 vma = __i915_gem_vma_create(obj, vm, 2789 i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL); 2790 2791 return vma; 2792 } 2793 2794 struct i915_vma * 2795 i915_gem_obj_lookup_or_create_ggtt_vma(struct drm_i915_gem_object *obj, 2796 const struct i915_ggtt_view *view) 2797 { 2798 struct i915_address_space *ggtt = i915_obj_to_ggtt(obj); 2799 struct i915_vma *vma; 2800 2801 if (WARN_ON(!view)) 2802 return ERR_PTR(-EINVAL); 2803 2804 vma = i915_gem_obj_to_ggtt_view(obj, view); 2805 2806 if (IS_ERR(vma)) 2807 return vma; 2808 2809 if (!vma) 2810 vma = __i915_gem_vma_create(obj, ggtt, view); 2811 2812 return vma; 2813 2814 } 2815 2816 static void 2817 rotate_pages(dma_addr_t *in, unsigned int width, unsigned int height, 2818 struct sg_table *st) 2819 { 2820 unsigned int column, row; 2821 unsigned int src_idx; 2822 struct scatterlist *sg = st->sgl; 2823 2824 st->nents = 0; 2825 2826 for (column = 0; column < width; column++) { 2827 src_idx = width * (height - 1) + column; 2828 for (row = 0; row < height; row++) { 2829 st->nents++; 2830 /* We don't need the pages, but need to initialize 2831 * the entries so the sg list can be happily traversed. 2832 * The only thing we need are DMA addresses. 2833 */ 2834 sg_set_page(sg, NULL, PAGE_SIZE, 0); 2835 sg_dma_address(sg) = in[src_idx]; 2836 sg_dma_len(sg) = PAGE_SIZE; 2837 sg = sg_next(sg); 2838 src_idx -= width; 2839 } 2840 } 2841 } 2842 2843 static struct sg_table * 2844 intel_rotate_fb_obj_pages(struct i915_ggtt_view *ggtt_view, 2845 struct drm_i915_gem_object *obj) 2846 { 2847 struct intel_rotation_info *rot_info = &ggtt_view->rotation_info; 2848 unsigned int size_pages = rot_info->size >> PAGE_SHIFT; 2849 struct sg_page_iter sg_iter; 2850 unsigned long i; 2851 dma_addr_t *page_addr_list; 2852 struct sg_table *st; 2853 int ret = -ENOMEM; 2854 2855 /* Allocate a temporary list of source pages for random access. */ 2856 page_addr_list = drm_malloc_ab(obj->base.size / PAGE_SIZE, 2857 sizeof(dma_addr_t)); 2858 if (!page_addr_list) 2859 return ERR_PTR(ret); 2860 2861 /* Allocate target SG list. */ 2862 st = kmalloc(sizeof(*st), M_DRM, M_WAITOK); 2863 if (!st) 2864 goto err_st_alloc; 2865 2866 ret = sg_alloc_table(st, size_pages, GFP_KERNEL); 2867 if (ret) 2868 goto err_sg_alloc; 2869 2870 /* Populate source page list from the object. */ 2871 i = 0; 2872 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) { 2873 page_addr_list[i] = sg_page_iter_dma_address(&sg_iter); 2874 i++; 2875 } 2876 2877 /* Rotate the pages. */ 2878 rotate_pages(page_addr_list, 2879 rot_info->width_pages, rot_info->height_pages, 2880 st); 2881 2882 DRM_DEBUG_KMS( 2883 "Created rotated page mapping for object size %zu (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %u pages).\n", 2884 obj->base.size, rot_info->pitch, rot_info->height, 2885 rot_info->pixel_format, rot_info->width_pages, 2886 rot_info->height_pages, size_pages); 2887 2888 drm_free_large(page_addr_list); 2889 2890 return st; 2891 2892 err_sg_alloc: 2893 kfree(st); 2894 err_st_alloc: 2895 drm_free_large(page_addr_list); 2896 2897 DRM_DEBUG_KMS( 2898 "Failed to create rotated mapping for object size %zu! (%d) (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %u pages)\n", 2899 obj->base.size, ret, rot_info->pitch, rot_info->height, 2900 rot_info->pixel_format, rot_info->width_pages, 2901 rot_info->height_pages, size_pages); 2902 return ERR_PTR(ret); 2903 } 2904 2905 static struct sg_table * 2906 intel_partial_pages(const struct i915_ggtt_view *view, 2907 struct drm_i915_gem_object *obj) 2908 { 2909 struct sg_table *st; 2910 struct scatterlist *sg; 2911 struct sg_page_iter obj_sg_iter; 2912 int ret = -ENOMEM; 2913 2914 st = kmalloc(sizeof(*st), M_DRM, M_WAITOK); 2915 if (!st) 2916 goto err_st_alloc; 2917 2918 ret = sg_alloc_table(st, view->params.partial.size, GFP_KERNEL); 2919 if (ret) 2920 goto err_sg_alloc; 2921 2922 sg = st->sgl; 2923 st->nents = 0; 2924 for_each_sg_page(obj->pages->sgl, &obj_sg_iter, obj->pages->nents, 2925 view->params.partial.offset) 2926 { 2927 if (st->nents >= view->params.partial.size) 2928 break; 2929 2930 sg_set_page(sg, NULL, PAGE_SIZE, 0); 2931 sg_dma_address(sg) = sg_page_iter_dma_address(&obj_sg_iter); 2932 sg_dma_len(sg) = PAGE_SIZE; 2933 2934 sg = sg_next(sg); 2935 st->nents++; 2936 } 2937 2938 return st; 2939 2940 err_sg_alloc: 2941 kfree(st); 2942 err_st_alloc: 2943 return ERR_PTR(ret); 2944 } 2945 2946 static int 2947 i915_get_ggtt_vma_pages(struct i915_vma *vma) 2948 { 2949 int ret = 0; 2950 2951 if (vma->ggtt_view.pages) 2952 return 0; 2953 2954 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) 2955 vma->ggtt_view.pages = vma->obj->pages; 2956 else if (vma->ggtt_view.type == I915_GGTT_VIEW_ROTATED) 2957 vma->ggtt_view.pages = 2958 intel_rotate_fb_obj_pages(&vma->ggtt_view, vma->obj); 2959 else if (vma->ggtt_view.type == I915_GGTT_VIEW_PARTIAL) 2960 vma->ggtt_view.pages = 2961 intel_partial_pages(&vma->ggtt_view, vma->obj); 2962 else 2963 WARN_ONCE(1, "GGTT view %u not implemented!\n", 2964 vma->ggtt_view.type); 2965 2966 if (!vma->ggtt_view.pages) { 2967 DRM_ERROR("Failed to get pages for GGTT view type %u!\n", 2968 vma->ggtt_view.type); 2969 ret = -EINVAL; 2970 } else if (IS_ERR(vma->ggtt_view.pages)) { 2971 ret = PTR_ERR(vma->ggtt_view.pages); 2972 vma->ggtt_view.pages = NULL; 2973 DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n", 2974 vma->ggtt_view.type, ret); 2975 } 2976 2977 return ret; 2978 } 2979 2980 /** 2981 * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space. 2982 * @vma: VMA to map 2983 * @cache_level: mapping cache level 2984 * @flags: flags like global or local mapping 2985 * 2986 * DMA addresses are taken from the scatter-gather table of this object (or of 2987 * this VMA in case of non-default GGTT views) and PTE entries set up. 2988 * Note that DMA addresses are also the only part of the SG table we care about. 2989 */ 2990 int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level, 2991 u32 flags) 2992 { 2993 int ret; 2994 u32 bind_flags; 2995 2996 if (WARN_ON(flags == 0)) 2997 return -EINVAL; 2998 2999 bind_flags = 0; 3000 if (flags & PIN_GLOBAL) 3001 bind_flags |= GLOBAL_BIND; 3002 if (flags & PIN_USER) 3003 bind_flags |= LOCAL_BIND; 3004 3005 if (flags & PIN_UPDATE) 3006 bind_flags |= vma->bound; 3007 else 3008 bind_flags &= ~vma->bound; 3009 3010 if (bind_flags == 0) 3011 return 0; 3012 3013 if (vma->bound == 0 && vma->vm->allocate_va_range) { 3014 trace_i915_va_alloc(vma->vm, 3015 vma->node.start, 3016 vma->node.size, 3017 VM_TO_TRACE_NAME(vma->vm)); 3018 3019 /* XXX: i915_vma_pin() will fix this +- hack */ 3020 vma->pin_count++; 3021 ret = vma->vm->allocate_va_range(vma->vm, 3022 vma->node.start, 3023 vma->node.size); 3024 vma->pin_count--; 3025 if (ret) 3026 return ret; 3027 } 3028 3029 ret = vma->vm->bind_vma(vma, cache_level, bind_flags); 3030 if (ret) 3031 return ret; 3032 3033 vma->bound |= bind_flags; 3034 3035 return 0; 3036 } 3037 3038 /** 3039 * i915_ggtt_view_size - Get the size of a GGTT view. 3040 * @obj: Object the view is of. 3041 * @view: The view in question. 3042 * 3043 * @return The size of the GGTT view in bytes. 3044 */ 3045 size_t 3046 i915_ggtt_view_size(struct drm_i915_gem_object *obj, 3047 const struct i915_ggtt_view *view) 3048 { 3049 if (view->type == I915_GGTT_VIEW_NORMAL) { 3050 return obj->base.size; 3051 } else if (view->type == I915_GGTT_VIEW_ROTATED) { 3052 return view->rotation_info.size; 3053 } else if (view->type == I915_GGTT_VIEW_PARTIAL) { 3054 return view->params.partial.size << PAGE_SHIFT; 3055 } else { 3056 WARN_ONCE(1, "GGTT view %u not implemented!\n", view->type); 3057 return obj->base.size; 3058 } 3059 } 3060