1 /* 2 * Copyright © 2010 Daniel Vetter 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <drm/drmP.h> 26 #include <drm/i915_drm.h> 27 #include "i915_drv.h" 28 #include "intel_drv.h" 29 30 #include <linux/highmem.h> 31 32 /* PPGTT stuff */ 33 #define GEN6_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0xff0)) 34 35 #define GEN6_PDE_VALID (1 << 0) 36 /* gen6+ has bit 11-4 for physical addr bit 39-32 */ 37 #define GEN6_PDE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) 38 39 #define GEN6_PTE_VALID (1 << 0) 40 #define GEN6_PTE_UNCACHED (1 << 1) 41 #define HSW_PTE_UNCACHED (0) 42 #define GEN6_PTE_CACHE_LLC (2 << 1) 43 #define GEN6_PTE_CACHE_LLC_MLC (3 << 1) 44 #define GEN6_PTE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) 45 46 static gen6_gtt_pte_t gen6_pte_encode(struct drm_device *dev, 47 dma_addr_t addr, 48 enum i915_cache_level level) 49 { 50 gen6_gtt_pte_t pte = GEN6_PTE_VALID; 51 pte |= GEN6_PTE_ADDR_ENCODE(addr); 52 53 switch (level) { 54 case I915_CACHE_LLC_MLC: 55 pte |= GEN6_PTE_CACHE_LLC_MLC; 56 break; 57 case I915_CACHE_LLC: 58 pte |= GEN6_PTE_CACHE_LLC; 59 break; 60 case I915_CACHE_NONE: 61 pte |= GEN6_PTE_UNCACHED; 62 break; 63 default: 64 BUG(); 65 } 66 67 return pte; 68 } 69 70 #define BYT_PTE_WRITEABLE (1 << 1) 71 #define BYT_PTE_SNOOPED_BY_CPU_CACHES (1 << 2) 72 73 static gen6_gtt_pte_t byt_pte_encode(struct drm_device *dev, 74 dma_addr_t addr, 75 enum i915_cache_level level) 76 { 77 gen6_gtt_pte_t pte = GEN6_PTE_VALID; 78 pte |= GEN6_PTE_ADDR_ENCODE(addr); 79 80 /* Mark the page as writeable. Other platforms don't have a 81 * setting for read-only/writable, so this matches that behavior. 82 */ 83 pte |= BYT_PTE_WRITEABLE; 84 85 if (level != I915_CACHE_NONE) 86 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES; 87 88 return pte; 89 } 90 91 static gen6_gtt_pte_t hsw_pte_encode(struct drm_device *dev, 92 dma_addr_t addr, 93 enum i915_cache_level level) 94 { 95 gen6_gtt_pte_t pte = GEN6_PTE_VALID; 96 pte |= GEN6_PTE_ADDR_ENCODE(addr); 97 98 if (level != I915_CACHE_NONE) 99 pte |= GEN6_PTE_CACHE_LLC; 100 101 return pte; 102 } 103 104 static int gen6_ppgtt_enable(struct drm_device *dev) 105 { 106 drm_i915_private_t *dev_priv = dev->dev_private; 107 uint32_t pd_offset; 108 struct intel_ring_buffer *ring; 109 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; 110 uint32_t pd_entry, first_pd_entry_in_global_pt; 111 int i; 112 113 WARN_ON(ppgtt->pd_offset & 0x3f); 114 115 first_pd_entry_in_global_pt = 512 * 1024 - I915_PPGTT_PD_ENTRIES; 116 for (i = 0; i < ppgtt->num_pd_entries; i++) { 117 dma_addr_t pt_addr; 118 119 pt_addr = VM_PAGE_TO_PHYS(ppgtt->pt_pages[i]); 120 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr); 121 pd_entry |= GEN6_PDE_VALID; 122 123 intel_gtt_write(first_pd_entry_in_global_pt + i, pd_entry); 124 } 125 intel_gtt_read_pte(first_pd_entry_in_global_pt); 126 127 pd_offset = ppgtt->pd_offset; 128 pd_offset /= 64; /* in cachelines, */ 129 pd_offset <<= 16; 130 131 if (INTEL_INFO(dev)->gen == 6) { 132 uint32_t ecochk, gab_ctl, ecobits; 133 134 ecobits = I915_READ(GAC_ECO_BITS); 135 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT | 136 ECOBITS_PPGTT_CACHE64B); 137 138 gab_ctl = I915_READ(GAB_CTL); 139 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT); 140 141 ecochk = I915_READ(GAM_ECOCHK); 142 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | 143 ECOCHK_PPGTT_CACHE64B); 144 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); 145 } else if (INTEL_INFO(dev)->gen >= 7) { 146 uint32_t ecochk, ecobits; 147 148 ecobits = I915_READ(GAC_ECO_BITS); 149 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B); 150 151 ecochk = I915_READ(GAM_ECOCHK); 152 if (IS_HASWELL(dev)) { 153 ecochk |= ECOCHK_PPGTT_WB_HSW; 154 } else { 155 ecochk |= ECOCHK_PPGTT_LLC_IVB; 156 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB; 157 } 158 I915_WRITE(GAM_ECOCHK, ecochk); 159 /* GFX_MODE is per-ring on gen7+ */ 160 } 161 162 for_each_ring(ring, dev_priv, i) { 163 if (INTEL_INFO(dev)->gen >= 7) 164 I915_WRITE(RING_MODE_GEN7(ring), 165 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); 166 167 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); 168 I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset); 169 } 170 return 0; 171 } 172 173 /* PPGTT support for Sandybdrige/Gen6 and later */ 174 static void gen6_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt, 175 unsigned first_entry, 176 unsigned num_entries) 177 { 178 gen6_gtt_pte_t *pt_vaddr, scratch_pte; 179 unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES; 180 unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES; 181 unsigned last_pte, i; 182 183 scratch_pte = ppgtt->pte_encode(ppgtt->dev, 184 ppgtt->scratch_page_dma_addr, 185 I915_CACHE_LLC); 186 187 while (num_entries) { 188 last_pte = first_pte + num_entries; 189 if (last_pte > I915_PPGTT_PT_ENTRIES) 190 last_pte = I915_PPGTT_PT_ENTRIES; 191 192 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pt]); 193 194 for (i = first_pte; i < last_pte; i++) 195 pt_vaddr[i] = scratch_pte; 196 197 kunmap_atomic(pt_vaddr); 198 199 num_entries -= last_pte - first_pte; 200 first_pte = 0; 201 act_pt++; 202 } 203 } 204 205 static void gen6_ppgtt_insert_entries(struct i915_hw_ppgtt *ppgtt, 206 struct sg_table *pages, 207 unsigned first_entry, 208 enum i915_cache_level cache_level) 209 { 210 gen6_gtt_pte_t *pt_vaddr; 211 unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES; 212 unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES; 213 unsigned i, j, m, segment_len; 214 dma_addr_t page_addr; 215 struct scatterlist *sg; 216 217 /* init sg walking */ 218 sg = pages->sgl; 219 i = 0; 220 segment_len = sg_dma_len(sg) >> PAGE_SHIFT; 221 m = 0; 222 223 while (i < pages->nents) { 224 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pt]); 225 226 for (j = first_pte; j < I915_PPGTT_PT_ENTRIES; j++) { 227 page_addr = sg_dma_address(sg) + (m << PAGE_SHIFT); 228 pt_vaddr[j] = ppgtt->pte_encode(ppgtt->dev, page_addr, 229 cache_level); 230 231 /* grab the next page */ 232 if (++m == segment_len) { 233 if (++i == pages->nents) 234 break; 235 236 sg = sg_next(sg); 237 segment_len = sg_dma_len(sg) >> PAGE_SHIFT; 238 m = 0; 239 } 240 } 241 242 kunmap_atomic(pt_vaddr); 243 244 first_pte = 0; 245 act_pt++; 246 } 247 } 248 249 static void gen6_ppgtt_cleanup(struct i915_hw_ppgtt *ppgtt) 250 { 251 #if 0 252 int i; 253 254 if (ppgtt->pt_dma_addr) { 255 for (i = 0; i < ppgtt->num_pd_entries; i++) 256 pci_unmap_page(ppgtt->dev->pdev, 257 ppgtt->pt_dma_addr[i], 258 4096, PCI_DMA_BIDIRECTIONAL); 259 } 260 261 kfree(ppgtt->pt_dma_addr); 262 for (i = 0; i < ppgtt->num_pd_entries; i++) 263 __free_page(ppgtt->pt_pages[i]); 264 kfree(ppgtt->pt_pages); 265 kfree(ppgtt); 266 #endif 267 } 268 269 static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) 270 { 271 struct drm_device *dev = ppgtt->dev; 272 struct drm_i915_private *dev_priv = dev->dev_private; 273 unsigned first_pd_entry_in_global_pt; 274 int i; 275 int ret = -ENOMEM; 276 277 /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024 278 * entries. For aliasing ppgtt support we just steal them at the end for 279 * now. 280 */ 281 first_pd_entry_in_global_pt = gtt_total_entries(dev_priv->gtt); 282 283 if (IS_HASWELL(dev)) { 284 ppgtt->pte_encode = hsw_pte_encode; 285 } else if (IS_VALLEYVIEW(dev)) { 286 ppgtt->pte_encode = byt_pte_encode; 287 } else { 288 ppgtt->pte_encode = gen6_pte_encode; 289 } 290 ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES; 291 ppgtt->enable = gen6_ppgtt_enable; 292 ppgtt->clear_range = gen6_ppgtt_clear_range; 293 ppgtt->insert_entries = gen6_ppgtt_insert_entries; 294 ppgtt->cleanup = gen6_ppgtt_cleanup; 295 ppgtt->pt_pages = kzalloc(sizeof(struct vm_page *)*ppgtt->num_pd_entries, 296 GFP_KERNEL); 297 if (!ppgtt->pt_pages) 298 return -ENOMEM; 299 300 for (i = 0; i < ppgtt->num_pd_entries; i++) { 301 ppgtt->pt_pages[i] = vm_page_alloc(NULL, 0, 302 VM_ALLOC_NORMAL | VM_ALLOC_ZERO); 303 if (!ppgtt->pt_pages[i]) 304 goto err_pt_alloc; 305 } 306 307 ppgtt->clear_range(ppgtt, 0, 308 ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES); 309 310 ppgtt->pd_offset = first_pd_entry_in_global_pt * sizeof(gen6_gtt_pte_t); 311 312 return 0; 313 314 err_pt_alloc: 315 dev_priv->mm.aliasing_ppgtt = ppgtt; 316 i915_gem_cleanup_aliasing_ppgtt(dev); 317 318 return ret; 319 } 320 321 static int i915_gem_init_aliasing_ppgtt(struct drm_device *dev) 322 { 323 struct drm_i915_private *dev_priv = dev->dev_private; 324 struct i915_hw_ppgtt *ppgtt; 325 int ret; 326 327 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); 328 if (!ppgtt) 329 return -ENOMEM; 330 331 ppgtt->dev = dev; 332 ppgtt->scratch_page_dma_addr = dev_priv->gtt.scratch_page_dma; 333 334 if (INTEL_INFO(dev)->gen < 8) 335 ret = gen6_ppgtt_init(ppgtt); 336 else 337 BUG(); 338 339 if (ret) 340 kfree(ppgtt); 341 else 342 dev_priv->mm.aliasing_ppgtt = ppgtt; 343 344 return ret; 345 } 346 347 void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev) 348 { 349 struct drm_i915_private *dev_priv = dev->dev_private; 350 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; 351 352 if (!ppgtt) 353 return; 354 355 ppgtt->cleanup(ppgtt); 356 dev_priv->mm.aliasing_ppgtt = NULL; 357 } 358 359 static void 360 i915_ppgtt_insert_pages(struct i915_hw_ppgtt *ppgtt, unsigned first_entry, 361 unsigned num_entries, vm_page_t *pages, enum i915_cache_level cache_level) 362 { 363 uint32_t *pt_vaddr; 364 unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES; 365 unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES; 366 unsigned last_pte, i; 367 dma_addr_t page_addr; 368 369 while (num_entries) { 370 last_pte = first_pte + num_entries; 371 if (last_pte > I915_PPGTT_PT_ENTRIES) 372 last_pte = I915_PPGTT_PT_ENTRIES; 373 374 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]); 375 376 for (i = first_pte; i < last_pte; i++) { 377 page_addr = VM_PAGE_TO_PHYS(*pages); 378 pt_vaddr[i] = ppgtt->pte_encode(ppgtt->dev, page_addr, 379 cache_level); 380 381 pages++; 382 } 383 384 kunmap_atomic(pt_vaddr); 385 386 num_entries -= last_pte - first_pte; 387 first_pte = 0; 388 act_pd++; 389 } 390 } 391 392 void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt, 393 struct drm_i915_gem_object *obj, 394 enum i915_cache_level cache_level) 395 { 396 i915_ppgtt_insert_pages(ppgtt, obj->gtt_space->start >> PAGE_SHIFT, 397 obj->base.size >> PAGE_SHIFT, obj->pages, cache_level); 398 } 399 400 void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt, 401 struct drm_i915_gem_object *obj) 402 { 403 ppgtt->clear_range(ppgtt, 404 obj->gtt_space->start >> PAGE_SHIFT, 405 obj->base.size >> PAGE_SHIFT); 406 } 407 408 extern int intel_iommu_gfx_mapped; 409 /* Certain Gen5 chipsets require require idling the GPU before 410 * unmapping anything from the GTT when VT-d is enabled. 411 */ 412 static inline bool needs_idle_maps(struct drm_device *dev) 413 { 414 #ifdef CONFIG_INTEL_IOMMU 415 /* Query intel_iommu to see if we need the workaround. Presumably that 416 * was loaded first. 417 */ 418 if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped) 419 return true; 420 #endif 421 return false; 422 } 423 424 static bool do_idling(struct drm_i915_private *dev_priv) 425 { 426 bool ret = dev_priv->mm.interruptible; 427 428 if (unlikely(dev_priv->gtt.do_idle_maps)) { 429 dev_priv->mm.interruptible = false; 430 if (i915_gpu_idle(dev_priv->dev)) { 431 DRM_ERROR("Couldn't idle GPU\n"); 432 /* Wait a bit, in hopes it avoids the hang */ 433 udelay(10); 434 } 435 } 436 437 return ret; 438 } 439 440 static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible) 441 { 442 if (unlikely(dev_priv->gtt.do_idle_maps)) 443 dev_priv->mm.interruptible = interruptible; 444 } 445 446 void i915_gem_restore_gtt_mappings(struct drm_device *dev) 447 { 448 struct drm_i915_private *dev_priv = dev->dev_private; 449 struct drm_i915_gem_object *obj; 450 451 /* First fill our portion of the GTT with scratch pages */ 452 dev_priv->gtt.gtt_clear_range(dev, dev_priv->gtt.start / PAGE_SIZE, 453 dev_priv->gtt.total / PAGE_SIZE); 454 455 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { 456 i915_gem_clflush_object(obj); 457 i915_gem_gtt_bind_object(obj, obj->cache_level); 458 } 459 460 i915_gem_chipset_flush(dev); 461 } 462 463 #if 0 464 int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj) 465 { 466 if (obj->has_dma_mapping) 467 return 0; 468 469 if (!dma_map_sg(&obj->base.dev->pdev->dev, 470 obj->pages->sgl, obj->pages->nents, 471 PCI_DMA_BIDIRECTIONAL)) 472 return -ENOSPC; 473 474 return 0; 475 } 476 #endif 477 478 /* 479 * Binds an object into the global gtt with the specified cache level. The object 480 * will be accessible to the GPU via commands whose operands reference offsets 481 * within the global GTT as well as accessible by the GPU through the GMADR 482 * mapped BAR (dev_priv->mm.gtt->gtt). 483 */ 484 #if 0 485 static void gen6_ggtt_insert_entries(struct drm_device *dev, 486 struct sg_table *st, 487 unsigned int first_entry, 488 enum i915_cache_level level) 489 { 490 struct drm_i915_private *dev_priv = dev->dev_private; 491 gen6_gtt_pte_t __iomem *gtt_entries = 492 (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry; 493 int i = 0; 494 struct sg_page_iter sg_iter; 495 dma_addr_t addr; 496 497 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) { 498 addr = sg_page_iter_dma_address(&sg_iter); 499 iowrite32(dev_priv->gtt.pte_encode(dev, addr, level), 500 >t_entries[i]); 501 i++; 502 } 503 504 /* XXX: This serves as a posting read to make sure that the PTE has 505 * actually been updated. There is some concern that even though 506 * registers and PTEs are within the same BAR that they are potentially 507 * of NUMA access patterns. Therefore, even with the way we assume 508 * hardware should work, we must keep this posting read for paranoia. 509 */ 510 if (i != 0) 511 WARN_ON(readl(>t_entries[i-1]) 512 != dev_priv->gtt.pte_encode(dev, addr, level)); 513 514 /* This next bit makes the above posting read even more important. We 515 * want to flush the TLBs only after we're certain all the PTE updates 516 * have finished. 517 */ 518 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 519 POSTING_READ(GFX_FLSH_CNTL_GEN6); 520 } 521 522 static void gen6_ggtt_clear_range(struct drm_device *dev, 523 unsigned int first_entry, 524 unsigned int num_entries) 525 { 526 struct drm_i915_private *dev_priv = dev->dev_private; 527 gen6_gtt_pte_t scratch_pte, __iomem *gtt_base = 528 (gen6_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry; 529 const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry; 530 int i; 531 532 if (WARN(num_entries > max_entries, 533 "First entry = %d; Num entries = %d (max=%d)\n", 534 first_entry, num_entries, max_entries)) 535 num_entries = max_entries; 536 537 scratch_pte = dev_priv->gtt.pte_encode(dev, 538 dev_priv->gtt.scratch_page_dma, 539 I915_CACHE_LLC); 540 for (i = 0; i < num_entries; i++) 541 iowrite32(scratch_pte, >t_base[i]); 542 readl(gtt_base); 543 } 544 #endif 545 546 static void i915_ggtt_insert_entries(struct drm_device *dev, 547 struct sg_table *st, 548 unsigned int pg_start, 549 enum i915_cache_level cache_level) 550 { 551 #if 0 552 unsigned int flags = (cache_level == I915_CACHE_NONE) ? 553 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; 554 555 intel_gtt_insert_sg_entries(st, pg_start, flags); 556 #endif 557 } 558 559 static void i915_ggtt_clear_range(struct drm_device *dev, 560 unsigned int first_entry, 561 unsigned int num_entries) 562 { 563 intel_gtt_clear_range(first_entry, num_entries); 564 } 565 566 void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj, 567 enum i915_cache_level cache_level) 568 { 569 unsigned int flags = (cache_level == I915_CACHE_NONE) ? 570 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; 571 intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT, 572 obj->base.size >> PAGE_SHIFT, obj->pages, flags); 573 574 obj->has_global_gtt_mapping = 1; 575 } 576 577 void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj) 578 { 579 struct drm_device *dev = obj->base.dev; 580 struct drm_i915_private *dev_priv = dev->dev_private; 581 582 dev_priv->gtt.gtt_clear_range(obj->base.dev, 583 obj->gtt_space->start >> PAGE_SHIFT, 584 obj->base.size >> PAGE_SHIFT); 585 586 obj->has_global_gtt_mapping = 0; 587 } 588 589 void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) 590 { 591 struct drm_device *dev = obj->base.dev; 592 struct drm_i915_private *dev_priv = dev->dev_private; 593 bool interruptible; 594 595 interruptible = do_idling(dev_priv); 596 597 #if 0 598 if (!obj->has_dma_mapping) 599 dma_unmap_sg(&dev->pdev->dev, 600 obj->pages->sgl, obj->pages->nents, 601 PCI_DMA_BIDIRECTIONAL); 602 #endif 603 604 undo_idling(dev_priv, interruptible); 605 } 606 607 static void i915_gtt_color_adjust(struct drm_mm_node *node, 608 unsigned long color, 609 unsigned long *start, 610 unsigned long *end) 611 { 612 if (node->color != color) 613 *start += 4096; 614 615 if (!list_empty(&node->node_list)) { 616 node = list_entry(node->node_list.next, 617 struct drm_mm_node, 618 node_list); 619 if (node->allocated && node->color != color) 620 *end -= 4096; 621 } 622 } 623 void i915_gem_setup_global_gtt(struct drm_device *dev, 624 unsigned long start, 625 unsigned long mappable_end, 626 unsigned long end) 627 { 628 /* Let GEM Manage all of the aperture. 629 * 630 * However, leave one page at the end still bound to the scratch page. 631 * There are a number of places where the hardware apparently prefetches 632 * past the end of the object, and we've seen multiple hangs with the 633 * GPU head pointer stuck in a batchbuffer bound at the last page of the 634 * aperture. One page should be enough to keep any prefetching inside 635 * of the aperture. 636 */ 637 drm_i915_private_t *dev_priv = dev->dev_private; 638 unsigned long mappable; 639 int error; 640 641 BUG_ON(mappable_end > end); 642 643 mappable = min(end, mappable_end) - start; 644 645 /* Substract the guard page ... */ 646 drm_mm_init(&dev_priv->mm.gtt_space, start, end - start); 647 if (!HAS_LLC(dev)) 648 dev_priv->mm.gtt_space.color_adjust = i915_gtt_color_adjust; 649 650 dev_priv->gtt.start = start; 651 dev_priv->gtt.mappable_end = mappable_end; 652 dev_priv->gtt.total = end - start; 653 654 /* ... but ensure that we clear the entire range. */ 655 intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE); 656 device_printf(dev->dev, 657 "taking over the fictitious range 0x%lx-0x%lx\n", 658 dev->agp->base + start, dev->agp->base + start + mappable); 659 error = -vm_phys_fictitious_reg_range(dev->agp->base + start, 660 dev->agp->base + start + mappable, VM_MEMATTR_WRITE_COMBINING); 661 } 662 663 static bool 664 intel_enable_ppgtt(struct drm_device *dev) 665 { 666 if (i915_enable_ppgtt >= 0) 667 return i915_enable_ppgtt; 668 669 #ifdef CONFIG_INTEL_IOMMU 670 /* Disable ppgtt on SNB if VT-d is on. */ 671 if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) 672 return false; 673 #endif 674 675 return true; 676 } 677 678 void i915_gem_init_global_gtt(struct drm_device *dev) 679 { 680 struct drm_i915_private *dev_priv = dev->dev_private; 681 unsigned long gtt_size, mappable_size; 682 683 gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT; 684 mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT; 685 686 if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) { 687 int ret; 688 689 if (INTEL_INFO(dev)->gen <= 7) { 690 /* PPGTT pdes are stolen from global gtt ptes, so shrink the 691 * aperture accordingly when using aliasing ppgtt. */ 692 gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE; 693 } 694 695 i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size); 696 697 ret = i915_gem_init_aliasing_ppgtt(dev); 698 if (!ret) 699 return; 700 701 DRM_ERROR("Aliased PPGTT setup failed %d\n", ret); 702 drm_mm_takedown(&dev_priv->mm.gtt_space); 703 gtt_size += I915_PPGTT_PD_ENTRIES*PAGE_SIZE; 704 } 705 i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size); 706 } 707 708 #if 0 709 static int gen6_gmch_probe(struct drm_device *dev, 710 size_t *gtt_total, 711 size_t *stolen, 712 phys_addr_t *mappable_base, 713 unsigned long *mappable_end) 714 { 715 struct drm_i915_private *dev_priv = dev->dev_private; 716 phys_addr_t gtt_bus_addr; 717 unsigned int gtt_size; 718 u16 snb_gmch_ctl; 719 int ret; 720 721 *mappable_base = pci_resource_start(dev->pdev, 2); 722 *mappable_end = pci_resource_len(dev->pdev, 2); 723 724 /* 64/512MB is the current min/max we actually know of, but this is just 725 * a coarse sanity check. 726 */ 727 if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) { 728 DRM_ERROR("Unknown GMADR size (%lx)\n", 729 dev_priv->gtt.mappable_end); 730 return -ENXIO; 731 } 732 733 if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40))) 734 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40)); 735 pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); 736 gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl); 737 738 if (IS_GEN7(dev) && !IS_VALLEYVIEW(dev)) 739 *stolen = gen7_get_stolen_size(snb_gmch_ctl); 740 else 741 *stolen = gen6_get_stolen_size(snb_gmch_ctl); 742 743 *gtt_total = (gtt_size / sizeof(gen6_gtt_pte_t)) << PAGE_SHIFT; 744 745 /* For Modern GENs the PTEs and register space are split in the BAR */ 746 gtt_bus_addr = pci_resource_start(dev->pdev, 0) + 747 (pci_resource_len(dev->pdev, 0) / 2); 748 749 dev_priv->gtt.gsm = ioremap_wc(gtt_bus_addr, gtt_size); 750 if (!dev_priv->gtt.gsm) { 751 DRM_ERROR("Failed to map the gtt page table\n"); 752 return -ENOMEM; 753 } 754 755 ret = setup_scratch_page(dev); 756 if (ret) 757 DRM_ERROR("Scratch setup failed\n"); 758 759 dev_priv->gtt.gtt_clear_range = gen6_ggtt_clear_range; 760 dev_priv->gtt.gtt_insert_entries = gen6_ggtt_insert_entries; 761 762 return 0; 763 } 764 765 static void gen6_gmch_remove(struct drm_device *dev) 766 { 767 struct drm_i915_private *dev_priv = dev->dev_private; 768 iounmap(dev_priv->gtt.gsm); 769 teardown_scratch_page(dev_priv->dev); 770 } 771 #endif 772 773 static int i915_gmch_probe(struct drm_device *dev, 774 size_t *gtt_total, 775 size_t *stolen, 776 phys_addr_t *mappable_base, 777 unsigned long *mappable_end) 778 { 779 return 0; 780 } 781 782 static void i915_gmch_remove(struct drm_device *dev) 783 { 784 } 785 786 int i915_gem_gtt_init(struct drm_device *dev) 787 { 788 struct drm_i915_private *dev_priv = dev->dev_private; 789 790 /* XXX Legacy agp stuff */ 791 dev_priv->mm.gtt = intel_gtt_get(); 792 if (!dev_priv->mm.gtt) { 793 DRM_ERROR("Failed to initialize GTT\n"); 794 return -ENODEV; 795 } 796 797 if (INTEL_INFO(dev)->gen <= 5 || 1) { 798 dev_priv->gtt.gtt_probe = i915_gmch_probe; 799 dev_priv->gtt.gtt_remove = i915_gmch_remove; 800 801 dev_priv->gtt.do_idle_maps = needs_idle_maps(dev); 802 803 dev_priv->gtt.gtt_clear_range = i915_ggtt_clear_range; 804 dev_priv->gtt.gtt_insert_entries = i915_ggtt_insert_entries; 805 #if 0 806 } else { 807 dev_priv->gtt.gtt_probe = gen6_gmch_probe; 808 dev_priv->gtt.gtt_remove = gen6_gmch_remove; 809 #endif 810 if (IS_HASWELL(dev)) { 811 dev_priv->gtt.pte_encode = hsw_pte_encode; 812 } else if (IS_VALLEYVIEW(dev)) { 813 dev_priv->gtt.pte_encode = byt_pte_encode; 814 } else { 815 dev_priv->gtt.pte_encode = gen6_pte_encode; 816 } 817 818 return 0; 819 } 820 821 dev_priv->mm.gtt = kmalloc(sizeof(*dev_priv->mm.gtt), M_DRM, M_WAITOK | M_ZERO); 822 if (!dev_priv->mm.gtt) 823 return -ENOMEM; 824 825 /* GMADR is the PCI mmio aperture into the global GTT. */ 826 DRM_INFO("Memory usable by graphics device = %zdM\n", 827 dev_priv->gtt.total >> 20); 828 DRM_DEBUG_DRIVER("GMADR size = %ldM\n", 829 dev_priv->gtt.mappable_end >> 20); 830 DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", 831 dev_priv->gtt.stolen_size >> 20); 832 833 return 0; 834 } 835