1 /* i915_drv.c -- i830,i845,i855,i865,i915 driver -*- linux-c -*- 2 */ 3 /* 4 * 5 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas. 6 * All Rights Reserved. 7 * 8 * Permission is hereby granted, free of charge, to any person obtaining a 9 * copy of this software and associated documentation files (the 10 * "Software"), to deal in the Software without restriction, including 11 * without limitation the rights to use, copy, modify, merge, publish, 12 * distribute, sub license, and/or sell copies of the Software, and to 13 * permit persons to whom the Software is furnished to do so, subject to 14 * the following conditions: 15 * 16 * The above copyright notice and this permission notice (including the 17 * next paragraph) shall be included in all copies or substantial portions 18 * of the Software. 19 * 20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 22 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 23 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR 24 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 25 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 26 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 27 * 28 */ 29 30 #ifdef __DragonFly__ 31 #include "opt_drm.h" /* for VGA_SWITCHEROO */ 32 #endif 33 34 #include <linux/acpi.h> 35 #include <linux/device.h> 36 #include <linux/module.h> 37 #include <linux/pci.h> 38 #include <linux/pm.h> 39 #include <linux/pm_runtime.h> 40 #include <linux/slab.h> 41 #include <linux/vgaarb.h> 42 #include <linux/vga_switcheroo.h> 43 #include <acpi/video.h> 44 45 #include <drm/drmP.h> 46 #include <drm/drm_crtc_helper.h> 47 #include <drm/i915_drm.h> 48 49 #include "i915_drv.h" 50 #include "i915_trace.h" 51 #include "i915_vgpu.h" 52 #include "intel_drv.h" 53 54 static struct drm_driver driver; 55 56 static unsigned int i915_load_fail_count; 57 58 bool __i915_inject_load_failure(const char *func, int line) 59 { 60 if (i915_load_fail_count >= i915.inject_load_failure) 61 return false; 62 63 if (++i915_load_fail_count == i915.inject_load_failure) { 64 DRM_INFO("Injecting failure at checkpoint %u [%s:%d]\n", 65 i915.inject_load_failure, func, line); 66 return true; 67 } 68 69 return false; 70 } 71 72 #define FDO_BUG_URL "https://bugs.freedesktop.org/enter_bug.cgi?product=DRI" 73 #define FDO_BUG_MSG "Please file a bug at " FDO_BUG_URL " against DRM/Intel " \ 74 "providing the dmesg log by booting with drm.debug=0xf" 75 76 void 77 __i915_printk(struct drm_i915_private *dev_priv, const char *level, 78 const char *fmt, ...) 79 { 80 static bool shown_bug_once; 81 struct device *dev = dev_priv->drm.dev; 82 bool is_error = level[1] <= KERN_ERR[1]; 83 bool is_debug = level[1] == KERN_DEBUG[1]; 84 struct va_format vaf; 85 va_list args; 86 87 if (is_debug && !(drm_debug & DRM_UT_DRIVER)) 88 return; 89 90 va_start(args, fmt); 91 92 vaf.fmt = fmt; 93 vaf.va = &args; 94 95 dev_printk(level, dev, "[" DRM_NAME ":%ps] %pV", 96 __builtin_return_address(0), &vaf); 97 98 if (is_error && !shown_bug_once) { 99 #if 0 100 dev_notice(dev, "%s", FDO_BUG_MSG); 101 #endif 102 shown_bug_once = true; 103 } 104 105 va_end(args); 106 } 107 108 static bool i915_error_injected(struct drm_i915_private *dev_priv) 109 { 110 return i915.inject_load_failure && 111 i915_load_fail_count == i915.inject_load_failure; 112 } 113 114 #define i915_load_error(dev_priv, fmt, ...) \ 115 __i915_printk(dev_priv, \ 116 i915_error_injected(dev_priv) ? KERN_DEBUG : KERN_ERR, \ 117 fmt, ##__VA_ARGS__) 118 119 120 static enum intel_pch intel_virt_detect_pch(struct drm_device *dev) 121 { 122 enum intel_pch ret = PCH_NOP; 123 124 /* 125 * In a virtualized passthrough environment we can be in a 126 * setup where the ISA bridge is not able to be passed through. 127 * In this case, a south bridge can be emulated and we have to 128 * make an educated guess as to which PCH is really there. 129 */ 130 131 if (IS_GEN5(dev)) { 132 ret = PCH_IBX; 133 DRM_DEBUG_KMS("Assuming Ibex Peak PCH\n"); 134 } else if (IS_GEN6(dev) || IS_IVYBRIDGE(dev)) { 135 ret = PCH_CPT; 136 DRM_DEBUG_KMS("Assuming CouarPoint PCH\n"); 137 } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) { 138 ret = PCH_LPT; 139 DRM_DEBUG_KMS("Assuming LynxPoint PCH\n"); 140 } else if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) { 141 ret = PCH_SPT; 142 DRM_DEBUG_KMS("Assuming SunrisePoint PCH\n"); 143 } 144 145 return ret; 146 } 147 148 static void intel_detect_pch(struct drm_device *dev) 149 { 150 struct drm_i915_private *dev_priv = to_i915(dev); 151 device_t pch = NULL; 152 struct pci_devinfo *di = NULL; 153 154 /* In all current cases, num_pipes is equivalent to the PCH_NOP setting 155 * (which really amounts to a PCH but no South Display). 156 */ 157 if (INTEL_INFO(dev)->num_pipes == 0) { 158 dev_priv->pch_type = PCH_NOP; 159 return; 160 } 161 162 /* XXX The ISA bridge probe causes some old Core2 machines to hang */ 163 if (INTEL_INFO(dev)->gen < 5) 164 return; 165 166 /* 167 * The reason to probe ISA bridge instead of Dev31:Fun0 is to 168 * make graphics device passthrough work easy for VMM, that only 169 * need to expose ISA bridge to let driver know the real hardware 170 * underneath. This is a requirement from virtualization team. 171 * 172 * In some virtualized environments (e.g. XEN), there is irrelevant 173 * ISA bridge in the system. To work reliably, we should scan trhough 174 * all the ISA bridge devices and check for the first match, instead 175 * of only checking the first one. 176 */ 177 while ((pch = pci_iterate_class(&di, PCIC_BRIDGE, PCIS_BRIDGE_ISA))) { 178 if (pci_get_vendor(pch) == PCI_VENDOR_ID_INTEL) { 179 unsigned short id = pci_get_device(pch) & INTEL_PCH_DEVICE_ID_MASK; 180 dev_priv->pch_id = id; 181 182 if (id == INTEL_PCH_IBX_DEVICE_ID_TYPE) { 183 dev_priv->pch_type = PCH_IBX; 184 DRM_DEBUG_KMS("Found Ibex Peak PCH\n"); 185 WARN_ON(!IS_GEN5(dev)); 186 } else if (id == INTEL_PCH_CPT_DEVICE_ID_TYPE) { 187 dev_priv->pch_type = PCH_CPT; 188 DRM_DEBUG_KMS("Found CougarPoint PCH\n"); 189 WARN_ON(!(IS_GEN6(dev) || IS_IVYBRIDGE(dev))); 190 } else if (id == INTEL_PCH_PPT_DEVICE_ID_TYPE) { 191 /* PantherPoint is CPT compatible */ 192 dev_priv->pch_type = PCH_CPT; 193 DRM_DEBUG_KMS("Found PantherPoint PCH\n"); 194 WARN_ON(!(IS_GEN6(dev) || IS_IVYBRIDGE(dev))); 195 } else if (id == INTEL_PCH_LPT_DEVICE_ID_TYPE) { 196 dev_priv->pch_type = PCH_LPT; 197 DRM_DEBUG_KMS("Found LynxPoint PCH\n"); 198 WARN_ON(!IS_HASWELL(dev) && !IS_BROADWELL(dev)); 199 WARN_ON(IS_HSW_ULT(dev) || IS_BDW_ULT(dev)); 200 } else if (id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) { 201 dev_priv->pch_type = PCH_LPT; 202 DRM_DEBUG_KMS("Found LynxPoint LP PCH\n"); 203 WARN_ON(!IS_HASWELL(dev) && !IS_BROADWELL(dev)); 204 WARN_ON(!IS_HSW_ULT(dev) && !IS_BDW_ULT(dev)); 205 } else if (id == INTEL_PCH_SPT_DEVICE_ID_TYPE) { 206 dev_priv->pch_type = PCH_SPT; 207 DRM_DEBUG_KMS("Found SunrisePoint PCH\n"); 208 WARN_ON(!IS_SKYLAKE(dev) && 209 !IS_KABYLAKE(dev)); 210 } else if (id == INTEL_PCH_SPT_LP_DEVICE_ID_TYPE) { 211 dev_priv->pch_type = PCH_SPT; 212 DRM_DEBUG_KMS("Found SunrisePoint LP PCH\n"); 213 WARN_ON(!IS_SKYLAKE(dev) && 214 !IS_KABYLAKE(dev)); 215 } else if (id == INTEL_PCH_KBP_DEVICE_ID_TYPE) { 216 dev_priv->pch_type = PCH_KBP; 217 DRM_DEBUG_KMS("Found KabyPoint PCH\n"); 218 WARN_ON(!IS_KABYLAKE(dev)); 219 } else if ((id == INTEL_PCH_P2X_DEVICE_ID_TYPE) || 220 (id == INTEL_PCH_P3X_DEVICE_ID_TYPE) || 221 ((id == INTEL_PCH_QEMU_DEVICE_ID_TYPE) && 222 1)) { 223 dev_priv->pch_type = intel_virt_detect_pch(dev); 224 } else 225 continue; 226 227 break; 228 } 229 } 230 if (!pch) 231 DRM_DEBUG_KMS("No PCH found.\n"); 232 233 #if 0 234 pci_dev_put(pch); 235 #endif 236 } 237 238 bool i915_semaphore_is_enabled(struct drm_i915_private *dev_priv) 239 { 240 if (INTEL_GEN(dev_priv) < 6) 241 return false; 242 243 if (i915.semaphores >= 0) 244 return i915.semaphores; 245 246 /* TODO: make semaphores and Execlists play nicely together */ 247 if (i915.enable_execlists) 248 return false; 249 250 #ifdef CONFIG_INTEL_IOMMU 251 /* Enable semaphores on SNB when IO remapping is off */ 252 if (IS_GEN6(dev_priv) && intel_iommu_gfx_mapped) 253 return false; 254 #endif 255 256 return true; 257 } 258 259 static int i915_getparam(struct drm_device *dev, void *data, 260 struct drm_file *file_priv) 261 { 262 struct drm_i915_private *dev_priv = to_i915(dev); 263 drm_i915_getparam_t *param = data; 264 int value; 265 266 switch (param->param) { 267 case I915_PARAM_IRQ_ACTIVE: 268 case I915_PARAM_ALLOW_BATCHBUFFER: 269 case I915_PARAM_LAST_DISPATCH: 270 /* Reject all old ums/dri params. */ 271 return -ENODEV; 272 case I915_PARAM_CHIPSET_ID: 273 value = dev->pdev->device; 274 break; 275 case I915_PARAM_REVISION: 276 value = dev->pdev->revision; 277 break; 278 case I915_PARAM_HAS_GEM: 279 value = 1; 280 break; 281 case I915_PARAM_NUM_FENCES_AVAIL: 282 value = dev_priv->num_fence_regs; 283 break; 284 case I915_PARAM_HAS_OVERLAY: 285 value = dev_priv->overlay ? 1 : 0; 286 break; 287 case I915_PARAM_HAS_PAGEFLIPPING: 288 value = 1; 289 break; 290 case I915_PARAM_HAS_EXECBUF2: 291 /* depends on GEM */ 292 value = 1; 293 break; 294 case I915_PARAM_HAS_BSD: 295 value = intel_engine_initialized(&dev_priv->engine[VCS]); 296 break; 297 case I915_PARAM_HAS_BLT: 298 value = intel_engine_initialized(&dev_priv->engine[BCS]); 299 break; 300 case I915_PARAM_HAS_VEBOX: 301 value = intel_engine_initialized(&dev_priv->engine[VECS]); 302 break; 303 case I915_PARAM_HAS_BSD2: 304 value = intel_engine_initialized(&dev_priv->engine[VCS2]); 305 break; 306 case I915_PARAM_HAS_RELAXED_FENCING: 307 value = 1; 308 break; 309 case I915_PARAM_HAS_COHERENT_RINGS: 310 value = 1; 311 break; 312 case I915_PARAM_HAS_EXEC_CONSTANTS: 313 value = INTEL_INFO(dev)->gen >= 4; 314 break; 315 case I915_PARAM_HAS_RELAXED_DELTA: 316 value = 1; 317 break; 318 case I915_PARAM_HAS_GEN7_SOL_RESET: 319 value = 1; 320 break; 321 case I915_PARAM_HAS_LLC: 322 value = HAS_LLC(dev); 323 break; 324 case I915_PARAM_HAS_WT: 325 value = HAS_WT(dev); 326 break; 327 case I915_PARAM_HAS_ALIASING_PPGTT: 328 value = USES_PPGTT(dev); 329 break; 330 case I915_PARAM_HAS_WAIT_TIMEOUT: 331 value = 1; 332 break; 333 case I915_PARAM_HAS_SEMAPHORES: 334 value = i915_semaphore_is_enabled(dev_priv); 335 break; 336 case I915_PARAM_HAS_PINNED_BATCHES: 337 value = 1; 338 break; 339 case I915_PARAM_HAS_EXEC_NO_RELOC: 340 value = 1; 341 break; 342 case I915_PARAM_HAS_EXEC_HANDLE_LUT: 343 value = 1; 344 break; 345 case I915_PARAM_CMD_PARSER_VERSION: 346 value = i915_cmd_parser_get_version(dev_priv); 347 break; 348 case I915_PARAM_HAS_COHERENT_PHYS_GTT: 349 value = 1; 350 break; 351 case I915_PARAM_SUBSLICE_TOTAL: 352 value = INTEL_INFO(dev)->subslice_total; 353 if (!value) 354 return -ENODEV; 355 break; 356 case I915_PARAM_EU_TOTAL: 357 value = INTEL_INFO(dev)->eu_total; 358 if (!value) 359 return -ENODEV; 360 break; 361 case I915_PARAM_HAS_GPU_RESET: 362 value = i915.enable_hangcheck && intel_has_gpu_reset(dev_priv); 363 break; 364 case I915_PARAM_HAS_RESOURCE_STREAMER: 365 value = HAS_RESOURCE_STREAMER(dev); 366 break; 367 case I915_PARAM_HAS_EXEC_SOFTPIN: 368 value = 1; 369 break; 370 case I915_PARAM_HAS_POOLED_EU: 371 value = HAS_POOLED_EU(dev); 372 break; 373 case I915_PARAM_MIN_EU_IN_POOL: 374 value = INTEL_INFO(dev)->min_eu_in_pool; 375 break; 376 default: 377 DRM_DEBUG("Unknown parameter %d\n", param->param); 378 return -EINVAL; 379 } 380 381 if (put_user(value, param->value)) 382 return -EFAULT; 383 384 return 0; 385 } 386 387 static int i915_get_bridge_dev(struct drm_device *dev) 388 { 389 struct drm_i915_private *dev_priv = to_i915(dev); 390 static struct pci_dev i915_bridge_dev; 391 392 i915_bridge_dev.dev.bsddev = pci_find_dbsf(0, 0, 0, 0); 393 if (!i915_bridge_dev.dev.bsddev) { 394 DRM_ERROR("bridge device not found\n"); 395 return -1; 396 } 397 398 dev_priv->bridge_dev = &i915_bridge_dev; 399 return 0; 400 } 401 402 /* Allocate space for the MCH regs if needed, return nonzero on error */ 403 static int 404 intel_alloc_mchbar_resource(struct drm_device *dev) 405 { 406 struct drm_i915_private *dev_priv = to_i915(dev); 407 int reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915; 408 u32 temp_lo, temp_hi = 0; 409 u64 mchbar_addr; 410 device_t vga; 411 412 if (INTEL_INFO(dev)->gen >= 4) 413 pci_read_config_dword(dev_priv->bridge_dev, reg + 4, &temp_hi); 414 pci_read_config_dword(dev_priv->bridge_dev, reg, &temp_lo); 415 mchbar_addr = ((u64)temp_hi << 32) | temp_lo; 416 417 /* If ACPI doesn't have it, assume we need to allocate it ourselves */ 418 #ifdef CONFIG_PNP 419 if (mchbar_addr && 420 pnp_range_reserved(mchbar_addr, mchbar_addr + MCHBAR_SIZE)) 421 return 0; 422 #endif 423 424 /* Get some space for it */ 425 vga = device_get_parent(dev->dev->bsddev); 426 dev_priv->mch_res_rid = 0x100; 427 dev_priv->mch_res = BUS_ALLOC_RESOURCE(device_get_parent(vga), 428 dev->dev->bsddev, SYS_RES_MEMORY, &dev_priv->mch_res_rid, 0, ~0UL, 429 MCHBAR_SIZE, RF_ACTIVE | RF_SHAREABLE, -1); 430 if (dev_priv->mch_res == NULL) { 431 DRM_ERROR("failed mchbar resource alloc\n"); 432 return (-ENOMEM); 433 } 434 435 if (INTEL_INFO(dev)->gen >= 4) 436 pci_write_config_dword(dev_priv->bridge_dev, reg + 4, 437 upper_32_bits(rman_get_start(dev_priv->mch_res))); 438 439 pci_write_config_dword(dev_priv->bridge_dev, reg, 440 lower_32_bits(rman_get_start(dev_priv->mch_res))); 441 return 0; 442 } 443 444 /* Setup MCHBAR if possible, return true if we should disable it again */ 445 static void 446 intel_setup_mchbar(struct drm_device *dev) 447 { 448 struct drm_i915_private *dev_priv = to_i915(dev); 449 int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915; 450 u32 temp; 451 bool enabled; 452 453 if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) 454 return; 455 456 dev_priv->mchbar_need_disable = false; 457 458 if (IS_I915G(dev) || IS_I915GM(dev)) { 459 pci_read_config_dword(dev_priv->bridge_dev, DEVEN, &temp); 460 enabled = !!(temp & DEVEN_MCHBAR_EN); 461 } else { 462 pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp); 463 enabled = temp & 1; 464 } 465 466 /* If it's already enabled, don't have to do anything */ 467 if (enabled) 468 return; 469 470 if (intel_alloc_mchbar_resource(dev)) 471 return; 472 473 dev_priv->mchbar_need_disable = true; 474 475 /* Space is allocated or reserved, so enable it. */ 476 if (IS_I915G(dev) || IS_I915GM(dev)) { 477 pci_write_config_dword(dev_priv->bridge_dev, DEVEN, 478 temp | DEVEN_MCHBAR_EN); 479 } else { 480 pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp); 481 pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg, temp | 1); 482 } 483 } 484 485 static void 486 intel_teardown_mchbar(struct drm_device *dev) 487 { 488 struct drm_i915_private *dev_priv = to_i915(dev); 489 int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915; 490 device_t vga; 491 492 if (dev_priv->mchbar_need_disable) { 493 if (IS_I915G(dev) || IS_I915GM(dev)) { 494 u32 deven_val; 495 496 pci_read_config_dword(dev_priv->bridge_dev, DEVEN, 497 &deven_val); 498 deven_val &= ~DEVEN_MCHBAR_EN; 499 pci_write_config_dword(dev_priv->bridge_dev, DEVEN, 500 deven_val); 501 } else { 502 u32 mchbar_val; 503 504 pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, 505 &mchbar_val); 506 mchbar_val &= ~1; 507 pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg, 508 mchbar_val); 509 } 510 } 511 512 if (dev_priv->mch_res != NULL) { 513 vga = device_get_parent(dev->dev->bsddev); 514 BUS_DEACTIVATE_RESOURCE(device_get_parent(vga), dev->dev->bsddev, 515 SYS_RES_MEMORY, dev_priv->mch_res_rid, dev_priv->mch_res); 516 BUS_RELEASE_RESOURCE(device_get_parent(vga), dev->dev->bsddev, 517 SYS_RES_MEMORY, dev_priv->mch_res_rid, dev_priv->mch_res); 518 dev_priv->mch_res = NULL; 519 } 520 } 521 522 #if 0 523 /* true = enable decode, false = disable decoder */ 524 static unsigned int i915_vga_set_decode(void *cookie, bool state) 525 { 526 struct drm_device *dev = cookie; 527 528 intel_modeset_vga_set_state(dev, state); 529 if (state) 530 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM | 531 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; 532 else 533 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; 534 } 535 536 static void i915_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state) 537 { 538 struct drm_device *dev = pci_get_drvdata(pdev); 539 pm_message_t pmm = { .event = PM_EVENT_SUSPEND }; 540 541 if (state == VGA_SWITCHEROO_ON) { 542 pr_info("switched on\n"); 543 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; 544 /* i915 resume handler doesn't set to D0 */ 545 pci_set_power_state(dev->pdev, PCI_D0); 546 i915_resume_switcheroo(dev); 547 dev->switch_power_state = DRM_SWITCH_POWER_ON; 548 } else { 549 pr_info("switched off\n"); 550 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; 551 i915_suspend_switcheroo(dev, pmm); 552 dev->switch_power_state = DRM_SWITCH_POWER_OFF; 553 } 554 } 555 556 static bool i915_switcheroo_can_switch(struct pci_dev *pdev) 557 { 558 struct drm_device *dev = pci_get_drvdata(pdev); 559 560 /* 561 * FIXME: open_count is protected by drm_global_mutex but that would lead to 562 * locking inversion with the driver load path. And the access here is 563 * completely racy anyway. So don't bother with locking for now. 564 */ 565 return dev->open_count == 0; 566 } 567 568 static const struct vga_switcheroo_client_ops i915_switcheroo_ops = { 569 .set_gpu_state = i915_switcheroo_set_state, 570 .reprobe = NULL, 571 .can_switch = i915_switcheroo_can_switch, 572 }; 573 #endif 574 575 static void i915_gem_fini(struct drm_device *dev) 576 { 577 struct drm_i915_private *dev_priv = to_i915(dev); 578 579 /* 580 * Neither the BIOS, ourselves or any other kernel 581 * expects the system to be in execlists mode on startup, 582 * so we need to reset the GPU back to legacy mode. And the only 583 * known way to disable logical contexts is through a GPU reset. 584 * 585 * So in order to leave the system in a known default configuration, 586 * always reset the GPU upon unload. Afterwards we then clean up the 587 * GEM state tracking, flushing off the requests and leaving the 588 * system in a known idle state. 589 * 590 * Note that is of the upmost importance that the GPU is idle and 591 * all stray writes are flushed *before* we dismantle the backing 592 * storage for the pinned objects. 593 * 594 * However, since we are uncertain that reseting the GPU on older 595 * machines is a good idea, we don't - just in case it leaves the 596 * machine in an unusable condition. 597 */ 598 if (HAS_HW_CONTEXTS(dev)) { 599 int reset = intel_gpu_reset(dev_priv, ALL_ENGINES); 600 WARN_ON(reset && reset != -ENODEV); 601 } 602 603 mutex_lock(&dev->struct_mutex); 604 i915_gem_reset(dev); 605 i915_gem_cleanup_engines(dev); 606 i915_gem_context_fini(dev); 607 mutex_unlock(&dev->struct_mutex); 608 609 WARN_ON(!list_empty(&to_i915(dev)->context_list)); 610 } 611 612 static int i915_load_modeset_init(struct drm_device *dev) 613 { 614 struct drm_i915_private *dev_priv = to_i915(dev); 615 int ret; 616 617 if (i915_inject_load_failure()) 618 return -ENODEV; 619 620 ret = intel_bios_init(dev_priv); 621 if (ret) 622 DRM_INFO("failed to find VBIOS tables\n"); 623 624 /* If we have > 1 VGA cards, then we need to arbitrate access 625 * to the common VGA resources. 626 * 627 * If we are a secondary display controller (!PCI_DISPLAY_CLASS_VGA), 628 * then we do not take part in VGA arbitration and the 629 * vga_client_register() fails with -ENODEV. 630 */ 631 #if 0 632 ret = vga_client_register(dev->pdev, dev, NULL, i915_vga_set_decode); 633 if (ret && ret != -ENODEV) 634 goto out; 635 636 intel_register_dsm_handler(); 637 638 ret = vga_switcheroo_register_client(dev->pdev, &i915_switcheroo_ops, false); 639 if (ret) 640 goto cleanup_vga_client; 641 #endif 642 643 /* must happen before intel_power_domains_init_hw() on VLV/CHV */ 644 intel_update_rawclk(dev_priv); 645 646 intel_power_domains_init_hw(dev_priv, false); 647 648 intel_csr_ucode_init(dev_priv); 649 650 ret = intel_irq_install(dev_priv); 651 if (ret) 652 goto cleanup_csr; 653 654 intel_setup_gmbus(dev); 655 656 /* Important: The output setup functions called by modeset_init need 657 * working irqs for e.g. gmbus and dp aux transfers. */ 658 intel_modeset_init(dev); 659 660 intel_guc_init(dev); 661 662 ret = i915_gem_init(dev); 663 if (ret) 664 goto cleanup_irq; 665 666 intel_modeset_gem_init(dev); 667 668 if (INTEL_INFO(dev)->num_pipes == 0) 669 return 0; 670 671 ret = intel_fbdev_init(dev); 672 if (ret) 673 goto cleanup_gem; 674 675 /* Only enable hotplug handling once the fbdev is fully set up. */ 676 intel_hpd_init(dev_priv); 677 678 drm_kms_helper_poll_init(dev); 679 680 #ifdef __DragonFly__ 681 /* 682 * If we are dealing with dual GPU machines the vga_switcheroo module 683 * has been loaded. Machines with dual GPUs have an integrated graphics 684 * device (IGD), which we assume is an Intel device. The other, the 685 * discrete device (DIS), is either an NVidia or a Radeon device. For 686 * now we will force switch the gmux so the intel driver outputs 687 * both to the laptop panel and the external monitor. 688 * 689 * DragonFly does not have an nvidia native driver yet. In the future, 690 * we will check for the radeon device: if present, we will leave 691 * the gmux switch as it is, so the user can choose between the IGD and 692 * the DIS using the /dev/vga_switcheroo device. 693 */ 694 if (vga_switcheroo_handler_flags() & VGA_SWITCHEROO_CAN_SWITCH_DDC) { 695 ret = vga_switcheroo_force_migd(); 696 if (ret) { 697 DRM_INFO("could not switch gmux to IGD\n"); 698 } 699 } 700 #endif 701 702 return 0; 703 704 cleanup_gem: 705 i915_gem_fini(dev); 706 cleanup_irq: 707 intel_guc_fini(dev); 708 drm_irq_uninstall(dev); 709 intel_teardown_gmbus(dev); 710 cleanup_csr: 711 intel_csr_ucode_fini(dev_priv); 712 intel_power_domains_fini(dev_priv); 713 #if 0 714 vga_switcheroo_unregister_client(dev->pdev); 715 cleanup_vga_client: 716 vga_client_register(dev->pdev, NULL, NULL, NULL); 717 out: 718 #endif 719 return ret; 720 } 721 722 #if IS_ENABLED(CONFIG_FB) 723 static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv) 724 { 725 struct apertures_struct *ap; 726 struct pci_dev *pdev = dev_priv->drm.pdev; 727 struct i915_ggtt *ggtt = &dev_priv->ggtt; 728 bool primary; 729 int ret; 730 731 ap = alloc_apertures(1); 732 if (!ap) 733 return -ENOMEM; 734 735 ap->ranges[0].base = ggtt->mappable_base; 736 ap->ranges[0].size = ggtt->mappable_end; 737 738 primary = 739 pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW; 740 741 ret = remove_conflicting_framebuffers(ap, "inteldrmfb", primary); 742 743 kfree(ap); 744 745 return ret; 746 } 747 #else 748 static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv) 749 { 750 return 0; 751 } 752 #endif 753 754 #if !defined(CONFIG_VGA_CONSOLE) 755 static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv) 756 { 757 return 0; 758 } 759 #elif !defined(CONFIG_DUMMY_CONSOLE) 760 static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv) 761 { 762 return -ENODEV; 763 } 764 #else 765 static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv) 766 { 767 int ret = 0; 768 769 DRM_INFO("Replacing VGA console driver\n"); 770 771 console_lock(); 772 if (con_is_bound(&vga_con)) 773 ret = do_take_over_console(&dummy_con, 0, MAX_NR_CONSOLES - 1, 1); 774 if (ret == 0) { 775 ret = do_unregister_con_driver(&vga_con); 776 777 /* Ignore "already unregistered". */ 778 if (ret == -ENODEV) 779 ret = 0; 780 } 781 console_unlock(); 782 783 return ret; 784 } 785 #endif 786 787 static void intel_init_dpio(struct drm_i915_private *dev_priv) 788 { 789 /* 790 * IOSF_PORT_DPIO is used for VLV x2 PHY (DP/HDMI B and C), 791 * CHV x1 PHY (DP/HDMI D) 792 * IOSF_PORT_DPIO_2 is used for CHV x2 PHY (DP/HDMI B and C) 793 */ 794 if (IS_CHERRYVIEW(dev_priv)) { 795 DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO_2; 796 DPIO_PHY_IOSF_PORT(DPIO_PHY1) = IOSF_PORT_DPIO; 797 } else if (IS_VALLEYVIEW(dev_priv)) { 798 DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO; 799 } 800 } 801 802 static int i915_workqueues_init(struct drm_i915_private *dev_priv) 803 { 804 /* 805 * The i915 workqueue is primarily used for batched retirement of 806 * requests (and thus managing bo) once the task has been completed 807 * by the GPU. i915_gem_retire_requests() is called directly when we 808 * need high-priority retirement, such as waiting for an explicit 809 * bo. 810 * 811 * It is also used for periodic low-priority events, such as 812 * idle-timers and recording error state. 813 * 814 * All tasks on the workqueue are expected to acquire the dev mutex 815 * so there is no point in running more than one instance of the 816 * workqueue at any time. Use an ordered one. 817 */ 818 dev_priv->wq = alloc_ordered_workqueue("i915", 0); 819 if (dev_priv->wq == NULL) 820 goto out_err; 821 822 dev_priv->hotplug.dp_wq = alloc_ordered_workqueue("i915-dp", 0); 823 if (dev_priv->hotplug.dp_wq == NULL) 824 goto out_free_wq; 825 826 return 0; 827 828 out_free_wq: 829 destroy_workqueue(dev_priv->wq); 830 out_err: 831 DRM_ERROR("Failed to allocate workqueues.\n"); 832 833 return -ENOMEM; 834 } 835 836 static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv) 837 { 838 destroy_workqueue(dev_priv->hotplug.dp_wq); 839 destroy_workqueue(dev_priv->wq); 840 } 841 842 /** 843 * i915_driver_init_early - setup state not requiring device access 844 * @dev_priv: device private 845 * 846 * Initialize everything that is a "SW-only" state, that is state not 847 * requiring accessing the device or exposing the driver via kernel internal 848 * or userspace interfaces. Example steps belonging here: lock initialization, 849 * system memory allocation, setting up device specific attributes and 850 * function hooks not requiring accessing the device. 851 */ 852 static int i915_driver_init_early(struct drm_i915_private *dev_priv, 853 const struct pci_device_id *ent) 854 { 855 const struct intel_device_info *match_info = 856 (struct intel_device_info *)ent->driver_data; 857 struct intel_device_info *device_info; 858 int ret = 0; 859 860 if (i915_inject_load_failure()) 861 return -ENODEV; 862 863 /* Setup the write-once "constant" device info */ 864 device_info = mkwrite_device_info(dev_priv); 865 memcpy(device_info, match_info, sizeof(*device_info)); 866 device_info->device_id = dev_priv->drm.pdev->device; 867 868 BUG_ON(device_info->gen > sizeof(device_info->gen_mask) * BITS_PER_BYTE); 869 device_info->gen_mask = BIT(device_info->gen - 1); 870 871 lockinit(&dev_priv->irq_lock, "userirq", 0, LK_CANRECURSE); 872 lockinit(&dev_priv->gpu_error.lock, "915err", 0, LK_CANRECURSE); 873 lockinit(&dev_priv->backlight_lock, "i915bl", 0, LK_CANRECURSE); 874 lockinit(&dev_priv->uncore.lock, "915gt", 0, LK_CANRECURSE); 875 lockinit(&dev_priv->mm.object_stat_lock, "i915osl", 0, 0); 876 lockinit(&dev_priv->mmio_flip_lock, "i915mfl", 0, 0); 877 lockinit(&dev_priv->sb_lock, "i915sbl", 0, LK_CANRECURSE); 878 lockinit(&dev_priv->modeset_restore_lock, "i915mrl", 0, LK_CANRECURSE); 879 lockinit(&dev_priv->av_mutex, "i915am", 0, LK_CANRECURSE); 880 lockinit(&dev_priv->wm.wm_mutex, "i915wm", 0, LK_CANRECURSE); 881 lockinit(&dev_priv->pps_mutex, "i915pm", 0, LK_CANRECURSE); 882 883 ret = i915_workqueues_init(dev_priv); 884 if (ret < 0) 885 return ret; 886 887 ret = intel_gvt_init(dev_priv); 888 if (ret < 0) 889 goto err_workqueues; 890 891 /* This must be called before any calls to HAS_PCH_* */ 892 intel_detect_pch(&dev_priv->drm); 893 894 intel_pm_setup(&dev_priv->drm); 895 intel_init_dpio(dev_priv); 896 intel_power_domains_init(dev_priv); 897 intel_irq_init(dev_priv); 898 intel_init_display_hooks(dev_priv); 899 intel_init_clock_gating_hooks(dev_priv); 900 intel_init_audio_hooks(dev_priv); 901 i915_gem_load_init(&dev_priv->drm); 902 903 intel_display_crc_init(&dev_priv->drm); 904 905 intel_device_info_dump(dev_priv); 906 907 /* Not all pre-production machines fall into this category, only the 908 * very first ones. Almost everything should work, except for maybe 909 * suspend/resume. And we don't implement workarounds that affect only 910 * pre-production machines. */ 911 if (IS_HSW_EARLY_SDV(dev_priv)) 912 DRM_INFO("This is an early pre-production Haswell machine. " 913 "It may not be fully functional.\n"); 914 915 return 0; 916 917 err_workqueues: 918 i915_workqueues_cleanup(dev_priv); 919 return ret; 920 } 921 922 /** 923 * i915_driver_cleanup_early - cleanup the setup done in i915_driver_init_early() 924 * @dev_priv: device private 925 */ 926 static void i915_driver_cleanup_early(struct drm_i915_private *dev_priv) 927 { 928 i915_gem_load_cleanup(&dev_priv->drm); 929 i915_workqueues_cleanup(dev_priv); 930 } 931 932 static int i915_mmio_setup(struct drm_device *dev) 933 { 934 struct drm_i915_private *dev_priv = to_i915(dev); 935 int mmio_bar; 936 int mmio_size; 937 938 mmio_bar = IS_GEN2(dev) ? 1 : 0; 939 /* 940 * Before gen4, the registers and the GTT are behind different BARs. 941 * However, from gen4 onwards, the registers and the GTT are shared 942 * in the same BAR, so we want to restrict this ioremap from 943 * clobbering the GTT which we want ioremap_wc instead. Fortunately, 944 * the register BAR remains the same size for all the earlier 945 * generations up to Ironlake. 946 */ 947 if (INTEL_INFO(dev)->gen < 5) 948 mmio_size = 512 * 1024; 949 else 950 mmio_size = 2 * 1024 * 1024; 951 dev_priv->regs = pci_iomap(dev->pdev, mmio_bar, mmio_size); 952 if (dev_priv->regs == NULL) { 953 DRM_ERROR("failed to map registers\n"); 954 955 return -EIO; 956 } 957 958 /* Try to make sure MCHBAR is enabled before poking at it */ 959 intel_setup_mchbar(dev); 960 961 return 0; 962 } 963 964 static void i915_mmio_cleanup(struct drm_device *dev) 965 { 966 #if 0 967 struct drm_i915_private *dev_priv = to_i915(dev); 968 #endif 969 970 intel_teardown_mchbar(dev); 971 #if 0 972 pci_iounmap(dev->pdev, dev_priv->regs); 973 #endif 974 } 975 976 /** 977 * i915_driver_init_mmio - setup device MMIO 978 * @dev_priv: device private 979 * 980 * Setup minimal device state necessary for MMIO accesses later in the 981 * initialization sequence. The setup here should avoid any other device-wide 982 * side effects or exposing the driver via kernel internal or user space 983 * interfaces. 984 */ 985 static int i915_driver_init_mmio(struct drm_i915_private *dev_priv) 986 { 987 struct drm_device *dev = &dev_priv->drm; 988 int ret; 989 990 if (i915_inject_load_failure()) 991 return -ENODEV; 992 993 if (i915_get_bridge_dev(dev)) 994 return -EIO; 995 996 ret = i915_mmio_setup(dev); 997 if (ret < 0) 998 goto put_bridge; 999 1000 intel_uncore_init(dev_priv); 1001 1002 return 0; 1003 1004 put_bridge: 1005 pci_dev_put(dev_priv->bridge_dev); 1006 1007 return ret; 1008 } 1009 1010 /** 1011 * i915_driver_cleanup_mmio - cleanup the setup done in i915_driver_init_mmio() 1012 * @dev_priv: device private 1013 */ 1014 static void i915_driver_cleanup_mmio(struct drm_i915_private *dev_priv) 1015 { 1016 struct drm_device *dev = &dev_priv->drm; 1017 1018 intel_uncore_fini(dev_priv); 1019 i915_mmio_cleanup(dev); 1020 pci_dev_put(dev_priv->bridge_dev); 1021 } 1022 1023 static void intel_sanitize_options(struct drm_i915_private *dev_priv) 1024 { 1025 i915.enable_execlists = 1026 intel_sanitize_enable_execlists(dev_priv, 1027 i915.enable_execlists); 1028 1029 /* 1030 * i915.enable_ppgtt is read-only, so do an early pass to validate the 1031 * user's requested state against the hardware/driver capabilities. We 1032 * do this now so that we can print out any log messages once rather 1033 * than every time we check intel_enable_ppgtt(). 1034 */ 1035 i915.enable_ppgtt = 1036 intel_sanitize_enable_ppgtt(dev_priv, i915.enable_ppgtt); 1037 DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt); 1038 } 1039 1040 /** 1041 * i915_driver_init_hw - setup state requiring device access 1042 * @dev_priv: device private 1043 * 1044 * Setup state that requires accessing the device, but doesn't require 1045 * exposing the driver via kernel internal or userspace interfaces. 1046 */ 1047 static int i915_driver_init_hw(struct drm_i915_private *dev_priv) 1048 { 1049 struct drm_device *dev = &dev_priv->drm; 1050 struct i915_ggtt *ggtt = &dev_priv->ggtt; 1051 uint32_t aperture_size; 1052 int ret; 1053 1054 if (i915_inject_load_failure()) 1055 return -ENODEV; 1056 1057 intel_device_info_runtime_init(dev_priv); 1058 1059 intel_sanitize_options(dev_priv); 1060 1061 ret = i915_ggtt_init_hw(dev); 1062 if (ret) 1063 return ret; 1064 1065 ret = i915_ggtt_enable_hw(dev); 1066 if (ret) { 1067 DRM_ERROR("failed to enable GGTT\n"); 1068 goto out_ggtt; 1069 } 1070 1071 /* WARNING: Apparently we must kick fbdev drivers before vgacon, 1072 * otherwise the vga fbdev driver falls over. */ 1073 ret = i915_kick_out_firmware_fb(dev_priv); 1074 if (ret) { 1075 DRM_ERROR("failed to remove conflicting framebuffer drivers\n"); 1076 goto out_ggtt; 1077 } 1078 1079 ret = i915_kick_out_vgacon(dev_priv); 1080 if (ret) { 1081 DRM_ERROR("failed to remove conflicting VGA console\n"); 1082 goto out_ggtt; 1083 } 1084 1085 #if 0 1086 pci_set_master(dev->pdev); 1087 1088 /* overlay on gen2 is broken and can't address above 1G */ 1089 if (IS_GEN2(dev)) { 1090 ret = dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(30)); 1091 if (ret) { 1092 DRM_ERROR("failed to set DMA mask\n"); 1093 1094 goto out_ggtt; 1095 } 1096 } 1097 1098 1099 /* 965GM sometimes incorrectly writes to hardware status page (HWS) 1100 * using 32bit addressing, overwriting memory if HWS is located 1101 * above 4GB. 1102 * 1103 * The documentation also mentions an issue with undefined 1104 * behaviour if any general state is accessed within a page above 4GB, 1105 * which also needs to be handled carefully. 1106 */ 1107 if (IS_BROADWATER(dev) || IS_CRESTLINE(dev)) { 1108 ret = dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(32)); 1109 1110 if (ret) { 1111 DRM_ERROR("failed to set DMA mask\n"); 1112 1113 goto out_ggtt; 1114 } 1115 } 1116 #endif 1117 1118 aperture_size = ggtt->mappable_end; 1119 1120 ggtt->mappable = 1121 io_mapping_create_wc(ggtt->mappable_base, 1122 aperture_size); 1123 if (!ggtt->mappable) { 1124 ret = -EIO; 1125 goto out_ggtt; 1126 } 1127 1128 ggtt->mtrr = arch_phys_wc_add(ggtt->mappable_base, 1129 aperture_size); 1130 1131 pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, 1132 PM_QOS_DEFAULT_VALUE); 1133 1134 intel_uncore_sanitize(dev_priv); 1135 1136 intel_opregion_setup(dev_priv); 1137 1138 i915_gem_load_init_fences(dev_priv); 1139 1140 /* On the 945G/GM, the chipset reports the MSI capability on the 1141 * integrated graphics even though the support isn't actually there 1142 * according to the published specs. It doesn't appear to function 1143 * correctly in testing on 945G. 1144 * This may be a side effect of MSI having been made available for PEG 1145 * and the registers being closely associated. 1146 * 1147 * According to chipset errata, on the 965GM, MSI interrupts may 1148 * be lost or delayed, but we use them anyways to avoid 1149 * stuck interrupts on some machines. 1150 */ 1151 #if 0 1152 if (!IS_I945G(dev) && !IS_I945GM(dev)) { 1153 if (pci_enable_msi(dev->pdev) < 0) 1154 DRM_DEBUG_DRIVER("can't enable MSI"); 1155 } 1156 #endif 1157 1158 return 0; 1159 1160 out_ggtt: 1161 i915_ggtt_cleanup_hw(dev); 1162 1163 return ret; 1164 } 1165 1166 /** 1167 * i915_driver_cleanup_hw - cleanup the setup done in i915_driver_init_hw() 1168 * @dev_priv: device private 1169 */ 1170 static void i915_driver_cleanup_hw(struct drm_i915_private *dev_priv) 1171 { 1172 struct drm_device *dev = &dev_priv->drm; 1173 struct i915_ggtt *ggtt = &dev_priv->ggtt; 1174 1175 #if 0 1176 if (dev->pdev->msi_enabled) 1177 pci_disable_msi(dev->pdev); 1178 #endif 1179 1180 pm_qos_remove_request(&dev_priv->pm_qos); 1181 arch_phys_wc_del(ggtt->mtrr); 1182 io_mapping_free(ggtt->mappable); 1183 i915_ggtt_cleanup_hw(dev); 1184 } 1185 1186 /** 1187 * i915_driver_register - register the driver with the rest of the system 1188 * @dev_priv: device private 1189 * 1190 * Perform any steps necessary to make the driver available via kernel 1191 * internal or userspace interfaces. 1192 */ 1193 static void i915_driver_register(struct drm_i915_private *dev_priv) 1194 { 1195 struct drm_device *dev = &dev_priv->drm; 1196 1197 i915_gem_shrinker_init(dev_priv); 1198 1199 /* 1200 * Notify a valid surface after modesetting, 1201 * when running inside a VM. 1202 */ 1203 if (intel_vgpu_active(dev_priv)) 1204 I915_WRITE(vgtif_reg(display_ready), VGT_DRV_DISPLAY_READY); 1205 1206 /* Reveal our presence to userspace */ 1207 if (drm_dev_register(dev, 0) == 0) { 1208 i915_debugfs_register(dev_priv); 1209 i915_setup_sysfs(dev); 1210 } else 1211 DRM_ERROR("Failed to register driver for userspace access!\n"); 1212 1213 if (INTEL_INFO(dev_priv)->num_pipes) { 1214 /* Must be done after probing outputs */ 1215 intel_opregion_register(dev_priv); 1216 acpi_video_register(); 1217 } 1218 1219 if (IS_GEN5(dev_priv)) 1220 intel_gpu_ips_init(dev_priv); 1221 1222 i915_audio_component_init(dev_priv); 1223 1224 /* 1225 * Some ports require correctly set-up hpd registers for detection to 1226 * work properly (leading to ghost connected connector status), e.g. VGA 1227 * on gm45. Hence we can only set up the initial fbdev config after hpd 1228 * irqs are fully enabled. We do it last so that the async config 1229 * cannot run before the connectors are registered. 1230 */ 1231 intel_fbdev_initial_config_async(dev); 1232 } 1233 1234 /** 1235 * i915_driver_unregister - cleanup the registration done in i915_driver_regiser() 1236 * @dev_priv: device private 1237 */ 1238 static void i915_driver_unregister(struct drm_i915_private *dev_priv) 1239 { 1240 i915_audio_component_cleanup(dev_priv); 1241 1242 intel_gpu_ips_teardown(); 1243 acpi_video_unregister(); 1244 intel_opregion_unregister(dev_priv); 1245 1246 i915_teardown_sysfs(&dev_priv->drm); 1247 i915_debugfs_unregister(dev_priv); 1248 drm_dev_unregister(&dev_priv->drm); 1249 1250 i915_gem_shrinker_cleanup(dev_priv); 1251 } 1252 1253 /** 1254 * i915_driver_load - setup chip and create an initial config 1255 * @dev: DRM device 1256 * @flags: startup flags 1257 * 1258 * The driver load routine has to do several things: 1259 * - drive output discovery via intel_modeset_init() 1260 * - initialize the memory manager 1261 * - allocate initial config memory 1262 * - setup the DRM framebuffer with the allocated memory 1263 */ 1264 int i915_driver_load(struct pci_dev *pdev, const struct pci_device_id *ent); 1265 int i915_driver_load(struct pci_dev *pdev, const struct pci_device_id *ent) 1266 { 1267 struct drm_i915_private *dev_priv; 1268 int ret; 1269 1270 if (i915.nuclear_pageflip) 1271 driver.driver_features |= DRIVER_ATOMIC; 1272 1273 ret = -ENOMEM; 1274 dev_priv = kzalloc(sizeof(*dev_priv), GFP_KERNEL); 1275 if (dev_priv) 1276 ret = drm_dev_init(&dev_priv->drm, &driver, &pdev->dev); 1277 if (ret) { 1278 dev_printk(KERN_ERR, &pdev->dev, 1279 "[" DRM_NAME ":%s] allocation failed\n", __func__); 1280 kfree(dev_priv); 1281 return ret; 1282 } 1283 1284 dev_priv->drm.pdev = pdev; 1285 dev_priv->drm.dev_private = dev_priv; 1286 1287 #if 0 1288 ret = pci_enable_device(pdev); 1289 if (ret) 1290 goto out_free_priv; 1291 #endif 1292 1293 pci_set_drvdata(pdev, &dev_priv->drm); 1294 1295 ret = i915_driver_init_early(dev_priv, ent); 1296 if (ret < 0) 1297 goto out_pci_disable; 1298 1299 intel_runtime_pm_get(dev_priv); 1300 1301 ret = i915_driver_init_mmio(dev_priv); 1302 if (ret < 0) 1303 goto out_runtime_pm_put; 1304 1305 ret = i915_driver_init_hw(dev_priv); 1306 if (ret < 0) 1307 goto out_cleanup_mmio; 1308 1309 /* 1310 * TODO: move the vblank init and parts of modeset init steps into one 1311 * of the i915_driver_init_/i915_driver_register functions according 1312 * to the role/effect of the given init step. 1313 */ 1314 if (INTEL_INFO(dev_priv)->num_pipes) { 1315 ret = drm_vblank_init(&dev_priv->drm, 1316 INTEL_INFO(dev_priv)->num_pipes); 1317 if (ret) 1318 goto out_cleanup_hw; 1319 } 1320 1321 ret = i915_load_modeset_init(&dev_priv->drm); 1322 if (ret < 0) 1323 goto out_cleanup_vblank; 1324 1325 i915_driver_register(dev_priv); 1326 1327 intel_runtime_pm_enable(dev_priv); 1328 1329 /* Everything is in place, we can now relax! */ 1330 DRM_INFO("Initialized %s %d.%d.%d %s for %s on minor %d\n", 1331 driver.name, driver.major, driver.minor, driver.patchlevel, 1332 driver.date, pci_name(pdev), dev_priv->drm.primary->index); 1333 1334 intel_runtime_pm_put(dev_priv); 1335 1336 return 0; 1337 1338 out_cleanup_vblank: 1339 drm_vblank_cleanup(&dev_priv->drm); 1340 out_cleanup_hw: 1341 i915_driver_cleanup_hw(dev_priv); 1342 out_cleanup_mmio: 1343 i915_driver_cleanup_mmio(dev_priv); 1344 out_runtime_pm_put: 1345 intel_runtime_pm_put(dev_priv); 1346 i915_driver_cleanup_early(dev_priv); 1347 out_pci_disable: 1348 #if 0 1349 pci_disable_device(pdev); 1350 out_free_priv: 1351 #endif 1352 i915_load_error(dev_priv, "Device initialization failed (%d)\n", ret); 1353 drm_dev_unref(&dev_priv->drm); 1354 return ret; 1355 } 1356 1357 void i915_driver_unload(struct drm_device *dev); 1358 void i915_driver_unload(struct drm_device *dev) 1359 { 1360 struct drm_i915_private *dev_priv = to_i915(dev); 1361 1362 intel_fbdev_fini(dev); 1363 1364 if (i915_gem_suspend(dev)) 1365 DRM_ERROR("failed to idle hardware; continuing to unload!\n"); 1366 1367 intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); 1368 1369 i915_driver_unregister(dev_priv); 1370 1371 drm_vblank_cleanup(dev); 1372 1373 intel_modeset_cleanup(dev); 1374 1375 /* 1376 * free the memory space allocated for the child device 1377 * config parsed from VBT 1378 */ 1379 if (dev_priv->vbt.child_dev && dev_priv->vbt.child_dev_num) { 1380 kfree(dev_priv->vbt.child_dev); 1381 dev_priv->vbt.child_dev = NULL; 1382 dev_priv->vbt.child_dev_num = 0; 1383 } 1384 kfree(dev_priv->vbt.sdvo_lvds_vbt_mode); 1385 dev_priv->vbt.sdvo_lvds_vbt_mode = NULL; 1386 kfree(dev_priv->vbt.lfp_lvds_vbt_mode); 1387 dev_priv->vbt.lfp_lvds_vbt_mode = NULL; 1388 1389 #if 0 1390 vga_switcheroo_unregister_client(dev->pdev); 1391 vga_client_register(dev->pdev, NULL, NULL, NULL); 1392 #endif 1393 1394 intel_csr_ucode_fini(dev_priv); 1395 1396 /* Free error state after interrupts are fully disabled. */ 1397 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work); 1398 i915_destroy_error_state(dev); 1399 1400 /* Flush any outstanding unpin_work. */ 1401 flush_workqueue(dev_priv->wq); 1402 1403 intel_guc_fini(dev); 1404 i915_gem_fini(dev); 1405 intel_fbc_cleanup_cfb(dev_priv); 1406 1407 intel_power_domains_fini(dev_priv); 1408 1409 i915_driver_cleanup_hw(dev_priv); 1410 i915_driver_cleanup_mmio(dev_priv); 1411 1412 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT); 1413 1414 i915_driver_cleanup_early(dev_priv); 1415 } 1416 1417 static int i915_driver_open(struct drm_device *dev, struct drm_file *file) 1418 { 1419 int ret; 1420 1421 ret = i915_gem_open(dev, file); 1422 if (ret) 1423 return ret; 1424 1425 return 0; 1426 } 1427 1428 /** 1429 * i915_driver_lastclose - clean up after all DRM clients have exited 1430 * @dev: DRM device 1431 * 1432 * Take care of cleaning up after all DRM clients have exited. In the 1433 * mode setting case, we want to restore the kernel's initial mode (just 1434 * in case the last client left us in a bad state). 1435 * 1436 * Additionally, in the non-mode setting case, we'll tear down the GTT 1437 * and DMA structures, since the kernel won't be using them, and clea 1438 * up any GEM state. 1439 */ 1440 static void i915_driver_lastclose(struct drm_device *dev) 1441 { 1442 intel_fbdev_restore_mode(dev); 1443 #if 0 1444 vga_switcheroo_process_delayed_switch(); 1445 #endif 1446 } 1447 1448 static void i915_driver_preclose(struct drm_device *dev, struct drm_file *file) 1449 { 1450 mutex_lock(&dev->struct_mutex); 1451 i915_gem_context_close(dev, file); 1452 i915_gem_release(dev, file); 1453 mutex_unlock(&dev->struct_mutex); 1454 } 1455 1456 static void i915_driver_postclose(struct drm_device *dev, struct drm_file *file) 1457 { 1458 struct drm_i915_file_private *file_priv = file->driver_priv; 1459 1460 kfree(file_priv); 1461 } 1462 1463 static void intel_suspend_encoders(struct drm_i915_private *dev_priv) 1464 { 1465 struct drm_device *dev = &dev_priv->drm; 1466 struct intel_encoder *encoder; 1467 1468 drm_modeset_lock_all(dev); 1469 for_each_intel_encoder(dev, encoder) 1470 if (encoder->suspend) 1471 encoder->suspend(encoder); 1472 drm_modeset_unlock_all(dev); 1473 } 1474 1475 static int vlv_resume_prepare(struct drm_i915_private *dev_priv, 1476 bool rpm_resume); 1477 static int vlv_suspend_complete(struct drm_i915_private *dev_priv); 1478 1479 static bool suspend_to_idle(struct drm_i915_private *dev_priv) 1480 { 1481 #if IS_ENABLED(CONFIG_ACPI_SLEEP) 1482 if (acpi_target_system_state() < ACPI_STATE_S3) 1483 return true; 1484 #endif 1485 return false; 1486 } 1487 1488 static int i915_drm_suspend(struct drm_device *dev) 1489 { 1490 struct drm_i915_private *dev_priv = to_i915(dev); 1491 pci_power_t opregion_target_state; 1492 int error; 1493 1494 /* ignore lid events during suspend */ 1495 mutex_lock(&dev_priv->modeset_restore_lock); 1496 dev_priv->modeset_restore = MODESET_SUSPENDED; 1497 mutex_unlock(&dev_priv->modeset_restore_lock); 1498 1499 disable_rpm_wakeref_asserts(dev_priv); 1500 1501 /* We do a lot of poking in a lot of registers, make sure they work 1502 * properly. */ 1503 intel_display_set_init_power(dev_priv, true); 1504 1505 drm_kms_helper_poll_disable(dev); 1506 1507 #if 0 1508 pci_save_state(dev->pdev); 1509 #endif 1510 1511 error = i915_gem_suspend(dev); 1512 if (error) { 1513 dev_err(&dev->pdev->dev, 1514 "GEM idle failed, resume might fail\n"); 1515 goto out; 1516 } 1517 1518 intel_guc_suspend(dev); 1519 1520 intel_suspend_gt_powersave(dev_priv); 1521 1522 intel_display_suspend(dev); 1523 1524 intel_dp_mst_suspend(dev); 1525 1526 intel_runtime_pm_disable_interrupts(dev_priv); 1527 intel_hpd_cancel_work(dev_priv); 1528 1529 intel_suspend_encoders(dev_priv); 1530 1531 intel_suspend_hw(dev); 1532 1533 i915_gem_suspend_gtt_mappings(dev); 1534 1535 i915_save_state(dev); 1536 1537 opregion_target_state = suspend_to_idle(dev_priv) ? PCI_D1 : PCI_D3cold; 1538 intel_opregion_notify_adapter(dev_priv, opregion_target_state); 1539 1540 intel_uncore_forcewake_reset(dev_priv, false); 1541 intel_opregion_unregister(dev_priv); 1542 1543 #if 0 1544 intel_fbdev_set_suspend(dev, FBINFO_STATE_SUSPENDED, true); 1545 #endif 1546 1547 dev_priv->suspend_count++; 1548 1549 intel_csr_ucode_suspend(dev_priv); 1550 1551 out: 1552 enable_rpm_wakeref_asserts(dev_priv); 1553 1554 return error; 1555 } 1556 1557 static int i915_drm_suspend_late(struct drm_device *drm_dev, bool hibernation) 1558 { 1559 struct drm_i915_private *dev_priv = to_i915(drm_dev); 1560 bool fw_csr; 1561 int ret; 1562 1563 disable_rpm_wakeref_asserts(dev_priv); 1564 1565 intel_display_set_init_power(dev_priv, false); 1566 1567 fw_csr = !IS_BROXTON(dev_priv) && 1568 suspend_to_idle(dev_priv) && dev_priv->csr.dmc_payload; 1569 /* 1570 * In case of firmware assisted context save/restore don't manually 1571 * deinit the power domains. This also means the CSR/DMC firmware will 1572 * stay active, it will power down any HW resources as required and 1573 * also enable deeper system power states that would be blocked if the 1574 * firmware was inactive. 1575 */ 1576 if (!fw_csr) 1577 intel_power_domains_suspend(dev_priv); 1578 1579 ret = 0; 1580 if (IS_BROXTON(dev_priv)) 1581 bxt_enable_dc9(dev_priv); 1582 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) 1583 hsw_enable_pc8(dev_priv); 1584 else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) 1585 ret = vlv_suspend_complete(dev_priv); 1586 1587 if (ret) { 1588 DRM_ERROR("Suspend complete failed: %d\n", ret); 1589 if (!fw_csr) 1590 intel_power_domains_init_hw(dev_priv, true); 1591 1592 goto out; 1593 } 1594 1595 #if 0 1596 pci_disable_device(drm_dev->pdev); 1597 /* 1598 * During hibernation on some platforms the BIOS may try to access 1599 * the device even though it's already in D3 and hang the machine. So 1600 * leave the device in D0 on those platforms and hope the BIOS will 1601 * power down the device properly. The issue was seen on multiple old 1602 * GENs with different BIOS vendors, so having an explicit blacklist 1603 * is inpractical; apply the workaround on everything pre GEN6. The 1604 * platforms where the issue was seen: 1605 * Lenovo Thinkpad X301, X61s, X60, T60, X41 1606 * Fujitsu FSC S7110 1607 * Acer Aspire 1830T 1608 */ 1609 if (!(hibernation && INTEL_INFO(dev_priv)->gen < 6)) 1610 pci_set_power_state(drm_dev->pdev, PCI_D3hot); 1611 #endif 1612 1613 dev_priv->suspended_to_idle = suspend_to_idle(dev_priv); 1614 1615 out: 1616 enable_rpm_wakeref_asserts(dev_priv); 1617 1618 return ret; 1619 } 1620 1621 int i915_suspend_switcheroo(device_t kdev) 1622 { 1623 struct drm_softc *softc = device_get_softc(kdev); 1624 struct drm_device *dev = softc->drm_driver_data; 1625 int error; 1626 1627 if (!dev) { 1628 DRM_ERROR("dev: %p\n", dev); 1629 DRM_ERROR("DRM not initialized, aborting suspend.\n"); 1630 return -ENODEV; 1631 } 1632 1633 #if 0 1634 if (WARN_ON_ONCE(state.event != PM_EVENT_SUSPEND && 1635 state.event != PM_EVENT_FREEZE)) 1636 return -EINVAL; 1637 #endif 1638 1639 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 1640 return 0; 1641 1642 error = i915_drm_suspend(dev); 1643 if (error) 1644 return error; 1645 1646 return i915_drm_suspend_late(dev, false); 1647 } 1648 1649 static int i915_drm_resume(struct drm_device *dev) 1650 { 1651 struct drm_i915_private *dev_priv = to_i915(dev); 1652 int ret; 1653 1654 disable_rpm_wakeref_asserts(dev_priv); 1655 1656 ret = i915_ggtt_enable_hw(dev); 1657 if (ret) 1658 DRM_ERROR("failed to re-enable GGTT\n"); 1659 1660 intel_csr_ucode_resume(dev_priv); 1661 1662 mutex_lock(&dev->struct_mutex); 1663 i915_gem_restore_gtt_mappings(dev); 1664 mutex_unlock(&dev->struct_mutex); 1665 1666 i915_restore_state(dev); 1667 intel_opregion_setup(dev_priv); 1668 1669 intel_init_pch_refclk(dev); 1670 drm_mode_config_reset(dev); 1671 1672 /* 1673 * Interrupts have to be enabled before any batches are run. If not the 1674 * GPU will hang. i915_gem_init_hw() will initiate batches to 1675 * update/restore the context. 1676 * 1677 * Modeset enabling in intel_modeset_init_hw() also needs working 1678 * interrupts. 1679 */ 1680 intel_runtime_pm_enable_interrupts(dev_priv); 1681 1682 mutex_lock(&dev->struct_mutex); 1683 if (i915_gem_init_hw(dev)) { 1684 DRM_ERROR("failed to re-initialize GPU, declaring wedged!\n"); 1685 atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter); 1686 } 1687 mutex_unlock(&dev->struct_mutex); 1688 1689 intel_guc_resume(dev); 1690 1691 intel_modeset_init_hw(dev); 1692 1693 spin_lock_irq(&dev_priv->irq_lock); 1694 if (dev_priv->display.hpd_irq_setup) 1695 dev_priv->display.hpd_irq_setup(dev_priv); 1696 spin_unlock_irq(&dev_priv->irq_lock); 1697 1698 intel_dp_mst_resume(dev); 1699 1700 intel_display_resume(dev); 1701 1702 /* 1703 * ... but also need to make sure that hotplug processing 1704 * doesn't cause havoc. Like in the driver load code we don't 1705 * bother with the tiny race here where we might loose hotplug 1706 * notifications. 1707 * */ 1708 intel_hpd_init(dev_priv); 1709 /* Config may have changed between suspend and resume */ 1710 drm_helper_hpd_irq_event(dev); 1711 1712 intel_opregion_register(dev_priv); 1713 1714 intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING, false); 1715 1716 mutex_lock(&dev_priv->modeset_restore_lock); 1717 dev_priv->modeset_restore = MODESET_DONE; 1718 mutex_unlock(&dev_priv->modeset_restore_lock); 1719 1720 intel_opregion_notify_adapter(dev_priv, PCI_D0); 1721 1722 drm_kms_helper_poll_enable(dev); 1723 1724 enable_rpm_wakeref_asserts(dev_priv); 1725 1726 return 0; 1727 } 1728 1729 static int i915_drm_resume_early(struct drm_device *dev) 1730 { 1731 struct drm_i915_private *dev_priv = to_i915(dev); 1732 int ret = 0; 1733 1734 /* 1735 * We have a resume ordering issue with the snd-hda driver also 1736 * requiring our device to be power up. Due to the lack of a 1737 * parent/child relationship we currently solve this with an early 1738 * resume hook. 1739 * 1740 * FIXME: This should be solved with a special hdmi sink device or 1741 * similar so that power domains can be employed. 1742 */ 1743 1744 /* 1745 * Note that we need to set the power state explicitly, since we 1746 * powered off the device during freeze and the PCI core won't power 1747 * it back up for us during thaw. Powering off the device during 1748 * freeze is not a hard requirement though, and during the 1749 * suspend/resume phases the PCI core makes sure we get here with the 1750 * device powered on. So in case we change our freeze logic and keep 1751 * the device powered we can also remove the following set power state 1752 * call. 1753 */ 1754 #if 0 1755 ret = pci_set_power_state(dev->pdev, PCI_D0); 1756 if (ret) { 1757 DRM_ERROR("failed to set PCI D0 power state (%d)\n", ret); 1758 goto out; 1759 } 1760 1761 /* 1762 * Note that pci_enable_device() first enables any parent bridge 1763 * device and only then sets the power state for this device. The 1764 * bridge enabling is a nop though, since bridge devices are resumed 1765 * first. The order of enabling power and enabling the device is 1766 * imposed by the PCI core as described above, so here we preserve the 1767 * same order for the freeze/thaw phases. 1768 * 1769 * TODO: eventually we should remove pci_disable_device() / 1770 * pci_enable_enable_device() from suspend/resume. Due to how they 1771 * depend on the device enable refcount we can't anyway depend on them 1772 * disabling/enabling the device. 1773 */ 1774 if (pci_enable_device(dev->pdev)) { 1775 ret = -EIO; 1776 goto out; 1777 } 1778 1779 pci_set_master(dev->pdev); 1780 #endif 1781 1782 disable_rpm_wakeref_asserts(dev_priv); 1783 1784 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) 1785 ret = vlv_resume_prepare(dev_priv, false); 1786 if (ret) 1787 DRM_ERROR("Resume prepare failed: %d, continuing anyway\n", 1788 ret); 1789 1790 intel_uncore_early_sanitize(dev_priv, true); 1791 1792 if (IS_BROXTON(dev_priv)) { 1793 if (!dev_priv->suspended_to_idle) 1794 gen9_sanitize_dc_state(dev_priv); 1795 bxt_disable_dc9(dev_priv); 1796 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { 1797 hsw_disable_pc8(dev_priv); 1798 } 1799 1800 intel_uncore_sanitize(dev_priv); 1801 1802 if (IS_BROXTON(dev_priv) || 1803 !(dev_priv->suspended_to_idle && dev_priv->csr.dmc_payload)) 1804 intel_power_domains_init_hw(dev_priv, true); 1805 1806 enable_rpm_wakeref_asserts(dev_priv); 1807 1808 #if 0 1809 out: 1810 #endif 1811 dev_priv->suspended_to_idle = false; 1812 1813 return ret; 1814 } 1815 1816 int i915_resume_switcheroo(struct drm_device *dev) 1817 { 1818 int ret; 1819 1820 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 1821 return 0; 1822 1823 ret = i915_drm_resume_early(dev); 1824 if (ret) 1825 return ret; 1826 1827 return i915_drm_resume(dev); 1828 } 1829 1830 /** 1831 * i915_reset - reset chip after a hang 1832 * @dev: drm device to reset 1833 * 1834 * Reset the chip. Useful if a hang is detected. Returns zero on successful 1835 * reset or otherwise an error code. 1836 * 1837 * Procedure is fairly simple: 1838 * - reset the chip using the reset reg 1839 * - re-init context state 1840 * - re-init hardware status page 1841 * - re-init ring buffer 1842 * - re-init interrupt state 1843 * - re-init display 1844 */ 1845 int i915_reset(struct drm_i915_private *dev_priv) 1846 { 1847 struct drm_device *dev = &dev_priv->drm; 1848 struct i915_gpu_error *error = &dev_priv->gpu_error; 1849 unsigned reset_counter; 1850 int ret; 1851 1852 intel_reset_gt_powersave(dev_priv); 1853 1854 mutex_lock(&dev->struct_mutex); 1855 1856 /* Clear any previous failed attempts at recovery. Time to try again. */ 1857 atomic_andnot(I915_WEDGED, &error->reset_counter); 1858 1859 /* Clear the reset-in-progress flag and increment the reset epoch. */ 1860 reset_counter = atomic_inc_return(&error->reset_counter); 1861 if (WARN_ON(__i915_reset_in_progress(reset_counter))) { 1862 ret = -EIO; 1863 goto error; 1864 } 1865 1866 pr_notice("drm/i915: Resetting chip after gpu hang\n"); 1867 1868 i915_gem_reset(dev); 1869 1870 ret = intel_gpu_reset(dev_priv, ALL_ENGINES); 1871 if (ret) { 1872 if (ret != -ENODEV) 1873 DRM_ERROR("Failed to reset chip: %i\n", ret); 1874 else 1875 DRM_DEBUG_DRIVER("GPU reset disabled\n"); 1876 goto error; 1877 } 1878 1879 intel_overlay_reset(dev_priv); 1880 1881 /* Ok, now get things going again... */ 1882 1883 /* 1884 * Everything depends on having the GTT running, so we need to start 1885 * there. Fortunately we don't need to do this unless we reset the 1886 * chip at a PCI level. 1887 * 1888 * Next we need to restore the context, but we don't use those 1889 * yet either... 1890 * 1891 * Ring buffer needs to be re-initialized in the KMS case, or if X 1892 * was running at the time of the reset (i.e. we weren't VT 1893 * switched away). 1894 */ 1895 ret = i915_gem_init_hw(dev); 1896 if (ret) { 1897 DRM_ERROR("Failed hw init on reset %d\n", ret); 1898 goto error; 1899 } 1900 1901 mutex_unlock(&dev->struct_mutex); 1902 1903 /* 1904 * rps/rc6 re-init is necessary to restore state lost after the 1905 * reset and the re-install of gt irqs. Skip for ironlake per 1906 * previous concerns that it doesn't respond well to some forms 1907 * of re-init after reset. 1908 */ 1909 if (INTEL_INFO(dev)->gen > 5) 1910 intel_enable_gt_powersave(dev_priv); 1911 1912 return 0; 1913 1914 error: 1915 atomic_or(I915_WEDGED, &error->reset_counter); 1916 mutex_unlock(&dev->struct_mutex); 1917 return ret; 1918 } 1919 1920 #if 0 1921 static int i915_pm_suspend(struct device *dev) 1922 { 1923 struct pci_dev *pdev = to_pci_dev(dev); 1924 struct drm_device *drm_dev = pci_get_drvdata(pdev); 1925 1926 if (!drm_dev) { 1927 dev_err(dev, "DRM not initialized, aborting suspend.\n"); 1928 return -ENODEV; 1929 } 1930 1931 if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF) 1932 return 0; 1933 1934 return i915_drm_suspend(drm_dev); 1935 } 1936 1937 static int i915_pm_suspend_late(struct device *dev) 1938 { 1939 struct drm_device *drm_dev = &dev_to_i915(dev)->drm; 1940 1941 /* 1942 * We have a suspend ordering issue with the snd-hda driver also 1943 * requiring our device to be power up. Due to the lack of a 1944 * parent/child relationship we currently solve this with an late 1945 * suspend hook. 1946 * 1947 * FIXME: This should be solved with a special hdmi sink device or 1948 * similar so that power domains can be employed. 1949 */ 1950 if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF) 1951 return 0; 1952 1953 return i915_drm_suspend_late(drm_dev, false); 1954 } 1955 1956 static int i915_pm_poweroff_late(struct device *dev) 1957 { 1958 struct drm_device *drm_dev = &dev_to_i915(dev)->drm; 1959 1960 if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF) 1961 return 0; 1962 1963 return i915_drm_suspend_late(drm_dev, true); 1964 } 1965 1966 static int i915_pm_resume_early(struct device *dev) 1967 { 1968 struct drm_device *drm_dev = &dev_to_i915(dev)->drm; 1969 1970 if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF) 1971 return 0; 1972 1973 return i915_drm_resume_early(drm_dev); 1974 } 1975 1976 static int i915_pm_resume(struct device *dev) 1977 { 1978 struct drm_device *drm_dev = &dev_to_i915(dev)->drm; 1979 1980 if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF) 1981 return 0; 1982 1983 return i915_drm_resume(drm_dev); 1984 } 1985 1986 /* freeze: before creating the hibernation_image */ 1987 static int i915_pm_freeze(struct device *dev) 1988 { 1989 return i915_pm_suspend(dev); 1990 } 1991 1992 static int i915_pm_freeze_late(struct device *dev) 1993 { 1994 int ret; 1995 1996 ret = i915_pm_suspend_late(dev); 1997 if (ret) 1998 return ret; 1999 2000 ret = i915_gem_freeze_late(dev_to_i915(dev)); 2001 if (ret) 2002 return ret; 2003 2004 return 0; 2005 } 2006 2007 /* thaw: called after creating the hibernation image, but before turning off. */ 2008 static int i915_pm_thaw_early(struct device *dev) 2009 { 2010 return i915_pm_resume_early(dev); 2011 } 2012 2013 static int i915_pm_thaw(struct device *dev) 2014 { 2015 return i915_pm_resume(dev); 2016 } 2017 2018 /* restore: called after loading the hibernation image. */ 2019 static int i915_pm_restore_early(struct device *dev) 2020 { 2021 return i915_pm_resume_early(dev); 2022 } 2023 2024 static int i915_pm_restore(struct device *dev) 2025 { 2026 return i915_pm_resume(dev); 2027 } 2028 #endif 2029 2030 /* 2031 * Save all Gunit registers that may be lost after a D3 and a subsequent 2032 * S0i[R123] transition. The list of registers needing a save/restore is 2033 * defined in the VLV2_S0IXRegs document. This documents marks all Gunit 2034 * registers in the following way: 2035 * - Driver: saved/restored by the driver 2036 * - Punit : saved/restored by the Punit firmware 2037 * - No, w/o marking: no need to save/restore, since the register is R/O or 2038 * used internally by the HW in a way that doesn't depend 2039 * keeping the content across a suspend/resume. 2040 * - Debug : used for debugging 2041 * 2042 * We save/restore all registers marked with 'Driver', with the following 2043 * exceptions: 2044 * - Registers out of use, including also registers marked with 'Debug'. 2045 * These have no effect on the driver's operation, so we don't save/restore 2046 * them to reduce the overhead. 2047 * - Registers that are fully setup by an initialization function called from 2048 * the resume path. For example many clock gating and RPS/RC6 registers. 2049 * - Registers that provide the right functionality with their reset defaults. 2050 * 2051 * TODO: Except for registers that based on the above 3 criteria can be safely 2052 * ignored, we save/restore all others, practically treating the HW context as 2053 * a black-box for the driver. Further investigation is needed to reduce the 2054 * saved/restored registers even further, by following the same 3 criteria. 2055 */ 2056 static void vlv_save_gunit_s0ix_state(struct drm_i915_private *dev_priv) 2057 { 2058 struct vlv_s0ix_state *s = &dev_priv->vlv_s0ix_state; 2059 int i; 2060 2061 /* GAM 0x4000-0x4770 */ 2062 s->wr_watermark = I915_READ(GEN7_WR_WATERMARK); 2063 s->gfx_prio_ctrl = I915_READ(GEN7_GFX_PRIO_CTRL); 2064 s->arb_mode = I915_READ(ARB_MODE); 2065 s->gfx_pend_tlb0 = I915_READ(GEN7_GFX_PEND_TLB0); 2066 s->gfx_pend_tlb1 = I915_READ(GEN7_GFX_PEND_TLB1); 2067 2068 for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++) 2069 s->lra_limits[i] = I915_READ(GEN7_LRA_LIMITS(i)); 2070 2071 s->media_max_req_count = I915_READ(GEN7_MEDIA_MAX_REQ_COUNT); 2072 s->gfx_max_req_count = I915_READ(GEN7_GFX_MAX_REQ_COUNT); 2073 2074 s->render_hwsp = I915_READ(RENDER_HWS_PGA_GEN7); 2075 s->ecochk = I915_READ(GAM_ECOCHK); 2076 s->bsd_hwsp = I915_READ(BSD_HWS_PGA_GEN7); 2077 s->blt_hwsp = I915_READ(BLT_HWS_PGA_GEN7); 2078 2079 s->tlb_rd_addr = I915_READ(GEN7_TLB_RD_ADDR); 2080 2081 /* MBC 0x9024-0x91D0, 0x8500 */ 2082 s->g3dctl = I915_READ(VLV_G3DCTL); 2083 s->gsckgctl = I915_READ(VLV_GSCKGCTL); 2084 s->mbctl = I915_READ(GEN6_MBCTL); 2085 2086 /* GCP 0x9400-0x9424, 0x8100-0x810C */ 2087 s->ucgctl1 = I915_READ(GEN6_UCGCTL1); 2088 s->ucgctl3 = I915_READ(GEN6_UCGCTL3); 2089 s->rcgctl1 = I915_READ(GEN6_RCGCTL1); 2090 s->rcgctl2 = I915_READ(GEN6_RCGCTL2); 2091 s->rstctl = I915_READ(GEN6_RSTCTL); 2092 s->misccpctl = I915_READ(GEN7_MISCCPCTL); 2093 2094 /* GPM 0xA000-0xAA84, 0x8000-0x80FC */ 2095 s->gfxpause = I915_READ(GEN6_GFXPAUSE); 2096 s->rpdeuhwtc = I915_READ(GEN6_RPDEUHWTC); 2097 s->rpdeuc = I915_READ(GEN6_RPDEUC); 2098 s->ecobus = I915_READ(ECOBUS); 2099 s->pwrdwnupctl = I915_READ(VLV_PWRDWNUPCTL); 2100 s->rp_down_timeout = I915_READ(GEN6_RP_DOWN_TIMEOUT); 2101 s->rp_deucsw = I915_READ(GEN6_RPDEUCSW); 2102 s->rcubmabdtmr = I915_READ(GEN6_RCUBMABDTMR); 2103 s->rcedata = I915_READ(VLV_RCEDATA); 2104 s->spare2gh = I915_READ(VLV_SPAREG2H); 2105 2106 /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */ 2107 s->gt_imr = I915_READ(GTIMR); 2108 s->gt_ier = I915_READ(GTIER); 2109 s->pm_imr = I915_READ(GEN6_PMIMR); 2110 s->pm_ier = I915_READ(GEN6_PMIER); 2111 2112 for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++) 2113 s->gt_scratch[i] = I915_READ(GEN7_GT_SCRATCH(i)); 2114 2115 /* GT SA CZ domain, 0x100000-0x138124 */ 2116 s->tilectl = I915_READ(TILECTL); 2117 s->gt_fifoctl = I915_READ(GTFIFOCTL); 2118 s->gtlc_wake_ctrl = I915_READ(VLV_GTLC_WAKE_CTRL); 2119 s->gtlc_survive = I915_READ(VLV_GTLC_SURVIVABILITY_REG); 2120 s->pmwgicz = I915_READ(VLV_PMWGICZ); 2121 2122 /* Gunit-Display CZ domain, 0x182028-0x1821CF */ 2123 s->gu_ctl0 = I915_READ(VLV_GU_CTL0); 2124 s->gu_ctl1 = I915_READ(VLV_GU_CTL1); 2125 s->pcbr = I915_READ(VLV_PCBR); 2126 s->clock_gate_dis2 = I915_READ(VLV_GUNIT_CLOCK_GATE2); 2127 2128 /* 2129 * Not saving any of: 2130 * DFT, 0x9800-0x9EC0 2131 * SARB, 0xB000-0xB1FC 2132 * GAC, 0x5208-0x524C, 0x14000-0x14C000 2133 * PCI CFG 2134 */ 2135 } 2136 2137 static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *dev_priv) 2138 { 2139 struct vlv_s0ix_state *s = &dev_priv->vlv_s0ix_state; 2140 u32 val; 2141 int i; 2142 2143 /* GAM 0x4000-0x4770 */ 2144 I915_WRITE(GEN7_WR_WATERMARK, s->wr_watermark); 2145 I915_WRITE(GEN7_GFX_PRIO_CTRL, s->gfx_prio_ctrl); 2146 I915_WRITE(ARB_MODE, s->arb_mode | (0xffff << 16)); 2147 I915_WRITE(GEN7_GFX_PEND_TLB0, s->gfx_pend_tlb0); 2148 I915_WRITE(GEN7_GFX_PEND_TLB1, s->gfx_pend_tlb1); 2149 2150 for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++) 2151 I915_WRITE(GEN7_LRA_LIMITS(i), s->lra_limits[i]); 2152 2153 I915_WRITE(GEN7_MEDIA_MAX_REQ_COUNT, s->media_max_req_count); 2154 I915_WRITE(GEN7_GFX_MAX_REQ_COUNT, s->gfx_max_req_count); 2155 2156 I915_WRITE(RENDER_HWS_PGA_GEN7, s->render_hwsp); 2157 I915_WRITE(GAM_ECOCHK, s->ecochk); 2158 I915_WRITE(BSD_HWS_PGA_GEN7, s->bsd_hwsp); 2159 I915_WRITE(BLT_HWS_PGA_GEN7, s->blt_hwsp); 2160 2161 I915_WRITE(GEN7_TLB_RD_ADDR, s->tlb_rd_addr); 2162 2163 /* MBC 0x9024-0x91D0, 0x8500 */ 2164 I915_WRITE(VLV_G3DCTL, s->g3dctl); 2165 I915_WRITE(VLV_GSCKGCTL, s->gsckgctl); 2166 I915_WRITE(GEN6_MBCTL, s->mbctl); 2167 2168 /* GCP 0x9400-0x9424, 0x8100-0x810C */ 2169 I915_WRITE(GEN6_UCGCTL1, s->ucgctl1); 2170 I915_WRITE(GEN6_UCGCTL3, s->ucgctl3); 2171 I915_WRITE(GEN6_RCGCTL1, s->rcgctl1); 2172 I915_WRITE(GEN6_RCGCTL2, s->rcgctl2); 2173 I915_WRITE(GEN6_RSTCTL, s->rstctl); 2174 I915_WRITE(GEN7_MISCCPCTL, s->misccpctl); 2175 2176 /* GPM 0xA000-0xAA84, 0x8000-0x80FC */ 2177 I915_WRITE(GEN6_GFXPAUSE, s->gfxpause); 2178 I915_WRITE(GEN6_RPDEUHWTC, s->rpdeuhwtc); 2179 I915_WRITE(GEN6_RPDEUC, s->rpdeuc); 2180 I915_WRITE(ECOBUS, s->ecobus); 2181 I915_WRITE(VLV_PWRDWNUPCTL, s->pwrdwnupctl); 2182 I915_WRITE(GEN6_RP_DOWN_TIMEOUT,s->rp_down_timeout); 2183 I915_WRITE(GEN6_RPDEUCSW, s->rp_deucsw); 2184 I915_WRITE(GEN6_RCUBMABDTMR, s->rcubmabdtmr); 2185 I915_WRITE(VLV_RCEDATA, s->rcedata); 2186 I915_WRITE(VLV_SPAREG2H, s->spare2gh); 2187 2188 /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */ 2189 I915_WRITE(GTIMR, s->gt_imr); 2190 I915_WRITE(GTIER, s->gt_ier); 2191 I915_WRITE(GEN6_PMIMR, s->pm_imr); 2192 I915_WRITE(GEN6_PMIER, s->pm_ier); 2193 2194 for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++) 2195 I915_WRITE(GEN7_GT_SCRATCH(i), s->gt_scratch[i]); 2196 2197 /* GT SA CZ domain, 0x100000-0x138124 */ 2198 I915_WRITE(TILECTL, s->tilectl); 2199 I915_WRITE(GTFIFOCTL, s->gt_fifoctl); 2200 /* 2201 * Preserve the GT allow wake and GFX force clock bit, they are not 2202 * be restored, as they are used to control the s0ix suspend/resume 2203 * sequence by the caller. 2204 */ 2205 val = I915_READ(VLV_GTLC_WAKE_CTRL); 2206 val &= VLV_GTLC_ALLOWWAKEREQ; 2207 val |= s->gtlc_wake_ctrl & ~VLV_GTLC_ALLOWWAKEREQ; 2208 I915_WRITE(VLV_GTLC_WAKE_CTRL, val); 2209 2210 val = I915_READ(VLV_GTLC_SURVIVABILITY_REG); 2211 val &= VLV_GFX_CLK_FORCE_ON_BIT; 2212 val |= s->gtlc_survive & ~VLV_GFX_CLK_FORCE_ON_BIT; 2213 I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val); 2214 2215 I915_WRITE(VLV_PMWGICZ, s->pmwgicz); 2216 2217 /* Gunit-Display CZ domain, 0x182028-0x1821CF */ 2218 I915_WRITE(VLV_GU_CTL0, s->gu_ctl0); 2219 I915_WRITE(VLV_GU_CTL1, s->gu_ctl1); 2220 I915_WRITE(VLV_PCBR, s->pcbr); 2221 I915_WRITE(VLV_GUNIT_CLOCK_GATE2, s->clock_gate_dis2); 2222 } 2223 2224 int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool force_on) 2225 { 2226 u32 val; 2227 int err; 2228 2229 val = I915_READ(VLV_GTLC_SURVIVABILITY_REG); 2230 val &= ~VLV_GFX_CLK_FORCE_ON_BIT; 2231 if (force_on) 2232 val |= VLV_GFX_CLK_FORCE_ON_BIT; 2233 I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val); 2234 2235 if (!force_on) 2236 return 0; 2237 2238 err = intel_wait_for_register(dev_priv, 2239 VLV_GTLC_SURVIVABILITY_REG, 2240 VLV_GFX_CLK_STATUS_BIT, 2241 VLV_GFX_CLK_STATUS_BIT, 2242 20); 2243 if (err) 2244 DRM_ERROR("timeout waiting for GFX clock force-on (%08x)\n", 2245 I915_READ(VLV_GTLC_SURVIVABILITY_REG)); 2246 2247 return err; 2248 } 2249 2250 static int vlv_allow_gt_wake(struct drm_i915_private *dev_priv, bool allow) 2251 { 2252 u32 val; 2253 int err = 0; 2254 2255 val = I915_READ(VLV_GTLC_WAKE_CTRL); 2256 val &= ~VLV_GTLC_ALLOWWAKEREQ; 2257 if (allow) 2258 val |= VLV_GTLC_ALLOWWAKEREQ; 2259 I915_WRITE(VLV_GTLC_WAKE_CTRL, val); 2260 POSTING_READ(VLV_GTLC_WAKE_CTRL); 2261 2262 err = intel_wait_for_register(dev_priv, 2263 VLV_GTLC_PW_STATUS, 2264 VLV_GTLC_ALLOWWAKEACK, 2265 allow, 2266 1); 2267 if (err) 2268 DRM_ERROR("timeout disabling GT waking\n"); 2269 2270 return err; 2271 } 2272 2273 static int vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv, 2274 bool wait_for_on) 2275 { 2276 u32 mask; 2277 u32 val; 2278 int err; 2279 2280 mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK; 2281 val = wait_for_on ? mask : 0; 2282 if ((I915_READ(VLV_GTLC_PW_STATUS) & mask) == val) 2283 return 0; 2284 2285 DRM_DEBUG_KMS("waiting for GT wells to go %s (%08x)\n", 2286 onoff(wait_for_on), 2287 I915_READ(VLV_GTLC_PW_STATUS)); 2288 2289 /* 2290 * RC6 transitioning can be delayed up to 2 msec (see 2291 * valleyview_enable_rps), use 3 msec for safety. 2292 */ 2293 err = intel_wait_for_register(dev_priv, 2294 VLV_GTLC_PW_STATUS, mask, val, 2295 3); 2296 if (err) 2297 DRM_ERROR("timeout waiting for GT wells to go %s\n", 2298 onoff(wait_for_on)); 2299 2300 return err; 2301 } 2302 2303 static void vlv_check_no_gt_access(struct drm_i915_private *dev_priv) 2304 { 2305 if (!(I915_READ(VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEERR)) 2306 return; 2307 2308 DRM_DEBUG_DRIVER("GT register access while GT waking disabled\n"); 2309 I915_WRITE(VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR); 2310 } 2311 2312 static int vlv_suspend_complete(struct drm_i915_private *dev_priv) 2313 { 2314 u32 mask; 2315 int err; 2316 2317 /* 2318 * Bspec defines the following GT well on flags as debug only, so 2319 * don't treat them as hard failures. 2320 */ 2321 (void)vlv_wait_for_gt_wells(dev_priv, false); 2322 2323 mask = VLV_GTLC_RENDER_CTX_EXISTS | VLV_GTLC_MEDIA_CTX_EXISTS; 2324 WARN_ON((I915_READ(VLV_GTLC_WAKE_CTRL) & mask) != mask); 2325 2326 vlv_check_no_gt_access(dev_priv); 2327 2328 err = vlv_force_gfx_clock(dev_priv, true); 2329 if (err) 2330 goto err1; 2331 2332 err = vlv_allow_gt_wake(dev_priv, false); 2333 if (err) 2334 goto err2; 2335 2336 if (!IS_CHERRYVIEW(dev_priv)) 2337 vlv_save_gunit_s0ix_state(dev_priv); 2338 2339 err = vlv_force_gfx_clock(dev_priv, false); 2340 if (err) 2341 goto err2; 2342 2343 return 0; 2344 2345 err2: 2346 /* For safety always re-enable waking and disable gfx clock forcing */ 2347 vlv_allow_gt_wake(dev_priv, true); 2348 err1: 2349 vlv_force_gfx_clock(dev_priv, false); 2350 2351 return err; 2352 } 2353 2354 static int vlv_resume_prepare(struct drm_i915_private *dev_priv, 2355 bool rpm_resume) 2356 { 2357 struct drm_device *dev = &dev_priv->drm; 2358 int err; 2359 int ret; 2360 2361 /* 2362 * If any of the steps fail just try to continue, that's the best we 2363 * can do at this point. Return the first error code (which will also 2364 * leave RPM permanently disabled). 2365 */ 2366 ret = vlv_force_gfx_clock(dev_priv, true); 2367 2368 if (!IS_CHERRYVIEW(dev_priv)) 2369 vlv_restore_gunit_s0ix_state(dev_priv); 2370 2371 err = vlv_allow_gt_wake(dev_priv, true); 2372 if (!ret) 2373 ret = err; 2374 2375 err = vlv_force_gfx_clock(dev_priv, false); 2376 if (!ret) 2377 ret = err; 2378 2379 vlv_check_no_gt_access(dev_priv); 2380 2381 if (rpm_resume) { 2382 intel_init_clock_gating(dev); 2383 i915_gem_restore_fences(dev); 2384 } 2385 2386 return ret; 2387 } 2388 2389 #if 0 2390 static int intel_runtime_suspend(struct device *device) 2391 { 2392 struct pci_dev *pdev = to_pci_dev(device); 2393 struct drm_device *dev = pci_get_drvdata(pdev); 2394 struct drm_i915_private *dev_priv = to_i915(dev); 2395 int ret; 2396 2397 if (WARN_ON_ONCE(!(dev_priv->rps.enabled && intel_enable_rc6()))) 2398 return -ENODEV; 2399 2400 if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev))) 2401 return -ENODEV; 2402 2403 DRM_DEBUG_KMS("Suspending device\n"); 2404 2405 /* 2406 * We could deadlock here in case another thread holding struct_mutex 2407 * calls RPM suspend concurrently, since the RPM suspend will wait 2408 * first for this RPM suspend to finish. In this case the concurrent 2409 * RPM resume will be followed by its RPM suspend counterpart. Still 2410 * for consistency return -EAGAIN, which will reschedule this suspend. 2411 */ 2412 if (!mutex_trylock(&dev->struct_mutex)) { 2413 DRM_DEBUG_KMS("device lock contention, deffering suspend\n"); 2414 /* 2415 * Bump the expiration timestamp, otherwise the suspend won't 2416 * be rescheduled. 2417 */ 2418 pm_runtime_mark_last_busy(device); 2419 2420 return -EAGAIN; 2421 } 2422 2423 disable_rpm_wakeref_asserts(dev_priv); 2424 2425 /* 2426 * We are safe here against re-faults, since the fault handler takes 2427 * an RPM reference. 2428 */ 2429 i915_gem_release_all_mmaps(dev_priv); 2430 mutex_unlock(&dev->struct_mutex); 2431 2432 intel_guc_suspend(dev); 2433 2434 intel_runtime_pm_disable_interrupts(dev_priv); 2435 2436 ret = 0; 2437 if (IS_BROXTON(dev_priv)) { 2438 bxt_display_core_uninit(dev_priv); 2439 bxt_enable_dc9(dev_priv); 2440 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { 2441 hsw_enable_pc8(dev_priv); 2442 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { 2443 ret = vlv_suspend_complete(dev_priv); 2444 } 2445 2446 if (ret) { 2447 DRM_ERROR("Runtime suspend failed, disabling it (%d)\n", ret); 2448 intel_runtime_pm_enable_interrupts(dev_priv); 2449 2450 enable_rpm_wakeref_asserts(dev_priv); 2451 2452 return ret; 2453 } 2454 2455 intel_uncore_forcewake_reset(dev_priv, false); 2456 2457 enable_rpm_wakeref_asserts(dev_priv); 2458 WARN_ON_ONCE(atomic_read(&dev_priv->pm.wakeref_count)); 2459 2460 if (intel_uncore_arm_unclaimed_mmio_detection(dev_priv)) 2461 DRM_ERROR("Unclaimed access detected prior to suspending\n"); 2462 2463 dev_priv->pm.suspended = true; 2464 2465 /* 2466 * FIXME: We really should find a document that references the arguments 2467 * used below! 2468 */ 2469 if (IS_BROADWELL(dev_priv)) { 2470 /* 2471 * On Broadwell, if we use PCI_D1 the PCH DDI ports will stop 2472 * being detected, and the call we do at intel_runtime_resume() 2473 * won't be able to restore them. Since PCI_D3hot matches the 2474 * actual specification and appears to be working, use it. 2475 */ 2476 intel_opregion_notify_adapter(dev_priv, PCI_D3hot); 2477 } else { 2478 /* 2479 * current versions of firmware which depend on this opregion 2480 * notification have repurposed the D1 definition to mean 2481 * "runtime suspended" vs. what you would normally expect (D3) 2482 * to distinguish it from notifications that might be sent via 2483 * the suspend path. 2484 */ 2485 intel_opregion_notify_adapter(dev_priv, PCI_D1); 2486 } 2487 2488 assert_forcewakes_inactive(dev_priv); 2489 2490 if (!IS_VALLEYVIEW(dev_priv) || !IS_CHERRYVIEW(dev_priv)) 2491 intel_hpd_poll_init(dev_priv); 2492 2493 DRM_DEBUG_KMS("Device suspended\n"); 2494 return 0; 2495 } 2496 2497 static int intel_runtime_resume(struct device *device) 2498 { 2499 struct pci_dev *pdev = to_pci_dev(device); 2500 struct drm_device *dev = pci_get_drvdata(pdev); 2501 struct drm_i915_private *dev_priv = to_i915(dev); 2502 int ret = 0; 2503 2504 if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev))) 2505 return -ENODEV; 2506 2507 DRM_DEBUG_KMS("Resuming device\n"); 2508 2509 WARN_ON_ONCE(atomic_read(&dev_priv->pm.wakeref_count)); 2510 disable_rpm_wakeref_asserts(dev_priv); 2511 2512 intel_opregion_notify_adapter(dev_priv, PCI_D0); 2513 dev_priv->pm.suspended = false; 2514 if (intel_uncore_unclaimed_mmio(dev_priv)) 2515 DRM_DEBUG_DRIVER("Unclaimed access during suspend, bios?\n"); 2516 2517 intel_guc_resume(dev); 2518 2519 if (IS_GEN6(dev_priv)) 2520 intel_init_pch_refclk(dev); 2521 2522 if (IS_BROXTON(dev)) { 2523 bxt_disable_dc9(dev_priv); 2524 bxt_display_core_init(dev_priv, true); 2525 if (dev_priv->csr.dmc_payload && 2526 (dev_priv->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC5)) 2527 gen9_enable_dc5(dev_priv); 2528 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { 2529 hsw_disable_pc8(dev_priv); 2530 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { 2531 ret = vlv_resume_prepare(dev_priv, true); 2532 } 2533 2534 /* 2535 * No point of rolling back things in case of an error, as the best 2536 * we can do is to hope that things will still work (and disable RPM). 2537 */ 2538 i915_gem_init_swizzling(dev); 2539 gen6_update_ring_freq(dev_priv); 2540 2541 intel_runtime_pm_enable_interrupts(dev_priv); 2542 2543 /* 2544 * On VLV/CHV display interrupts are part of the display 2545 * power well, so hpd is reinitialized from there. For 2546 * everyone else do it here. 2547 */ 2548 if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) 2549 intel_hpd_init(dev_priv); 2550 2551 enable_rpm_wakeref_asserts(dev_priv); 2552 2553 if (ret) 2554 DRM_ERROR("Runtime resume failed, disabling it (%d)\n", ret); 2555 else 2556 DRM_DEBUG_KMS("Device resumed\n"); 2557 2558 return ret; 2559 } 2560 2561 const struct dev_pm_ops i915_pm_ops = { 2562 /* 2563 * S0ix (via system suspend) and S3 event handlers [PMSG_SUSPEND, 2564 * PMSG_RESUME] 2565 */ 2566 .suspend = i915_pm_suspend, 2567 .suspend_late = i915_pm_suspend_late, 2568 .resume_early = i915_pm_resume_early, 2569 .resume = i915_pm_resume, 2570 2571 /* 2572 * S4 event handlers 2573 * @freeze, @freeze_late : called (1) before creating the 2574 * hibernation image [PMSG_FREEZE] and 2575 * (2) after rebooting, before restoring 2576 * the image [PMSG_QUIESCE] 2577 * @thaw, @thaw_early : called (1) after creating the hibernation 2578 * image, before writing it [PMSG_THAW] 2579 * and (2) after failing to create or 2580 * restore the image [PMSG_RECOVER] 2581 * @poweroff, @poweroff_late: called after writing the hibernation 2582 * image, before rebooting [PMSG_HIBERNATE] 2583 * @restore, @restore_early : called after rebooting and restoring the 2584 * hibernation image [PMSG_RESTORE] 2585 */ 2586 .freeze = i915_pm_freeze, 2587 .freeze_late = i915_pm_freeze_late, 2588 .thaw_early = i915_pm_thaw_early, 2589 .thaw = i915_pm_thaw, 2590 .poweroff = i915_pm_suspend, 2591 .poweroff_late = i915_pm_poweroff_late, 2592 .restore_early = i915_pm_restore_early, 2593 .restore = i915_pm_restore, 2594 2595 /* S0ix (via runtime suspend) event handlers */ 2596 .runtime_suspend = intel_runtime_suspend, 2597 .runtime_resume = intel_runtime_resume, 2598 }; 2599 2600 static const struct vm_operations_struct i915_gem_vm_ops = { 2601 .fault = i915_gem_fault, 2602 .open = drm_gem_vm_open, 2603 .close = drm_gem_vm_close, 2604 }; 2605 #endif 2606 2607 static struct cdev_pager_ops i915_gem_vm_ops = { 2608 .cdev_pg_fault = i915_gem_fault, 2609 .cdev_pg_ctor = i915_gem_pager_ctor, 2610 .cdev_pg_dtor = i915_gem_pager_dtor 2611 }; 2612 2613 static const struct file_operations i915_driver_fops = { 2614 .owner = THIS_MODULE, 2615 #if 0 2616 .open = drm_open, 2617 .release = drm_release, 2618 .unlocked_ioctl = drm_ioctl, 2619 .mmap = drm_gem_mmap, 2620 .poll = drm_poll, 2621 .read = drm_read, 2622 #ifdef CONFIG_COMPAT 2623 .compat_ioctl = i915_compat_ioctl, 2624 #endif 2625 .llseek = noop_llseek, 2626 #endif 2627 }; 2628 2629 static int 2630 i915_gem_reject_pin_ioctl(struct drm_device *dev, void *data, 2631 struct drm_file *file) 2632 { 2633 return -ENODEV; 2634 } 2635 2636 static const struct drm_ioctl_desc i915_ioctls[] = { 2637 DRM_IOCTL_DEF_DRV(I915_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 2638 DRM_IOCTL_DEF_DRV(I915_FLUSH, drm_noop, DRM_AUTH), 2639 DRM_IOCTL_DEF_DRV(I915_FLIP, drm_noop, DRM_AUTH), 2640 DRM_IOCTL_DEF_DRV(I915_BATCHBUFFER, drm_noop, DRM_AUTH), 2641 DRM_IOCTL_DEF_DRV(I915_IRQ_EMIT, drm_noop, DRM_AUTH), 2642 DRM_IOCTL_DEF_DRV(I915_IRQ_WAIT, drm_noop, DRM_AUTH), 2643 DRM_IOCTL_DEF_DRV(I915_GETPARAM, i915_getparam, DRM_AUTH|DRM_RENDER_ALLOW), 2644 DRM_IOCTL_DEF_DRV(I915_SETPARAM, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 2645 DRM_IOCTL_DEF_DRV(I915_ALLOC, drm_noop, DRM_AUTH), 2646 DRM_IOCTL_DEF_DRV(I915_FREE, drm_noop, DRM_AUTH), 2647 DRM_IOCTL_DEF_DRV(I915_INIT_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 2648 DRM_IOCTL_DEF_DRV(I915_CMDBUFFER, drm_noop, DRM_AUTH), 2649 DRM_IOCTL_DEF_DRV(I915_DESTROY_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 2650 DRM_IOCTL_DEF_DRV(I915_SET_VBLANK_PIPE, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 2651 DRM_IOCTL_DEF_DRV(I915_GET_VBLANK_PIPE, drm_noop, DRM_AUTH), 2652 DRM_IOCTL_DEF_DRV(I915_VBLANK_SWAP, drm_noop, DRM_AUTH), 2653 DRM_IOCTL_DEF_DRV(I915_HWS_ADDR, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 2654 DRM_IOCTL_DEF_DRV(I915_GEM_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 2655 DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER, i915_gem_execbuffer, DRM_AUTH), 2656 DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER2, i915_gem_execbuffer2, DRM_AUTH|DRM_RENDER_ALLOW), 2657 DRM_IOCTL_DEF_DRV(I915_GEM_PIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY), 2658 DRM_IOCTL_DEF_DRV(I915_GEM_UNPIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY), 2659 DRM_IOCTL_DEF_DRV(I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), 2660 DRM_IOCTL_DEF_DRV(I915_GEM_SET_CACHING, i915_gem_set_caching_ioctl, DRM_RENDER_ALLOW), 2661 DRM_IOCTL_DEF_DRV(I915_GEM_GET_CACHING, i915_gem_get_caching_ioctl, DRM_RENDER_ALLOW), 2662 DRM_IOCTL_DEF_DRV(I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), 2663 DRM_IOCTL_DEF_DRV(I915_GEM_ENTERVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 2664 DRM_IOCTL_DEF_DRV(I915_GEM_LEAVEVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 2665 DRM_IOCTL_DEF_DRV(I915_GEM_CREATE, i915_gem_create_ioctl, DRM_RENDER_ALLOW), 2666 DRM_IOCTL_DEF_DRV(I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_RENDER_ALLOW), 2667 DRM_IOCTL_DEF_DRV(I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_RENDER_ALLOW), 2668 DRM_IOCTL_DEF_DRV(I915_GEM_MMAP, i915_gem_mmap_ioctl, DRM_RENDER_ALLOW), 2669 DRM_IOCTL_DEF_DRV(I915_GEM_MMAP_GTT, i915_gem_mmap_gtt_ioctl, DRM_RENDER_ALLOW), 2670 DRM_IOCTL_DEF_DRV(I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, DRM_RENDER_ALLOW), 2671 DRM_IOCTL_DEF_DRV(I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, DRM_RENDER_ALLOW), 2672 DRM_IOCTL_DEF_DRV(I915_GEM_SET_TILING, i915_gem_set_tiling, DRM_RENDER_ALLOW), 2673 DRM_IOCTL_DEF_DRV(I915_GEM_GET_TILING, i915_gem_get_tiling, DRM_RENDER_ALLOW), 2674 DRM_IOCTL_DEF_DRV(I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, DRM_RENDER_ALLOW), 2675 DRM_IOCTL_DEF_DRV(I915_GET_PIPE_FROM_CRTC_ID, intel_get_pipe_from_crtc_id, 0), 2676 DRM_IOCTL_DEF_DRV(I915_GEM_MADVISE, i915_gem_madvise_ioctl, DRM_RENDER_ALLOW), 2677 DRM_IOCTL_DEF_DRV(I915_OVERLAY_PUT_IMAGE, intel_overlay_put_image_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW), 2678 DRM_IOCTL_DEF_DRV(I915_OVERLAY_ATTRS, intel_overlay_attrs_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW), 2679 DRM_IOCTL_DEF_DRV(I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey, DRM_MASTER|DRM_CONTROL_ALLOW), 2680 DRM_IOCTL_DEF_DRV(I915_GET_SPRITE_COLORKEY, drm_noop, DRM_MASTER|DRM_CONTROL_ALLOW), 2681 DRM_IOCTL_DEF_DRV(I915_GEM_WAIT, i915_gem_wait_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), 2682 DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_CREATE, i915_gem_context_create_ioctl, DRM_RENDER_ALLOW), 2683 DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_RENDER_ALLOW), 2684 DRM_IOCTL_DEF_DRV(I915_REG_READ, i915_reg_read_ioctl, DRM_RENDER_ALLOW), 2685 DRM_IOCTL_DEF_DRV(I915_GET_RESET_STATS, i915_gem_context_reset_stats_ioctl, DRM_RENDER_ALLOW), 2686 #if 0 2687 DRM_IOCTL_DEF_DRV(I915_GEM_USERPTR, i915_gem_userptr_ioctl, DRM_RENDER_ALLOW), 2688 #endif 2689 DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_GETPARAM, i915_gem_context_getparam_ioctl, DRM_RENDER_ALLOW), 2690 DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_SETPARAM, i915_gem_context_setparam_ioctl, DRM_RENDER_ALLOW), 2691 }; 2692 2693 static int i915_sysctl_init(struct drm_device *dev, struct sysctl_ctx_list *ctx, 2694 struct sysctl_oid *top) 2695 { 2696 return drm_add_busid_modesetting(dev, ctx, top); 2697 } 2698 2699 static struct drm_driver driver = { 2700 /* Don't use MTRRs here; the Xserver or userspace app should 2701 * deal with them for Intel hardware. 2702 */ 2703 .driver_features = 2704 DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED | DRIVER_GEM | 2705 DRIVER_RENDER | DRIVER_MODESET, 2706 .open = i915_driver_open, 2707 .lastclose = i915_driver_lastclose, 2708 .preclose = i915_driver_preclose, 2709 .postclose = i915_driver_postclose, 2710 .set_busid = drm_pci_set_busid, 2711 2712 .gem_free_object = i915_gem_free_object, 2713 .gem_vm_ops = &i915_gem_vm_ops, 2714 2715 .prime_handle_to_fd = drm_gem_prime_handle_to_fd, 2716 .prime_fd_to_handle = drm_gem_prime_fd_to_handle, 2717 .gem_prime_export = i915_gem_prime_export, 2718 .gem_prime_import = i915_gem_prime_import, 2719 2720 .dumb_create = i915_gem_dumb_create, 2721 .dumb_map_offset = i915_gem_mmap_gtt, 2722 .dumb_destroy = drm_gem_dumb_destroy, 2723 .ioctls = i915_ioctls, 2724 .num_ioctls = ARRAY_SIZE(i915_ioctls), 2725 .fops = &i915_driver_fops, 2726 .name = DRIVER_NAME, 2727 .desc = DRIVER_DESC, 2728 .date = DRIVER_DATE, 2729 .major = DRIVER_MAJOR, 2730 .minor = DRIVER_MINOR, 2731 .patchlevel = DRIVER_PATCHLEVEL, 2732 #ifdef __DragonFly__ 2733 .sysctl_init = i915_sysctl_init, 2734 #endif 2735 }; 2736