1 /* i915_drv.c -- i830,i845,i855,i865,i915 driver -*- linux-c -*- 2 */ 3 /* 4 * 5 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas. 6 * All Rights Reserved. 7 * 8 * Permission is hereby granted, free of charge, to any person obtaining a 9 * copy of this software and associated documentation files (the 10 * "Software"), to deal in the Software without restriction, including 11 * without limitation the rights to use, copy, modify, merge, publish, 12 * distribute, sub license, and/or sell copies of the Software, and to 13 * permit persons to whom the Software is furnished to do so, subject to 14 * the following conditions: 15 * 16 * The above copyright notice and this permission notice (including the 17 * next paragraph) shall be included in all copies or substantial portions 18 * of the Software. 19 * 20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 22 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 23 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR 24 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 25 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 26 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 27 * 28 */ 29 30 #ifdef __DragonFly__ 31 #include "opt_drm.h" /* for VGA_SWITCHEROO */ 32 #endif 33 34 #include <linux/acpi.h> 35 #include <linux/device.h> 36 #include <linux/oom.h> 37 #include <linux/module.h> 38 #include <linux/pci.h> 39 #include <linux/pm.h> 40 #include <linux/pm_runtime.h> 41 #include <linux/pnp.h> 42 #include <linux/slab.h> 43 #include <linux/vgaarb.h> 44 #include <linux/vga_switcheroo.h> 45 #include <linux/vt.h> 46 #include <acpi/video.h> 47 48 #include <drm/drmP.h> 49 #include <drm/drm_crtc_helper.h> 50 #include <drm/i915_drm.h> 51 52 #include "i915_drv.h" 53 #include "i915_trace.h" 54 #include "i915_vgpu.h" 55 #include "intel_drv.h" 56 57 static struct drm_driver driver; 58 59 static unsigned int i915_load_fail_count; 60 61 bool __i915_inject_load_failure(const char *func, int line) 62 { 63 if (i915_load_fail_count >= i915.inject_load_failure) 64 return false; 65 66 if (++i915_load_fail_count == i915.inject_load_failure) { 67 DRM_INFO("Injecting failure at checkpoint %u [%s:%d]\n", 68 i915.inject_load_failure, func, line); 69 return true; 70 } 71 72 return false; 73 } 74 75 #define FDO_BUG_URL "https://bugs.freedesktop.org/enter_bug.cgi?product=DRI" 76 #define FDO_BUG_MSG "Please file a bug at " FDO_BUG_URL " against DRM/Intel " \ 77 "providing the dmesg log by booting with drm.debug=0xf" 78 79 void 80 __i915_printk(struct drm_i915_private *dev_priv, const char *level, 81 const char *fmt, ...) 82 { 83 static bool shown_bug_once; 84 struct device *kdev = dev_priv->drm.dev; 85 bool is_error = level[1] <= KERN_ERR[1]; 86 bool is_debug = level[1] == KERN_DEBUG[1]; 87 struct va_format vaf; 88 va_list args; 89 90 if (is_debug && !(drm_debug & DRM_UT_DRIVER)) 91 return; 92 93 va_start(args, fmt); 94 95 vaf.fmt = fmt; 96 vaf.va = &args; 97 98 dev_printk(level, kdev, "[" DRM_NAME ":%ps] %pV", 99 __builtin_return_address(0), &vaf); 100 101 if (is_error && !shown_bug_once) { 102 dev_notice(kdev, "%s", FDO_BUG_MSG); 103 shown_bug_once = true; 104 } 105 106 va_end(args); 107 } 108 109 static bool i915_error_injected(struct drm_i915_private *dev_priv) 110 { 111 return i915.inject_load_failure && 112 i915_load_fail_count == i915.inject_load_failure; 113 } 114 115 #define i915_load_error(dev_priv, fmt, ...) \ 116 __i915_printk(dev_priv, \ 117 i915_error_injected(dev_priv) ? KERN_DEBUG : KERN_ERR, \ 118 fmt, ##__VA_ARGS__) 119 120 121 static enum intel_pch intel_virt_detect_pch(struct drm_i915_private *dev_priv) 122 { 123 enum intel_pch ret = PCH_NOP; 124 125 /* 126 * In a virtualized passthrough environment we can be in a 127 * setup where the ISA bridge is not able to be passed through. 128 * In this case, a south bridge can be emulated and we have to 129 * make an educated guess as to which PCH is really there. 130 */ 131 132 if (IS_GEN5(dev_priv)) { 133 ret = PCH_IBX; 134 DRM_DEBUG_KMS("Assuming Ibex Peak PCH\n"); 135 } else if (IS_GEN6(dev_priv) || IS_IVYBRIDGE(dev_priv)) { 136 ret = PCH_CPT; 137 DRM_DEBUG_KMS("Assuming CouarPoint PCH\n"); 138 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { 139 ret = PCH_LPT; 140 DRM_DEBUG_KMS("Assuming LynxPoint PCH\n"); 141 } else if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) { 142 ret = PCH_SPT; 143 DRM_DEBUG_KMS("Assuming SunrisePoint PCH\n"); 144 } 145 146 return ret; 147 } 148 149 static void intel_detect_pch(struct drm_device *dev) 150 { 151 struct drm_i915_private *dev_priv = to_i915(dev); 152 device_t pch = NULL; 153 struct pci_devinfo *di = NULL; 154 155 /* In all current cases, num_pipes is equivalent to the PCH_NOP setting 156 * (which really amounts to a PCH but no South Display). 157 */ 158 if (INTEL_INFO(dev_priv)->num_pipes == 0) { 159 dev_priv->pch_type = PCH_NOP; 160 return; 161 } 162 163 /* XXX The ISA bridge probe causes some old Core2 machines to hang */ 164 if (INTEL_INFO(dev_priv)->gen < 5) 165 return; 166 167 /* 168 * The reason to probe ISA bridge instead of Dev31:Fun0 is to 169 * make graphics device passthrough work easy for VMM, that only 170 * need to expose ISA bridge to let driver know the real hardware 171 * underneath. This is a requirement from virtualization team. 172 * 173 * In some virtualized environments (e.g. XEN), there is irrelevant 174 * ISA bridge in the system. To work reliably, we should scan trhough 175 * all the ISA bridge devices and check for the first match, instead 176 * of only checking the first one. 177 */ 178 while ((pch = pci_iterate_class(&di, PCIC_BRIDGE, PCIS_BRIDGE_ISA))) { 179 if (pci_get_vendor(pch) == PCI_VENDOR_ID_INTEL) { 180 unsigned short id = pci_get_device(pch) & INTEL_PCH_DEVICE_ID_MASK; 181 dev_priv->pch_id = id; 182 183 if (id == INTEL_PCH_IBX_DEVICE_ID_TYPE) { 184 dev_priv->pch_type = PCH_IBX; 185 DRM_DEBUG_KMS("Found Ibex Peak PCH\n"); 186 WARN_ON(!IS_GEN5(dev_priv)); 187 } else if (id == INTEL_PCH_CPT_DEVICE_ID_TYPE) { 188 dev_priv->pch_type = PCH_CPT; 189 DRM_DEBUG_KMS("Found CougarPoint PCH\n"); 190 WARN_ON(!(IS_GEN6(dev_priv) || 191 IS_IVYBRIDGE(dev_priv))); 192 } else if (id == INTEL_PCH_PPT_DEVICE_ID_TYPE) { 193 /* PantherPoint is CPT compatible */ 194 dev_priv->pch_type = PCH_CPT; 195 DRM_DEBUG_KMS("Found PantherPoint PCH\n"); 196 WARN_ON(!(IS_GEN6(dev_priv) || 197 IS_IVYBRIDGE(dev_priv))); 198 } else if (id == INTEL_PCH_LPT_DEVICE_ID_TYPE) { 199 dev_priv->pch_type = PCH_LPT; 200 DRM_DEBUG_KMS("Found LynxPoint PCH\n"); 201 WARN_ON(!IS_HASWELL(dev_priv) && 202 !IS_BROADWELL(dev_priv)); 203 WARN_ON(IS_HSW_ULT(dev_priv) || 204 IS_BDW_ULT(dev_priv)); 205 } else if (id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) { 206 dev_priv->pch_type = PCH_LPT; 207 DRM_DEBUG_KMS("Found LynxPoint LP PCH\n"); 208 WARN_ON(!IS_HASWELL(dev_priv) && 209 !IS_BROADWELL(dev_priv)); 210 WARN_ON(!IS_HSW_ULT(dev_priv) && 211 !IS_BDW_ULT(dev_priv)); 212 } else if (id == INTEL_PCH_SPT_DEVICE_ID_TYPE) { 213 dev_priv->pch_type = PCH_SPT; 214 DRM_DEBUG_KMS("Found SunrisePoint PCH\n"); 215 WARN_ON(!IS_SKYLAKE(dev_priv) && 216 !IS_KABYLAKE(dev_priv)); 217 } else if (id == INTEL_PCH_SPT_LP_DEVICE_ID_TYPE) { 218 dev_priv->pch_type = PCH_SPT; 219 DRM_DEBUG_KMS("Found SunrisePoint LP PCH\n"); 220 WARN_ON(!IS_SKYLAKE(dev_priv) && 221 !IS_KABYLAKE(dev_priv)); 222 } else if (id == INTEL_PCH_KBP_DEVICE_ID_TYPE) { 223 dev_priv->pch_type = PCH_KBP; 224 DRM_DEBUG_KMS("Found KabyPoint PCH\n"); 225 WARN_ON(!IS_SKYLAKE(dev_priv) && 226 !IS_KABYLAKE(dev_priv)); 227 } else if ((id == INTEL_PCH_P2X_DEVICE_ID_TYPE) || 228 (id == INTEL_PCH_P3X_DEVICE_ID_TYPE) || 229 ((id == INTEL_PCH_QEMU_DEVICE_ID_TYPE) && 230 1)) { 231 dev_priv->pch_type = 232 intel_virt_detect_pch(dev_priv); 233 } else 234 continue; 235 236 break; 237 } 238 } 239 if (!pch) 240 DRM_DEBUG_KMS("No PCH found.\n"); 241 242 #if 0 243 pci_dev_put(pch); 244 #endif 245 } 246 247 static int i915_getparam(struct drm_device *dev, void *data, 248 struct drm_file *file_priv) 249 { 250 struct drm_i915_private *dev_priv = to_i915(dev); 251 struct pci_dev *pdev = dev_priv->drm.pdev; 252 drm_i915_getparam_t *param = data; 253 int value; 254 255 switch (param->param) { 256 case I915_PARAM_IRQ_ACTIVE: 257 case I915_PARAM_ALLOW_BATCHBUFFER: 258 case I915_PARAM_LAST_DISPATCH: 259 case I915_PARAM_HAS_EXEC_CONSTANTS: 260 /* Reject all old ums/dri params. */ 261 return -ENODEV; 262 case I915_PARAM_CHIPSET_ID: 263 value = pdev->device; 264 break; 265 case I915_PARAM_REVISION: 266 value = pdev->revision; 267 break; 268 case I915_PARAM_NUM_FENCES_AVAIL: 269 value = dev_priv->num_fence_regs; 270 break; 271 case I915_PARAM_HAS_OVERLAY: 272 value = dev_priv->overlay ? 1 : 0; 273 break; 274 case I915_PARAM_HAS_BSD: 275 value = !!dev_priv->engine[VCS]; 276 break; 277 case I915_PARAM_HAS_BLT: 278 value = !!dev_priv->engine[BCS]; 279 break; 280 case I915_PARAM_HAS_VEBOX: 281 value = !!dev_priv->engine[VECS]; 282 break; 283 case I915_PARAM_HAS_BSD2: 284 value = !!dev_priv->engine[VCS2]; 285 break; 286 case I915_PARAM_HAS_LLC: 287 value = HAS_LLC(dev_priv); 288 break; 289 case I915_PARAM_HAS_WT: 290 value = HAS_WT(dev_priv); 291 break; 292 case I915_PARAM_HAS_ALIASING_PPGTT: 293 value = USES_PPGTT(dev_priv); 294 break; 295 case I915_PARAM_HAS_SEMAPHORES: 296 value = i915.semaphores; 297 break; 298 #if 0 299 case I915_PARAM_HAS_SECURE_BATCHES: 300 value = capable(CAP_SYS_ADMIN); 301 break; 302 #endif 303 case I915_PARAM_CMD_PARSER_VERSION: 304 value = i915_cmd_parser_get_version(dev_priv); 305 break; 306 case I915_PARAM_SUBSLICE_TOTAL: 307 value = sseu_subslice_total(&INTEL_INFO(dev_priv)->sseu); 308 if (!value) 309 return -ENODEV; 310 break; 311 case I915_PARAM_EU_TOTAL: 312 value = INTEL_INFO(dev_priv)->sseu.eu_total; 313 if (!value) 314 return -ENODEV; 315 break; 316 case I915_PARAM_HAS_GPU_RESET: 317 value = i915.enable_hangcheck && intel_has_gpu_reset(dev_priv); 318 break; 319 case I915_PARAM_HAS_RESOURCE_STREAMER: 320 value = HAS_RESOURCE_STREAMER(dev_priv); 321 break; 322 case I915_PARAM_HAS_POOLED_EU: 323 value = HAS_POOLED_EU(dev_priv); 324 break; 325 case I915_PARAM_MIN_EU_IN_POOL: 326 value = INTEL_INFO(dev_priv)->sseu.min_eu_in_pool; 327 break; 328 case I915_PARAM_MMAP_GTT_VERSION: 329 /* Though we've started our numbering from 1, and so class all 330 * earlier versions as 0, in effect their value is undefined as 331 * the ioctl will report EINVAL for the unknown param! 332 */ 333 value = i915_gem_mmap_gtt_version(); 334 break; 335 case I915_PARAM_HAS_SCHEDULER: 336 value = dev_priv->engine[RCS] && 337 dev_priv->engine[RCS]->schedule; 338 break; 339 #if 0 340 case I915_PARAM_MMAP_VERSION: 341 /* Remember to bump this if the version changes! */ 342 #endif 343 case I915_PARAM_HAS_GEM: 344 case I915_PARAM_HAS_PAGEFLIPPING: 345 case I915_PARAM_HAS_EXECBUF2: /* depends on GEM */ 346 case I915_PARAM_HAS_RELAXED_FENCING: 347 case I915_PARAM_HAS_COHERENT_RINGS: 348 case I915_PARAM_HAS_RELAXED_DELTA: 349 case I915_PARAM_HAS_GEN7_SOL_RESET: 350 case I915_PARAM_HAS_WAIT_TIMEOUT: 351 #if 0 352 case I915_PARAM_HAS_PRIME_VMAP_FLUSH: 353 #endif 354 case I915_PARAM_HAS_PINNED_BATCHES: 355 case I915_PARAM_HAS_EXEC_NO_RELOC: 356 case I915_PARAM_HAS_EXEC_HANDLE_LUT: 357 case I915_PARAM_HAS_COHERENT_PHYS_GTT: 358 case I915_PARAM_HAS_EXEC_SOFTPIN: 359 /* For the time being all of these are always true; 360 * if some supported hardware does not have one of these 361 * features this value needs to be provided from 362 * INTEL_INFO(), a feature macro, or similar. 363 */ 364 value = 1; 365 break; 366 default: 367 DRM_DEBUG("Unknown parameter %d\n", param->param); 368 return -EINVAL; 369 } 370 371 if (put_user(value, param->value)) 372 return -EFAULT; 373 374 return 0; 375 } 376 377 static int i915_get_bridge_dev(struct drm_device *dev) 378 { 379 struct drm_i915_private *dev_priv = to_i915(dev); 380 static struct pci_dev i915_bridge_dev; 381 382 i915_bridge_dev.dev.bsddev = pci_find_dbsf(0, 0, 0, 0); 383 if (!i915_bridge_dev.dev.bsddev) { 384 DRM_ERROR("bridge device not found\n"); 385 return -1; 386 } 387 388 dev_priv->bridge_dev = &i915_bridge_dev; 389 return 0; 390 } 391 392 /* Allocate space for the MCH regs if needed, return nonzero on error */ 393 static int 394 intel_alloc_mchbar_resource(struct drm_device *dev) 395 { 396 struct drm_i915_private *dev_priv = to_i915(dev); 397 int reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915; 398 u32 temp_lo, temp_hi = 0; 399 u64 mchbar_addr; 400 device_t vga; 401 402 if (INTEL_GEN(dev_priv) >= 4) 403 pci_read_config_dword(dev_priv->bridge_dev, reg + 4, &temp_hi); 404 pci_read_config_dword(dev_priv->bridge_dev, reg, &temp_lo); 405 mchbar_addr = ((u64)temp_hi << 32) | temp_lo; 406 407 /* If ACPI doesn't have it, assume we need to allocate it ourselves */ 408 #ifdef CONFIG_PNP 409 if (mchbar_addr && 410 pnp_range_reserved(mchbar_addr, mchbar_addr + MCHBAR_SIZE)) 411 return 0; 412 #endif 413 414 /* Get some space for it */ 415 vga = device_get_parent(dev->dev->bsddev); 416 dev_priv->mch_res_rid = 0x100; 417 dev_priv->mch_res = BUS_ALLOC_RESOURCE(device_get_parent(vga), 418 dev->dev->bsddev, SYS_RES_MEMORY, &dev_priv->mch_res_rid, 0, ~0UL, 419 MCHBAR_SIZE, RF_ACTIVE | RF_SHAREABLE, -1); 420 if (dev_priv->mch_res == NULL) { 421 DRM_ERROR("failed mchbar resource alloc\n"); 422 return (-ENOMEM); 423 } 424 425 if (INTEL_GEN(dev_priv) >= 4) 426 pci_write_config_dword(dev_priv->bridge_dev, reg + 4, 427 upper_32_bits(rman_get_start(dev_priv->mch_res))); 428 429 pci_write_config_dword(dev_priv->bridge_dev, reg, 430 lower_32_bits(rman_get_start(dev_priv->mch_res))); 431 return 0; 432 } 433 434 /* Setup MCHBAR if possible, return true if we should disable it again */ 435 static void 436 intel_setup_mchbar(struct drm_device *dev) 437 { 438 struct drm_i915_private *dev_priv = to_i915(dev); 439 int mchbar_reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915; 440 u32 temp; 441 bool enabled; 442 443 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) 444 return; 445 446 dev_priv->mchbar_need_disable = false; 447 448 if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) { 449 pci_read_config_dword(dev_priv->bridge_dev, DEVEN, &temp); 450 enabled = !!(temp & DEVEN_MCHBAR_EN); 451 } else { 452 pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp); 453 enabled = temp & 1; 454 } 455 456 /* If it's already enabled, don't have to do anything */ 457 if (enabled) 458 return; 459 460 if (intel_alloc_mchbar_resource(dev)) 461 return; 462 463 dev_priv->mchbar_need_disable = true; 464 465 /* Space is allocated or reserved, so enable it. */ 466 if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) { 467 pci_write_config_dword(dev_priv->bridge_dev, DEVEN, 468 temp | DEVEN_MCHBAR_EN); 469 } else { 470 pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp); 471 pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg, temp | 1); 472 } 473 } 474 475 static void 476 intel_teardown_mchbar(struct drm_device *dev) 477 { 478 struct drm_i915_private *dev_priv = to_i915(dev); 479 int mchbar_reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915; 480 device_t vga; 481 482 if (dev_priv->mchbar_need_disable) { 483 if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) { 484 u32 deven_val; 485 486 pci_read_config_dword(dev_priv->bridge_dev, DEVEN, 487 &deven_val); 488 deven_val &= ~DEVEN_MCHBAR_EN; 489 pci_write_config_dword(dev_priv->bridge_dev, DEVEN, 490 deven_val); 491 } else { 492 u32 mchbar_val; 493 494 pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, 495 &mchbar_val); 496 mchbar_val &= ~1; 497 pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg, 498 mchbar_val); 499 } 500 } 501 502 if (dev_priv->mch_res != NULL) { 503 vga = device_get_parent(dev->dev->bsddev); 504 BUS_DEACTIVATE_RESOURCE(device_get_parent(vga), dev->dev->bsddev, 505 SYS_RES_MEMORY, dev_priv->mch_res_rid, dev_priv->mch_res); 506 BUS_RELEASE_RESOURCE(device_get_parent(vga), dev->dev->bsddev, 507 SYS_RES_MEMORY, dev_priv->mch_res_rid, dev_priv->mch_res); 508 dev_priv->mch_res = NULL; 509 } 510 } 511 512 #if 0 513 /* true = enable decode, false = disable decoder */ 514 static unsigned int i915_vga_set_decode(void *cookie, bool state) 515 { 516 struct drm_device *dev = cookie; 517 518 intel_modeset_vga_set_state(to_i915(dev), state); 519 if (state) 520 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM | 521 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; 522 else 523 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; 524 } 525 526 static void i915_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state) 527 { 528 struct drm_device *dev = pci_get_drvdata(pdev); 529 pm_message_t pmm = { .event = PM_EVENT_SUSPEND }; 530 531 if (state == VGA_SWITCHEROO_ON) { 532 pr_info("switched on\n"); 533 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; 534 /* i915 resume handler doesn't set to D0 */ 535 pci_set_power_state(pdev, PCI_D0); 536 i915_resume_switcheroo(dev); 537 dev->switch_power_state = DRM_SWITCH_POWER_ON; 538 } else { 539 pr_info("switched off\n"); 540 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; 541 i915_suspend_switcheroo(dev, pmm); 542 dev->switch_power_state = DRM_SWITCH_POWER_OFF; 543 } 544 } 545 546 static bool i915_switcheroo_can_switch(struct pci_dev *pdev) 547 { 548 struct drm_device *dev = pci_get_drvdata(pdev); 549 550 /* 551 * FIXME: open_count is protected by drm_global_mutex but that would lead to 552 * locking inversion with the driver load path. And the access here is 553 * completely racy anyway. So don't bother with locking for now. 554 */ 555 return dev->open_count == 0; 556 } 557 558 static const struct vga_switcheroo_client_ops i915_switcheroo_ops = { 559 .set_gpu_state = i915_switcheroo_set_state, 560 .reprobe = NULL, 561 .can_switch = i915_switcheroo_can_switch, 562 }; 563 #endif 564 565 static void i915_gem_fini(struct drm_i915_private *dev_priv) 566 { 567 mutex_lock(&dev_priv->drm.struct_mutex); 568 i915_gem_cleanup_engines(&dev_priv->drm); 569 i915_gem_context_fini(&dev_priv->drm); 570 mutex_unlock(&dev_priv->drm.struct_mutex); 571 572 rcu_barrier(); 573 flush_work(&dev_priv->mm.free_work); 574 575 WARN_ON(!list_empty(&dev_priv->context_list)); 576 } 577 578 static int i915_load_modeset_init(struct drm_device *dev) 579 { 580 struct drm_i915_private *dev_priv = to_i915(dev); 581 int ret; 582 583 if (i915_inject_load_failure()) 584 return -ENODEV; 585 586 ret = intel_bios_init(dev_priv); 587 if (ret) 588 DRM_INFO("failed to find VBIOS tables\n"); 589 590 /* If we have > 1 VGA cards, then we need to arbitrate access 591 * to the common VGA resources. 592 * 593 * If we are a secondary display controller (!PCI_DISPLAY_CLASS_VGA), 594 * then we do not take part in VGA arbitration and the 595 * vga_client_register() fails with -ENODEV. 596 */ 597 #if 0 598 ret = vga_client_register(dev->pdev, dev, NULL, i915_vga_set_decode); 599 if (ret && ret != -ENODEV) 600 goto out; 601 602 intel_register_dsm_handler(); 603 604 ret = vga_switcheroo_register_client(dev->pdev, &i915_switcheroo_ops, false); 605 if (ret) 606 goto cleanup_vga_client; 607 #endif 608 609 /* must happen before intel_power_domains_init_hw() on VLV/CHV */ 610 intel_update_rawclk(dev_priv); 611 612 intel_power_domains_init_hw(dev_priv, false); 613 614 intel_csr_ucode_init(dev_priv); 615 616 ret = intel_irq_install(dev_priv); 617 if (ret) 618 goto cleanup_csr; 619 620 intel_setup_gmbus(dev); 621 622 /* Important: The output setup functions called by modeset_init need 623 * working irqs for e.g. gmbus and dp aux transfers. */ 624 ret = intel_modeset_init(dev); 625 if (ret) 626 goto cleanup_irq; 627 628 intel_guc_init(dev); 629 630 ret = i915_gem_init(dev); 631 if (ret) 632 goto cleanup_irq; 633 634 intel_modeset_gem_init(dev); 635 636 if (INTEL_INFO(dev_priv)->num_pipes == 0) 637 return 0; 638 639 ret = intel_fbdev_init(dev); 640 if (ret) 641 goto cleanup_gem; 642 643 /* Only enable hotplug handling once the fbdev is fully set up. */ 644 intel_hpd_init(dev_priv); 645 646 drm_kms_helper_poll_init(dev); 647 648 #ifdef __DragonFly__ 649 /* 650 * If we are dealing with dual GPU machines the vga_switcheroo module 651 * has been loaded. Machines with dual GPUs have an integrated graphics 652 * device (IGD), which we assume is an Intel device. The other, the 653 * discrete device (DIS), is either an NVidia or a Radeon device. For 654 * now we will force switch the gmux so the intel driver outputs 655 * both to the laptop panel and the external monitor. 656 * 657 * DragonFly does not have an nvidia native driver yet. In the future, 658 * we will check for the radeon device: if present, we will leave 659 * the gmux switch as it is, so the user can choose between the IGD and 660 * the DIS using the /dev/vga_switcheroo device. 661 */ 662 if (vga_switcheroo_handler_flags() & VGA_SWITCHEROO_CAN_SWITCH_DDC) { 663 ret = vga_switcheroo_force_migd(); 664 if (ret) { 665 DRM_INFO("could not switch gmux to IGD\n"); 666 } 667 } 668 #endif 669 670 return 0; 671 672 cleanup_gem: 673 if (i915_gem_suspend(dev)) 674 DRM_ERROR("failed to idle hardware; continuing to unload!\n"); 675 i915_gem_fini(dev_priv); 676 cleanup_irq: 677 intel_guc_fini(dev); 678 drm_irq_uninstall(dev); 679 intel_teardown_gmbus(dev); 680 cleanup_csr: 681 intel_csr_ucode_fini(dev_priv); 682 intel_power_domains_fini(dev_priv); 683 #if 0 684 vga_switcheroo_unregister_client(dev->pdev); 685 cleanup_vga_client: 686 vga_client_register(dev->pdev, NULL, NULL, NULL); 687 out: 688 #endif 689 return ret; 690 } 691 692 #if IS_ENABLED(CONFIG_FB) 693 static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv) 694 { 695 struct apertures_struct *ap; 696 struct pci_dev *pdev = dev_priv->drm.pdev; 697 struct i915_ggtt *ggtt = &dev_priv->ggtt; 698 bool primary; 699 int ret; 700 701 ap = alloc_apertures(1); 702 if (!ap) 703 return -ENOMEM; 704 705 ap->ranges[0].base = ggtt->mappable_base; 706 ap->ranges[0].size = ggtt->mappable_end; 707 708 primary = 709 pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW; 710 711 ret = drm_fb_helper_remove_conflicting_framebuffers(ap, "inteldrmfb", primary); 712 713 kfree(ap); 714 715 return ret; 716 } 717 #else 718 static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv) 719 { 720 return 0; 721 } 722 #endif 723 724 #if !defined(CONFIG_VGA_CONSOLE) 725 static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv) 726 { 727 return 0; 728 } 729 #elif !defined(CONFIG_DUMMY_CONSOLE) 730 static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv) 731 { 732 return -ENODEV; 733 } 734 #else 735 static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv) 736 { 737 int ret = 0; 738 739 DRM_INFO("Replacing VGA console driver\n"); 740 741 console_lock(); 742 if (con_is_bound(&vga_con)) 743 ret = do_take_over_console(&dummy_con, 0, MAX_NR_CONSOLES - 1, 1); 744 if (ret == 0) { 745 ret = do_unregister_con_driver(&vga_con); 746 747 /* Ignore "already unregistered". */ 748 if (ret == -ENODEV) 749 ret = 0; 750 } 751 console_unlock(); 752 753 return ret; 754 } 755 #endif 756 757 static void intel_init_dpio(struct drm_i915_private *dev_priv) 758 { 759 /* 760 * IOSF_PORT_DPIO is used for VLV x2 PHY (DP/HDMI B and C), 761 * CHV x1 PHY (DP/HDMI D) 762 * IOSF_PORT_DPIO_2 is used for CHV x2 PHY (DP/HDMI B and C) 763 */ 764 if (IS_CHERRYVIEW(dev_priv)) { 765 DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO_2; 766 DPIO_PHY_IOSF_PORT(DPIO_PHY1) = IOSF_PORT_DPIO; 767 } else if (IS_VALLEYVIEW(dev_priv)) { 768 DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO; 769 } 770 } 771 772 static int i915_workqueues_init(struct drm_i915_private *dev_priv) 773 { 774 /* 775 * The i915 workqueue is primarily used for batched retirement of 776 * requests (and thus managing bo) once the task has been completed 777 * by the GPU. i915_gem_retire_requests() is called directly when we 778 * need high-priority retirement, such as waiting for an explicit 779 * bo. 780 * 781 * It is also used for periodic low-priority events, such as 782 * idle-timers and recording error state. 783 * 784 * All tasks on the workqueue are expected to acquire the dev mutex 785 * so there is no point in running more than one instance of the 786 * workqueue at any time. Use an ordered one. 787 */ 788 dev_priv->wq = alloc_ordered_workqueue("i915", 0); 789 if (dev_priv->wq == NULL) 790 goto out_err; 791 792 dev_priv->hotplug.dp_wq = alloc_ordered_workqueue("i915-dp", 0); 793 if (dev_priv->hotplug.dp_wq == NULL) 794 goto out_free_wq; 795 796 return 0; 797 798 out_free_wq: 799 destroy_workqueue(dev_priv->wq); 800 out_err: 801 DRM_ERROR("Failed to allocate workqueues.\n"); 802 803 return -ENOMEM; 804 } 805 806 static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv) 807 { 808 destroy_workqueue(dev_priv->hotplug.dp_wq); 809 destroy_workqueue(dev_priv->wq); 810 } 811 812 /* 813 * We don't keep the workarounds for pre-production hardware, so we expect our 814 * driver to fail on these machines in one way or another. A little warning on 815 * dmesg may help both the user and the bug triagers. 816 */ 817 static void intel_detect_preproduction_hw(struct drm_i915_private *dev_priv) 818 { 819 if (IS_HSW_EARLY_SDV(dev_priv) || 820 IS_SKL_REVID(dev_priv, 0, SKL_REVID_F0)) 821 DRM_ERROR("This is a pre-production stepping. " 822 "It may not be fully functional.\n"); 823 } 824 825 /** 826 * i915_driver_init_early - setup state not requiring device access 827 * @dev_priv: device private 828 * 829 * Initialize everything that is a "SW-only" state, that is state not 830 * requiring accessing the device or exposing the driver via kernel internal 831 * or userspace interfaces. Example steps belonging here: lock initialization, 832 * system memory allocation, setting up device specific attributes and 833 * function hooks not requiring accessing the device. 834 */ 835 static int i915_driver_init_early(struct drm_i915_private *dev_priv, 836 const struct pci_device_id *ent) 837 { 838 const struct intel_device_info *match_info = 839 (struct intel_device_info *)ent->driver_data; 840 struct intel_device_info *device_info; 841 int ret = 0; 842 843 if (i915_inject_load_failure()) 844 return -ENODEV; 845 846 /* Setup the write-once "constant" device info */ 847 device_info = mkwrite_device_info(dev_priv); 848 memcpy(device_info, match_info, sizeof(*device_info)); 849 device_info->device_id = dev_priv->drm.pdev->device; 850 851 BUG_ON(device_info->gen > sizeof(device_info->gen_mask) * BITS_PER_BYTE); 852 device_info->gen_mask = BIT(device_info->gen - 1); 853 854 lockinit(&dev_priv->irq_lock, "userirq", 0, 0); 855 lockinit(&dev_priv->gpu_error.lock, "915err", 0, 0); 856 lockinit(&dev_priv->backlight_lock, "i915bl", 0, LK_CANRECURSE); 857 lockinit(&dev_priv->uncore.lock, "915gt", 0, 0); 858 lockinit(&dev_priv->mm.object_stat_lock, "i915osl", 0, 0); 859 lockinit(&dev_priv->mmio_flip_lock, "i915mfl", 0, 0); 860 lockinit(&dev_priv->sb_lock, "i915sbl", 0, LK_CANRECURSE); 861 lockinit(&dev_priv->modeset_restore_lock, "i915mrl", 0, LK_CANRECURSE); 862 lockinit(&dev_priv->av_mutex, "i915am", 0, LK_CANRECURSE); 863 lockinit(&dev_priv->wm.wm_mutex, "i915wm", 0, LK_CANRECURSE); 864 lockinit(&dev_priv->pps_mutex, "i915pm", 0, LK_CANRECURSE); 865 866 i915_memcpy_init_early(dev_priv); 867 868 ret = i915_workqueues_init(dev_priv); 869 if (ret < 0) 870 return ret; 871 872 ret = intel_gvt_init(dev_priv); 873 if (ret < 0) 874 goto err_workqueues; 875 876 /* This must be called before any calls to HAS_PCH_* */ 877 intel_detect_pch(&dev_priv->drm); 878 879 intel_pm_setup(&dev_priv->drm); 880 intel_init_dpio(dev_priv); 881 intel_power_domains_init(dev_priv); 882 intel_irq_init(dev_priv); 883 intel_hangcheck_init(dev_priv); 884 intel_init_display_hooks(dev_priv); 885 intel_init_clock_gating_hooks(dev_priv); 886 intel_init_audio_hooks(dev_priv); 887 ret = i915_gem_load_init(&dev_priv->drm); 888 if (ret < 0) 889 goto err_gvt; 890 891 intel_display_crc_init(dev_priv); 892 893 intel_device_info_dump(dev_priv); 894 895 intel_detect_preproduction_hw(dev_priv); 896 897 return 0; 898 899 err_gvt: 900 intel_gvt_cleanup(dev_priv); 901 err_workqueues: 902 i915_workqueues_cleanup(dev_priv); 903 return ret; 904 } 905 906 /** 907 * i915_driver_cleanup_early - cleanup the setup done in i915_driver_init_early() 908 * @dev_priv: device private 909 */ 910 static void i915_driver_cleanup_early(struct drm_i915_private *dev_priv) 911 { 912 i915_gem_load_cleanup(&dev_priv->drm); 913 i915_workqueues_cleanup(dev_priv); 914 } 915 916 static int i915_mmio_setup(struct drm_device *dev) 917 { 918 struct drm_i915_private *dev_priv = to_i915(dev); 919 struct pci_dev *pdev = dev_priv->drm.pdev; 920 int mmio_bar; 921 int mmio_size; 922 923 mmio_bar = IS_GEN2(dev_priv) ? 1 : 0; 924 /* 925 * Before gen4, the registers and the GTT are behind different BARs. 926 * However, from gen4 onwards, the registers and the GTT are shared 927 * in the same BAR, so we want to restrict this ioremap from 928 * clobbering the GTT which we want ioremap_wc instead. Fortunately, 929 * the register BAR remains the same size for all the earlier 930 * generations up to Ironlake. 931 */ 932 if (INTEL_GEN(dev_priv) < 5) 933 mmio_size = 512 * 1024; 934 else 935 mmio_size = 2 * 1024 * 1024; 936 dev_priv->regs = pci_iomap(pdev, mmio_bar, mmio_size); 937 if (dev_priv->regs == NULL) { 938 DRM_ERROR("failed to map registers\n"); 939 940 return -EIO; 941 } 942 943 /* Try to make sure MCHBAR is enabled before poking at it */ 944 intel_setup_mchbar(dev); 945 946 return 0; 947 } 948 949 static void i915_mmio_cleanup(struct drm_device *dev) 950 { 951 #if 0 952 struct drm_i915_private *dev_priv = to_i915(dev); 953 #endif 954 955 intel_teardown_mchbar(dev); 956 #if 0 957 pci_iounmap(pdev, dev_priv->regs); 958 #endif 959 } 960 961 /** 962 * i915_driver_init_mmio - setup device MMIO 963 * @dev_priv: device private 964 * 965 * Setup minimal device state necessary for MMIO accesses later in the 966 * initialization sequence. The setup here should avoid any other device-wide 967 * side effects or exposing the driver via kernel internal or user space 968 * interfaces. 969 */ 970 static int i915_driver_init_mmio(struct drm_i915_private *dev_priv) 971 { 972 struct drm_device *dev = &dev_priv->drm; 973 int ret; 974 975 if (i915_inject_load_failure()) 976 return -ENODEV; 977 978 if (i915_get_bridge_dev(dev)) 979 return -EIO; 980 981 ret = i915_mmio_setup(dev); 982 if (ret < 0) 983 goto put_bridge; 984 985 intel_uncore_init(dev_priv); 986 987 return 0; 988 989 put_bridge: 990 pci_dev_put(dev_priv->bridge_dev); 991 992 return ret; 993 } 994 995 /** 996 * i915_driver_cleanup_mmio - cleanup the setup done in i915_driver_init_mmio() 997 * @dev_priv: device private 998 */ 999 static void i915_driver_cleanup_mmio(struct drm_i915_private *dev_priv) 1000 { 1001 struct drm_device *dev = &dev_priv->drm; 1002 1003 intel_uncore_fini(dev_priv); 1004 i915_mmio_cleanup(dev); 1005 pci_dev_put(dev_priv->bridge_dev); 1006 } 1007 1008 static void intel_sanitize_options(struct drm_i915_private *dev_priv) 1009 { 1010 i915.enable_execlists = 1011 intel_sanitize_enable_execlists(dev_priv, 1012 i915.enable_execlists); 1013 1014 /* 1015 * i915.enable_ppgtt is read-only, so do an early pass to validate the 1016 * user's requested state against the hardware/driver capabilities. We 1017 * do this now so that we can print out any log messages once rather 1018 * than every time we check intel_enable_ppgtt(). 1019 */ 1020 i915.enable_ppgtt = 1021 intel_sanitize_enable_ppgtt(dev_priv, i915.enable_ppgtt); 1022 DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt); 1023 1024 i915.semaphores = intel_sanitize_semaphores(dev_priv, i915.semaphores); 1025 DRM_DEBUG_DRIVER("use GPU sempahores? %s\n", yesno(i915.semaphores)); 1026 } 1027 1028 /** 1029 * i915_driver_init_hw - setup state requiring device access 1030 * @dev_priv: device private 1031 * 1032 * Setup state that requires accessing the device, but doesn't require 1033 * exposing the driver via kernel internal or userspace interfaces. 1034 */ 1035 static int i915_driver_init_hw(struct drm_i915_private *dev_priv) 1036 { 1037 struct pci_dev *pdev = dev_priv->drm.pdev; 1038 int ret; 1039 1040 if (i915_inject_load_failure()) 1041 return -ENODEV; 1042 1043 intel_device_info_runtime_init(dev_priv); 1044 1045 intel_sanitize_options(dev_priv); 1046 1047 ret = i915_ggtt_probe_hw(dev_priv); 1048 if (ret) 1049 return ret; 1050 1051 /* WARNING: Apparently we must kick fbdev drivers before vgacon, 1052 * otherwise the vga fbdev driver falls over. */ 1053 ret = i915_kick_out_firmware_fb(dev_priv); 1054 if (ret) { 1055 DRM_ERROR("failed to remove conflicting framebuffer drivers\n"); 1056 goto out_ggtt; 1057 } 1058 1059 ret = i915_kick_out_vgacon(dev_priv); 1060 if (ret) { 1061 DRM_ERROR("failed to remove conflicting VGA console\n"); 1062 goto out_ggtt; 1063 } 1064 1065 ret = i915_ggtt_init_hw(dev_priv); 1066 if (ret) 1067 return ret; 1068 1069 ret = i915_ggtt_enable_hw(dev_priv); 1070 if (ret) { 1071 DRM_ERROR("failed to enable GGTT\n"); 1072 goto out_ggtt; 1073 } 1074 1075 pci_set_master(pdev); 1076 1077 #if 0 1078 /* overlay on gen2 is broken and can't address above 1G */ 1079 if (IS_GEN2(dev_priv)) { 1080 ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(30)); 1081 if (ret) { 1082 DRM_ERROR("failed to set DMA mask\n"); 1083 1084 goto out_ggtt; 1085 } 1086 } 1087 1088 /* 965GM sometimes incorrectly writes to hardware status page (HWS) 1089 * using 32bit addressing, overwriting memory if HWS is located 1090 * above 4GB. 1091 * 1092 * The documentation also mentions an issue with undefined 1093 * behaviour if any general state is accessed within a page above 4GB, 1094 * which also needs to be handled carefully. 1095 */ 1096 if (IS_BROADWATER(dev_priv) || IS_CRESTLINE(dev_priv)) { 1097 ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32)); 1098 1099 if (ret) { 1100 DRM_ERROR("failed to set DMA mask\n"); 1101 1102 goto out_ggtt; 1103 } 1104 } 1105 #endif 1106 1107 pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, 1108 PM_QOS_DEFAULT_VALUE); 1109 1110 intel_uncore_sanitize(dev_priv); 1111 1112 intel_opregion_setup(dev_priv); 1113 1114 i915_gem_load_init_fences(dev_priv); 1115 1116 /* On the 945G/GM, the chipset reports the MSI capability on the 1117 * integrated graphics even though the support isn't actually there 1118 * according to the published specs. It doesn't appear to function 1119 * correctly in testing on 945G. 1120 * This may be a side effect of MSI having been made available for PEG 1121 * and the registers being closely associated. 1122 * 1123 * According to chipset errata, on the 965GM, MSI interrupts may 1124 * be lost or delayed, but we use them anyways to avoid 1125 * stuck interrupts on some machines. 1126 */ 1127 #if 0 1128 if (!IS_I945G(dev_priv) && !IS_I945GM(dev_priv)) { 1129 if (pci_enable_msi(pdev) < 0) 1130 DRM_DEBUG_DRIVER("can't enable MSI"); 1131 } 1132 #endif 1133 1134 return 0; 1135 1136 out_ggtt: 1137 i915_ggtt_cleanup_hw(dev_priv); 1138 1139 return ret; 1140 } 1141 1142 /** 1143 * i915_driver_cleanup_hw - cleanup the setup done in i915_driver_init_hw() 1144 * @dev_priv: device private 1145 */ 1146 static void i915_driver_cleanup_hw(struct drm_i915_private *dev_priv) 1147 { 1148 1149 #if 0 1150 if (dev->pdev->msi_enabled) 1151 pci_disable_msi(dev->pdev); 1152 #endif 1153 1154 pm_qos_remove_request(&dev_priv->pm_qos); 1155 i915_ggtt_cleanup_hw(dev_priv); 1156 } 1157 1158 /** 1159 * i915_driver_register - register the driver with the rest of the system 1160 * @dev_priv: device private 1161 * 1162 * Perform any steps necessary to make the driver available via kernel 1163 * internal or userspace interfaces. 1164 */ 1165 static void i915_driver_register(struct drm_i915_private *dev_priv) 1166 { 1167 struct drm_device *dev = &dev_priv->drm; 1168 1169 i915_gem_shrinker_init(dev_priv); 1170 1171 /* 1172 * Notify a valid surface after modesetting, 1173 * when running inside a VM. 1174 */ 1175 if (intel_vgpu_active(dev_priv)) 1176 I915_WRITE(vgtif_reg(display_ready), VGT_DRV_DISPLAY_READY); 1177 1178 /* Reveal our presence to userspace */ 1179 if (drm_dev_register(dev, 0) == 0) { 1180 i915_debugfs_register(dev_priv); 1181 i915_guc_register(dev_priv); 1182 i915_setup_sysfs(dev_priv); 1183 } else 1184 DRM_ERROR("Failed to register driver for userspace access!\n"); 1185 1186 if (INTEL_INFO(dev_priv)->num_pipes) { 1187 /* Must be done after probing outputs */ 1188 intel_opregion_register(dev_priv); 1189 acpi_video_register(); 1190 } 1191 1192 if (IS_GEN5(dev_priv)) 1193 intel_gpu_ips_init(dev_priv); 1194 1195 i915_audio_component_init(dev_priv); 1196 1197 /* 1198 * Some ports require correctly set-up hpd registers for detection to 1199 * work properly (leading to ghost connected connector status), e.g. VGA 1200 * on gm45. Hence we can only set up the initial fbdev config after hpd 1201 * irqs are fully enabled. We do it last so that the async config 1202 * cannot run before the connectors are registered. 1203 */ 1204 intel_fbdev_initial_config_async(dev); 1205 } 1206 1207 /** 1208 * i915_driver_unregister - cleanup the registration done in i915_driver_regiser() 1209 * @dev_priv: device private 1210 */ 1211 static void i915_driver_unregister(struct drm_i915_private *dev_priv) 1212 { 1213 i915_audio_component_cleanup(dev_priv); 1214 1215 intel_gpu_ips_teardown(); 1216 acpi_video_unregister(); 1217 intel_opregion_unregister(dev_priv); 1218 1219 i915_teardown_sysfs(dev_priv); 1220 i915_guc_unregister(dev_priv); 1221 i915_debugfs_unregister(dev_priv); 1222 drm_dev_unregister(&dev_priv->drm); 1223 1224 i915_gem_shrinker_cleanup(dev_priv); 1225 } 1226 1227 /** 1228 * i915_driver_load - setup chip and create an initial config 1229 * @pdev: PCI device 1230 * @ent: matching PCI ID entry 1231 * 1232 * The driver load routine has to do several things: 1233 * - drive output discovery via intel_modeset_init() 1234 * - initialize the memory manager 1235 * - allocate initial config memory 1236 * - setup the DRM framebuffer with the allocated memory 1237 */ 1238 int i915_driver_load(struct pci_dev *pdev, const struct pci_device_id *ent) 1239 { 1240 struct drm_i915_private *dev_priv; 1241 int ret; 1242 1243 if (i915.nuclear_pageflip) 1244 driver.driver_features |= DRIVER_ATOMIC; 1245 1246 ret = -ENOMEM; 1247 dev_priv = kzalloc(sizeof(*dev_priv), GFP_KERNEL); 1248 if (dev_priv) 1249 ret = drm_dev_init(&dev_priv->drm, &driver, &pdev->dev); 1250 if (ret) { 1251 dev_printk(KERN_ERR, &pdev->dev, 1252 "[" DRM_NAME ":%s] allocation failed\n", __func__); 1253 kfree(dev_priv); 1254 return ret; 1255 } 1256 1257 dev_priv->drm.pdev = pdev; 1258 dev_priv->drm.dev_private = dev_priv; 1259 1260 #if 0 1261 ret = pci_enable_device(pdev); 1262 if (ret) 1263 goto out_free_priv; 1264 #endif 1265 1266 pci_set_drvdata(pdev, &dev_priv->drm); 1267 1268 ret = i915_driver_init_early(dev_priv, ent); 1269 if (ret < 0) 1270 goto out_pci_disable; 1271 1272 intel_runtime_pm_get(dev_priv); 1273 1274 ret = i915_driver_init_mmio(dev_priv); 1275 if (ret < 0) 1276 goto out_runtime_pm_put; 1277 1278 ret = i915_driver_init_hw(dev_priv); 1279 if (ret < 0) 1280 goto out_cleanup_mmio; 1281 1282 /* 1283 * TODO: move the vblank init and parts of modeset init steps into one 1284 * of the i915_driver_init_/i915_driver_register functions according 1285 * to the role/effect of the given init step. 1286 */ 1287 if (INTEL_INFO(dev_priv)->num_pipes) { 1288 ret = drm_vblank_init(&dev_priv->drm, 1289 INTEL_INFO(dev_priv)->num_pipes); 1290 if (ret) 1291 goto out_cleanup_hw; 1292 } 1293 1294 ret = i915_load_modeset_init(&dev_priv->drm); 1295 if (ret < 0) 1296 goto out_cleanup_vblank; 1297 1298 i915_driver_register(dev_priv); 1299 1300 intel_runtime_pm_enable(dev_priv); 1301 1302 /* Everything is in place, we can now relax! */ 1303 DRM_INFO("Initialized %s %d.%d.%d %s for %s on minor %d\n", 1304 driver.name, driver.major, driver.minor, driver.patchlevel, 1305 driver.date, pci_name(pdev), dev_priv->drm.primary->index); 1306 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG)) 1307 DRM_INFO("DRM_I915_DEBUG enabled\n"); 1308 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) 1309 DRM_INFO("DRM_I915_DEBUG_GEM enabled\n"); 1310 1311 intel_runtime_pm_put(dev_priv); 1312 1313 return 0; 1314 1315 out_cleanup_vblank: 1316 drm_vblank_cleanup(&dev_priv->drm); 1317 out_cleanup_hw: 1318 i915_driver_cleanup_hw(dev_priv); 1319 out_cleanup_mmio: 1320 i915_driver_cleanup_mmio(dev_priv); 1321 out_runtime_pm_put: 1322 intel_runtime_pm_put(dev_priv); 1323 i915_driver_cleanup_early(dev_priv); 1324 out_pci_disable: 1325 #if 0 1326 pci_disable_device(pdev); 1327 out_free_priv: 1328 #endif 1329 i915_load_error(dev_priv, "Device initialization failed (%d)\n", ret); 1330 drm_dev_unref(&dev_priv->drm); 1331 return ret; 1332 } 1333 1334 void i915_driver_unload(struct drm_device *dev) 1335 { 1336 struct drm_i915_private *dev_priv = to_i915(dev); 1337 1338 intel_fbdev_fini(dev); 1339 1340 if (i915_gem_suspend(dev)) 1341 DRM_ERROR("failed to idle hardware; continuing to unload!\n"); 1342 1343 intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); 1344 1345 i915_driver_unregister(dev_priv); 1346 1347 drm_vblank_cleanup(dev); 1348 1349 intel_modeset_cleanup(dev); 1350 1351 /* 1352 * free the memory space allocated for the child device 1353 * config parsed from VBT 1354 */ 1355 if (dev_priv->vbt.child_dev && dev_priv->vbt.child_dev_num) { 1356 kfree(dev_priv->vbt.child_dev); 1357 dev_priv->vbt.child_dev = NULL; 1358 dev_priv->vbt.child_dev_num = 0; 1359 } 1360 kfree(dev_priv->vbt.sdvo_lvds_vbt_mode); 1361 dev_priv->vbt.sdvo_lvds_vbt_mode = NULL; 1362 kfree(dev_priv->vbt.lfp_lvds_vbt_mode); 1363 dev_priv->vbt.lfp_lvds_vbt_mode = NULL; 1364 1365 #if 0 1366 vga_switcheroo_unregister_client(dev->pdev); 1367 vga_client_register(dev->pdev, NULL, NULL, NULL); 1368 #endif 1369 1370 intel_csr_ucode_fini(dev_priv); 1371 1372 /* Free error state after interrupts are fully disabled. */ 1373 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work); 1374 i915_destroy_error_state(dev); 1375 1376 /* Flush any outstanding unpin_work. */ 1377 drain_workqueue(dev_priv->wq); 1378 1379 intel_guc_fini(dev); 1380 i915_gem_fini(dev_priv); 1381 intel_fbc_cleanup_cfb(dev_priv); 1382 1383 intel_power_domains_fini(dev_priv); 1384 1385 i915_driver_cleanup_hw(dev_priv); 1386 i915_driver_cleanup_mmio(dev_priv); 1387 1388 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT); 1389 1390 i915_driver_cleanup_early(dev_priv); 1391 } 1392 1393 static int i915_driver_open(struct drm_device *dev, struct drm_file *file) 1394 { 1395 int ret; 1396 1397 ret = i915_gem_open(dev, file); 1398 if (ret) 1399 return ret; 1400 1401 return 0; 1402 } 1403 1404 /** 1405 * i915_driver_lastclose - clean up after all DRM clients have exited 1406 * @dev: DRM device 1407 * 1408 * Take care of cleaning up after all DRM clients have exited. In the 1409 * mode setting case, we want to restore the kernel's initial mode (just 1410 * in case the last client left us in a bad state). 1411 * 1412 * Additionally, in the non-mode setting case, we'll tear down the GTT 1413 * and DMA structures, since the kernel won't be using them, and clea 1414 * up any GEM state. 1415 */ 1416 static void i915_driver_lastclose(struct drm_device *dev) 1417 { 1418 intel_fbdev_restore_mode(dev); 1419 #if 0 1420 vga_switcheroo_process_delayed_switch(); 1421 #endif 1422 } 1423 1424 static void i915_driver_preclose(struct drm_device *dev, struct drm_file *file) 1425 { 1426 mutex_lock(&dev->struct_mutex); 1427 i915_gem_context_close(dev, file); 1428 i915_gem_release(dev, file); 1429 mutex_unlock(&dev->struct_mutex); 1430 } 1431 1432 static void i915_driver_postclose(struct drm_device *dev, struct drm_file *file) 1433 { 1434 struct drm_i915_file_private *file_priv = file->driver_priv; 1435 1436 kfree(file_priv); 1437 } 1438 1439 static void intel_suspend_encoders(struct drm_i915_private *dev_priv) 1440 { 1441 struct drm_device *dev = &dev_priv->drm; 1442 struct intel_encoder *encoder; 1443 1444 drm_modeset_lock_all(dev); 1445 for_each_intel_encoder(dev, encoder) 1446 if (encoder->suspend) 1447 encoder->suspend(encoder); 1448 drm_modeset_unlock_all(dev); 1449 } 1450 1451 static int vlv_resume_prepare(struct drm_i915_private *dev_priv, 1452 bool rpm_resume); 1453 static int vlv_suspend_complete(struct drm_i915_private *dev_priv); 1454 1455 static bool suspend_to_idle(struct drm_i915_private *dev_priv) 1456 { 1457 #if IS_ENABLED(CONFIG_ACPI_SLEEP) 1458 if (acpi_target_system_state() < ACPI_STATE_S3) 1459 return true; 1460 #endif 1461 return false; 1462 } 1463 1464 static int i915_drm_suspend(struct drm_device *dev) 1465 { 1466 struct drm_i915_private *dev_priv = to_i915(dev); 1467 struct pci_dev *pdev = dev_priv->drm.pdev; 1468 pci_power_t opregion_target_state; 1469 int error; 1470 1471 /* ignore lid events during suspend */ 1472 mutex_lock(&dev_priv->modeset_restore_lock); 1473 dev_priv->modeset_restore = MODESET_SUSPENDED; 1474 mutex_unlock(&dev_priv->modeset_restore_lock); 1475 1476 disable_rpm_wakeref_asserts(dev_priv); 1477 1478 /* We do a lot of poking in a lot of registers, make sure they work 1479 * properly. */ 1480 intel_display_set_init_power(dev_priv, true); 1481 1482 drm_kms_helper_poll_disable(dev); 1483 1484 #if 0 1485 pci_save_state(pdev); 1486 #endif 1487 1488 error = i915_gem_suspend(dev); 1489 if (error) { 1490 dev_err(&pdev->dev, 1491 "GEM idle failed, resume might fail\n"); 1492 goto out; 1493 } 1494 1495 intel_guc_suspend(dev); 1496 1497 intel_display_suspend(dev); 1498 1499 intel_dp_mst_suspend(dev); 1500 1501 intel_runtime_pm_disable_interrupts(dev_priv); 1502 intel_hpd_cancel_work(dev_priv); 1503 1504 intel_suspend_encoders(dev_priv); 1505 1506 intel_suspend_hw(dev_priv); 1507 1508 i915_gem_suspend_gtt_mappings(dev_priv); 1509 1510 i915_save_state(dev); 1511 1512 opregion_target_state = suspend_to_idle(dev_priv) ? PCI_D1 : PCI_D3cold; 1513 intel_opregion_notify_adapter(dev_priv, opregion_target_state); 1514 1515 intel_uncore_forcewake_reset(dev_priv, false); 1516 intel_opregion_unregister(dev_priv); 1517 1518 #if 0 1519 intel_fbdev_set_suspend(dev, FBINFO_STATE_SUSPENDED, true); 1520 #endif 1521 1522 dev_priv->suspend_count++; 1523 1524 intel_csr_ucode_suspend(dev_priv); 1525 1526 out: 1527 enable_rpm_wakeref_asserts(dev_priv); 1528 1529 return error; 1530 } 1531 1532 static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation) 1533 { 1534 struct drm_i915_private *dev_priv = to_i915(dev); 1535 bool fw_csr; 1536 int ret; 1537 1538 disable_rpm_wakeref_asserts(dev_priv); 1539 1540 intel_display_set_init_power(dev_priv, false); 1541 1542 fw_csr = !IS_BROXTON(dev_priv) && 1543 suspend_to_idle(dev_priv) && dev_priv->csr.dmc_payload; 1544 /* 1545 * In case of firmware assisted context save/restore don't manually 1546 * deinit the power domains. This also means the CSR/DMC firmware will 1547 * stay active, it will power down any HW resources as required and 1548 * also enable deeper system power states that would be blocked if the 1549 * firmware was inactive. 1550 */ 1551 if (!fw_csr) 1552 intel_power_domains_suspend(dev_priv); 1553 1554 ret = 0; 1555 if (IS_BROXTON(dev_priv)) 1556 bxt_enable_dc9(dev_priv); 1557 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) 1558 hsw_enable_pc8(dev_priv); 1559 else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) 1560 ret = vlv_suspend_complete(dev_priv); 1561 1562 if (ret) { 1563 DRM_ERROR("Suspend complete failed: %d\n", ret); 1564 if (!fw_csr) 1565 intel_power_domains_init_hw(dev_priv, true); 1566 1567 goto out; 1568 } 1569 1570 #if 0 1571 pci_disable_device(drm_dev->pdev); 1572 /* 1573 * During hibernation on some platforms the BIOS may try to access 1574 * the device even though it's already in D3 and hang the machine. So 1575 * leave the device in D0 on those platforms and hope the BIOS will 1576 * power down the device properly. The issue was seen on multiple old 1577 * GENs with different BIOS vendors, so having an explicit blacklist 1578 * is inpractical; apply the workaround on everything pre GEN6. The 1579 * platforms where the issue was seen: 1580 * Lenovo Thinkpad X301, X61s, X60, T60, X41 1581 * Fujitsu FSC S7110 1582 * Acer Aspire 1830T 1583 */ 1584 if (!(hibernation && INTEL_GEN(dev_priv) < 6)) 1585 pci_set_power_state(drm_dev->pdev, PCI_D3hot); 1586 #endif 1587 1588 dev_priv->suspended_to_idle = suspend_to_idle(dev_priv); 1589 1590 out: 1591 enable_rpm_wakeref_asserts(dev_priv); 1592 1593 return ret; 1594 } 1595 1596 int i915_suspend_switcheroo(device_t kdev) 1597 { 1598 struct drm_softc *softc = device_get_softc(kdev); 1599 struct drm_device *dev = softc->drm_driver_data; 1600 int error; 1601 1602 if (!dev) { 1603 DRM_ERROR("dev: %p\n", dev); 1604 DRM_ERROR("DRM not initialized, aborting suspend.\n"); 1605 return -ENODEV; 1606 } 1607 1608 #if 0 1609 if (WARN_ON_ONCE(state.event != PM_EVENT_SUSPEND && 1610 state.event != PM_EVENT_FREEZE)) 1611 return -EINVAL; 1612 #endif 1613 1614 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 1615 return 0; 1616 1617 error = i915_drm_suspend(dev); 1618 if (error) 1619 return error; 1620 1621 return i915_drm_suspend_late(dev, false); 1622 } 1623 1624 static int i915_drm_resume(struct drm_device *dev) 1625 { 1626 struct drm_i915_private *dev_priv = to_i915(dev); 1627 int ret; 1628 1629 disable_rpm_wakeref_asserts(dev_priv); 1630 intel_sanitize_gt_powersave(dev_priv); 1631 1632 ret = i915_ggtt_enable_hw(dev_priv); 1633 if (ret) 1634 DRM_ERROR("failed to re-enable GGTT\n"); 1635 1636 intel_csr_ucode_resume(dev_priv); 1637 1638 i915_gem_resume(dev); 1639 1640 i915_restore_state(dev); 1641 intel_pps_unlock_regs_wa(dev_priv); 1642 intel_opregion_setup(dev_priv); 1643 1644 intel_init_pch_refclk(dev); 1645 1646 /* 1647 * Interrupts have to be enabled before any batches are run. If not the 1648 * GPU will hang. i915_gem_init_hw() will initiate batches to 1649 * update/restore the context. 1650 * 1651 * drm_mode_config_reset() needs AUX interrupts. 1652 * 1653 * Modeset enabling in intel_modeset_init_hw() also needs working 1654 * interrupts. 1655 */ 1656 intel_runtime_pm_enable_interrupts(dev_priv); 1657 1658 drm_mode_config_reset(dev); 1659 1660 mutex_lock(&dev->struct_mutex); 1661 if (i915_gem_init_hw(dev)) { 1662 DRM_ERROR("failed to re-initialize GPU, declaring wedged!\n"); 1663 i915_gem_set_wedged(dev_priv); 1664 } 1665 mutex_unlock(&dev->struct_mutex); 1666 1667 intel_guc_resume(dev); 1668 1669 intel_modeset_init_hw(dev); 1670 1671 spin_lock_irq(&dev_priv->irq_lock); 1672 if (dev_priv->display.hpd_irq_setup) 1673 dev_priv->display.hpd_irq_setup(dev_priv); 1674 spin_unlock_irq(&dev_priv->irq_lock); 1675 1676 intel_dp_mst_resume(dev); 1677 1678 intel_display_resume(dev); 1679 1680 drm_kms_helper_poll_enable(dev); 1681 1682 /* 1683 * ... but also need to make sure that hotplug processing 1684 * doesn't cause havoc. Like in the driver load code we don't 1685 * bother with the tiny race here where we might loose hotplug 1686 * notifications. 1687 * */ 1688 intel_hpd_init(dev_priv); 1689 1690 intel_opregion_register(dev_priv); 1691 1692 intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING, false); 1693 1694 mutex_lock(&dev_priv->modeset_restore_lock); 1695 dev_priv->modeset_restore = MODESET_DONE; 1696 mutex_unlock(&dev_priv->modeset_restore_lock); 1697 1698 intel_opregion_notify_adapter(dev_priv, PCI_D0); 1699 1700 intel_autoenable_gt_powersave(dev_priv); 1701 1702 enable_rpm_wakeref_asserts(dev_priv); 1703 1704 return 0; 1705 } 1706 1707 static int i915_drm_resume_early(struct drm_device *dev) 1708 { 1709 struct drm_i915_private *dev_priv = to_i915(dev); 1710 struct pci_dev *pdev = dev_priv->drm.pdev; 1711 int ret = 0; 1712 1713 /* 1714 * We have a resume ordering issue with the snd-hda driver also 1715 * requiring our device to be power up. Due to the lack of a 1716 * parent/child relationship we currently solve this with an early 1717 * resume hook. 1718 * 1719 * FIXME: This should be solved with a special hdmi sink device or 1720 * similar so that power domains can be employed. 1721 */ 1722 1723 /* 1724 * Note that we need to set the power state explicitly, since we 1725 * powered off the device during freeze and the PCI core won't power 1726 * it back up for us during thaw. Powering off the device during 1727 * freeze is not a hard requirement though, and during the 1728 * suspend/resume phases the PCI core makes sure we get here with the 1729 * device powered on. So in case we change our freeze logic and keep 1730 * the device powered we can also remove the following set power state 1731 * call. 1732 */ 1733 #if 0 1734 ret = pci_set_power_state(dev->pdev, PCI_D0); 1735 if (ret) { 1736 DRM_ERROR("failed to set PCI D0 power state (%d)\n", ret); 1737 goto out; 1738 } 1739 1740 /* 1741 * Note that pci_enable_device() first enables any parent bridge 1742 * device and only then sets the power state for this device. The 1743 * bridge enabling is a nop though, since bridge devices are resumed 1744 * first. The order of enabling power and enabling the device is 1745 * imposed by the PCI core as described above, so here we preserve the 1746 * same order for the freeze/thaw phases. 1747 * 1748 * TODO: eventually we should remove pci_disable_device() / 1749 * pci_enable_enable_device() from suspend/resume. Due to how they 1750 * depend on the device enable refcount we can't anyway depend on them 1751 * disabling/enabling the device. 1752 */ 1753 if (pci_enable_device(dev->pdev)) { 1754 ret = -EIO; 1755 goto out; 1756 } 1757 #endif 1758 1759 pci_set_master(pdev); 1760 1761 disable_rpm_wakeref_asserts(dev_priv); 1762 1763 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) 1764 ret = vlv_resume_prepare(dev_priv, false); 1765 if (ret) 1766 DRM_ERROR("Resume prepare failed: %d, continuing anyway\n", 1767 ret); 1768 1769 intel_uncore_early_sanitize(dev_priv, true); 1770 1771 if (IS_BROXTON(dev_priv)) { 1772 if (!dev_priv->suspended_to_idle) 1773 gen9_sanitize_dc_state(dev_priv); 1774 bxt_disable_dc9(dev_priv); 1775 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { 1776 hsw_disable_pc8(dev_priv); 1777 } 1778 1779 intel_uncore_sanitize(dev_priv); 1780 1781 if (IS_BROXTON(dev_priv) || 1782 !(dev_priv->suspended_to_idle && dev_priv->csr.dmc_payload)) 1783 intel_power_domains_init_hw(dev_priv, true); 1784 1785 enable_rpm_wakeref_asserts(dev_priv); 1786 1787 #if 0 1788 out: 1789 #endif 1790 dev_priv->suspended_to_idle = false; 1791 1792 return ret; 1793 } 1794 1795 int i915_resume_switcheroo(struct drm_device *dev) 1796 { 1797 int ret; 1798 1799 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 1800 return 0; 1801 1802 ret = i915_drm_resume_early(dev); 1803 if (ret) 1804 return ret; 1805 1806 return i915_drm_resume(dev); 1807 } 1808 1809 static void disable_engines_irq(struct drm_i915_private *dev_priv) 1810 { 1811 struct intel_engine_cs *engine; 1812 enum intel_engine_id id; 1813 1814 /* Ensure irq handler finishes, and not run again. */ 1815 disable_irq(dev_priv->drm.irq); 1816 for_each_engine(engine, dev_priv, id) 1817 tasklet_kill(&engine->irq_tasklet); 1818 } 1819 1820 static void enable_engines_irq(struct drm_i915_private *dev_priv) 1821 { 1822 enable_irq(dev_priv->drm.irq); 1823 } 1824 1825 /** 1826 * i915_reset - reset chip after a hang 1827 * @dev: drm device to reset 1828 * 1829 * Reset the chip. Useful if a hang is detected. Marks the device as wedged 1830 * on failure. 1831 * 1832 * Caller must hold the struct_mutex. 1833 * 1834 * Procedure is fairly simple: 1835 * - reset the chip using the reset reg 1836 * - re-init context state 1837 * - re-init hardware status page 1838 * - re-init ring buffer 1839 * - re-init interrupt state 1840 * - re-init display 1841 */ 1842 void i915_reset(struct drm_i915_private *dev_priv) 1843 { 1844 struct drm_device *dev = &dev_priv->drm; 1845 struct i915_gpu_error *error = &dev_priv->gpu_error; 1846 int ret; 1847 1848 lockdep_assert_held(&dev->struct_mutex); 1849 1850 if (!test_and_clear_bit(I915_RESET_IN_PROGRESS, &error->flags)) 1851 return; 1852 1853 /* Clear any previous failed attempts at recovery. Time to try again. */ 1854 __clear_bit(I915_WEDGED, &error->flags); 1855 error->reset_count++; 1856 1857 pr_notice("drm/i915: Resetting chip after gpu hang\n"); 1858 1859 disable_engines_irq(dev_priv); 1860 ret = intel_gpu_reset(dev_priv, ALL_ENGINES); 1861 enable_engines_irq(dev_priv); 1862 1863 if (ret) { 1864 if (ret != -ENODEV) 1865 DRM_ERROR("Failed to reset chip: %i\n", ret); 1866 else 1867 DRM_DEBUG_DRIVER("GPU reset disabled\n"); 1868 goto error; 1869 } 1870 1871 i915_gem_reset(dev_priv); 1872 intel_overlay_reset(dev_priv); 1873 1874 /* Ok, now get things going again... */ 1875 1876 /* 1877 * Everything depends on having the GTT running, so we need to start 1878 * there. Fortunately we don't need to do this unless we reset the 1879 * chip at a PCI level. 1880 * 1881 * Next we need to restore the context, but we don't use those 1882 * yet either... 1883 * 1884 * Ring buffer needs to be re-initialized in the KMS case, or if X 1885 * was running at the time of the reset (i.e. we weren't VT 1886 * switched away). 1887 */ 1888 ret = i915_gem_init_hw(dev); 1889 if (ret) { 1890 DRM_ERROR("Failed hw init on reset %d\n", ret); 1891 goto error; 1892 } 1893 1894 wakeup: 1895 wake_up_bit(&error->flags, I915_RESET_IN_PROGRESS); 1896 return; 1897 1898 error: 1899 i915_gem_set_wedged(dev_priv); 1900 goto wakeup; 1901 } 1902 1903 #if 0 1904 static int i915_pm_suspend(struct device *kdev) 1905 { 1906 struct pci_dev *pdev = to_pci_dev(kdev); 1907 struct drm_device *dev = pci_get_drvdata(pdev); 1908 1909 if (!dev) { 1910 dev_err(kdev, "DRM not initialized, aborting suspend.\n"); 1911 return -ENODEV; 1912 } 1913 1914 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 1915 return 0; 1916 1917 return i915_drm_suspend(dev); 1918 } 1919 1920 static int i915_pm_suspend_late(struct device *kdev) 1921 { 1922 struct drm_device *dev = &kdev_to_i915(kdev)->drm; 1923 1924 1925 /* 1926 * We have a suspend ordering issue with the snd-hda driver also 1927 * requiring our device to be power up. Due to the lack of a 1928 * parent/child relationship we currently solve this with an late 1929 * suspend hook. 1930 * 1931 * FIXME: This should be solved with a special hdmi sink device or 1932 * similar so that power domains can be employed. 1933 */ 1934 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 1935 return 0; 1936 1937 return i915_drm_suspend_late(dev, false); 1938 } 1939 1940 static int i915_pm_poweroff_late(struct device *kdev) 1941 { 1942 struct drm_device *dev = &kdev_to_i915(kdev)->drm; 1943 1944 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 1945 return 0; 1946 1947 return i915_drm_suspend_late(dev, true); 1948 } 1949 1950 static int i915_pm_resume_early(struct device *kdev) 1951 { 1952 struct drm_device *dev = &kdev_to_i915(kdev)->drm; 1953 1954 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 1955 return 0; 1956 1957 return i915_drm_resume_early(dev); 1958 } 1959 1960 static int i915_pm_resume(struct device *kdev) 1961 { 1962 struct drm_device *dev = &kdev_to_i915(kdev)->drm; 1963 1964 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 1965 return 0; 1966 1967 return i915_drm_resume(dev); 1968 } 1969 1970 /* freeze: before creating the hibernation_image */ 1971 static int i915_pm_freeze(struct device *kdev) 1972 { 1973 int ret; 1974 1975 ret = i915_pm_suspend(kdev); 1976 if (ret) 1977 return ret; 1978 1979 ret = i915_gem_freeze(kdev_to_i915(kdev)); 1980 if (ret) 1981 return ret; 1982 1983 return 0; 1984 } 1985 1986 static int i915_pm_freeze_late(struct device *kdev) 1987 { 1988 int ret; 1989 1990 ret = i915_pm_suspend_late(kdev); 1991 if (ret) 1992 return ret; 1993 1994 ret = i915_gem_freeze_late(kdev_to_i915(kdev)); 1995 if (ret) 1996 return ret; 1997 1998 return 0; 1999 } 2000 2001 /* thaw: called after creating the hibernation image, but before turning off. */ 2002 static int i915_pm_thaw_early(struct device *kdev) 2003 { 2004 return i915_pm_resume_early(kdev); 2005 } 2006 2007 static int i915_pm_thaw(struct device *kdev) 2008 { 2009 return i915_pm_resume(kdev); 2010 } 2011 2012 /* restore: called after loading the hibernation image. */ 2013 static int i915_pm_restore_early(struct device *kdev) 2014 { 2015 return i915_pm_resume_early(kdev); 2016 } 2017 2018 static int i915_pm_restore(struct device *kdev) 2019 { 2020 return i915_pm_resume(kdev); 2021 } 2022 #endif 2023 2024 /* 2025 * Save all Gunit registers that may be lost after a D3 and a subsequent 2026 * S0i[R123] transition. The list of registers needing a save/restore is 2027 * defined in the VLV2_S0IXRegs document. This documents marks all Gunit 2028 * registers in the following way: 2029 * - Driver: saved/restored by the driver 2030 * - Punit : saved/restored by the Punit firmware 2031 * - No, w/o marking: no need to save/restore, since the register is R/O or 2032 * used internally by the HW in a way that doesn't depend 2033 * keeping the content across a suspend/resume. 2034 * - Debug : used for debugging 2035 * 2036 * We save/restore all registers marked with 'Driver', with the following 2037 * exceptions: 2038 * - Registers out of use, including also registers marked with 'Debug'. 2039 * These have no effect on the driver's operation, so we don't save/restore 2040 * them to reduce the overhead. 2041 * - Registers that are fully setup by an initialization function called from 2042 * the resume path. For example many clock gating and RPS/RC6 registers. 2043 * - Registers that provide the right functionality with their reset defaults. 2044 * 2045 * TODO: Except for registers that based on the above 3 criteria can be safely 2046 * ignored, we save/restore all others, practically treating the HW context as 2047 * a black-box for the driver. Further investigation is needed to reduce the 2048 * saved/restored registers even further, by following the same 3 criteria. 2049 */ 2050 static void vlv_save_gunit_s0ix_state(struct drm_i915_private *dev_priv) 2051 { 2052 struct vlv_s0ix_state *s = &dev_priv->vlv_s0ix_state; 2053 int i; 2054 2055 /* GAM 0x4000-0x4770 */ 2056 s->wr_watermark = I915_READ(GEN7_WR_WATERMARK); 2057 s->gfx_prio_ctrl = I915_READ(GEN7_GFX_PRIO_CTRL); 2058 s->arb_mode = I915_READ(ARB_MODE); 2059 s->gfx_pend_tlb0 = I915_READ(GEN7_GFX_PEND_TLB0); 2060 s->gfx_pend_tlb1 = I915_READ(GEN7_GFX_PEND_TLB1); 2061 2062 for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++) 2063 s->lra_limits[i] = I915_READ(GEN7_LRA_LIMITS(i)); 2064 2065 s->media_max_req_count = I915_READ(GEN7_MEDIA_MAX_REQ_COUNT); 2066 s->gfx_max_req_count = I915_READ(GEN7_GFX_MAX_REQ_COUNT); 2067 2068 s->render_hwsp = I915_READ(RENDER_HWS_PGA_GEN7); 2069 s->ecochk = I915_READ(GAM_ECOCHK); 2070 s->bsd_hwsp = I915_READ(BSD_HWS_PGA_GEN7); 2071 s->blt_hwsp = I915_READ(BLT_HWS_PGA_GEN7); 2072 2073 s->tlb_rd_addr = I915_READ(GEN7_TLB_RD_ADDR); 2074 2075 /* MBC 0x9024-0x91D0, 0x8500 */ 2076 s->g3dctl = I915_READ(VLV_G3DCTL); 2077 s->gsckgctl = I915_READ(VLV_GSCKGCTL); 2078 s->mbctl = I915_READ(GEN6_MBCTL); 2079 2080 /* GCP 0x9400-0x9424, 0x8100-0x810C */ 2081 s->ucgctl1 = I915_READ(GEN6_UCGCTL1); 2082 s->ucgctl3 = I915_READ(GEN6_UCGCTL3); 2083 s->rcgctl1 = I915_READ(GEN6_RCGCTL1); 2084 s->rcgctl2 = I915_READ(GEN6_RCGCTL2); 2085 s->rstctl = I915_READ(GEN6_RSTCTL); 2086 s->misccpctl = I915_READ(GEN7_MISCCPCTL); 2087 2088 /* GPM 0xA000-0xAA84, 0x8000-0x80FC */ 2089 s->gfxpause = I915_READ(GEN6_GFXPAUSE); 2090 s->rpdeuhwtc = I915_READ(GEN6_RPDEUHWTC); 2091 s->rpdeuc = I915_READ(GEN6_RPDEUC); 2092 s->ecobus = I915_READ(ECOBUS); 2093 s->pwrdwnupctl = I915_READ(VLV_PWRDWNUPCTL); 2094 s->rp_down_timeout = I915_READ(GEN6_RP_DOWN_TIMEOUT); 2095 s->rp_deucsw = I915_READ(GEN6_RPDEUCSW); 2096 s->rcubmabdtmr = I915_READ(GEN6_RCUBMABDTMR); 2097 s->rcedata = I915_READ(VLV_RCEDATA); 2098 s->spare2gh = I915_READ(VLV_SPAREG2H); 2099 2100 /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */ 2101 s->gt_imr = I915_READ(GTIMR); 2102 s->gt_ier = I915_READ(GTIER); 2103 s->pm_imr = I915_READ(GEN6_PMIMR); 2104 s->pm_ier = I915_READ(GEN6_PMIER); 2105 2106 for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++) 2107 s->gt_scratch[i] = I915_READ(GEN7_GT_SCRATCH(i)); 2108 2109 /* GT SA CZ domain, 0x100000-0x138124 */ 2110 s->tilectl = I915_READ(TILECTL); 2111 s->gt_fifoctl = I915_READ(GTFIFOCTL); 2112 s->gtlc_wake_ctrl = I915_READ(VLV_GTLC_WAKE_CTRL); 2113 s->gtlc_survive = I915_READ(VLV_GTLC_SURVIVABILITY_REG); 2114 s->pmwgicz = I915_READ(VLV_PMWGICZ); 2115 2116 /* Gunit-Display CZ domain, 0x182028-0x1821CF */ 2117 s->gu_ctl0 = I915_READ(VLV_GU_CTL0); 2118 s->gu_ctl1 = I915_READ(VLV_GU_CTL1); 2119 s->pcbr = I915_READ(VLV_PCBR); 2120 s->clock_gate_dis2 = I915_READ(VLV_GUNIT_CLOCK_GATE2); 2121 2122 /* 2123 * Not saving any of: 2124 * DFT, 0x9800-0x9EC0 2125 * SARB, 0xB000-0xB1FC 2126 * GAC, 0x5208-0x524C, 0x14000-0x14C000 2127 * PCI CFG 2128 */ 2129 } 2130 2131 static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *dev_priv) 2132 { 2133 struct vlv_s0ix_state *s = &dev_priv->vlv_s0ix_state; 2134 u32 val; 2135 int i; 2136 2137 /* GAM 0x4000-0x4770 */ 2138 I915_WRITE(GEN7_WR_WATERMARK, s->wr_watermark); 2139 I915_WRITE(GEN7_GFX_PRIO_CTRL, s->gfx_prio_ctrl); 2140 I915_WRITE(ARB_MODE, s->arb_mode | (0xffff << 16)); 2141 I915_WRITE(GEN7_GFX_PEND_TLB0, s->gfx_pend_tlb0); 2142 I915_WRITE(GEN7_GFX_PEND_TLB1, s->gfx_pend_tlb1); 2143 2144 for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++) 2145 I915_WRITE(GEN7_LRA_LIMITS(i), s->lra_limits[i]); 2146 2147 I915_WRITE(GEN7_MEDIA_MAX_REQ_COUNT, s->media_max_req_count); 2148 I915_WRITE(GEN7_GFX_MAX_REQ_COUNT, s->gfx_max_req_count); 2149 2150 I915_WRITE(RENDER_HWS_PGA_GEN7, s->render_hwsp); 2151 I915_WRITE(GAM_ECOCHK, s->ecochk); 2152 I915_WRITE(BSD_HWS_PGA_GEN7, s->bsd_hwsp); 2153 I915_WRITE(BLT_HWS_PGA_GEN7, s->blt_hwsp); 2154 2155 I915_WRITE(GEN7_TLB_RD_ADDR, s->tlb_rd_addr); 2156 2157 /* MBC 0x9024-0x91D0, 0x8500 */ 2158 I915_WRITE(VLV_G3DCTL, s->g3dctl); 2159 I915_WRITE(VLV_GSCKGCTL, s->gsckgctl); 2160 I915_WRITE(GEN6_MBCTL, s->mbctl); 2161 2162 /* GCP 0x9400-0x9424, 0x8100-0x810C */ 2163 I915_WRITE(GEN6_UCGCTL1, s->ucgctl1); 2164 I915_WRITE(GEN6_UCGCTL3, s->ucgctl3); 2165 I915_WRITE(GEN6_RCGCTL1, s->rcgctl1); 2166 I915_WRITE(GEN6_RCGCTL2, s->rcgctl2); 2167 I915_WRITE(GEN6_RSTCTL, s->rstctl); 2168 I915_WRITE(GEN7_MISCCPCTL, s->misccpctl); 2169 2170 /* GPM 0xA000-0xAA84, 0x8000-0x80FC */ 2171 I915_WRITE(GEN6_GFXPAUSE, s->gfxpause); 2172 I915_WRITE(GEN6_RPDEUHWTC, s->rpdeuhwtc); 2173 I915_WRITE(GEN6_RPDEUC, s->rpdeuc); 2174 I915_WRITE(ECOBUS, s->ecobus); 2175 I915_WRITE(VLV_PWRDWNUPCTL, s->pwrdwnupctl); 2176 I915_WRITE(GEN6_RP_DOWN_TIMEOUT,s->rp_down_timeout); 2177 I915_WRITE(GEN6_RPDEUCSW, s->rp_deucsw); 2178 I915_WRITE(GEN6_RCUBMABDTMR, s->rcubmabdtmr); 2179 I915_WRITE(VLV_RCEDATA, s->rcedata); 2180 I915_WRITE(VLV_SPAREG2H, s->spare2gh); 2181 2182 /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */ 2183 I915_WRITE(GTIMR, s->gt_imr); 2184 I915_WRITE(GTIER, s->gt_ier); 2185 I915_WRITE(GEN6_PMIMR, s->pm_imr); 2186 I915_WRITE(GEN6_PMIER, s->pm_ier); 2187 2188 for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++) 2189 I915_WRITE(GEN7_GT_SCRATCH(i), s->gt_scratch[i]); 2190 2191 /* GT SA CZ domain, 0x100000-0x138124 */ 2192 I915_WRITE(TILECTL, s->tilectl); 2193 I915_WRITE(GTFIFOCTL, s->gt_fifoctl); 2194 /* 2195 * Preserve the GT allow wake and GFX force clock bit, they are not 2196 * be restored, as they are used to control the s0ix suspend/resume 2197 * sequence by the caller. 2198 */ 2199 val = I915_READ(VLV_GTLC_WAKE_CTRL); 2200 val &= VLV_GTLC_ALLOWWAKEREQ; 2201 val |= s->gtlc_wake_ctrl & ~VLV_GTLC_ALLOWWAKEREQ; 2202 I915_WRITE(VLV_GTLC_WAKE_CTRL, val); 2203 2204 val = I915_READ(VLV_GTLC_SURVIVABILITY_REG); 2205 val &= VLV_GFX_CLK_FORCE_ON_BIT; 2206 val |= s->gtlc_survive & ~VLV_GFX_CLK_FORCE_ON_BIT; 2207 I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val); 2208 2209 I915_WRITE(VLV_PMWGICZ, s->pmwgicz); 2210 2211 /* Gunit-Display CZ domain, 0x182028-0x1821CF */ 2212 I915_WRITE(VLV_GU_CTL0, s->gu_ctl0); 2213 I915_WRITE(VLV_GU_CTL1, s->gu_ctl1); 2214 I915_WRITE(VLV_PCBR, s->pcbr); 2215 I915_WRITE(VLV_GUNIT_CLOCK_GATE2, s->clock_gate_dis2); 2216 } 2217 2218 int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool force_on) 2219 { 2220 u32 val; 2221 int err; 2222 2223 val = I915_READ(VLV_GTLC_SURVIVABILITY_REG); 2224 val &= ~VLV_GFX_CLK_FORCE_ON_BIT; 2225 if (force_on) 2226 val |= VLV_GFX_CLK_FORCE_ON_BIT; 2227 I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val); 2228 2229 if (!force_on) 2230 return 0; 2231 2232 err = intel_wait_for_register(dev_priv, 2233 VLV_GTLC_SURVIVABILITY_REG, 2234 VLV_GFX_CLK_STATUS_BIT, 2235 VLV_GFX_CLK_STATUS_BIT, 2236 20); 2237 if (err) 2238 DRM_ERROR("timeout waiting for GFX clock force-on (%08x)\n", 2239 I915_READ(VLV_GTLC_SURVIVABILITY_REG)); 2240 2241 return err; 2242 } 2243 2244 static int vlv_allow_gt_wake(struct drm_i915_private *dev_priv, bool allow) 2245 { 2246 u32 val; 2247 int err = 0; 2248 2249 val = I915_READ(VLV_GTLC_WAKE_CTRL); 2250 val &= ~VLV_GTLC_ALLOWWAKEREQ; 2251 if (allow) 2252 val |= VLV_GTLC_ALLOWWAKEREQ; 2253 I915_WRITE(VLV_GTLC_WAKE_CTRL, val); 2254 POSTING_READ(VLV_GTLC_WAKE_CTRL); 2255 2256 err = intel_wait_for_register(dev_priv, 2257 VLV_GTLC_PW_STATUS, 2258 VLV_GTLC_ALLOWWAKEACK, 2259 allow, 2260 1); 2261 if (err) 2262 DRM_ERROR("timeout disabling GT waking\n"); 2263 2264 return err; 2265 } 2266 2267 static int vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv, 2268 bool wait_for_on) 2269 { 2270 u32 mask; 2271 u32 val; 2272 int err; 2273 2274 mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK; 2275 val = wait_for_on ? mask : 0; 2276 if ((I915_READ(VLV_GTLC_PW_STATUS) & mask) == val) 2277 return 0; 2278 2279 DRM_DEBUG_KMS("waiting for GT wells to go %s (%08x)\n", 2280 onoff(wait_for_on), 2281 I915_READ(VLV_GTLC_PW_STATUS)); 2282 2283 /* 2284 * RC6 transitioning can be delayed up to 2 msec (see 2285 * valleyview_enable_rps), use 3 msec for safety. 2286 */ 2287 err = intel_wait_for_register(dev_priv, 2288 VLV_GTLC_PW_STATUS, mask, val, 2289 3); 2290 if (err) 2291 DRM_ERROR("timeout waiting for GT wells to go %s\n", 2292 onoff(wait_for_on)); 2293 2294 return err; 2295 } 2296 2297 static void vlv_check_no_gt_access(struct drm_i915_private *dev_priv) 2298 { 2299 if (!(I915_READ(VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEERR)) 2300 return; 2301 2302 DRM_DEBUG_DRIVER("GT register access while GT waking disabled\n"); 2303 I915_WRITE(VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR); 2304 } 2305 2306 static int vlv_suspend_complete(struct drm_i915_private *dev_priv) 2307 { 2308 u32 mask; 2309 int err; 2310 2311 /* 2312 * Bspec defines the following GT well on flags as debug only, so 2313 * don't treat them as hard failures. 2314 */ 2315 (void)vlv_wait_for_gt_wells(dev_priv, false); 2316 2317 mask = VLV_GTLC_RENDER_CTX_EXISTS | VLV_GTLC_MEDIA_CTX_EXISTS; 2318 WARN_ON((I915_READ(VLV_GTLC_WAKE_CTRL) & mask) != mask); 2319 2320 vlv_check_no_gt_access(dev_priv); 2321 2322 err = vlv_force_gfx_clock(dev_priv, true); 2323 if (err) 2324 goto err1; 2325 2326 err = vlv_allow_gt_wake(dev_priv, false); 2327 if (err) 2328 goto err2; 2329 2330 if (!IS_CHERRYVIEW(dev_priv)) 2331 vlv_save_gunit_s0ix_state(dev_priv); 2332 2333 err = vlv_force_gfx_clock(dev_priv, false); 2334 if (err) 2335 goto err2; 2336 2337 return 0; 2338 2339 err2: 2340 /* For safety always re-enable waking and disable gfx clock forcing */ 2341 vlv_allow_gt_wake(dev_priv, true); 2342 err1: 2343 vlv_force_gfx_clock(dev_priv, false); 2344 2345 return err; 2346 } 2347 2348 static int vlv_resume_prepare(struct drm_i915_private *dev_priv, 2349 bool rpm_resume) 2350 { 2351 int err; 2352 int ret; 2353 2354 /* 2355 * If any of the steps fail just try to continue, that's the best we 2356 * can do at this point. Return the first error code (which will also 2357 * leave RPM permanently disabled). 2358 */ 2359 ret = vlv_force_gfx_clock(dev_priv, true); 2360 2361 if (!IS_CHERRYVIEW(dev_priv)) 2362 vlv_restore_gunit_s0ix_state(dev_priv); 2363 2364 err = vlv_allow_gt_wake(dev_priv, true); 2365 if (!ret) 2366 ret = err; 2367 2368 err = vlv_force_gfx_clock(dev_priv, false); 2369 if (!ret) 2370 ret = err; 2371 2372 vlv_check_no_gt_access(dev_priv); 2373 2374 if (rpm_resume) 2375 intel_init_clock_gating(dev_priv); 2376 2377 return ret; 2378 } 2379 2380 #if 0 2381 static int intel_runtime_suspend(struct device *device) 2382 { 2383 struct pci_dev *pdev = to_pci_dev(device); 2384 struct drm_device *dev = pci_get_drvdata(pdev); 2385 struct drm_i915_private *dev_priv = to_i915(dev); 2386 int ret; 2387 2388 if (WARN_ON_ONCE(!(dev_priv->rps.enabled && intel_enable_rc6()))) 2389 return -ENODEV; 2390 2391 if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev_priv))) 2392 return -ENODEV; 2393 2394 DRM_DEBUG_KMS("Suspending device\n"); 2395 2396 disable_rpm_wakeref_asserts(dev_priv); 2397 2398 /* 2399 * We are safe here against re-faults, since the fault handler takes 2400 * an RPM reference. 2401 */ 2402 i915_gem_runtime_suspend(dev_priv); 2403 2404 intel_guc_suspend(dev); 2405 2406 intel_runtime_pm_disable_interrupts(dev_priv); 2407 2408 ret = 0; 2409 if (IS_BROXTON(dev_priv)) { 2410 bxt_display_core_uninit(dev_priv); 2411 bxt_enable_dc9(dev_priv); 2412 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { 2413 hsw_enable_pc8(dev_priv); 2414 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { 2415 ret = vlv_suspend_complete(dev_priv); 2416 } 2417 2418 if (ret) { 2419 DRM_ERROR("Runtime suspend failed, disabling it (%d)\n", ret); 2420 intel_runtime_pm_enable_interrupts(dev_priv); 2421 2422 enable_rpm_wakeref_asserts(dev_priv); 2423 2424 return ret; 2425 } 2426 2427 intel_uncore_forcewake_reset(dev_priv, false); 2428 2429 enable_rpm_wakeref_asserts(dev_priv); 2430 WARN_ON_ONCE(atomic_read(&dev_priv->pm.wakeref_count)); 2431 2432 if (intel_uncore_arm_unclaimed_mmio_detection(dev_priv)) 2433 DRM_ERROR("Unclaimed access detected prior to suspending\n"); 2434 2435 dev_priv->pm.suspended = true; 2436 2437 /* 2438 * FIXME: We really should find a document that references the arguments 2439 * used below! 2440 */ 2441 if (IS_BROADWELL(dev_priv)) { 2442 /* 2443 * On Broadwell, if we use PCI_D1 the PCH DDI ports will stop 2444 * being detected, and the call we do at intel_runtime_resume() 2445 * won't be able to restore them. Since PCI_D3hot matches the 2446 * actual specification and appears to be working, use it. 2447 */ 2448 intel_opregion_notify_adapter(dev_priv, PCI_D3hot); 2449 } else { 2450 /* 2451 * current versions of firmware which depend on this opregion 2452 * notification have repurposed the D1 definition to mean 2453 * "runtime suspended" vs. what you would normally expect (D3) 2454 * to distinguish it from notifications that might be sent via 2455 * the suspend path. 2456 */ 2457 intel_opregion_notify_adapter(dev_priv, PCI_D1); 2458 } 2459 2460 assert_forcewakes_inactive(dev_priv); 2461 2462 if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) 2463 intel_hpd_poll_init(dev_priv); 2464 2465 DRM_DEBUG_KMS("Device suspended\n"); 2466 return 0; 2467 } 2468 2469 static int intel_runtime_resume(struct device *kdev) 2470 { 2471 struct pci_dev *pdev = to_pci_dev(kdev); 2472 struct drm_device *dev = pci_get_drvdata(pdev); 2473 struct drm_i915_private *dev_priv = to_i915(dev); 2474 int ret = 0; 2475 2476 if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev_priv))) 2477 return -ENODEV; 2478 2479 DRM_DEBUG_KMS("Resuming device\n"); 2480 2481 WARN_ON_ONCE(atomic_read(&dev_priv->pm.wakeref_count)); 2482 disable_rpm_wakeref_asserts(dev_priv); 2483 2484 intel_opregion_notify_adapter(dev_priv, PCI_D0); 2485 dev_priv->pm.suspended = false; 2486 if (intel_uncore_unclaimed_mmio(dev_priv)) 2487 DRM_DEBUG_DRIVER("Unclaimed access during suspend, bios?\n"); 2488 2489 intel_guc_resume(dev); 2490 2491 if (IS_GEN6(dev_priv)) 2492 intel_init_pch_refclk(dev); 2493 2494 if (IS_BROXTON(dev_priv)) { 2495 bxt_disable_dc9(dev_priv); 2496 bxt_display_core_init(dev_priv, true); 2497 if (dev_priv->csr.dmc_payload && 2498 (dev_priv->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC5)) 2499 gen9_enable_dc5(dev_priv); 2500 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { 2501 hsw_disable_pc8(dev_priv); 2502 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { 2503 ret = vlv_resume_prepare(dev_priv, true); 2504 } 2505 2506 /* 2507 * No point of rolling back things in case of an error, as the best 2508 * we can do is to hope that things will still work (and disable RPM). 2509 */ 2510 i915_gem_init_swizzling(dev_priv); 2511 i915_gem_restore_fences(dev_priv); 2512 2513 intel_runtime_pm_enable_interrupts(dev_priv); 2514 2515 /* 2516 * On VLV/CHV display interrupts are part of the display 2517 * power well, so hpd is reinitialized from there. For 2518 * everyone else do it here. 2519 */ 2520 if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) 2521 intel_hpd_init(dev_priv); 2522 2523 enable_rpm_wakeref_asserts(dev_priv); 2524 2525 if (ret) 2526 DRM_ERROR("Runtime resume failed, disabling it (%d)\n", ret); 2527 else 2528 DRM_DEBUG_KMS("Device resumed\n"); 2529 2530 return ret; 2531 } 2532 2533 const struct dev_pm_ops i915_pm_ops = { 2534 /* 2535 * S0ix (via system suspend) and S3 event handlers [PMSG_SUSPEND, 2536 * PMSG_RESUME] 2537 */ 2538 .suspend = i915_pm_suspend, 2539 .suspend_late = i915_pm_suspend_late, 2540 .resume_early = i915_pm_resume_early, 2541 .resume = i915_pm_resume, 2542 2543 /* 2544 * S4 event handlers 2545 * @freeze, @freeze_late : called (1) before creating the 2546 * hibernation image [PMSG_FREEZE] and 2547 * (2) after rebooting, before restoring 2548 * the image [PMSG_QUIESCE] 2549 * @thaw, @thaw_early : called (1) after creating the hibernation 2550 * image, before writing it [PMSG_THAW] 2551 * and (2) after failing to create or 2552 * restore the image [PMSG_RECOVER] 2553 * @poweroff, @poweroff_late: called after writing the hibernation 2554 * image, before rebooting [PMSG_HIBERNATE] 2555 * @restore, @restore_early : called after rebooting and restoring the 2556 * hibernation image [PMSG_RESTORE] 2557 */ 2558 .freeze = i915_pm_freeze, 2559 .freeze_late = i915_pm_freeze_late, 2560 .thaw_early = i915_pm_thaw_early, 2561 .thaw = i915_pm_thaw, 2562 .poweroff = i915_pm_suspend, 2563 .poweroff_late = i915_pm_poweroff_late, 2564 .restore_early = i915_pm_restore_early, 2565 .restore = i915_pm_restore, 2566 2567 /* S0ix (via runtime suspend) event handlers */ 2568 .runtime_suspend = intel_runtime_suspend, 2569 .runtime_resume = intel_runtime_resume, 2570 }; 2571 2572 static const struct vm_operations_struct i915_gem_vm_ops = { 2573 .fault = i915_gem_fault, 2574 .open = drm_gem_vm_open, 2575 .close = drm_gem_vm_close, 2576 }; 2577 #endif 2578 2579 static struct cdev_pager_ops i915_gem_vm_ops = { 2580 .cdev_pg_fault = i915_gem_fault, 2581 .cdev_pg_ctor = i915_gem_pager_ctor, 2582 .cdev_pg_dtor = i915_gem_pager_dtor 2583 }; 2584 2585 static const struct file_operations i915_driver_fops = { 2586 .owner = THIS_MODULE, 2587 #if 0 2588 .open = drm_open, 2589 .release = drm_release, 2590 .unlocked_ioctl = drm_ioctl, 2591 .mmap = drm_gem_mmap, 2592 .poll = drm_poll, 2593 .read = drm_read, 2594 .compat_ioctl = i915_compat_ioctl, 2595 .llseek = noop_llseek, 2596 #endif 2597 }; 2598 2599 static int 2600 i915_gem_reject_pin_ioctl(struct drm_device *dev, void *data, 2601 struct drm_file *file) 2602 { 2603 return -ENODEV; 2604 } 2605 2606 static const struct drm_ioctl_desc i915_ioctls[] = { 2607 DRM_IOCTL_DEF_DRV(I915_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 2608 DRM_IOCTL_DEF_DRV(I915_FLUSH, drm_noop, DRM_AUTH), 2609 DRM_IOCTL_DEF_DRV(I915_FLIP, drm_noop, DRM_AUTH), 2610 DRM_IOCTL_DEF_DRV(I915_BATCHBUFFER, drm_noop, DRM_AUTH), 2611 DRM_IOCTL_DEF_DRV(I915_IRQ_EMIT, drm_noop, DRM_AUTH), 2612 DRM_IOCTL_DEF_DRV(I915_IRQ_WAIT, drm_noop, DRM_AUTH), 2613 DRM_IOCTL_DEF_DRV(I915_GETPARAM, i915_getparam, DRM_AUTH|DRM_RENDER_ALLOW), 2614 DRM_IOCTL_DEF_DRV(I915_SETPARAM, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 2615 DRM_IOCTL_DEF_DRV(I915_ALLOC, drm_noop, DRM_AUTH), 2616 DRM_IOCTL_DEF_DRV(I915_FREE, drm_noop, DRM_AUTH), 2617 DRM_IOCTL_DEF_DRV(I915_INIT_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 2618 DRM_IOCTL_DEF_DRV(I915_CMDBUFFER, drm_noop, DRM_AUTH), 2619 DRM_IOCTL_DEF_DRV(I915_DESTROY_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 2620 DRM_IOCTL_DEF_DRV(I915_SET_VBLANK_PIPE, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 2621 DRM_IOCTL_DEF_DRV(I915_GET_VBLANK_PIPE, drm_noop, DRM_AUTH), 2622 DRM_IOCTL_DEF_DRV(I915_VBLANK_SWAP, drm_noop, DRM_AUTH), 2623 DRM_IOCTL_DEF_DRV(I915_HWS_ADDR, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 2624 DRM_IOCTL_DEF_DRV(I915_GEM_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 2625 DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER, i915_gem_execbuffer, DRM_AUTH), 2626 DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER2, i915_gem_execbuffer2, DRM_AUTH|DRM_RENDER_ALLOW), 2627 DRM_IOCTL_DEF_DRV(I915_GEM_PIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY), 2628 DRM_IOCTL_DEF_DRV(I915_GEM_UNPIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY), 2629 DRM_IOCTL_DEF_DRV(I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), 2630 DRM_IOCTL_DEF_DRV(I915_GEM_SET_CACHING, i915_gem_set_caching_ioctl, DRM_RENDER_ALLOW), 2631 DRM_IOCTL_DEF_DRV(I915_GEM_GET_CACHING, i915_gem_get_caching_ioctl, DRM_RENDER_ALLOW), 2632 DRM_IOCTL_DEF_DRV(I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), 2633 DRM_IOCTL_DEF_DRV(I915_GEM_ENTERVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 2634 DRM_IOCTL_DEF_DRV(I915_GEM_LEAVEVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 2635 DRM_IOCTL_DEF_DRV(I915_GEM_CREATE, i915_gem_create_ioctl, DRM_RENDER_ALLOW), 2636 DRM_IOCTL_DEF_DRV(I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_RENDER_ALLOW), 2637 DRM_IOCTL_DEF_DRV(I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_RENDER_ALLOW), 2638 DRM_IOCTL_DEF_DRV(I915_GEM_MMAP, i915_gem_mmap_ioctl, DRM_RENDER_ALLOW), 2639 DRM_IOCTL_DEF_DRV(I915_GEM_MMAP_GTT, i915_gem_mmap_gtt_ioctl, DRM_RENDER_ALLOW), 2640 DRM_IOCTL_DEF_DRV(I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, DRM_RENDER_ALLOW), 2641 DRM_IOCTL_DEF_DRV(I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, DRM_RENDER_ALLOW), 2642 DRM_IOCTL_DEF_DRV(I915_GEM_SET_TILING, i915_gem_set_tiling, DRM_RENDER_ALLOW), 2643 DRM_IOCTL_DEF_DRV(I915_GEM_GET_TILING, i915_gem_get_tiling, DRM_RENDER_ALLOW), 2644 DRM_IOCTL_DEF_DRV(I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, DRM_RENDER_ALLOW), 2645 DRM_IOCTL_DEF_DRV(I915_GET_PIPE_FROM_CRTC_ID, intel_get_pipe_from_crtc_id, 0), 2646 DRM_IOCTL_DEF_DRV(I915_GEM_MADVISE, i915_gem_madvise_ioctl, DRM_RENDER_ALLOW), 2647 DRM_IOCTL_DEF_DRV(I915_OVERLAY_PUT_IMAGE, intel_overlay_put_image_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW), 2648 DRM_IOCTL_DEF_DRV(I915_OVERLAY_ATTRS, intel_overlay_attrs_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW), 2649 DRM_IOCTL_DEF_DRV(I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey, DRM_MASTER|DRM_CONTROL_ALLOW), 2650 DRM_IOCTL_DEF_DRV(I915_GET_SPRITE_COLORKEY, drm_noop, DRM_MASTER|DRM_CONTROL_ALLOW), 2651 DRM_IOCTL_DEF_DRV(I915_GEM_WAIT, i915_gem_wait_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), 2652 DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_CREATE, i915_gem_context_create_ioctl, DRM_RENDER_ALLOW), 2653 DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_RENDER_ALLOW), 2654 DRM_IOCTL_DEF_DRV(I915_REG_READ, i915_reg_read_ioctl, DRM_RENDER_ALLOW), 2655 DRM_IOCTL_DEF_DRV(I915_GET_RESET_STATS, i915_gem_context_reset_stats_ioctl, DRM_RENDER_ALLOW), 2656 #if 0 2657 DRM_IOCTL_DEF_DRV(I915_GEM_USERPTR, i915_gem_userptr_ioctl, DRM_RENDER_ALLOW), 2658 #endif 2659 DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_GETPARAM, i915_gem_context_getparam_ioctl, DRM_RENDER_ALLOW), 2660 DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_SETPARAM, i915_gem_context_setparam_ioctl, DRM_RENDER_ALLOW), 2661 }; 2662 2663 static int i915_sysctl_init(struct drm_device *dev, struct sysctl_ctx_list *ctx, 2664 struct sysctl_oid *top) 2665 { 2666 return drm_add_busid_modesetting(dev, ctx, top); 2667 } 2668 2669 static struct drm_driver driver = { 2670 /* Don't use MTRRs here; the Xserver or userspace app should 2671 * deal with them for Intel hardware. 2672 */ 2673 .driver_features = 2674 DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED | DRIVER_GEM | 2675 DRIVER_RENDER | DRIVER_MODESET, 2676 .open = i915_driver_open, 2677 .lastclose = i915_driver_lastclose, 2678 .preclose = i915_driver_preclose, 2679 .postclose = i915_driver_postclose, 2680 .set_busid = drm_pci_set_busid, 2681 2682 .gem_close_object = i915_gem_close_object, 2683 .gem_free_object_unlocked = i915_gem_free_object, 2684 .gem_vm_ops = &i915_gem_vm_ops, 2685 2686 .prime_handle_to_fd = drm_gem_prime_handle_to_fd, 2687 .prime_fd_to_handle = drm_gem_prime_fd_to_handle, 2688 .gem_prime_export = i915_gem_prime_export, 2689 .gem_prime_import = i915_gem_prime_import, 2690 2691 .dumb_create = i915_gem_dumb_create, 2692 .dumb_map_offset = i915_gem_mmap_gtt, 2693 .dumb_destroy = drm_gem_dumb_destroy, 2694 .ioctls = i915_ioctls, 2695 .num_ioctls = ARRAY_SIZE(i915_ioctls), 2696 .fops = &i915_driver_fops, 2697 .name = DRIVER_NAME, 2698 .desc = DRIVER_DESC, 2699 .date = DRIVER_DATE, 2700 .major = DRIVER_MAJOR, 2701 .minor = DRIVER_MINOR, 2702 .patchlevel = DRIVER_PATCHLEVEL, 2703 #ifdef __DragonFly__ 2704 .sysctl_init = i915_sysctl_init, 2705 #endif 2706 }; 2707