1 /* i915_drv.c -- i830,i845,i855,i865,i915 driver -*- linux-c -*- 2 */ 3 /* 4 * 5 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas. 6 * All Rights Reserved. 7 * 8 * Permission is hereby granted, free of charge, to any person obtaining a 9 * copy of this software and associated documentation files (the 10 * "Software"), to deal in the Software without restriction, including 11 * without limitation the rights to use, copy, modify, merge, publish, 12 * distribute, sub license, and/or sell copies of the Software, and to 13 * permit persons to whom the Software is furnished to do so, subject to 14 * the following conditions: 15 * 16 * The above copyright notice and this permission notice (including the 17 * next paragraph) shall be included in all copies or substantial portions 18 * of the Software. 19 * 20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 22 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 23 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR 24 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 25 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 26 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 27 * 28 */ 29 30 #ifdef __DragonFly__ 31 #include "opt_drm.h" /* for VGA_SWITCHEROO */ 32 #endif 33 34 #include <linux/acpi.h> 35 #include <linux/device.h> 36 #include <linux/oom.h> 37 #include <linux/module.h> 38 #include <linux/pci.h> 39 #include <linux/pm.h> 40 #include <linux/pm_runtime.h> 41 #include <linux/pnp.h> 42 #include <linux/slab.h> 43 #include <linux/vgaarb.h> 44 #include <linux/vga_switcheroo.h> 45 #include <linux/vt.h> 46 #include <acpi/video.h> 47 48 #include <drm/drmP.h> 49 #include <drm/drm_crtc_helper.h> 50 #include <drm/drm_atomic_helper.h> 51 #include <drm/i915_drm.h> 52 53 #include "i915_drv.h" 54 #include "i915_trace.h" 55 #include "i915_vgpu.h" 56 #include "intel_drv.h" 57 #include "intel_uc.h" 58 59 static struct drm_driver driver; 60 61 static unsigned int i915_load_fail_count; 62 63 bool __i915_inject_load_failure(const char *func, int line) 64 { 65 if (i915_load_fail_count >= i915_modparams.inject_load_failure) 66 return false; 67 68 if (++i915_load_fail_count == i915_modparams.inject_load_failure) { 69 DRM_INFO("Injecting failure at checkpoint %u [%s:%d]\n", 70 i915_modparams.inject_load_failure, func, line); 71 return true; 72 } 73 74 return false; 75 } 76 77 #define FDO_BUG_URL "https://bugs.freedesktop.org/enter_bug.cgi?product=DRI" 78 #define FDO_BUG_MSG "Please file a bug at " FDO_BUG_URL " against DRM/Intel " \ 79 "providing the dmesg log by booting with drm.debug=0xf" 80 81 void 82 __i915_printk(struct drm_i915_private *dev_priv, const char *level, 83 const char *fmt, ...) 84 { 85 static bool shown_bug_once; 86 struct device *kdev = dev_priv->drm.dev; 87 bool is_error = level[1] <= KERN_ERR[1]; 88 bool is_debug = level[1] == KERN_DEBUG[1]; 89 struct va_format vaf; 90 va_list args; 91 92 if (is_debug && !(drm_debug & DRM_UT_DRIVER)) 93 return; 94 95 va_start(args, fmt); 96 97 vaf.fmt = fmt; 98 vaf.va = &args; 99 100 dev_printk(level, kdev, "[" DRM_NAME ":%ps] %pV", 101 __builtin_return_address(0), &vaf); 102 103 if (is_error && !shown_bug_once) { 104 dev_notice(kdev, "%s", FDO_BUG_MSG); 105 shown_bug_once = true; 106 } 107 108 va_end(args); 109 } 110 111 static bool i915_error_injected(struct drm_i915_private *dev_priv) 112 { 113 return i915_modparams.inject_load_failure && 114 i915_load_fail_count == i915_modparams.inject_load_failure; 115 } 116 117 #define i915_load_error(dev_priv, fmt, ...) \ 118 __i915_printk(dev_priv, \ 119 i915_error_injected(dev_priv) ? KERN_DEBUG : KERN_ERR, \ 120 fmt, ##__VA_ARGS__) 121 122 123 static enum intel_pch intel_virt_detect_pch(struct drm_i915_private *dev_priv) 124 { 125 enum intel_pch ret = PCH_NOP; 126 127 /* 128 * In a virtualized passthrough environment we can be in a 129 * setup where the ISA bridge is not able to be passed through. 130 * In this case, a south bridge can be emulated and we have to 131 * make an educated guess as to which PCH is really there. 132 */ 133 134 if (IS_GEN5(dev_priv)) { 135 ret = PCH_IBX; 136 DRM_DEBUG_KMS("Assuming Ibex Peak PCH\n"); 137 } else if (IS_GEN6(dev_priv) || IS_IVYBRIDGE(dev_priv)) { 138 ret = PCH_CPT; 139 DRM_DEBUG_KMS("Assuming CougarPoint PCH\n"); 140 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { 141 ret = PCH_LPT; 142 if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv)) 143 dev_priv->pch_id = INTEL_PCH_LPT_LP_DEVICE_ID_TYPE; 144 else 145 dev_priv->pch_id = INTEL_PCH_LPT_DEVICE_ID_TYPE; 146 DRM_DEBUG_KMS("Assuming LynxPoint PCH\n"); 147 } else if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) { 148 ret = PCH_SPT; 149 DRM_DEBUG_KMS("Assuming SunrisePoint PCH\n"); 150 } else if (IS_COFFEELAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) { 151 ret = PCH_CNP; 152 DRM_DEBUG_KMS("Assuming CannonPoint PCH\n"); 153 } 154 155 return ret; 156 } 157 158 static void intel_detect_pch(struct drm_i915_private *dev_priv) 159 { 160 device_t pch = NULL; 161 struct pci_devinfo *di = NULL; 162 163 /* In all current cases, num_pipes is equivalent to the PCH_NOP setting 164 * (which really amounts to a PCH but no South Display). 165 */ 166 if (INTEL_INFO(dev_priv)->num_pipes == 0) { 167 dev_priv->pch_type = PCH_NOP; 168 return; 169 } 170 171 /* XXX The ISA bridge probe causes some old Core2 machines to hang */ 172 if (INTEL_INFO(dev_priv)->gen < 5) 173 return; 174 175 /* 176 * The reason to probe ISA bridge instead of Dev31:Fun0 is to 177 * make graphics device passthrough work easy for VMM, that only 178 * need to expose ISA bridge to let driver know the real hardware 179 * underneath. This is a requirement from virtualization team. 180 * 181 * In some virtualized environments (e.g. XEN), there is irrelevant 182 * ISA bridge in the system. To work reliably, we should scan trhough 183 * all the ISA bridge devices and check for the first match, instead 184 * of only checking the first one. 185 */ 186 while ((pch = pci_iterate_class(&di, PCIC_BRIDGE, PCIS_BRIDGE_ISA))) { 187 if (pci_get_vendor(pch) == PCI_VENDOR_ID_INTEL) { 188 unsigned short id = pci_get_device(pch) & INTEL_PCH_DEVICE_ID_MASK; 189 190 dev_priv->pch_id = id; 191 192 if (id == INTEL_PCH_IBX_DEVICE_ID_TYPE) { 193 dev_priv->pch_type = PCH_IBX; 194 DRM_DEBUG_KMS("Found Ibex Peak PCH\n"); 195 WARN_ON(!IS_GEN5(dev_priv)); 196 } else if (id == INTEL_PCH_CPT_DEVICE_ID_TYPE) { 197 dev_priv->pch_type = PCH_CPT; 198 DRM_DEBUG_KMS("Found CougarPoint PCH\n"); 199 WARN_ON(!IS_GEN6(dev_priv) && 200 !IS_IVYBRIDGE(dev_priv)); 201 } else if (id == INTEL_PCH_PPT_DEVICE_ID_TYPE) { 202 /* PantherPoint is CPT compatible */ 203 dev_priv->pch_type = PCH_CPT; 204 DRM_DEBUG_KMS("Found PantherPoint PCH\n"); 205 WARN_ON(!IS_GEN6(dev_priv) && 206 !IS_IVYBRIDGE(dev_priv)); 207 } else if (id == INTEL_PCH_LPT_DEVICE_ID_TYPE) { 208 dev_priv->pch_type = PCH_LPT; 209 DRM_DEBUG_KMS("Found LynxPoint PCH\n"); 210 WARN_ON(!IS_HASWELL(dev_priv) && 211 !IS_BROADWELL(dev_priv)); 212 WARN_ON(IS_HSW_ULT(dev_priv) || 213 IS_BDW_ULT(dev_priv)); 214 } else if (id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) { 215 dev_priv->pch_type = PCH_LPT; 216 DRM_DEBUG_KMS("Found LynxPoint LP PCH\n"); 217 WARN_ON(!IS_HASWELL(dev_priv) && 218 !IS_BROADWELL(dev_priv)); 219 WARN_ON(!IS_HSW_ULT(dev_priv) && 220 !IS_BDW_ULT(dev_priv)); 221 } else if (id == INTEL_PCH_WPT_DEVICE_ID_TYPE) { 222 /* WildcatPoint is LPT compatible */ 223 dev_priv->pch_type = PCH_LPT; 224 DRM_DEBUG_KMS("Found WildcatPoint PCH\n"); 225 WARN_ON(!IS_HASWELL(dev_priv) && 226 !IS_BROADWELL(dev_priv)); 227 WARN_ON(IS_HSW_ULT(dev_priv) || 228 IS_BDW_ULT(dev_priv)); 229 } else if (id == INTEL_PCH_WPT_LP_DEVICE_ID_TYPE) { 230 /* WildcatPoint is LPT compatible */ 231 dev_priv->pch_type = PCH_LPT; 232 DRM_DEBUG_KMS("Found WildcatPoint LP PCH\n"); 233 WARN_ON(!IS_HASWELL(dev_priv) && 234 !IS_BROADWELL(dev_priv)); 235 WARN_ON(!IS_HSW_ULT(dev_priv) && 236 !IS_BDW_ULT(dev_priv)); 237 } else if (id == INTEL_PCH_SPT_DEVICE_ID_TYPE) { 238 dev_priv->pch_type = PCH_SPT; 239 DRM_DEBUG_KMS("Found SunrisePoint PCH\n"); 240 WARN_ON(!IS_SKYLAKE(dev_priv) && 241 !IS_KABYLAKE(dev_priv)); 242 } else if (id == INTEL_PCH_SPT_LP_DEVICE_ID_TYPE) { 243 dev_priv->pch_type = PCH_SPT; 244 DRM_DEBUG_KMS("Found SunrisePoint LP PCH\n"); 245 WARN_ON(!IS_SKYLAKE(dev_priv) && 246 !IS_KABYLAKE(dev_priv)); 247 } else if (id == INTEL_PCH_KBP_DEVICE_ID_TYPE) { 248 dev_priv->pch_type = PCH_KBP; 249 DRM_DEBUG_KMS("Found Kaby Lake PCH (KBP)\n"); 250 WARN_ON(!IS_SKYLAKE(dev_priv) && 251 !IS_KABYLAKE(dev_priv) && 252 !IS_COFFEELAKE(dev_priv)); 253 } else if (id == INTEL_PCH_CNP_DEVICE_ID_TYPE) { 254 dev_priv->pch_type = PCH_CNP; 255 DRM_DEBUG_KMS("Found Cannon Lake PCH (CNP)\n"); 256 WARN_ON(!IS_CANNONLAKE(dev_priv) && 257 !IS_COFFEELAKE(dev_priv)); 258 } else if (id == INTEL_PCH_CNP_LP_DEVICE_ID_TYPE) { 259 dev_priv->pch_type = PCH_CNP; 260 DRM_DEBUG_KMS("Found Cannon Lake LP PCH (CNP-LP)\n"); 261 WARN_ON(!IS_CANNONLAKE(dev_priv) && 262 !IS_COFFEELAKE(dev_priv)); 263 } else if (id == INTEL_PCH_P2X_DEVICE_ID_TYPE || 264 id == INTEL_PCH_P3X_DEVICE_ID_TYPE || 265 (id == INTEL_PCH_QEMU_DEVICE_ID_TYPE && 266 1)) { 267 dev_priv->pch_type = 268 intel_virt_detect_pch(dev_priv); 269 } else 270 continue; 271 272 break; 273 } 274 } 275 if (!pch) 276 DRM_DEBUG_KMS("No PCH found.\n"); 277 278 #if 0 279 pci_dev_put(pch); 280 #endif 281 } 282 283 static int i915_getparam(struct drm_device *dev, void *data, 284 struct drm_file *file_priv) 285 { 286 struct drm_i915_private *dev_priv = to_i915(dev); 287 struct pci_dev *pdev = dev_priv->drm.pdev; 288 drm_i915_getparam_t *param = data; 289 int value; 290 291 switch (param->param) { 292 case I915_PARAM_IRQ_ACTIVE: 293 case I915_PARAM_ALLOW_BATCHBUFFER: 294 case I915_PARAM_LAST_DISPATCH: 295 case I915_PARAM_HAS_EXEC_CONSTANTS: 296 /* Reject all old ums/dri params. */ 297 return -ENODEV; 298 case I915_PARAM_CHIPSET_ID: 299 value = pdev->device; 300 break; 301 case I915_PARAM_REVISION: 302 value = pdev->revision; 303 break; 304 case I915_PARAM_NUM_FENCES_AVAIL: 305 value = dev_priv->num_fence_regs; 306 break; 307 case I915_PARAM_HAS_OVERLAY: 308 value = dev_priv->overlay ? 1 : 0; 309 break; 310 case I915_PARAM_HAS_BSD: 311 value = !!dev_priv->engine[VCS]; 312 break; 313 case I915_PARAM_HAS_BLT: 314 value = !!dev_priv->engine[BCS]; 315 break; 316 case I915_PARAM_HAS_VEBOX: 317 value = !!dev_priv->engine[VECS]; 318 break; 319 case I915_PARAM_HAS_BSD2: 320 value = !!dev_priv->engine[VCS2]; 321 break; 322 case I915_PARAM_HAS_LLC: 323 value = HAS_LLC(dev_priv); 324 break; 325 case I915_PARAM_HAS_WT: 326 value = HAS_WT(dev_priv); 327 break; 328 case I915_PARAM_HAS_ALIASING_PPGTT: 329 value = USES_PPGTT(dev_priv); 330 break; 331 case I915_PARAM_HAS_SEMAPHORES: 332 value = i915_modparams.semaphores; 333 break; 334 #if 0 335 case I915_PARAM_HAS_SECURE_BATCHES: 336 value = capable(CAP_SYS_ADMIN); 337 break; 338 #endif 339 case I915_PARAM_CMD_PARSER_VERSION: 340 value = i915_cmd_parser_get_version(dev_priv); 341 break; 342 case I915_PARAM_SUBSLICE_TOTAL: 343 value = sseu_subslice_total(&INTEL_INFO(dev_priv)->sseu); 344 if (!value) 345 return -ENODEV; 346 break; 347 case I915_PARAM_EU_TOTAL: 348 value = INTEL_INFO(dev_priv)->sseu.eu_total; 349 if (!value) 350 return -ENODEV; 351 break; 352 case I915_PARAM_HAS_GPU_RESET: 353 value = i915_modparams.enable_hangcheck && 354 intel_has_gpu_reset(dev_priv); 355 if (value && intel_has_reset_engine(dev_priv)) 356 value = 2; 357 break; 358 case I915_PARAM_HAS_RESOURCE_STREAMER: 359 value = HAS_RESOURCE_STREAMER(dev_priv); 360 break; 361 case I915_PARAM_HAS_POOLED_EU: 362 value = HAS_POOLED_EU(dev_priv); 363 break; 364 case I915_PARAM_MIN_EU_IN_POOL: 365 value = INTEL_INFO(dev_priv)->sseu.min_eu_in_pool; 366 break; 367 case I915_PARAM_HUC_STATUS: 368 intel_runtime_pm_get(dev_priv); 369 value = I915_READ(HUC_STATUS2) & HUC_FW_VERIFIED; 370 intel_runtime_pm_put(dev_priv); 371 break; 372 case I915_PARAM_MMAP_GTT_VERSION: 373 /* Though we've started our numbering from 1, and so class all 374 * earlier versions as 0, in effect their value is undefined as 375 * the ioctl will report EINVAL for the unknown param! 376 */ 377 value = i915_gem_mmap_gtt_version(); 378 break; 379 case I915_PARAM_HAS_SCHEDULER: 380 value = 0; 381 if (dev_priv->engine[RCS] && dev_priv->engine[RCS]->schedule) { 382 value |= I915_SCHEDULER_CAP_ENABLED; 383 value |= I915_SCHEDULER_CAP_PRIORITY; 384 385 if (INTEL_INFO(dev_priv)->has_logical_ring_preemption && 386 i915_modparams.enable_execlists && 387 !i915_modparams.enable_guc_submission) 388 value |= I915_SCHEDULER_CAP_PREEMPTION; 389 } 390 break; 391 case I915_PARAM_MMAP_VERSION: 392 /* Remember to bump this if the version changes! */ 393 case I915_PARAM_HAS_GEM: 394 case I915_PARAM_HAS_PAGEFLIPPING: 395 case I915_PARAM_HAS_EXECBUF2: /* depends on GEM */ 396 case I915_PARAM_HAS_RELAXED_FENCING: 397 case I915_PARAM_HAS_COHERENT_RINGS: 398 case I915_PARAM_HAS_RELAXED_DELTA: 399 case I915_PARAM_HAS_GEN7_SOL_RESET: 400 case I915_PARAM_HAS_WAIT_TIMEOUT: 401 #if 0 402 case I915_PARAM_HAS_PRIME_VMAP_FLUSH: 403 #endif 404 case I915_PARAM_HAS_PINNED_BATCHES: 405 case I915_PARAM_HAS_EXEC_NO_RELOC: 406 case I915_PARAM_HAS_EXEC_HANDLE_LUT: 407 case I915_PARAM_HAS_COHERENT_PHYS_GTT: 408 case I915_PARAM_HAS_EXEC_SOFTPIN: 409 case I915_PARAM_HAS_EXEC_ASYNC: 410 case I915_PARAM_HAS_EXEC_FENCE: 411 case I915_PARAM_HAS_EXEC_CAPTURE: 412 case I915_PARAM_HAS_EXEC_BATCH_FIRST: 413 case I915_PARAM_HAS_EXEC_FENCE_ARRAY: 414 /* For the time being all of these are always true; 415 * if some supported hardware does not have one of these 416 * features this value needs to be provided from 417 * INTEL_INFO(), a feature macro, or similar. 418 */ 419 value = 1; 420 break; 421 case I915_PARAM_SLICE_MASK: 422 value = INTEL_INFO(dev_priv)->sseu.slice_mask; 423 if (!value) 424 return -ENODEV; 425 break; 426 case I915_PARAM_SUBSLICE_MASK: 427 value = INTEL_INFO(dev_priv)->sseu.subslice_mask; 428 if (!value) 429 return -ENODEV; 430 break; 431 default: 432 DRM_DEBUG("Unknown parameter %d\n", param->param); 433 return -EINVAL; 434 } 435 436 if (put_user(value, param->value)) 437 return -EFAULT; 438 439 return 0; 440 } 441 442 static int i915_get_bridge_dev(struct drm_i915_private *dev_priv) 443 { 444 static struct pci_dev i915_bridge_dev; 445 446 i915_bridge_dev.dev.bsddev = pci_find_dbsf(0, 0, 0, 0); 447 if (!i915_bridge_dev.dev.bsddev) { 448 DRM_ERROR("bridge device not found\n"); 449 return -1; 450 } 451 452 dev_priv->bridge_dev = &i915_bridge_dev; 453 return 0; 454 } 455 456 /* Allocate space for the MCH regs if needed, return nonzero on error */ 457 static int 458 intel_alloc_mchbar_resource(struct drm_i915_private *dev_priv) 459 { 460 int reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915; 461 u32 temp_lo, temp_hi = 0; 462 u64 mchbar_addr; 463 device_t bsddev, vga; 464 465 if (INTEL_GEN(dev_priv) >= 4) 466 pci_read_config_dword(dev_priv->bridge_dev, reg + 4, &temp_hi); 467 pci_read_config_dword(dev_priv->bridge_dev, reg, &temp_lo); 468 mchbar_addr = ((u64)temp_hi << 32) | temp_lo; 469 470 /* If ACPI doesn't have it, assume we need to allocate it ourselves */ 471 #ifdef CONFIG_PNP 472 if (mchbar_addr && 473 pnp_range_reserved(mchbar_addr, mchbar_addr + MCHBAR_SIZE)) 474 return 0; 475 #endif 476 477 /* Get some space for it */ 478 bsddev = dev_priv->bridge_dev->dev.bsddev; 479 vga = device_get_parent(bsddev); 480 dev_priv->mch_res_rid = 0x100; 481 dev_priv->mch_res = BUS_ALLOC_RESOURCE(device_get_parent(vga), 482 bsddev, SYS_RES_MEMORY, &dev_priv->mch_res_rid, 0, ~0UL, 483 MCHBAR_SIZE, RF_ACTIVE | RF_SHAREABLE, -1); 484 if (dev_priv->mch_res == NULL) { 485 DRM_DEBUG_DRIVER("failed mchbar resource alloc\n"); 486 return (-ENOMEM); 487 } 488 489 if (INTEL_GEN(dev_priv) >= 4) 490 pci_write_config_dword(dev_priv->bridge_dev, reg + 4, 491 upper_32_bits(rman_get_start(dev_priv->mch_res))); 492 493 pci_write_config_dword(dev_priv->bridge_dev, reg, 494 lower_32_bits(rman_get_start(dev_priv->mch_res))); 495 return 0; 496 } 497 498 /* Setup MCHBAR if possible, return true if we should disable it again */ 499 static void 500 intel_setup_mchbar(struct drm_i915_private *dev_priv) 501 { 502 int mchbar_reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915; 503 u32 temp; 504 bool enabled; 505 506 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) 507 return; 508 509 dev_priv->mchbar_need_disable = false; 510 511 if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) { 512 pci_read_config_dword(dev_priv->bridge_dev, DEVEN, &temp); 513 enabled = !!(temp & DEVEN_MCHBAR_EN); 514 } else { 515 pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp); 516 enabled = temp & 1; 517 } 518 519 /* If it's already enabled, don't have to do anything */ 520 if (enabled) 521 return; 522 523 if (intel_alloc_mchbar_resource(dev_priv)) 524 return; 525 526 dev_priv->mchbar_need_disable = true; 527 528 /* Space is allocated or reserved, so enable it. */ 529 if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) { 530 pci_write_config_dword(dev_priv->bridge_dev, DEVEN, 531 temp | DEVEN_MCHBAR_EN); 532 } else { 533 pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp); 534 pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg, temp | 1); 535 } 536 } 537 538 static void 539 intel_teardown_mchbar(struct drm_i915_private *dev_priv) 540 { 541 int mchbar_reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915; 542 device_t bsddev, vga; 543 544 if (dev_priv->mchbar_need_disable) { 545 if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) { 546 u32 deven_val; 547 548 pci_read_config_dword(dev_priv->bridge_dev, DEVEN, 549 &deven_val); 550 deven_val &= ~DEVEN_MCHBAR_EN; 551 pci_write_config_dword(dev_priv->bridge_dev, DEVEN, 552 deven_val); 553 } else { 554 u32 mchbar_val; 555 556 pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, 557 &mchbar_val); 558 mchbar_val &= ~1; 559 pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg, 560 mchbar_val); 561 } 562 } 563 564 bsddev = dev_priv->bridge_dev->dev.bsddev; 565 if (dev_priv->mch_res != NULL) { 566 vga = device_get_parent(bsddev); 567 BUS_DEACTIVATE_RESOURCE(device_get_parent(vga), bsddev, 568 SYS_RES_MEMORY, dev_priv->mch_res_rid, dev_priv->mch_res); 569 BUS_RELEASE_RESOURCE(device_get_parent(vga), bsddev, 570 SYS_RES_MEMORY, dev_priv->mch_res_rid, dev_priv->mch_res); 571 dev_priv->mch_res = NULL; 572 } 573 } 574 575 #if 0 576 /* true = enable decode, false = disable decoder */ 577 static unsigned int i915_vga_set_decode(void *cookie, bool state) 578 { 579 struct drm_i915_private *dev_priv = cookie; 580 581 intel_modeset_vga_set_state(dev_priv, state); 582 if (state) 583 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM | 584 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; 585 else 586 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; 587 } 588 589 static int i915_resume_switcheroo(struct drm_device *dev); 590 static int i915_suspend_switcheroo(struct drm_device *dev, pm_message_t state); 591 592 static void i915_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state) 593 { 594 struct drm_device *dev = pci_get_drvdata(pdev); 595 pm_message_t pmm = { .event = PM_EVENT_SUSPEND }; 596 597 if (state == VGA_SWITCHEROO_ON) { 598 pr_info("switched on\n"); 599 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; 600 /* i915 resume handler doesn't set to D0 */ 601 pci_set_power_state(pdev, PCI_D0); 602 i915_resume_switcheroo(dev); 603 dev->switch_power_state = DRM_SWITCH_POWER_ON; 604 } else { 605 pr_info("switched off\n"); 606 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; 607 i915_suspend_switcheroo(dev, pmm); 608 dev->switch_power_state = DRM_SWITCH_POWER_OFF; 609 } 610 } 611 612 static bool i915_switcheroo_can_switch(struct pci_dev *pdev) 613 { 614 struct drm_device *dev = pci_get_drvdata(pdev); 615 616 /* 617 * FIXME: open_count is protected by drm_global_mutex but that would lead to 618 * locking inversion with the driver load path. And the access here is 619 * completely racy anyway. So don't bother with locking for now. 620 */ 621 return dev->open_count == 0; 622 } 623 624 static const struct vga_switcheroo_client_ops i915_switcheroo_ops = { 625 .set_gpu_state = i915_switcheroo_set_state, 626 .reprobe = NULL, 627 .can_switch = i915_switcheroo_can_switch, 628 }; 629 #endif 630 631 static void i915_gem_fini(struct drm_i915_private *dev_priv) 632 { 633 /* Flush any outstanding unpin_work. */ 634 i915_gem_drain_workqueue(dev_priv); 635 636 mutex_lock(&dev_priv->drm.struct_mutex); 637 intel_uc_fini_hw(dev_priv); 638 i915_gem_cleanup_engines(dev_priv); 639 i915_gem_contexts_fini(dev_priv); 640 mutex_unlock(&dev_priv->drm.struct_mutex); 641 642 i915_gem_cleanup_userptr(dev_priv); 643 644 i915_gem_drain_freed_objects(dev_priv); 645 646 WARN_ON(!list_empty(&dev_priv->contexts.list)); 647 } 648 649 static int i915_load_modeset_init(struct drm_device *dev) 650 { 651 struct drm_i915_private *dev_priv = to_i915(dev); 652 int ret; 653 654 if (i915_inject_load_failure()) 655 return -ENODEV; 656 657 intel_bios_init(dev_priv); 658 659 /* If we have > 1 VGA cards, then we need to arbitrate access 660 * to the common VGA resources. 661 * 662 * If we are a secondary display controller (!PCI_DISPLAY_CLASS_VGA), 663 * then we do not take part in VGA arbitration and the 664 * vga_client_register() fails with -ENODEV. 665 */ 666 #if 0 667 ret = vga_client_register(pdev, dev_priv, NULL, i915_vga_set_decode); 668 if (ret && ret != -ENODEV) 669 goto out; 670 671 intel_register_dsm_handler(); 672 673 ret = vga_switcheroo_register_client(pdev, &i915_switcheroo_ops, false); 674 if (ret) 675 goto cleanup_vga_client; 676 #endif 677 678 /* must happen before intel_power_domains_init_hw() on VLV/CHV */ 679 intel_update_rawclk(dev_priv); 680 681 intel_power_domains_init_hw(dev_priv, false); 682 683 intel_csr_ucode_init(dev_priv); 684 685 ret = intel_irq_install(dev_priv); 686 if (ret) 687 goto cleanup_csr; 688 689 intel_setup_gmbus(dev_priv); 690 691 /* Important: The output setup functions called by modeset_init need 692 * working irqs for e.g. gmbus and dp aux transfers. */ 693 ret = intel_modeset_init(dev); 694 if (ret) 695 goto cleanup_irq; 696 697 intel_uc_init_fw(dev_priv); 698 699 ret = i915_gem_init(dev_priv); 700 if (ret) 701 goto cleanup_uc; 702 703 intel_modeset_gem_init(dev); 704 705 if (INTEL_INFO(dev_priv)->num_pipes == 0) 706 return 0; 707 708 ret = intel_fbdev_init(dev); 709 if (ret) 710 goto cleanup_gem; 711 712 /* Only enable hotplug handling once the fbdev is fully set up. */ 713 intel_hpd_init(dev_priv); 714 715 drm_kms_helper_poll_init(dev); 716 717 #ifdef __DragonFly__ 718 /* 719 * If we are dealing with dual GPU machines the vga_switcheroo module 720 * has been loaded. Machines with dual GPUs have an integrated graphics 721 * device (IGD), which we assume is an Intel device. The other, the 722 * discrete device (DIS), is either an NVidia or a Radeon device. For 723 * now we will force switch the gmux so the intel driver outputs 724 * both to the laptop panel and the external monitor. 725 * 726 * DragonFly does not have an nvidia native driver yet. In the future, 727 * we will check for the radeon device: if present, we will leave 728 * the gmux switch as it is, so the user can choose between the IGD and 729 * the DIS using the /dev/vga_switcheroo device. 730 */ 731 if (vga_switcheroo_handler_flags() & VGA_SWITCHEROO_CAN_SWITCH_DDC) { 732 ret = vga_switcheroo_force_migd(); 733 if (ret) { 734 DRM_INFO("could not switch gmux to IGD\n"); 735 } 736 } 737 #endif 738 739 return 0; 740 741 cleanup_gem: 742 if (i915_gem_suspend(dev_priv)) 743 DRM_ERROR("failed to idle hardware; continuing to unload!\n"); 744 i915_gem_fini(dev_priv); 745 cleanup_uc: 746 intel_uc_fini_fw(dev_priv); 747 cleanup_irq: 748 drm_irq_uninstall(dev); 749 intel_teardown_gmbus(dev_priv); 750 cleanup_csr: 751 intel_csr_ucode_fini(dev_priv); 752 intel_power_domains_fini(dev_priv); 753 #if 0 754 vga_switcheroo_unregister_client(pdev); 755 cleanup_vga_client: 756 vga_client_register(pdev, NULL, NULL, NULL); 757 out: 758 #endif 759 return ret; 760 } 761 762 #ifdef __DragonFly__ 763 static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv) 764 { 765 return 0; 766 } 767 #else 768 static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv) 769 { 770 struct apertures_struct *ap; 771 struct pci_dev *pdev = dev_priv->drm.pdev; 772 struct i915_ggtt *ggtt = &dev_priv->ggtt; 773 bool primary; 774 int ret; 775 776 ap = alloc_apertures(1); 777 if (!ap) 778 return -ENOMEM; 779 780 ap->ranges[0].base = ggtt->mappable_base; 781 ap->ranges[0].size = ggtt->mappable_end; 782 783 primary = 784 pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW; 785 786 ret = drm_fb_helper_remove_conflicting_framebuffers(ap, "inteldrmfb", primary); 787 788 kfree(ap); 789 790 return ret; 791 } 792 #endif 793 794 #if !defined(CONFIG_VGA_CONSOLE) 795 static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv) 796 { 797 return 0; 798 } 799 #elif !defined(CONFIG_DUMMY_CONSOLE) 800 static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv) 801 { 802 return -ENODEV; 803 } 804 #else 805 static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv) 806 { 807 int ret = 0; 808 809 DRM_INFO("Replacing VGA console driver\n"); 810 811 console_lock(); 812 if (con_is_bound(&vga_con)) 813 ret = do_take_over_console(&dummy_con, 0, MAX_NR_CONSOLES - 1, 1); 814 if (ret == 0) { 815 ret = do_unregister_con_driver(&vga_con); 816 817 /* Ignore "already unregistered". */ 818 if (ret == -ENODEV) 819 ret = 0; 820 } 821 console_unlock(); 822 823 return ret; 824 } 825 #endif 826 827 static void intel_init_dpio(struct drm_i915_private *dev_priv) 828 { 829 /* 830 * IOSF_PORT_DPIO is used for VLV x2 PHY (DP/HDMI B and C), 831 * CHV x1 PHY (DP/HDMI D) 832 * IOSF_PORT_DPIO_2 is used for CHV x2 PHY (DP/HDMI B and C) 833 */ 834 if (IS_CHERRYVIEW(dev_priv)) { 835 DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO_2; 836 DPIO_PHY_IOSF_PORT(DPIO_PHY1) = IOSF_PORT_DPIO; 837 } else if (IS_VALLEYVIEW(dev_priv)) { 838 DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO; 839 } 840 } 841 842 static int i915_workqueues_init(struct drm_i915_private *dev_priv) 843 { 844 /* 845 * The i915 workqueue is primarily used for batched retirement of 846 * requests (and thus managing bo) once the task has been completed 847 * by the GPU. i915_gem_retire_requests() is called directly when we 848 * need high-priority retirement, such as waiting for an explicit 849 * bo. 850 * 851 * It is also used for periodic low-priority events, such as 852 * idle-timers and recording error state. 853 * 854 * All tasks on the workqueue are expected to acquire the dev mutex 855 * so there is no point in running more than one instance of the 856 * workqueue at any time. Use an ordered one. 857 */ 858 dev_priv->wq = alloc_ordered_workqueue("i915", 0); 859 if (dev_priv->wq == NULL) 860 goto out_err; 861 862 dev_priv->hotplug.dp_wq = alloc_ordered_workqueue("i915-dp", 0); 863 if (dev_priv->hotplug.dp_wq == NULL) 864 goto out_free_wq; 865 866 return 0; 867 868 out_free_wq: 869 destroy_workqueue(dev_priv->wq); 870 out_err: 871 DRM_ERROR("Failed to allocate workqueues.\n"); 872 873 return -ENOMEM; 874 } 875 876 static void i915_engines_cleanup(struct drm_i915_private *i915) 877 { 878 struct intel_engine_cs *engine; 879 enum intel_engine_id id; 880 881 for_each_engine(engine, i915, id) 882 kfree(engine); 883 } 884 885 static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv) 886 { 887 destroy_workqueue(dev_priv->hotplug.dp_wq); 888 destroy_workqueue(dev_priv->wq); 889 } 890 891 /* 892 * We don't keep the workarounds for pre-production hardware, so we expect our 893 * driver to fail on these machines in one way or another. A little warning on 894 * dmesg may help both the user and the bug triagers. 895 */ 896 static void intel_detect_preproduction_hw(struct drm_i915_private *dev_priv) 897 { 898 bool pre = false; 899 900 pre |= IS_HSW_EARLY_SDV(dev_priv); 901 pre |= IS_SKL_REVID(dev_priv, 0, SKL_REVID_F0); 902 pre |= IS_BXT_REVID(dev_priv, 0, BXT_REVID_B_LAST); 903 904 if (pre) { 905 DRM_ERROR("This is a pre-production stepping. " 906 "It may not be fully functional.\n"); 907 add_taint(TAINT_MACHINE_CHECK, LOCKDEP_STILL_OK); 908 } 909 } 910 911 /** 912 * i915_driver_init_early - setup state not requiring device access 913 * @dev_priv: device private 914 * 915 * Initialize everything that is a "SW-only" state, that is state not 916 * requiring accessing the device or exposing the driver via kernel internal 917 * or userspace interfaces. Example steps belonging here: lock initialization, 918 * system memory allocation, setting up device specific attributes and 919 * function hooks not requiring accessing the device. 920 */ 921 static int i915_driver_init_early(struct drm_i915_private *dev_priv, 922 const struct pci_device_id *ent) 923 { 924 const struct intel_device_info *match_info = 925 (struct intel_device_info *)ent->driver_data; 926 struct intel_device_info *device_info; 927 int ret = 0; 928 929 if (i915_inject_load_failure()) 930 return -ENODEV; 931 932 /* Setup the write-once "constant" device info */ 933 device_info = mkwrite_device_info(dev_priv); 934 memcpy(device_info, match_info, sizeof(*device_info)); 935 device_info->device_id = dev_priv->drm.pdev->device; 936 937 BUILD_BUG_ON(INTEL_MAX_PLATFORMS > 938 sizeof(device_info->platform_mask) * BITS_PER_BYTE); 939 device_info->platform_mask = BIT(device_info->platform); 940 941 BUG_ON(device_info->gen > sizeof(device_info->gen_mask) * BITS_PER_BYTE); 942 device_info->gen_mask = BIT(device_info->gen - 1); 943 944 lockinit(&dev_priv->irq_lock, "userirq", 0, 0); 945 lockinit(&dev_priv->gpu_error.lock, "915err", 0, 0); 946 lockinit(&dev_priv->backlight_lock, "i915bl", 0, LK_CANRECURSE); 947 lockinit(&dev_priv->uncore.lock, "915gt", 0, 0); 948 949 lockinit(&dev_priv->mm.object_stat_lock, "i915osl", 0, 0); 950 lockinit(&dev_priv->sb_lock, "i915sbl", 0, LK_CANRECURSE); 951 lockinit(&dev_priv->modeset_restore_lock, "i915mrl", 0, LK_CANRECURSE); 952 lockinit(&dev_priv->av_mutex, "i915am", 0, LK_CANRECURSE); 953 lockinit(&dev_priv->wm.wm_mutex, "i915wm", 0, LK_CANRECURSE); 954 lockinit(&dev_priv->pps_mutex, "i915pm", 0, LK_CANRECURSE); 955 956 intel_uc_init_early(dev_priv); 957 i915_memcpy_init_early(dev_priv); 958 959 ret = i915_workqueues_init(dev_priv); 960 if (ret < 0) 961 goto err_engines; 962 963 /* This must be called before any calls to HAS_PCH_* */ 964 intel_detect_pch(dev_priv); 965 966 intel_pm_setup(dev_priv); 967 intel_init_dpio(dev_priv); 968 intel_power_domains_init(dev_priv); 969 intel_irq_init(dev_priv); 970 intel_hangcheck_init(dev_priv); 971 intel_init_display_hooks(dev_priv); 972 intel_init_clock_gating_hooks(dev_priv); 973 intel_init_audio_hooks(dev_priv); 974 ret = i915_gem_load_init(dev_priv); 975 if (ret < 0) 976 goto err_irq; 977 978 intel_display_crc_init(dev_priv); 979 980 intel_device_info_dump(dev_priv); 981 982 intel_detect_preproduction_hw(dev_priv); 983 984 i915_perf_init(dev_priv); 985 986 return 0; 987 988 err_irq: 989 intel_irq_fini(dev_priv); 990 i915_workqueues_cleanup(dev_priv); 991 err_engines: 992 i915_engines_cleanup(dev_priv); 993 return ret; 994 } 995 996 /** 997 * i915_driver_cleanup_early - cleanup the setup done in i915_driver_init_early() 998 * @dev_priv: device private 999 */ 1000 static void i915_driver_cleanup_early(struct drm_i915_private *dev_priv) 1001 { 1002 i915_perf_fini(dev_priv); 1003 i915_gem_load_cleanup(dev_priv); 1004 intel_irq_fini(dev_priv); 1005 i915_workqueues_cleanup(dev_priv); 1006 i915_engines_cleanup(dev_priv); 1007 } 1008 1009 static int i915_mmio_setup(struct drm_i915_private *dev_priv) 1010 { 1011 struct pci_dev *pdev = dev_priv->drm.pdev; 1012 int mmio_bar; 1013 int mmio_size; 1014 1015 mmio_bar = IS_GEN2(dev_priv) ? 1 : 0; 1016 /* 1017 * Before gen4, the registers and the GTT are behind different BARs. 1018 * However, from gen4 onwards, the registers and the GTT are shared 1019 * in the same BAR, so we want to restrict this ioremap from 1020 * clobbering the GTT which we want ioremap_wc instead. Fortunately, 1021 * the register BAR remains the same size for all the earlier 1022 * generations up to Ironlake. 1023 */ 1024 if (INTEL_GEN(dev_priv) < 5) 1025 mmio_size = 512 * 1024; 1026 else 1027 mmio_size = 2 * 1024 * 1024; 1028 dev_priv->regs = pci_iomap(pdev, mmio_bar, mmio_size); 1029 if (dev_priv->regs == NULL) { 1030 DRM_ERROR("failed to map registers\n"); 1031 1032 return -EIO; 1033 } 1034 1035 /* Try to make sure MCHBAR is enabled before poking at it */ 1036 intel_setup_mchbar(dev_priv); 1037 1038 return 0; 1039 } 1040 1041 static void i915_mmio_cleanup(struct drm_i915_private *dev_priv) 1042 { 1043 #if 0 1044 struct pci_dev *pdev = dev_priv->drm.pdev; 1045 #endif 1046 1047 intel_teardown_mchbar(dev_priv); 1048 #if 0 1049 pci_iounmap(pdev, dev_priv->regs); 1050 #endif 1051 } 1052 1053 /** 1054 * i915_driver_init_mmio - setup device MMIO 1055 * @dev_priv: device private 1056 * 1057 * Setup minimal device state necessary for MMIO accesses later in the 1058 * initialization sequence. The setup here should avoid any other device-wide 1059 * side effects or exposing the driver via kernel internal or user space 1060 * interfaces. 1061 */ 1062 static int i915_driver_init_mmio(struct drm_i915_private *dev_priv) 1063 { 1064 int ret; 1065 1066 if (i915_inject_load_failure()) 1067 return -ENODEV; 1068 1069 if (i915_get_bridge_dev(dev_priv)) 1070 return -EIO; 1071 1072 ret = i915_mmio_setup(dev_priv); 1073 if (ret < 0) 1074 goto err_bridge; 1075 1076 intel_uncore_init(dev_priv); 1077 1078 intel_uc_init_mmio(dev_priv); 1079 1080 ret = intel_engines_init_mmio(dev_priv); 1081 if (ret) 1082 goto err_uncore; 1083 1084 i915_gem_init_mmio(dev_priv); 1085 1086 return 0; 1087 1088 err_uncore: 1089 intel_uncore_fini(dev_priv); 1090 err_bridge: 1091 pci_dev_put(dev_priv->bridge_dev); 1092 1093 return ret; 1094 } 1095 1096 /** 1097 * i915_driver_cleanup_mmio - cleanup the setup done in i915_driver_init_mmio() 1098 * @dev_priv: device private 1099 */ 1100 static void i915_driver_cleanup_mmio(struct drm_i915_private *dev_priv) 1101 { 1102 intel_uncore_fini(dev_priv); 1103 i915_mmio_cleanup(dev_priv); 1104 pci_dev_put(dev_priv->bridge_dev); 1105 } 1106 1107 static void intel_sanitize_options(struct drm_i915_private *dev_priv) 1108 { 1109 i915_modparams.enable_execlists = 1110 intel_sanitize_enable_execlists(dev_priv, 1111 i915_modparams.enable_execlists); 1112 1113 /* 1114 * i915.enable_ppgtt is read-only, so do an early pass to validate the 1115 * user's requested state against the hardware/driver capabilities. We 1116 * do this now so that we can print out any log messages once rather 1117 * than every time we check intel_enable_ppgtt(). 1118 */ 1119 i915_modparams.enable_ppgtt = 1120 intel_sanitize_enable_ppgtt(dev_priv, 1121 i915_modparams.enable_ppgtt); 1122 DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915_modparams.enable_ppgtt); 1123 1124 i915_modparams.semaphores = 1125 intel_sanitize_semaphores(dev_priv, i915_modparams.semaphores); 1126 DRM_DEBUG_DRIVER("use GPU semaphores? %s\n", 1127 yesno(i915_modparams.semaphores)); 1128 1129 intel_uc_sanitize_options(dev_priv); 1130 1131 intel_gvt_sanitize_options(dev_priv); 1132 } 1133 1134 /** 1135 * i915_driver_init_hw - setup state requiring device access 1136 * @dev_priv: device private 1137 * 1138 * Setup state that requires accessing the device, but doesn't require 1139 * exposing the driver via kernel internal or userspace interfaces. 1140 */ 1141 static int i915_driver_init_hw(struct drm_i915_private *dev_priv) 1142 { 1143 struct pci_dev *pdev = dev_priv->drm.pdev; 1144 int ret; 1145 1146 if (i915_inject_load_failure()) 1147 return -ENODEV; 1148 1149 intel_device_info_runtime_init(dev_priv); 1150 1151 intel_sanitize_options(dev_priv); 1152 1153 ret = i915_ggtt_probe_hw(dev_priv); 1154 if (ret) 1155 return ret; 1156 1157 /* WARNING: Apparently we must kick fbdev drivers before vgacon, 1158 * otherwise the vga fbdev driver falls over. */ 1159 ret = i915_kick_out_firmware_fb(dev_priv); 1160 if (ret) { 1161 DRM_ERROR("failed to remove conflicting framebuffer drivers\n"); 1162 goto out_ggtt; 1163 } 1164 1165 ret = i915_kick_out_vgacon(dev_priv); 1166 if (ret) { 1167 DRM_ERROR("failed to remove conflicting VGA console\n"); 1168 goto out_ggtt; 1169 } 1170 1171 ret = i915_ggtt_init_hw(dev_priv); 1172 if (ret) 1173 return ret; 1174 1175 ret = i915_ggtt_enable_hw(dev_priv); 1176 if (ret) { 1177 DRM_ERROR("failed to enable GGTT\n"); 1178 goto out_ggtt; 1179 } 1180 1181 pci_set_master(pdev); 1182 1183 #if 0 1184 /* overlay on gen2 is broken and can't address above 1G */ 1185 if (IS_GEN2(dev_priv)) { 1186 ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(30)); 1187 if (ret) { 1188 DRM_ERROR("failed to set DMA mask\n"); 1189 1190 goto out_ggtt; 1191 } 1192 } 1193 1194 /* 965GM sometimes incorrectly writes to hardware status page (HWS) 1195 * using 32bit addressing, overwriting memory if HWS is located 1196 * above 4GB. 1197 * 1198 * The documentation also mentions an issue with undefined 1199 * behaviour if any general state is accessed within a page above 4GB, 1200 * which also needs to be handled carefully. 1201 */ 1202 if (IS_I965G(dev_priv) || IS_I965GM(dev_priv)) { 1203 ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32)); 1204 1205 if (ret) { 1206 DRM_ERROR("failed to set DMA mask\n"); 1207 1208 goto out_ggtt; 1209 } 1210 } 1211 #endif 1212 1213 pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, 1214 PM_QOS_DEFAULT_VALUE); 1215 1216 intel_uncore_sanitize(dev_priv); 1217 1218 intel_opregion_setup(dev_priv); 1219 1220 i915_gem_load_init_fences(dev_priv); 1221 1222 /* On the 945G/GM, the chipset reports the MSI capability on the 1223 * integrated graphics even though the support isn't actually there 1224 * according to the published specs. It doesn't appear to function 1225 * correctly in testing on 945G. 1226 * This may be a side effect of MSI having been made available for PEG 1227 * and the registers being closely associated. 1228 * 1229 * According to chipset errata, on the 965GM, MSI interrupts may 1230 * be lost or delayed, and was defeatured. MSI interrupts seem to 1231 * get lost on g4x as well, and interrupt delivery seems to stay 1232 * properly dead afterwards. So we'll just disable them for all 1233 * pre-gen5 chipsets. 1234 */ 1235 #if 0 1236 if (INTEL_GEN(dev_priv) >= 5) { 1237 if (pci_enable_msi(pdev) < 0) 1238 DRM_DEBUG_DRIVER("can't enable MSI"); 1239 } 1240 #endif 1241 1242 ret = intel_gvt_init(dev_priv); 1243 if (ret) 1244 goto out_ggtt; 1245 1246 return 0; 1247 1248 out_ggtt: 1249 i915_ggtt_cleanup_hw(dev_priv); 1250 1251 return ret; 1252 } 1253 1254 /** 1255 * i915_driver_cleanup_hw - cleanup the setup done in i915_driver_init_hw() 1256 * @dev_priv: device private 1257 */ 1258 static void i915_driver_cleanup_hw(struct drm_i915_private *dev_priv) 1259 { 1260 #if 0 1261 struct pci_dev *pdev = dev_priv->drm.pdev; 1262 1263 if (pdev->msi_enabled) 1264 pci_disable_msi(pdev); 1265 #endif 1266 1267 pm_qos_remove_request(&dev_priv->pm_qos); 1268 i915_ggtt_cleanup_hw(dev_priv); 1269 } 1270 1271 /** 1272 * i915_driver_register - register the driver with the rest of the system 1273 * @dev_priv: device private 1274 * 1275 * Perform any steps necessary to make the driver available via kernel 1276 * internal or userspace interfaces. 1277 */ 1278 static void i915_driver_register(struct drm_i915_private *dev_priv) 1279 { 1280 struct drm_device *dev = &dev_priv->drm; 1281 1282 i915_gem_shrinker_init(dev_priv); 1283 1284 /* 1285 * Notify a valid surface after modesetting, 1286 * when running inside a VM. 1287 */ 1288 if (intel_vgpu_active(dev_priv)) 1289 I915_WRITE(vgtif_reg(display_ready), VGT_DRV_DISPLAY_READY); 1290 1291 /* Reveal our presence to userspace */ 1292 if (drm_dev_register(dev, 0) == 0) { 1293 i915_debugfs_register(dev_priv); 1294 i915_guc_log_register(dev_priv); 1295 i915_setup_sysfs(dev_priv); 1296 1297 /* Depends on sysfs having been initialized */ 1298 i915_perf_register(dev_priv); 1299 } else 1300 DRM_ERROR("Failed to register driver for userspace access!\n"); 1301 1302 if (INTEL_INFO(dev_priv)->num_pipes) { 1303 /* Must be done after probing outputs */ 1304 intel_opregion_register(dev_priv); 1305 acpi_video_register(); 1306 } 1307 1308 if (IS_GEN5(dev_priv)) 1309 intel_gpu_ips_init(dev_priv); 1310 1311 intel_audio_init(dev_priv); 1312 1313 /* 1314 * Some ports require correctly set-up hpd registers for detection to 1315 * work properly (leading to ghost connected connector status), e.g. VGA 1316 * on gm45. Hence we can only set up the initial fbdev config after hpd 1317 * irqs are fully enabled. We do it last so that the async config 1318 * cannot run before the connectors are registered. 1319 */ 1320 intel_fbdev_initial_config_async(dev); 1321 } 1322 1323 /** 1324 * i915_driver_unregister - cleanup the registration done in i915_driver_regiser() 1325 * @dev_priv: device private 1326 */ 1327 static void i915_driver_unregister(struct drm_i915_private *dev_priv) 1328 { 1329 intel_fbdev_unregister(dev_priv); 1330 intel_audio_deinit(dev_priv); 1331 1332 intel_gpu_ips_teardown(); 1333 acpi_video_unregister(); 1334 intel_opregion_unregister(dev_priv); 1335 1336 i915_perf_unregister(dev_priv); 1337 1338 i915_teardown_sysfs(dev_priv); 1339 i915_guc_log_unregister(dev_priv); 1340 drm_dev_unregister(&dev_priv->drm); 1341 1342 i915_gem_shrinker_cleanup(dev_priv); 1343 } 1344 1345 /** 1346 * i915_driver_load - setup chip and create an initial config 1347 * @pdev: PCI device 1348 * @ent: matching PCI ID entry 1349 * 1350 * The driver load routine has to do several things: 1351 * - drive output discovery via intel_modeset_init() 1352 * - initialize the memory manager 1353 * - allocate initial config memory 1354 * - setup the DRM framebuffer with the allocated memory 1355 */ 1356 int i915_driver_load(struct pci_dev *pdev, const struct pci_device_id *ent) 1357 { 1358 const struct intel_device_info *match_info = 1359 (struct intel_device_info *)ent->driver_data; 1360 struct drm_i915_private *dev_priv; 1361 int ret; 1362 1363 /* Enable nuclear pageflip on ILK+ */ 1364 if (!i915_modparams.nuclear_pageflip && match_info->gen < 5) 1365 driver.driver_features &= ~DRIVER_ATOMIC; 1366 1367 ret = -ENOMEM; 1368 dev_priv = kzalloc(sizeof(*dev_priv), GFP_KERNEL); 1369 if (dev_priv) 1370 ret = drm_dev_init(&dev_priv->drm, &driver, &pdev->dev); 1371 if (ret) { 1372 DRM_DEV_ERROR(&pdev->dev, "allocation failed\n"); 1373 goto out_free; 1374 } 1375 1376 dev_priv->drm.pdev = pdev; 1377 dev_priv->drm.dev_private = dev_priv; 1378 1379 #if 0 1380 ret = pci_enable_device(pdev); 1381 if (ret) 1382 goto out_fini; 1383 #endif 1384 1385 pci_set_drvdata(pdev, &dev_priv->drm); 1386 /* 1387 * Disable the system suspend direct complete optimization, which can 1388 * leave the device suspended skipping the driver's suspend handlers 1389 * if the device was already runtime suspended. This is needed due to 1390 * the difference in our runtime and system suspend sequence and 1391 * becaue the HDA driver may require us to enable the audio power 1392 * domain during system suspend. 1393 */ 1394 dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_NEVER_SKIP); 1395 1396 ret = i915_driver_init_early(dev_priv, ent); 1397 if (ret < 0) 1398 goto out_pci_disable; 1399 1400 intel_runtime_pm_get(dev_priv); 1401 1402 ret = i915_driver_init_mmio(dev_priv); 1403 if (ret < 0) 1404 goto out_runtime_pm_put; 1405 1406 ret = i915_driver_init_hw(dev_priv); 1407 if (ret < 0) 1408 goto out_cleanup_mmio; 1409 1410 /* 1411 * TODO: move the vblank init and parts of modeset init steps into one 1412 * of the i915_driver_init_/i915_driver_register functions according 1413 * to the role/effect of the given init step. 1414 */ 1415 if (INTEL_INFO(dev_priv)->num_pipes) { 1416 ret = drm_vblank_init(&dev_priv->drm, 1417 INTEL_INFO(dev_priv)->num_pipes); 1418 if (ret) 1419 goto out_cleanup_hw; 1420 } 1421 1422 ret = i915_load_modeset_init(&dev_priv->drm); 1423 if (ret < 0) 1424 goto out_cleanup_hw; 1425 1426 i915_driver_register(dev_priv); 1427 1428 intel_runtime_pm_enable(dev_priv); 1429 1430 intel_init_ipc(dev_priv); 1431 1432 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG)) 1433 DRM_INFO("DRM_I915_DEBUG enabled\n"); 1434 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) 1435 DRM_INFO("DRM_I915_DEBUG_GEM enabled\n"); 1436 1437 intel_runtime_pm_put(dev_priv); 1438 1439 return 0; 1440 1441 out_cleanup_hw: 1442 i915_driver_cleanup_hw(dev_priv); 1443 out_cleanup_mmio: 1444 i915_driver_cleanup_mmio(dev_priv); 1445 out_runtime_pm_put: 1446 intel_runtime_pm_put(dev_priv); 1447 i915_driver_cleanup_early(dev_priv); 1448 out_pci_disable: 1449 #if 0 1450 pci_disable_device(pdev); 1451 out_fini: 1452 #endif 1453 i915_load_error(dev_priv, "Device initialization failed (%d)\n", ret); 1454 drm_dev_fini(&dev_priv->drm); 1455 out_free: 1456 kfree(dev_priv); 1457 return ret; 1458 } 1459 1460 void i915_driver_unload(struct drm_device *dev) 1461 { 1462 struct drm_i915_private *dev_priv = to_i915(dev); 1463 1464 i915_driver_unregister(dev_priv); 1465 1466 if (i915_gem_suspend(dev_priv)) 1467 DRM_ERROR("failed to idle hardware; continuing to unload!\n"); 1468 1469 intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); 1470 1471 drm_atomic_helper_shutdown(dev); 1472 1473 intel_gvt_cleanup(dev_priv); 1474 1475 intel_modeset_cleanup(dev); 1476 1477 /* 1478 * free the memory space allocated for the child device 1479 * config parsed from VBT 1480 */ 1481 if (dev_priv->vbt.child_dev && dev_priv->vbt.child_dev_num) { 1482 kfree(dev_priv->vbt.child_dev); 1483 dev_priv->vbt.child_dev = NULL; 1484 dev_priv->vbt.child_dev_num = 0; 1485 } 1486 kfree(dev_priv->vbt.sdvo_lvds_vbt_mode); 1487 dev_priv->vbt.sdvo_lvds_vbt_mode = NULL; 1488 kfree(dev_priv->vbt.lfp_lvds_vbt_mode); 1489 dev_priv->vbt.lfp_lvds_vbt_mode = NULL; 1490 1491 #if 0 1492 vga_switcheroo_unregister_client(pdev); 1493 vga_client_register(pdev, NULL, NULL, NULL); 1494 #endif 1495 1496 intel_csr_ucode_fini(dev_priv); 1497 1498 /* Free error state after interrupts are fully disabled. */ 1499 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work); 1500 i915_reset_error_state(dev_priv); 1501 1502 i915_gem_fini(dev_priv); 1503 intel_uc_fini_fw(dev_priv); 1504 intel_fbc_cleanup_cfb(dev_priv); 1505 1506 intel_power_domains_fini(dev_priv); 1507 1508 i915_driver_cleanup_hw(dev_priv); 1509 i915_driver_cleanup_mmio(dev_priv); 1510 1511 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT); 1512 } 1513 1514 static void i915_driver_release(struct drm_device *dev) 1515 { 1516 struct drm_i915_private *dev_priv = to_i915(dev); 1517 1518 i915_driver_cleanup_early(dev_priv); 1519 drm_dev_fini(&dev_priv->drm); 1520 1521 kfree(dev_priv); 1522 } 1523 1524 static int i915_driver_open(struct drm_device *dev, struct drm_file *file) 1525 { 1526 struct drm_i915_private *i915 = to_i915(dev); 1527 int ret; 1528 1529 ret = i915_gem_open(i915, file); 1530 if (ret) 1531 return ret; 1532 1533 return 0; 1534 } 1535 1536 /** 1537 * i915_driver_lastclose - clean up after all DRM clients have exited 1538 * @dev: DRM device 1539 * 1540 * Take care of cleaning up after all DRM clients have exited. In the 1541 * mode setting case, we want to restore the kernel's initial mode (just 1542 * in case the last client left us in a bad state). 1543 * 1544 * Additionally, in the non-mode setting case, we'll tear down the GTT 1545 * and DMA structures, since the kernel won't be using them, and clea 1546 * up any GEM state. 1547 */ 1548 static void i915_driver_lastclose(struct drm_device *dev) 1549 { 1550 intel_fbdev_restore_mode(dev); 1551 #if 0 1552 vga_switcheroo_process_delayed_switch(); 1553 #endif 1554 } 1555 1556 static void i915_driver_postclose(struct drm_device *dev, struct drm_file *file) 1557 { 1558 struct drm_i915_file_private *file_priv = file->driver_priv; 1559 1560 mutex_lock(&dev->struct_mutex); 1561 i915_gem_context_close(file); 1562 i915_gem_release(dev, file); 1563 mutex_unlock(&dev->struct_mutex); 1564 1565 kfree(file_priv); 1566 } 1567 1568 #if 0 1569 static void intel_suspend_encoders(struct drm_i915_private *dev_priv) 1570 { 1571 struct drm_device *dev = &dev_priv->drm; 1572 struct intel_encoder *encoder; 1573 1574 drm_modeset_lock_all(dev); 1575 for_each_intel_encoder(dev, encoder) 1576 if (encoder->suspend) 1577 encoder->suspend(encoder); 1578 drm_modeset_unlock_all(dev); 1579 } 1580 1581 static int vlv_resume_prepare(struct drm_i915_private *dev_priv, 1582 bool rpm_resume); 1583 static int vlv_suspend_complete(struct drm_i915_private *dev_priv); 1584 1585 static bool suspend_to_idle(struct drm_i915_private *dev_priv) 1586 { 1587 #if IS_ENABLED(CONFIG_ACPI_SLEEP) 1588 if (acpi_target_system_state() < ACPI_STATE_S3) 1589 return true; 1590 #endif 1591 return false; 1592 } 1593 1594 static int i915_drm_suspend(struct drm_device *dev) 1595 { 1596 struct drm_i915_private *dev_priv = to_i915(dev); 1597 struct pci_dev *pdev = dev_priv->drm.pdev; 1598 pci_power_t opregion_target_state; 1599 int error; 1600 1601 /* ignore lid events during suspend */ 1602 mutex_lock(&dev_priv->modeset_restore_lock); 1603 dev_priv->modeset_restore = MODESET_SUSPENDED; 1604 mutex_unlock(&dev_priv->modeset_restore_lock); 1605 1606 disable_rpm_wakeref_asserts(dev_priv); 1607 1608 /* We do a lot of poking in a lot of registers, make sure they work 1609 * properly. */ 1610 intel_display_set_init_power(dev_priv, true); 1611 1612 drm_kms_helper_poll_disable(dev); 1613 1614 pci_save_state(pdev); 1615 1616 error = i915_gem_suspend(dev_priv); 1617 if (error) { 1618 dev_err(&pdev->dev, 1619 "GEM idle failed, resume might fail\n"); 1620 goto out; 1621 } 1622 1623 intel_display_suspend(dev); 1624 1625 intel_dp_mst_suspend(dev); 1626 1627 intel_runtime_pm_disable_interrupts(dev_priv); 1628 intel_hpd_cancel_work(dev_priv); 1629 1630 intel_suspend_encoders(dev_priv); 1631 1632 intel_suspend_hw(dev_priv); 1633 1634 i915_gem_suspend_gtt_mappings(dev_priv); 1635 1636 i915_save_state(dev_priv); 1637 1638 opregion_target_state = suspend_to_idle(dev_priv) ? PCI_D1 : PCI_D3cold; 1639 intel_opregion_notify_adapter(dev_priv, opregion_target_state); 1640 1641 intel_uncore_suspend(dev_priv); 1642 intel_opregion_unregister(dev_priv); 1643 1644 intel_fbdev_set_suspend(dev, FBINFO_STATE_SUSPENDED, true); 1645 1646 dev_priv->suspend_count++; 1647 1648 intel_csr_ucode_suspend(dev_priv); 1649 1650 out: 1651 enable_rpm_wakeref_asserts(dev_priv); 1652 1653 return error; 1654 } 1655 1656 static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation) 1657 { 1658 struct drm_i915_private *dev_priv = to_i915(dev); 1659 struct pci_dev *pdev = dev_priv->drm.pdev; 1660 bool fw_csr; 1661 int ret; 1662 1663 disable_rpm_wakeref_asserts(dev_priv); 1664 1665 intel_display_set_init_power(dev_priv, false); 1666 1667 fw_csr = !IS_GEN9_LP(dev_priv) && !hibernation && 1668 suspend_to_idle(dev_priv) && dev_priv->csr.dmc_payload; 1669 /* 1670 * In case of firmware assisted context save/restore don't manually 1671 * deinit the power domains. This also means the CSR/DMC firmware will 1672 * stay active, it will power down any HW resources as required and 1673 * also enable deeper system power states that would be blocked if the 1674 * firmware was inactive. 1675 */ 1676 if (!fw_csr) 1677 intel_power_domains_suspend(dev_priv); 1678 1679 ret = 0; 1680 if (IS_GEN9_LP(dev_priv)) 1681 bxt_enable_dc9(dev_priv); 1682 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) 1683 hsw_enable_pc8(dev_priv); 1684 else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) 1685 ret = vlv_suspend_complete(dev_priv); 1686 1687 if (ret) { 1688 DRM_ERROR("Suspend complete failed: %d\n", ret); 1689 if (!fw_csr) 1690 intel_power_domains_init_hw(dev_priv, true); 1691 1692 goto out; 1693 } 1694 1695 pci_disable_device(pdev); 1696 /* 1697 * During hibernation on some platforms the BIOS may try to access 1698 * the device even though it's already in D3 and hang the machine. So 1699 * leave the device in D0 on those platforms and hope the BIOS will 1700 * power down the device properly. The issue was seen on multiple old 1701 * GENs with different BIOS vendors, so having an explicit blacklist 1702 * is inpractical; apply the workaround on everything pre GEN6. The 1703 * platforms where the issue was seen: 1704 * Lenovo Thinkpad X301, X61s, X60, T60, X41 1705 * Fujitsu FSC S7110 1706 * Acer Aspire 1830T 1707 */ 1708 if (!(hibernation && INTEL_GEN(dev_priv) < 6)) 1709 pci_set_power_state(pdev, PCI_D3hot); 1710 1711 dev_priv->suspended_to_idle = suspend_to_idle(dev_priv); 1712 1713 out: 1714 enable_rpm_wakeref_asserts(dev_priv); 1715 1716 return ret; 1717 } 1718 1719 static int i915_suspend_switcheroo(struct drm_device *dev, pm_message_t state) 1720 { 1721 int error; 1722 1723 if (!dev) { 1724 DRM_ERROR("dev: %p\n", dev); 1725 DRM_ERROR("DRM not initialized, aborting suspend.\n"); 1726 return -ENODEV; 1727 } 1728 1729 if (WARN_ON_ONCE(state.event != PM_EVENT_SUSPEND && 1730 state.event != PM_EVENT_FREEZE)) 1731 return -EINVAL; 1732 1733 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 1734 return 0; 1735 1736 error = i915_drm_suspend(dev); 1737 if (error) 1738 return error; 1739 1740 return i915_drm_suspend_late(dev, false); 1741 } 1742 1743 static int i915_drm_resume(struct drm_device *dev) 1744 { 1745 struct drm_i915_private *dev_priv = to_i915(dev); 1746 int ret; 1747 1748 disable_rpm_wakeref_asserts(dev_priv); 1749 intel_sanitize_gt_powersave(dev_priv); 1750 1751 ret = i915_ggtt_enable_hw(dev_priv); 1752 if (ret) 1753 DRM_ERROR("failed to re-enable GGTT\n"); 1754 1755 intel_csr_ucode_resume(dev_priv); 1756 1757 i915_gem_resume(dev_priv); 1758 1759 i915_restore_state(dev_priv); 1760 intel_pps_unlock_regs_wa(dev_priv); 1761 intel_opregion_setup(dev_priv); 1762 1763 intel_init_pch_refclk(dev_priv); 1764 1765 /* 1766 * Interrupts have to be enabled before any batches are run. If not the 1767 * GPU will hang. i915_gem_init_hw() will initiate batches to 1768 * update/restore the context. 1769 * 1770 * drm_mode_config_reset() needs AUX interrupts. 1771 * 1772 * Modeset enabling in intel_modeset_init_hw() also needs working 1773 * interrupts. 1774 */ 1775 intel_runtime_pm_enable_interrupts(dev_priv); 1776 1777 drm_mode_config_reset(dev); 1778 1779 mutex_lock(&dev->struct_mutex); 1780 if (i915_gem_init_hw(dev_priv)) { 1781 DRM_ERROR("failed to re-initialize GPU, declaring wedged!\n"); 1782 i915_gem_set_wedged(dev_priv); 1783 } 1784 mutex_unlock(&dev->struct_mutex); 1785 1786 intel_guc_resume(dev_priv); 1787 1788 intel_modeset_init_hw(dev); 1789 intel_init_clock_gating(dev_priv); 1790 1791 spin_lock_irq(&dev_priv->irq_lock); 1792 if (dev_priv->display.hpd_irq_setup) 1793 dev_priv->display.hpd_irq_setup(dev_priv); 1794 spin_unlock_irq(&dev_priv->irq_lock); 1795 1796 intel_dp_mst_resume(dev); 1797 1798 intel_display_resume(dev); 1799 1800 drm_kms_helper_poll_enable(dev); 1801 1802 /* 1803 * ... but also need to make sure that hotplug processing 1804 * doesn't cause havoc. Like in the driver load code we don't 1805 * bother with the tiny race here where we might loose hotplug 1806 * notifications. 1807 * */ 1808 intel_hpd_init(dev_priv); 1809 1810 intel_opregion_register(dev_priv); 1811 1812 intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING, false); 1813 1814 mutex_lock(&dev_priv->modeset_restore_lock); 1815 dev_priv->modeset_restore = MODESET_DONE; 1816 mutex_unlock(&dev_priv->modeset_restore_lock); 1817 1818 intel_opregion_notify_adapter(dev_priv, PCI_D0); 1819 1820 intel_autoenable_gt_powersave(dev_priv); 1821 1822 enable_rpm_wakeref_asserts(dev_priv); 1823 1824 return 0; 1825 } 1826 1827 static int i915_drm_resume_early(struct drm_device *dev) 1828 { 1829 struct drm_i915_private *dev_priv = to_i915(dev); 1830 struct pci_dev *pdev = dev_priv->drm.pdev; 1831 int ret; 1832 1833 /* 1834 * We have a resume ordering issue with the snd-hda driver also 1835 * requiring our device to be power up. Due to the lack of a 1836 * parent/child relationship we currently solve this with an early 1837 * resume hook. 1838 * 1839 * FIXME: This should be solved with a special hdmi sink device or 1840 * similar so that power domains can be employed. 1841 */ 1842 1843 /* 1844 * Note that we need to set the power state explicitly, since we 1845 * powered off the device during freeze and the PCI core won't power 1846 * it back up for us during thaw. Powering off the device during 1847 * freeze is not a hard requirement though, and during the 1848 * suspend/resume phases the PCI core makes sure we get here with the 1849 * device powered on. So in case we change our freeze logic and keep 1850 * the device powered we can also remove the following set power state 1851 * call. 1852 */ 1853 ret = pci_set_power_state(pdev, PCI_D0); 1854 if (ret) { 1855 DRM_ERROR("failed to set PCI D0 power state (%d)\n", ret); 1856 goto out; 1857 } 1858 1859 /* 1860 * Note that pci_enable_device() first enables any parent bridge 1861 * device and only then sets the power state for this device. The 1862 * bridge enabling is a nop though, since bridge devices are resumed 1863 * first. The order of enabling power and enabling the device is 1864 * imposed by the PCI core as described above, so here we preserve the 1865 * same order for the freeze/thaw phases. 1866 * 1867 * TODO: eventually we should remove pci_disable_device() / 1868 * pci_enable_enable_device() from suspend/resume. Due to how they 1869 * depend on the device enable refcount we can't anyway depend on them 1870 * disabling/enabling the device. 1871 */ 1872 if (pci_enable_device(pdev)) { 1873 ret = -EIO; 1874 goto out; 1875 } 1876 1877 pci_set_master(pdev); 1878 1879 disable_rpm_wakeref_asserts(dev_priv); 1880 1881 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) 1882 ret = vlv_resume_prepare(dev_priv, false); 1883 if (ret) 1884 DRM_ERROR("Resume prepare failed: %d, continuing anyway\n", 1885 ret); 1886 1887 intel_uncore_resume_early(dev_priv); 1888 1889 if (IS_GEN9_LP(dev_priv)) { 1890 if (!dev_priv->suspended_to_idle) 1891 gen9_sanitize_dc_state(dev_priv); 1892 bxt_disable_dc9(dev_priv); 1893 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { 1894 hsw_disable_pc8(dev_priv); 1895 } 1896 1897 intel_uncore_sanitize(dev_priv); 1898 1899 if (IS_GEN9_LP(dev_priv) || 1900 !(dev_priv->suspended_to_idle && dev_priv->csr.dmc_payload)) 1901 intel_power_domains_init_hw(dev_priv, true); 1902 else 1903 intel_display_set_init_power(dev_priv, true); 1904 1905 i915_gem_sanitize(dev_priv); 1906 1907 enable_rpm_wakeref_asserts(dev_priv); 1908 1909 out: 1910 dev_priv->suspended_to_idle = false; 1911 1912 return ret; 1913 } 1914 1915 static int i915_resume_switcheroo(struct drm_device *dev) 1916 { 1917 int ret; 1918 1919 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 1920 return 0; 1921 1922 ret = i915_drm_resume_early(dev); 1923 if (ret) 1924 return ret; 1925 1926 return i915_drm_resume(dev); 1927 } 1928 #endif 1929 1930 /** 1931 * i915_reset - reset chip after a hang 1932 * @i915: #drm_i915_private to reset 1933 * @flags: Instructions 1934 * 1935 * Reset the chip. Useful if a hang is detected. Marks the device as wedged 1936 * on failure. 1937 * 1938 * Caller must hold the struct_mutex. 1939 * 1940 * Procedure is fairly simple: 1941 * - reset the chip using the reset reg 1942 * - re-init context state 1943 * - re-init hardware status page 1944 * - re-init ring buffer 1945 * - re-init interrupt state 1946 * - re-init display 1947 */ 1948 void i915_reset(struct drm_i915_private *i915, unsigned int flags) 1949 { 1950 struct i915_gpu_error *error = &i915->gpu_error; 1951 int ret; 1952 1953 lockdep_assert_held(&i915->drm.struct_mutex); 1954 GEM_BUG_ON(!test_bit(I915_RESET_BACKOFF, &error->flags)); 1955 1956 if (!test_bit(I915_RESET_HANDOFF, &error->flags)) 1957 return; 1958 1959 /* Clear any previous failed attempts at recovery. Time to try again. */ 1960 if (!i915_gem_unset_wedged(i915)) 1961 goto wakeup; 1962 1963 if (!(flags & I915_RESET_QUIET)) 1964 dev_notice(i915->drm.dev, "Resetting chip after gpu hang\n"); 1965 error->reset_count++; 1966 1967 disable_irq(i915->drm.irq); 1968 ret = i915_gem_reset_prepare(i915); 1969 if (ret) { 1970 DRM_ERROR("GPU recovery failed\n"); 1971 intel_gpu_reset(i915, ALL_ENGINES); 1972 goto error; 1973 } 1974 1975 ret = intel_gpu_reset(i915, ALL_ENGINES); 1976 if (ret) { 1977 if (ret != -ENODEV) 1978 DRM_ERROR("Failed to reset chip: %i\n", ret); 1979 else 1980 DRM_DEBUG_DRIVER("GPU reset disabled\n"); 1981 goto error; 1982 } 1983 1984 i915_gem_reset(i915); 1985 intel_overlay_reset(i915); 1986 1987 /* Ok, now get things going again... */ 1988 1989 /* 1990 * Everything depends on having the GTT running, so we need to start 1991 * there. 1992 */ 1993 ret = i915_ggtt_enable_hw(i915); 1994 if (ret) { 1995 DRM_ERROR("Failed to re-enable GGTT following reset %d\n", ret); 1996 goto error; 1997 } 1998 1999 /* 2000 * Next we need to restore the context, but we don't use those 2001 * yet either... 2002 * 2003 * Ring buffer needs to be re-initialized in the KMS case, or if X 2004 * was running at the time of the reset (i.e. we weren't VT 2005 * switched away). 2006 */ 2007 ret = i915_gem_init_hw(i915); 2008 if (ret) { 2009 DRM_ERROR("Failed hw init on reset %d\n", ret); 2010 goto error; 2011 } 2012 2013 i915_queue_hangcheck(i915); 2014 2015 finish: 2016 i915_gem_reset_finish(i915); 2017 enable_irq(i915->drm.irq); 2018 2019 wakeup: 2020 clear_bit(I915_RESET_HANDOFF, &error->flags); 2021 wake_up_bit(&error->flags, I915_RESET_HANDOFF); 2022 return; 2023 2024 error: 2025 i915_gem_set_wedged(i915); 2026 i915_gem_retire_requests(i915); 2027 goto finish; 2028 } 2029 2030 /** 2031 * i915_reset_engine - reset GPU engine to recover from a hang 2032 * @engine: engine to reset 2033 * @flags: options 2034 * 2035 * Reset a specific GPU engine. Useful if a hang is detected. 2036 * Returns zero on successful reset or otherwise an error code. 2037 * 2038 * Procedure is: 2039 * - identifies the request that caused the hang and it is dropped 2040 * - reset engine (which will force the engine to idle) 2041 * - re-init/configure engine 2042 */ 2043 int i915_reset_engine(struct intel_engine_cs *engine, unsigned int flags) 2044 { 2045 struct i915_gpu_error *error = &engine->i915->gpu_error; 2046 struct drm_i915_gem_request *active_request; 2047 int ret; 2048 2049 GEM_BUG_ON(!test_bit(I915_RESET_ENGINE + engine->id, &error->flags)); 2050 2051 if (!(flags & I915_RESET_QUIET)) { 2052 dev_notice(engine->i915->drm.dev, 2053 "Resetting %s after gpu hang\n", engine->name); 2054 } 2055 error->reset_engine_count[engine->id]++; 2056 2057 active_request = i915_gem_reset_prepare_engine(engine); 2058 if (IS_ERR(active_request)) { 2059 DRM_DEBUG_DRIVER("Previous reset failed, promote to full reset\n"); 2060 ret = PTR_ERR(active_request); 2061 goto out; 2062 } 2063 2064 ret = intel_gpu_reset(engine->i915, intel_engine_flag(engine)); 2065 if (ret) { 2066 /* If we fail here, we expect to fallback to a global reset */ 2067 DRM_DEBUG_DRIVER("Failed to reset %s, ret=%d\n", 2068 engine->name, ret); 2069 goto out; 2070 } 2071 2072 /* 2073 * The request that caused the hang is stuck on elsp, we know the 2074 * active request and can drop it, adjust head to skip the offending 2075 * request to resume executing remaining requests in the queue. 2076 */ 2077 i915_gem_reset_engine(engine, active_request); 2078 2079 /* 2080 * The engine and its registers (and workarounds in case of render) 2081 * have been reset to their default values. Follow the init_ring 2082 * process to program RING_MODE, HWSP and re-enable submission. 2083 */ 2084 ret = engine->init_hw(engine); 2085 if (ret) 2086 goto out; 2087 2088 out: 2089 i915_gem_reset_finish_engine(engine); 2090 return ret; 2091 } 2092 2093 #if 0 2094 static int i915_pm_suspend(struct device *kdev) 2095 { 2096 struct pci_dev *pdev = to_pci_dev(kdev); 2097 struct drm_device *dev = pci_get_drvdata(pdev); 2098 2099 if (!dev) { 2100 dev_err(kdev, "DRM not initialized, aborting suspend.\n"); 2101 return -ENODEV; 2102 } 2103 2104 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 2105 return 0; 2106 2107 return i915_drm_suspend(dev); 2108 } 2109 2110 static int i915_pm_suspend_late(struct device *kdev) 2111 { 2112 struct drm_device *dev = &kdev_to_i915(kdev)->drm; 2113 2114 /* 2115 * We have a suspend ordering issue with the snd-hda driver also 2116 * requiring our device to be power up. Due to the lack of a 2117 * parent/child relationship we currently solve this with an late 2118 * suspend hook. 2119 * 2120 * FIXME: This should be solved with a special hdmi sink device or 2121 * similar so that power domains can be employed. 2122 */ 2123 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 2124 return 0; 2125 2126 return i915_drm_suspend_late(dev, false); 2127 } 2128 2129 static int i915_pm_poweroff_late(struct device *kdev) 2130 { 2131 struct drm_device *dev = &kdev_to_i915(kdev)->drm; 2132 2133 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 2134 return 0; 2135 2136 return i915_drm_suspend_late(dev, true); 2137 } 2138 2139 static int i915_pm_resume_early(struct device *kdev) 2140 { 2141 struct drm_device *dev = &kdev_to_i915(kdev)->drm; 2142 2143 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 2144 return 0; 2145 2146 return i915_drm_resume_early(dev); 2147 } 2148 2149 static int i915_pm_resume(struct device *kdev) 2150 { 2151 struct drm_device *dev = &kdev_to_i915(kdev)->drm; 2152 2153 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 2154 return 0; 2155 2156 return i915_drm_resume(dev); 2157 } 2158 2159 /* freeze: before creating the hibernation_image */ 2160 static int i915_pm_freeze(struct device *kdev) 2161 { 2162 struct drm_device *dev = &kdev_to_i915(kdev)->drm; 2163 int ret; 2164 2165 if (dev->switch_power_state != DRM_SWITCH_POWER_OFF) { 2166 ret = i915_drm_suspend(dev); 2167 if (ret) 2168 return ret; 2169 } 2170 2171 ret = i915_gem_freeze(kdev_to_i915(kdev)); 2172 if (ret) 2173 return ret; 2174 2175 return 0; 2176 } 2177 2178 static int i915_pm_freeze_late(struct device *kdev) 2179 { 2180 struct drm_device *dev = &kdev_to_i915(kdev)->drm; 2181 int ret; 2182 2183 if (dev->switch_power_state != DRM_SWITCH_POWER_OFF) { 2184 ret = i915_drm_suspend_late(dev, true); 2185 if (ret) 2186 return ret; 2187 } 2188 2189 ret = i915_gem_freeze_late(kdev_to_i915(kdev)); 2190 if (ret) 2191 return ret; 2192 2193 return 0; 2194 } 2195 2196 /* thaw: called after creating the hibernation image, but before turning off. */ 2197 static int i915_pm_thaw_early(struct device *kdev) 2198 { 2199 return i915_pm_resume_early(kdev); 2200 } 2201 2202 static int i915_pm_thaw(struct device *kdev) 2203 { 2204 return i915_pm_resume(kdev); 2205 } 2206 2207 /* restore: called after loading the hibernation image. */ 2208 static int i915_pm_restore_early(struct device *kdev) 2209 { 2210 return i915_pm_resume_early(kdev); 2211 } 2212 2213 static int i915_pm_restore(struct device *kdev) 2214 { 2215 return i915_pm_resume(kdev); 2216 } 2217 2218 /* 2219 * Save all Gunit registers that may be lost after a D3 and a subsequent 2220 * S0i[R123] transition. The list of registers needing a save/restore is 2221 * defined in the VLV2_S0IXRegs document. This documents marks all Gunit 2222 * registers in the following way: 2223 * - Driver: saved/restored by the driver 2224 * - Punit : saved/restored by the Punit firmware 2225 * - No, w/o marking: no need to save/restore, since the register is R/O or 2226 * used internally by the HW in a way that doesn't depend 2227 * keeping the content across a suspend/resume. 2228 * - Debug : used for debugging 2229 * 2230 * We save/restore all registers marked with 'Driver', with the following 2231 * exceptions: 2232 * - Registers out of use, including also registers marked with 'Debug'. 2233 * These have no effect on the driver's operation, so we don't save/restore 2234 * them to reduce the overhead. 2235 * - Registers that are fully setup by an initialization function called from 2236 * the resume path. For example many clock gating and RPS/RC6 registers. 2237 * - Registers that provide the right functionality with their reset defaults. 2238 * 2239 * TODO: Except for registers that based on the above 3 criteria can be safely 2240 * ignored, we save/restore all others, practically treating the HW context as 2241 * a black-box for the driver. Further investigation is needed to reduce the 2242 * saved/restored registers even further, by following the same 3 criteria. 2243 */ 2244 static void vlv_save_gunit_s0ix_state(struct drm_i915_private *dev_priv) 2245 { 2246 struct vlv_s0ix_state *s = &dev_priv->vlv_s0ix_state; 2247 int i; 2248 2249 /* GAM 0x4000-0x4770 */ 2250 s->wr_watermark = I915_READ(GEN7_WR_WATERMARK); 2251 s->gfx_prio_ctrl = I915_READ(GEN7_GFX_PRIO_CTRL); 2252 s->arb_mode = I915_READ(ARB_MODE); 2253 s->gfx_pend_tlb0 = I915_READ(GEN7_GFX_PEND_TLB0); 2254 s->gfx_pend_tlb1 = I915_READ(GEN7_GFX_PEND_TLB1); 2255 2256 for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++) 2257 s->lra_limits[i] = I915_READ(GEN7_LRA_LIMITS(i)); 2258 2259 s->media_max_req_count = I915_READ(GEN7_MEDIA_MAX_REQ_COUNT); 2260 s->gfx_max_req_count = I915_READ(GEN7_GFX_MAX_REQ_COUNT); 2261 2262 s->render_hwsp = I915_READ(RENDER_HWS_PGA_GEN7); 2263 s->ecochk = I915_READ(GAM_ECOCHK); 2264 s->bsd_hwsp = I915_READ(BSD_HWS_PGA_GEN7); 2265 s->blt_hwsp = I915_READ(BLT_HWS_PGA_GEN7); 2266 2267 s->tlb_rd_addr = I915_READ(GEN7_TLB_RD_ADDR); 2268 2269 /* MBC 0x9024-0x91D0, 0x8500 */ 2270 s->g3dctl = I915_READ(VLV_G3DCTL); 2271 s->gsckgctl = I915_READ(VLV_GSCKGCTL); 2272 s->mbctl = I915_READ(GEN6_MBCTL); 2273 2274 /* GCP 0x9400-0x9424, 0x8100-0x810C */ 2275 s->ucgctl1 = I915_READ(GEN6_UCGCTL1); 2276 s->ucgctl3 = I915_READ(GEN6_UCGCTL3); 2277 s->rcgctl1 = I915_READ(GEN6_RCGCTL1); 2278 s->rcgctl2 = I915_READ(GEN6_RCGCTL2); 2279 s->rstctl = I915_READ(GEN6_RSTCTL); 2280 s->misccpctl = I915_READ(GEN7_MISCCPCTL); 2281 2282 /* GPM 0xA000-0xAA84, 0x8000-0x80FC */ 2283 s->gfxpause = I915_READ(GEN6_GFXPAUSE); 2284 s->rpdeuhwtc = I915_READ(GEN6_RPDEUHWTC); 2285 s->rpdeuc = I915_READ(GEN6_RPDEUC); 2286 s->ecobus = I915_READ(ECOBUS); 2287 s->pwrdwnupctl = I915_READ(VLV_PWRDWNUPCTL); 2288 s->rp_down_timeout = I915_READ(GEN6_RP_DOWN_TIMEOUT); 2289 s->rp_deucsw = I915_READ(GEN6_RPDEUCSW); 2290 s->rcubmabdtmr = I915_READ(GEN6_RCUBMABDTMR); 2291 s->rcedata = I915_READ(VLV_RCEDATA); 2292 s->spare2gh = I915_READ(VLV_SPAREG2H); 2293 2294 /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */ 2295 s->gt_imr = I915_READ(GTIMR); 2296 s->gt_ier = I915_READ(GTIER); 2297 s->pm_imr = I915_READ(GEN6_PMIMR); 2298 s->pm_ier = I915_READ(GEN6_PMIER); 2299 2300 for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++) 2301 s->gt_scratch[i] = I915_READ(GEN7_GT_SCRATCH(i)); 2302 2303 /* GT SA CZ domain, 0x100000-0x138124 */ 2304 s->tilectl = I915_READ(TILECTL); 2305 s->gt_fifoctl = I915_READ(GTFIFOCTL); 2306 s->gtlc_wake_ctrl = I915_READ(VLV_GTLC_WAKE_CTRL); 2307 s->gtlc_survive = I915_READ(VLV_GTLC_SURVIVABILITY_REG); 2308 s->pmwgicz = I915_READ(VLV_PMWGICZ); 2309 2310 /* Gunit-Display CZ domain, 0x182028-0x1821CF */ 2311 s->gu_ctl0 = I915_READ(VLV_GU_CTL0); 2312 s->gu_ctl1 = I915_READ(VLV_GU_CTL1); 2313 s->pcbr = I915_READ(VLV_PCBR); 2314 s->clock_gate_dis2 = I915_READ(VLV_GUNIT_CLOCK_GATE2); 2315 2316 /* 2317 * Not saving any of: 2318 * DFT, 0x9800-0x9EC0 2319 * SARB, 0xB000-0xB1FC 2320 * GAC, 0x5208-0x524C, 0x14000-0x14C000 2321 * PCI CFG 2322 */ 2323 } 2324 2325 static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *dev_priv) 2326 { 2327 struct vlv_s0ix_state *s = &dev_priv->vlv_s0ix_state; 2328 u32 val; 2329 int i; 2330 2331 /* GAM 0x4000-0x4770 */ 2332 I915_WRITE(GEN7_WR_WATERMARK, s->wr_watermark); 2333 I915_WRITE(GEN7_GFX_PRIO_CTRL, s->gfx_prio_ctrl); 2334 I915_WRITE(ARB_MODE, s->arb_mode | (0xffff << 16)); 2335 I915_WRITE(GEN7_GFX_PEND_TLB0, s->gfx_pend_tlb0); 2336 I915_WRITE(GEN7_GFX_PEND_TLB1, s->gfx_pend_tlb1); 2337 2338 for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++) 2339 I915_WRITE(GEN7_LRA_LIMITS(i), s->lra_limits[i]); 2340 2341 I915_WRITE(GEN7_MEDIA_MAX_REQ_COUNT, s->media_max_req_count); 2342 I915_WRITE(GEN7_GFX_MAX_REQ_COUNT, s->gfx_max_req_count); 2343 2344 I915_WRITE(RENDER_HWS_PGA_GEN7, s->render_hwsp); 2345 I915_WRITE(GAM_ECOCHK, s->ecochk); 2346 I915_WRITE(BSD_HWS_PGA_GEN7, s->bsd_hwsp); 2347 I915_WRITE(BLT_HWS_PGA_GEN7, s->blt_hwsp); 2348 2349 I915_WRITE(GEN7_TLB_RD_ADDR, s->tlb_rd_addr); 2350 2351 /* MBC 0x9024-0x91D0, 0x8500 */ 2352 I915_WRITE(VLV_G3DCTL, s->g3dctl); 2353 I915_WRITE(VLV_GSCKGCTL, s->gsckgctl); 2354 I915_WRITE(GEN6_MBCTL, s->mbctl); 2355 2356 /* GCP 0x9400-0x9424, 0x8100-0x810C */ 2357 I915_WRITE(GEN6_UCGCTL1, s->ucgctl1); 2358 I915_WRITE(GEN6_UCGCTL3, s->ucgctl3); 2359 I915_WRITE(GEN6_RCGCTL1, s->rcgctl1); 2360 I915_WRITE(GEN6_RCGCTL2, s->rcgctl2); 2361 I915_WRITE(GEN6_RSTCTL, s->rstctl); 2362 I915_WRITE(GEN7_MISCCPCTL, s->misccpctl); 2363 2364 /* GPM 0xA000-0xAA84, 0x8000-0x80FC */ 2365 I915_WRITE(GEN6_GFXPAUSE, s->gfxpause); 2366 I915_WRITE(GEN6_RPDEUHWTC, s->rpdeuhwtc); 2367 I915_WRITE(GEN6_RPDEUC, s->rpdeuc); 2368 I915_WRITE(ECOBUS, s->ecobus); 2369 I915_WRITE(VLV_PWRDWNUPCTL, s->pwrdwnupctl); 2370 I915_WRITE(GEN6_RP_DOWN_TIMEOUT,s->rp_down_timeout); 2371 I915_WRITE(GEN6_RPDEUCSW, s->rp_deucsw); 2372 I915_WRITE(GEN6_RCUBMABDTMR, s->rcubmabdtmr); 2373 I915_WRITE(VLV_RCEDATA, s->rcedata); 2374 I915_WRITE(VLV_SPAREG2H, s->spare2gh); 2375 2376 /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */ 2377 I915_WRITE(GTIMR, s->gt_imr); 2378 I915_WRITE(GTIER, s->gt_ier); 2379 I915_WRITE(GEN6_PMIMR, s->pm_imr); 2380 I915_WRITE(GEN6_PMIER, s->pm_ier); 2381 2382 for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++) 2383 I915_WRITE(GEN7_GT_SCRATCH(i), s->gt_scratch[i]); 2384 2385 /* GT SA CZ domain, 0x100000-0x138124 */ 2386 I915_WRITE(TILECTL, s->tilectl); 2387 I915_WRITE(GTFIFOCTL, s->gt_fifoctl); 2388 /* 2389 * Preserve the GT allow wake and GFX force clock bit, they are not 2390 * be restored, as they are used to control the s0ix suspend/resume 2391 * sequence by the caller. 2392 */ 2393 val = I915_READ(VLV_GTLC_WAKE_CTRL); 2394 val &= VLV_GTLC_ALLOWWAKEREQ; 2395 val |= s->gtlc_wake_ctrl & ~VLV_GTLC_ALLOWWAKEREQ; 2396 I915_WRITE(VLV_GTLC_WAKE_CTRL, val); 2397 2398 val = I915_READ(VLV_GTLC_SURVIVABILITY_REG); 2399 val &= VLV_GFX_CLK_FORCE_ON_BIT; 2400 val |= s->gtlc_survive & ~VLV_GFX_CLK_FORCE_ON_BIT; 2401 I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val); 2402 2403 I915_WRITE(VLV_PMWGICZ, s->pmwgicz); 2404 2405 /* Gunit-Display CZ domain, 0x182028-0x1821CF */ 2406 I915_WRITE(VLV_GU_CTL0, s->gu_ctl0); 2407 I915_WRITE(VLV_GU_CTL1, s->gu_ctl1); 2408 I915_WRITE(VLV_PCBR, s->pcbr); 2409 I915_WRITE(VLV_GUNIT_CLOCK_GATE2, s->clock_gate_dis2); 2410 } 2411 2412 static int vlv_wait_for_pw_status(struct drm_i915_private *dev_priv, 2413 u32 mask, u32 val) 2414 { 2415 /* The HW does not like us polling for PW_STATUS frequently, so 2416 * use the sleeping loop rather than risk the busy spin within 2417 * intel_wait_for_register(). 2418 * 2419 * Transitioning between RC6 states should be at most 2ms (see 2420 * valleyview_enable_rps) so use a 3ms timeout. 2421 */ 2422 return wait_for((I915_READ_NOTRACE(VLV_GTLC_PW_STATUS) & mask) == val, 2423 3); 2424 } 2425 #endif 2426 2427 int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool force_on) 2428 { 2429 u32 val; 2430 int err; 2431 2432 val = I915_READ(VLV_GTLC_SURVIVABILITY_REG); 2433 val &= ~VLV_GFX_CLK_FORCE_ON_BIT; 2434 if (force_on) 2435 val |= VLV_GFX_CLK_FORCE_ON_BIT; 2436 I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val); 2437 2438 if (!force_on) 2439 return 0; 2440 2441 err = intel_wait_for_register(dev_priv, 2442 VLV_GTLC_SURVIVABILITY_REG, 2443 VLV_GFX_CLK_STATUS_BIT, 2444 VLV_GFX_CLK_STATUS_BIT, 2445 20); 2446 if (err) 2447 DRM_ERROR("timeout waiting for GFX clock force-on (%08x)\n", 2448 I915_READ(VLV_GTLC_SURVIVABILITY_REG)); 2449 2450 return err; 2451 } 2452 2453 #if 0 2454 static int vlv_allow_gt_wake(struct drm_i915_private *dev_priv, bool allow) 2455 { 2456 u32 mask; 2457 u32 val; 2458 int err; 2459 2460 val = I915_READ(VLV_GTLC_WAKE_CTRL); 2461 val &= ~VLV_GTLC_ALLOWWAKEREQ; 2462 if (allow) 2463 val |= VLV_GTLC_ALLOWWAKEREQ; 2464 I915_WRITE(VLV_GTLC_WAKE_CTRL, val); 2465 POSTING_READ(VLV_GTLC_WAKE_CTRL); 2466 2467 mask = VLV_GTLC_ALLOWWAKEACK; 2468 val = allow ? mask : 0; 2469 2470 err = vlv_wait_for_pw_status(dev_priv, mask, val); 2471 if (err) 2472 DRM_ERROR("timeout disabling GT waking\n"); 2473 2474 return err; 2475 } 2476 2477 static void vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv, 2478 bool wait_for_on) 2479 { 2480 u32 mask; 2481 u32 val; 2482 2483 mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK; 2484 val = wait_for_on ? mask : 0; 2485 2486 /* 2487 * RC6 transitioning can be delayed up to 2 msec (see 2488 * valleyview_enable_rps), use 3 msec for safety. 2489 */ 2490 if (vlv_wait_for_pw_status(dev_priv, mask, val)) 2491 DRM_ERROR("timeout waiting for GT wells to go %s\n", 2492 onoff(wait_for_on)); 2493 } 2494 2495 static void vlv_check_no_gt_access(struct drm_i915_private *dev_priv) 2496 { 2497 if (!(I915_READ(VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEERR)) 2498 return; 2499 2500 DRM_DEBUG_DRIVER("GT register access while GT waking disabled\n"); 2501 I915_WRITE(VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR); 2502 } 2503 2504 static int vlv_suspend_complete(struct drm_i915_private *dev_priv) 2505 { 2506 u32 mask; 2507 int err; 2508 2509 /* 2510 * Bspec defines the following GT well on flags as debug only, so 2511 * don't treat them as hard failures. 2512 */ 2513 vlv_wait_for_gt_wells(dev_priv, false); 2514 2515 mask = VLV_GTLC_RENDER_CTX_EXISTS | VLV_GTLC_MEDIA_CTX_EXISTS; 2516 WARN_ON((I915_READ(VLV_GTLC_WAKE_CTRL) & mask) != mask); 2517 2518 vlv_check_no_gt_access(dev_priv); 2519 2520 err = vlv_force_gfx_clock(dev_priv, true); 2521 if (err) 2522 goto err1; 2523 2524 err = vlv_allow_gt_wake(dev_priv, false); 2525 if (err) 2526 goto err2; 2527 2528 if (!IS_CHERRYVIEW(dev_priv)) 2529 vlv_save_gunit_s0ix_state(dev_priv); 2530 2531 err = vlv_force_gfx_clock(dev_priv, false); 2532 if (err) 2533 goto err2; 2534 2535 return 0; 2536 2537 err2: 2538 /* For safety always re-enable waking and disable gfx clock forcing */ 2539 vlv_allow_gt_wake(dev_priv, true); 2540 err1: 2541 vlv_force_gfx_clock(dev_priv, false); 2542 2543 return err; 2544 } 2545 2546 static int vlv_resume_prepare(struct drm_i915_private *dev_priv, 2547 bool rpm_resume) 2548 { 2549 int err; 2550 int ret; 2551 2552 /* 2553 * If any of the steps fail just try to continue, that's the best we 2554 * can do at this point. Return the first error code (which will also 2555 * leave RPM permanently disabled). 2556 */ 2557 ret = vlv_force_gfx_clock(dev_priv, true); 2558 2559 if (!IS_CHERRYVIEW(dev_priv)) 2560 vlv_restore_gunit_s0ix_state(dev_priv); 2561 2562 err = vlv_allow_gt_wake(dev_priv, true); 2563 if (!ret) 2564 ret = err; 2565 2566 err = vlv_force_gfx_clock(dev_priv, false); 2567 if (!ret) 2568 ret = err; 2569 2570 vlv_check_no_gt_access(dev_priv); 2571 2572 if (rpm_resume) 2573 intel_init_clock_gating(dev_priv); 2574 2575 return ret; 2576 } 2577 2578 static int intel_runtime_suspend(struct device *kdev) 2579 { 2580 struct pci_dev *pdev = to_pci_dev(kdev); 2581 struct drm_device *dev = pci_get_drvdata(pdev); 2582 struct drm_i915_private *dev_priv = to_i915(dev); 2583 int ret; 2584 2585 if (WARN_ON_ONCE(!(dev_priv->gt_pm.rc6.enabled && intel_rc6_enabled()))) 2586 return -ENODEV; 2587 2588 if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev_priv))) 2589 return -ENODEV; 2590 2591 DRM_DEBUG_KMS("Suspending device\n"); 2592 2593 disable_rpm_wakeref_asserts(dev_priv); 2594 2595 /* 2596 * We are safe here against re-faults, since the fault handler takes 2597 * an RPM reference. 2598 */ 2599 i915_gem_runtime_suspend(dev_priv); 2600 2601 intel_guc_suspend(dev_priv); 2602 2603 intel_runtime_pm_disable_interrupts(dev_priv); 2604 2605 ret = 0; 2606 if (IS_GEN9_LP(dev_priv)) { 2607 bxt_display_core_uninit(dev_priv); 2608 bxt_enable_dc9(dev_priv); 2609 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { 2610 hsw_enable_pc8(dev_priv); 2611 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { 2612 ret = vlv_suspend_complete(dev_priv); 2613 } 2614 2615 if (ret) { 2616 DRM_ERROR("Runtime suspend failed, disabling it (%d)\n", ret); 2617 intel_runtime_pm_enable_interrupts(dev_priv); 2618 2619 enable_rpm_wakeref_asserts(dev_priv); 2620 2621 return ret; 2622 } 2623 2624 intel_uncore_suspend(dev_priv); 2625 2626 enable_rpm_wakeref_asserts(dev_priv); 2627 WARN_ON_ONCE(atomic_read(&dev_priv->runtime_pm.wakeref_count)); 2628 2629 if (intel_uncore_arm_unclaimed_mmio_detection(dev_priv)) 2630 DRM_ERROR("Unclaimed access detected prior to suspending\n"); 2631 2632 dev_priv->runtime_pm.suspended = true; 2633 2634 /* 2635 * FIXME: We really should find a document that references the arguments 2636 * used below! 2637 */ 2638 if (IS_BROADWELL(dev_priv)) { 2639 /* 2640 * On Broadwell, if we use PCI_D1 the PCH DDI ports will stop 2641 * being detected, and the call we do at intel_runtime_resume() 2642 * won't be able to restore them. Since PCI_D3hot matches the 2643 * actual specification and appears to be working, use it. 2644 */ 2645 intel_opregion_notify_adapter(dev_priv, PCI_D3hot); 2646 } else { 2647 /* 2648 * current versions of firmware which depend on this opregion 2649 * notification have repurposed the D1 definition to mean 2650 * "runtime suspended" vs. what you would normally expect (D3) 2651 * to distinguish it from notifications that might be sent via 2652 * the suspend path. 2653 */ 2654 intel_opregion_notify_adapter(dev_priv, PCI_D1); 2655 } 2656 2657 assert_forcewakes_inactive(dev_priv); 2658 2659 if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) 2660 intel_hpd_poll_init(dev_priv); 2661 2662 DRM_DEBUG_KMS("Device suspended\n"); 2663 return 0; 2664 } 2665 2666 static int intel_runtime_resume(struct device *kdev) 2667 { 2668 struct pci_dev *pdev = to_pci_dev(kdev); 2669 struct drm_device *dev = pci_get_drvdata(pdev); 2670 struct drm_i915_private *dev_priv = to_i915(dev); 2671 int ret = 0; 2672 2673 if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev_priv))) 2674 return -ENODEV; 2675 2676 DRM_DEBUG_KMS("Resuming device\n"); 2677 2678 WARN_ON_ONCE(atomic_read(&dev_priv->runtime_pm.wakeref_count)); 2679 disable_rpm_wakeref_asserts(dev_priv); 2680 2681 intel_opregion_notify_adapter(dev_priv, PCI_D0); 2682 dev_priv->runtime_pm.suspended = false; 2683 if (intel_uncore_unclaimed_mmio(dev_priv)) 2684 DRM_DEBUG_DRIVER("Unclaimed access during suspend, bios?\n"); 2685 2686 intel_guc_resume(dev_priv); 2687 2688 if (IS_GEN9_LP(dev_priv)) { 2689 bxt_disable_dc9(dev_priv); 2690 bxt_display_core_init(dev_priv, true); 2691 if (dev_priv->csr.dmc_payload && 2692 (dev_priv->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC5)) 2693 gen9_enable_dc5(dev_priv); 2694 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { 2695 hsw_disable_pc8(dev_priv); 2696 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { 2697 ret = vlv_resume_prepare(dev_priv, true); 2698 } 2699 2700 intel_uncore_runtime_resume(dev_priv); 2701 2702 /* 2703 * No point of rolling back things in case of an error, as the best 2704 * we can do is to hope that things will still work (and disable RPM). 2705 */ 2706 i915_gem_init_swizzling(dev_priv); 2707 i915_gem_restore_fences(dev_priv); 2708 2709 intel_runtime_pm_enable_interrupts(dev_priv); 2710 2711 /* 2712 * On VLV/CHV display interrupts are part of the display 2713 * power well, so hpd is reinitialized from there. For 2714 * everyone else do it here. 2715 */ 2716 if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) 2717 intel_hpd_init(dev_priv); 2718 2719 intel_enable_ipc(dev_priv); 2720 2721 enable_rpm_wakeref_asserts(dev_priv); 2722 2723 if (ret) 2724 DRM_ERROR("Runtime resume failed, disabling it (%d)\n", ret); 2725 else 2726 DRM_DEBUG_KMS("Device resumed\n"); 2727 2728 return ret; 2729 } 2730 2731 const struct dev_pm_ops i915_pm_ops = { 2732 /* 2733 * S0ix (via system suspend) and S3 event handlers [PMSG_SUSPEND, 2734 * PMSG_RESUME] 2735 */ 2736 .suspend = i915_pm_suspend, 2737 .suspend_late = i915_pm_suspend_late, 2738 .resume_early = i915_pm_resume_early, 2739 .resume = i915_pm_resume, 2740 2741 /* 2742 * S4 event handlers 2743 * @freeze, @freeze_late : called (1) before creating the 2744 * hibernation image [PMSG_FREEZE] and 2745 * (2) after rebooting, before restoring 2746 * the image [PMSG_QUIESCE] 2747 * @thaw, @thaw_early : called (1) after creating the hibernation 2748 * image, before writing it [PMSG_THAW] 2749 * and (2) after failing to create or 2750 * restore the image [PMSG_RECOVER] 2751 * @poweroff, @poweroff_late: called after writing the hibernation 2752 * image, before rebooting [PMSG_HIBERNATE] 2753 * @restore, @restore_early : called after rebooting and restoring the 2754 * hibernation image [PMSG_RESTORE] 2755 */ 2756 .freeze = i915_pm_freeze, 2757 .freeze_late = i915_pm_freeze_late, 2758 .thaw_early = i915_pm_thaw_early, 2759 .thaw = i915_pm_thaw, 2760 .poweroff = i915_pm_suspend, 2761 .poweroff_late = i915_pm_poweroff_late, 2762 .restore_early = i915_pm_restore_early, 2763 .restore = i915_pm_restore, 2764 2765 /* S0ix (via runtime suspend) event handlers */ 2766 .runtime_suspend = intel_runtime_suspend, 2767 .runtime_resume = intel_runtime_resume, 2768 }; 2769 2770 static const struct vm_operations_struct i915_gem_vm_ops = { 2771 .fault = i915_gem_fault, 2772 .open = drm_gem_vm_open, 2773 .close = drm_gem_vm_close, 2774 }; 2775 #endif 2776 2777 static struct cdev_pager_ops i915_gem_vm_ops = { 2778 .cdev_pg_fault = i915_gem_fault, 2779 .cdev_pg_ctor = i915_gem_pager_ctor, 2780 .cdev_pg_dtor = i915_gem_pager_dtor 2781 }; 2782 2783 static const struct file_operations i915_driver_fops = { 2784 .owner = THIS_MODULE, 2785 #if 0 2786 .open = drm_open, 2787 .release = drm_release, 2788 .unlocked_ioctl = drm_ioctl, 2789 .mmap = drm_gem_mmap, 2790 .poll = drm_poll, 2791 .read = drm_read, 2792 .compat_ioctl = i915_compat_ioctl, 2793 .llseek = noop_llseek, 2794 #endif 2795 }; 2796 2797 static int 2798 i915_gem_reject_pin_ioctl(struct drm_device *dev, void *data, 2799 struct drm_file *file) 2800 { 2801 return -ENODEV; 2802 } 2803 2804 static const struct drm_ioctl_desc i915_ioctls[] = { 2805 DRM_IOCTL_DEF_DRV(I915_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 2806 DRM_IOCTL_DEF_DRV(I915_FLUSH, drm_noop, DRM_AUTH), 2807 DRM_IOCTL_DEF_DRV(I915_FLIP, drm_noop, DRM_AUTH), 2808 DRM_IOCTL_DEF_DRV(I915_BATCHBUFFER, drm_noop, DRM_AUTH), 2809 DRM_IOCTL_DEF_DRV(I915_IRQ_EMIT, drm_noop, DRM_AUTH), 2810 DRM_IOCTL_DEF_DRV(I915_IRQ_WAIT, drm_noop, DRM_AUTH), 2811 DRM_IOCTL_DEF_DRV(I915_GETPARAM, i915_getparam, DRM_AUTH|DRM_RENDER_ALLOW), 2812 DRM_IOCTL_DEF_DRV(I915_SETPARAM, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 2813 DRM_IOCTL_DEF_DRV(I915_ALLOC, drm_noop, DRM_AUTH), 2814 DRM_IOCTL_DEF_DRV(I915_FREE, drm_noop, DRM_AUTH), 2815 DRM_IOCTL_DEF_DRV(I915_INIT_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 2816 DRM_IOCTL_DEF_DRV(I915_CMDBUFFER, drm_noop, DRM_AUTH), 2817 DRM_IOCTL_DEF_DRV(I915_DESTROY_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 2818 DRM_IOCTL_DEF_DRV(I915_SET_VBLANK_PIPE, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 2819 DRM_IOCTL_DEF_DRV(I915_GET_VBLANK_PIPE, drm_noop, DRM_AUTH), 2820 DRM_IOCTL_DEF_DRV(I915_VBLANK_SWAP, drm_noop, DRM_AUTH), 2821 DRM_IOCTL_DEF_DRV(I915_HWS_ADDR, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 2822 DRM_IOCTL_DEF_DRV(I915_GEM_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 2823 DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER, i915_gem_execbuffer, DRM_AUTH), 2824 DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER2_WR, i915_gem_execbuffer2, DRM_AUTH|DRM_RENDER_ALLOW), 2825 DRM_IOCTL_DEF_DRV(I915_GEM_PIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY), 2826 DRM_IOCTL_DEF_DRV(I915_GEM_UNPIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY), 2827 DRM_IOCTL_DEF_DRV(I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), 2828 DRM_IOCTL_DEF_DRV(I915_GEM_SET_CACHING, i915_gem_set_caching_ioctl, DRM_RENDER_ALLOW), 2829 DRM_IOCTL_DEF_DRV(I915_GEM_GET_CACHING, i915_gem_get_caching_ioctl, DRM_RENDER_ALLOW), 2830 DRM_IOCTL_DEF_DRV(I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), 2831 DRM_IOCTL_DEF_DRV(I915_GEM_ENTERVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 2832 DRM_IOCTL_DEF_DRV(I915_GEM_LEAVEVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 2833 DRM_IOCTL_DEF_DRV(I915_GEM_CREATE, i915_gem_create_ioctl, DRM_RENDER_ALLOW), 2834 DRM_IOCTL_DEF_DRV(I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_RENDER_ALLOW), 2835 DRM_IOCTL_DEF_DRV(I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_RENDER_ALLOW), 2836 DRM_IOCTL_DEF_DRV(I915_GEM_MMAP, i915_gem_mmap_ioctl, DRM_RENDER_ALLOW), 2837 DRM_IOCTL_DEF_DRV(I915_GEM_MMAP_GTT, i915_gem_mmap_gtt_ioctl, DRM_RENDER_ALLOW), 2838 DRM_IOCTL_DEF_DRV(I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, DRM_RENDER_ALLOW), 2839 DRM_IOCTL_DEF_DRV(I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, DRM_RENDER_ALLOW), 2840 DRM_IOCTL_DEF_DRV(I915_GEM_SET_TILING, i915_gem_set_tiling_ioctl, DRM_RENDER_ALLOW), 2841 DRM_IOCTL_DEF_DRV(I915_GEM_GET_TILING, i915_gem_get_tiling_ioctl, DRM_RENDER_ALLOW), 2842 DRM_IOCTL_DEF_DRV(I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, DRM_RENDER_ALLOW), 2843 DRM_IOCTL_DEF_DRV(I915_GET_PIPE_FROM_CRTC_ID, intel_get_pipe_from_crtc_id, 0), 2844 DRM_IOCTL_DEF_DRV(I915_GEM_MADVISE, i915_gem_madvise_ioctl, DRM_RENDER_ALLOW), 2845 DRM_IOCTL_DEF_DRV(I915_OVERLAY_PUT_IMAGE, intel_overlay_put_image_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW), 2846 DRM_IOCTL_DEF_DRV(I915_OVERLAY_ATTRS, intel_overlay_attrs_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW), 2847 DRM_IOCTL_DEF_DRV(I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey, DRM_MASTER|DRM_CONTROL_ALLOW), 2848 DRM_IOCTL_DEF_DRV(I915_GET_SPRITE_COLORKEY, drm_noop, DRM_MASTER|DRM_CONTROL_ALLOW), 2849 DRM_IOCTL_DEF_DRV(I915_GEM_WAIT, i915_gem_wait_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), 2850 DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_CREATE, i915_gem_context_create_ioctl, DRM_RENDER_ALLOW), 2851 DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_RENDER_ALLOW), 2852 DRM_IOCTL_DEF_DRV(I915_REG_READ, i915_reg_read_ioctl, DRM_RENDER_ALLOW), 2853 DRM_IOCTL_DEF_DRV(I915_GET_RESET_STATS, i915_gem_context_reset_stats_ioctl, DRM_RENDER_ALLOW), 2854 #if 0 2855 DRM_IOCTL_DEF_DRV(I915_GEM_USERPTR, i915_gem_userptr_ioctl, DRM_RENDER_ALLOW), 2856 #endif 2857 DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_GETPARAM, i915_gem_context_getparam_ioctl, DRM_RENDER_ALLOW), 2858 DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_SETPARAM, i915_gem_context_setparam_ioctl, DRM_RENDER_ALLOW), 2859 DRM_IOCTL_DEF_DRV(I915_PERF_OPEN, i915_perf_open_ioctl, DRM_RENDER_ALLOW), 2860 #if 0 2861 DRM_IOCTL_DEF_DRV(I915_PERF_ADD_CONFIG, i915_perf_add_config_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), 2862 DRM_IOCTL_DEF_DRV(I915_PERF_REMOVE_CONFIG, i915_perf_remove_config_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), 2863 #endif 2864 }; 2865 2866 static int i915_sysctl_init(struct drm_device *dev, struct sysctl_ctx_list *ctx, 2867 struct sysctl_oid *top) 2868 { 2869 return drm_add_busid_modesetting(dev, ctx, top); 2870 } 2871 2872 static struct drm_driver driver = { 2873 /* Don't use MTRRs here; the Xserver or userspace app should 2874 * deal with them for Intel hardware. 2875 */ 2876 .driver_features = 2877 DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED | DRIVER_GEM | DRIVER_PRIME | 2878 DRIVER_RENDER | DRIVER_MODESET | DRIVER_ATOMIC | DRIVER_SYNCOBJ, 2879 .release = i915_driver_release, 2880 .open = i915_driver_open, 2881 .lastclose = i915_driver_lastclose, 2882 .postclose = i915_driver_postclose, 2883 2884 .gem_close_object = i915_gem_close_object, 2885 .gem_free_object_unlocked = i915_gem_free_object, 2886 .gem_vm_ops = &i915_gem_vm_ops, 2887 2888 .prime_handle_to_fd = drm_gem_prime_handle_to_fd, 2889 .prime_fd_to_handle = drm_gem_prime_fd_to_handle, 2890 .gem_prime_export = i915_gem_prime_export, 2891 .gem_prime_import = i915_gem_prime_import, 2892 2893 .dumb_create = i915_gem_dumb_create, 2894 .dumb_map_offset = i915_gem_mmap_gtt, 2895 .ioctls = i915_ioctls, 2896 .num_ioctls = ARRAY_SIZE(i915_ioctls), 2897 .fops = &i915_driver_fops, 2898 .name = DRIVER_NAME, 2899 .desc = DRIVER_DESC, 2900 .date = DRIVER_DATE, 2901 .major = DRIVER_MAJOR, 2902 .minor = DRIVER_MINOR, 2903 .patchlevel = DRIVER_PATCHLEVEL, 2904 #ifdef __DragonFly__ 2905 .sysctl_init = i915_sysctl_init, 2906 #endif 2907 }; 2908 2909 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 2910 #include "selftests/mock_drm.c" 2911 #endif 2912