1 /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*- 2 */ 3 /* 4 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas. 5 * All Rights Reserved. 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the 9 * "Software"), to deal in the Software without restriction, including 10 * without limitation the rights to use, copy, modify, merge, publish, 11 * distribute, sub license, and/or sell copies of the Software, and to 12 * permit persons to whom the Software is furnished to do so, subject to 13 * the following conditions: 14 * 15 * The above copyright notice and this permission notice (including the 16 * next paragraph) shall be included in all copies or substantial portions 17 * of the Software. 18 * 19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 20 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 22 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR 23 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 24 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 25 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 26 * 27 */ 28 29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 30 31 #include <linux/sysrq.h> 32 #include <linux/slab.h> 33 #include <linux/circ_buf.h> 34 #include <drm/drmP.h> 35 #include <drm/i915_drm.h> 36 #include "i915_drv.h" 37 #include "i915_trace.h" 38 #include "intel_drv.h" 39 40 /** 41 * DOC: interrupt handling 42 * 43 * These functions provide the basic support for enabling and disabling the 44 * interrupt handling support. There's a lot more functionality in i915_irq.c 45 * and related files, but that will be described in separate chapters. 46 */ 47 48 static const u32 hpd_ilk[HPD_NUM_PINS] = { 49 [HPD_PORT_A] = DE_DP_A_HOTPLUG, 50 }; 51 52 static const u32 hpd_ivb[HPD_NUM_PINS] = { 53 [HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB, 54 }; 55 56 static const u32 hpd_bdw[HPD_NUM_PINS] = { 57 [HPD_PORT_A] = GEN8_PORT_DP_A_HOTPLUG, 58 }; 59 60 static const u32 hpd_ibx[HPD_NUM_PINS] = { 61 [HPD_CRT] = SDE_CRT_HOTPLUG, 62 [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG, 63 [HPD_PORT_B] = SDE_PORTB_HOTPLUG, 64 [HPD_PORT_C] = SDE_PORTC_HOTPLUG, 65 [HPD_PORT_D] = SDE_PORTD_HOTPLUG 66 }; 67 68 static const u32 hpd_cpt[HPD_NUM_PINS] = { 69 [HPD_CRT] = SDE_CRT_HOTPLUG_CPT, 70 [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT, 71 [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT, 72 [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT, 73 [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT 74 }; 75 76 static const u32 hpd_spt[HPD_NUM_PINS] = { 77 [HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT, 78 [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT, 79 [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT, 80 [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT, 81 [HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT 82 }; 83 84 static const u32 hpd_mask_i915[HPD_NUM_PINS] = { 85 [HPD_CRT] = CRT_HOTPLUG_INT_EN, 86 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN, 87 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN, 88 [HPD_PORT_B] = PORTB_HOTPLUG_INT_EN, 89 [HPD_PORT_C] = PORTC_HOTPLUG_INT_EN, 90 [HPD_PORT_D] = PORTD_HOTPLUG_INT_EN 91 }; 92 93 static const u32 hpd_status_g4x[HPD_NUM_PINS] = { 94 [HPD_CRT] = CRT_HOTPLUG_INT_STATUS, 95 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X, 96 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X, 97 [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS, 98 [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS, 99 [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS 100 }; 101 102 static const u32 hpd_status_i915[HPD_NUM_PINS] = { 103 [HPD_CRT] = CRT_HOTPLUG_INT_STATUS, 104 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915, 105 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915, 106 [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS, 107 [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS, 108 [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS 109 }; 110 111 /* BXT hpd list */ 112 static const u32 hpd_bxt[HPD_NUM_PINS] = { 113 [HPD_PORT_A] = BXT_DE_PORT_HP_DDIA, 114 [HPD_PORT_B] = BXT_DE_PORT_HP_DDIB, 115 [HPD_PORT_C] = BXT_DE_PORT_HP_DDIC 116 }; 117 118 /* IIR can theoretically queue up two events. Be paranoid. */ 119 #define GEN8_IRQ_RESET_NDX(type, which) do { \ 120 I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \ 121 POSTING_READ(GEN8_##type##_IMR(which)); \ 122 I915_WRITE(GEN8_##type##_IER(which), 0); \ 123 I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \ 124 POSTING_READ(GEN8_##type##_IIR(which)); \ 125 I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \ 126 POSTING_READ(GEN8_##type##_IIR(which)); \ 127 } while (0) 128 129 #define GEN5_IRQ_RESET(type) do { \ 130 I915_WRITE(type##IMR, 0xffffffff); \ 131 POSTING_READ(type##IMR); \ 132 I915_WRITE(type##IER, 0); \ 133 I915_WRITE(type##IIR, 0xffffffff); \ 134 POSTING_READ(type##IIR); \ 135 I915_WRITE(type##IIR, 0xffffffff); \ 136 POSTING_READ(type##IIR); \ 137 } while (0) 138 139 /* 140 * We should clear IMR at preinstall/uninstall, and just check at postinstall. 141 */ 142 static void gen5_assert_iir_is_zero(struct drm_i915_private *dev_priv, 143 i915_reg_t reg) 144 { 145 u32 val = I915_READ(reg); 146 147 if (val == 0) 148 return; 149 150 WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n", 151 i915_mmio_reg_offset(reg), val); 152 I915_WRITE(reg, 0xffffffff); 153 POSTING_READ(reg); 154 I915_WRITE(reg, 0xffffffff); 155 POSTING_READ(reg); 156 } 157 158 #define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \ 159 gen5_assert_iir_is_zero(dev_priv, GEN8_##type##_IIR(which)); \ 160 I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \ 161 I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \ 162 POSTING_READ(GEN8_##type##_IMR(which)); \ 163 } while (0) 164 165 #define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \ 166 gen5_assert_iir_is_zero(dev_priv, type##IIR); \ 167 I915_WRITE(type##IER, (ier_val)); \ 168 I915_WRITE(type##IMR, (imr_val)); \ 169 POSTING_READ(type##IMR); \ 170 } while (0) 171 172 static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir); 173 static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir); 174 175 /* For display hotplug interrupt */ 176 static inline void 177 i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv, 178 uint32_t mask, 179 uint32_t bits) 180 { 181 uint32_t val; 182 183 assert_spin_locked(&dev_priv->irq_lock); 184 WARN_ON(bits & ~mask); 185 186 val = I915_READ(PORT_HOTPLUG_EN); 187 val &= ~mask; 188 val |= bits; 189 I915_WRITE(PORT_HOTPLUG_EN, val); 190 } 191 192 /** 193 * i915_hotplug_interrupt_update - update hotplug interrupt enable 194 * @dev_priv: driver private 195 * @mask: bits to update 196 * @bits: bits to enable 197 * NOTE: the HPD enable bits are modified both inside and outside 198 * of an interrupt context. To avoid that read-modify-write cycles 199 * interfer, these bits are protected by a spinlock. Since this 200 * function is usually not called from a context where the lock is 201 * held already, this function acquires the lock itself. A non-locking 202 * version is also available. 203 */ 204 void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv, 205 uint32_t mask, 206 uint32_t bits) 207 { 208 spin_lock_irq(&dev_priv->irq_lock); 209 i915_hotplug_interrupt_update_locked(dev_priv, mask, bits); 210 spin_unlock_irq(&dev_priv->irq_lock); 211 } 212 213 /** 214 * ilk_update_display_irq - update DEIMR 215 * @dev_priv: driver private 216 * @interrupt_mask: mask of interrupt bits to update 217 * @enabled_irq_mask: mask of interrupt bits to enable 218 */ 219 void ilk_update_display_irq(struct drm_i915_private *dev_priv, 220 uint32_t interrupt_mask, 221 uint32_t enabled_irq_mask) 222 { 223 uint32_t new_val; 224 225 assert_spin_locked(&dev_priv->irq_lock); 226 227 WARN_ON(enabled_irq_mask & ~interrupt_mask); 228 229 if (WARN_ON(!intel_irqs_enabled(dev_priv))) 230 return; 231 232 new_val = dev_priv->irq_mask; 233 new_val &= ~interrupt_mask; 234 new_val |= (~enabled_irq_mask & interrupt_mask); 235 236 if (new_val != dev_priv->irq_mask) { 237 dev_priv->irq_mask = new_val; 238 I915_WRITE(DEIMR, dev_priv->irq_mask); 239 POSTING_READ(DEIMR); 240 } 241 } 242 243 /** 244 * ilk_update_gt_irq - update GTIMR 245 * @dev_priv: driver private 246 * @interrupt_mask: mask of interrupt bits to update 247 * @enabled_irq_mask: mask of interrupt bits to enable 248 */ 249 static void ilk_update_gt_irq(struct drm_i915_private *dev_priv, 250 uint32_t interrupt_mask, 251 uint32_t enabled_irq_mask) 252 { 253 assert_spin_locked(&dev_priv->irq_lock); 254 255 WARN_ON(enabled_irq_mask & ~interrupt_mask); 256 257 if (WARN_ON(!intel_irqs_enabled(dev_priv))) 258 return; 259 260 dev_priv->gt_irq_mask &= ~interrupt_mask; 261 dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask); 262 I915_WRITE(GTIMR, dev_priv->gt_irq_mask); 263 } 264 265 void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask) 266 { 267 ilk_update_gt_irq(dev_priv, mask, mask); 268 POSTING_READ_FW(GTIMR); 269 } 270 271 void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask) 272 { 273 ilk_update_gt_irq(dev_priv, mask, 0); 274 } 275 276 static i915_reg_t gen6_pm_iir(struct drm_i915_private *dev_priv) 277 { 278 return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR; 279 } 280 281 static i915_reg_t gen6_pm_imr(struct drm_i915_private *dev_priv) 282 { 283 return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR; 284 } 285 286 static i915_reg_t gen6_pm_ier(struct drm_i915_private *dev_priv) 287 { 288 return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER; 289 } 290 291 /** 292 * snb_update_pm_irq - update GEN6_PMIMR 293 * @dev_priv: driver private 294 * @interrupt_mask: mask of interrupt bits to update 295 * @enabled_irq_mask: mask of interrupt bits to enable 296 */ 297 static void snb_update_pm_irq(struct drm_i915_private *dev_priv, 298 uint32_t interrupt_mask, 299 uint32_t enabled_irq_mask) 300 { 301 uint32_t new_val; 302 303 WARN_ON(enabled_irq_mask & ~interrupt_mask); 304 305 assert_spin_locked(&dev_priv->irq_lock); 306 307 new_val = dev_priv->pm_imr; 308 new_val &= ~interrupt_mask; 309 new_val |= (~enabled_irq_mask & interrupt_mask); 310 311 if (new_val != dev_priv->pm_imr) { 312 dev_priv->pm_imr = new_val; 313 I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_imr); 314 POSTING_READ(gen6_pm_imr(dev_priv)); 315 } 316 } 317 318 void gen6_unmask_pm_irq(struct drm_i915_private *dev_priv, u32 mask) 319 { 320 if (WARN_ON(!intel_irqs_enabled(dev_priv))) 321 return; 322 323 snb_update_pm_irq(dev_priv, mask, mask); 324 } 325 326 static void __gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask) 327 { 328 snb_update_pm_irq(dev_priv, mask, 0); 329 } 330 331 void gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask) 332 { 333 if (WARN_ON(!intel_irqs_enabled(dev_priv))) 334 return; 335 336 __gen6_mask_pm_irq(dev_priv, mask); 337 } 338 339 void gen6_reset_pm_iir(struct drm_i915_private *dev_priv, u32 reset_mask) 340 { 341 i915_reg_t reg = gen6_pm_iir(dev_priv); 342 343 assert_spin_locked(&dev_priv->irq_lock); 344 345 I915_WRITE(reg, reset_mask); 346 I915_WRITE(reg, reset_mask); 347 POSTING_READ(reg); 348 } 349 350 void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, u32 enable_mask) 351 { 352 assert_spin_locked(&dev_priv->irq_lock); 353 354 dev_priv->pm_ier |= enable_mask; 355 I915_WRITE(gen6_pm_ier(dev_priv), dev_priv->pm_ier); 356 gen6_unmask_pm_irq(dev_priv, enable_mask); 357 /* unmask_pm_irq provides an implicit barrier (POSTING_READ) */ 358 } 359 360 void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, u32 disable_mask) 361 { 362 assert_spin_locked(&dev_priv->irq_lock); 363 364 dev_priv->pm_ier &= ~disable_mask; 365 __gen6_mask_pm_irq(dev_priv, disable_mask); 366 I915_WRITE(gen6_pm_ier(dev_priv), dev_priv->pm_ier); 367 /* though a barrier is missing here, but don't really need a one */ 368 } 369 370 void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv) 371 { 372 spin_lock_irq(&dev_priv->irq_lock); 373 gen6_reset_pm_iir(dev_priv, dev_priv->pm_rps_events); 374 dev_priv->rps.pm_iir = 0; 375 spin_unlock_irq(&dev_priv->irq_lock); 376 } 377 378 void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv) 379 { 380 if (READ_ONCE(dev_priv->rps.interrupts_enabled)) 381 return; 382 383 spin_lock_irq(&dev_priv->irq_lock); 384 WARN_ON_ONCE(dev_priv->rps.pm_iir); 385 WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events); 386 dev_priv->rps.interrupts_enabled = true; 387 gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events); 388 389 spin_unlock_irq(&dev_priv->irq_lock); 390 } 391 392 u32 gen6_sanitize_rps_pm_mask(struct drm_i915_private *dev_priv, u32 mask) 393 { 394 return (mask & ~dev_priv->rps.pm_intr_keep); 395 } 396 397 void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv) 398 { 399 if (!READ_ONCE(dev_priv->rps.interrupts_enabled)) 400 return; 401 402 spin_lock_irq(&dev_priv->irq_lock); 403 dev_priv->rps.interrupts_enabled = false; 404 405 I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0u)); 406 407 gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events); 408 409 spin_unlock_irq(&dev_priv->irq_lock); 410 synchronize_irq(dev_priv->drm.irq); 411 412 /* Now that we will not be generating any more work, flush any 413 * outsanding tasks. As we are called on the RPS idle path, 414 * we will reset the GPU to minimum frequencies, so the current 415 * state of the worker can be discarded. 416 */ 417 cancel_work_sync(&dev_priv->rps.work); 418 gen6_reset_rps_interrupts(dev_priv); 419 } 420 421 void gen9_reset_guc_interrupts(struct drm_i915_private *dev_priv) 422 { 423 spin_lock_irq(&dev_priv->irq_lock); 424 gen6_reset_pm_iir(dev_priv, dev_priv->pm_guc_events); 425 spin_unlock_irq(&dev_priv->irq_lock); 426 } 427 428 void gen9_enable_guc_interrupts(struct drm_i915_private *dev_priv) 429 { 430 spin_lock_irq(&dev_priv->irq_lock); 431 if (!dev_priv->guc.interrupts_enabled) { 432 WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & 433 dev_priv->pm_guc_events); 434 dev_priv->guc.interrupts_enabled = true; 435 gen6_enable_pm_irq(dev_priv, dev_priv->pm_guc_events); 436 } 437 spin_unlock_irq(&dev_priv->irq_lock); 438 } 439 440 void gen9_disable_guc_interrupts(struct drm_i915_private *dev_priv) 441 { 442 spin_lock_irq(&dev_priv->irq_lock); 443 dev_priv->guc.interrupts_enabled = false; 444 445 gen6_disable_pm_irq(dev_priv, dev_priv->pm_guc_events); 446 447 spin_unlock_irq(&dev_priv->irq_lock); 448 synchronize_irq(dev_priv->drm.irq); 449 450 gen9_reset_guc_interrupts(dev_priv); 451 } 452 453 /** 454 * bdw_update_port_irq - update DE port interrupt 455 * @dev_priv: driver private 456 * @interrupt_mask: mask of interrupt bits to update 457 * @enabled_irq_mask: mask of interrupt bits to enable 458 */ 459 static void bdw_update_port_irq(struct drm_i915_private *dev_priv, 460 uint32_t interrupt_mask, 461 uint32_t enabled_irq_mask) 462 { 463 uint32_t new_val; 464 uint32_t old_val; 465 466 assert_spin_locked(&dev_priv->irq_lock); 467 468 WARN_ON(enabled_irq_mask & ~interrupt_mask); 469 470 if (WARN_ON(!intel_irqs_enabled(dev_priv))) 471 return; 472 473 old_val = I915_READ(GEN8_DE_PORT_IMR); 474 475 new_val = old_val; 476 new_val &= ~interrupt_mask; 477 new_val |= (~enabled_irq_mask & interrupt_mask); 478 479 if (new_val != old_val) { 480 I915_WRITE(GEN8_DE_PORT_IMR, new_val); 481 POSTING_READ(GEN8_DE_PORT_IMR); 482 } 483 } 484 485 /** 486 * bdw_update_pipe_irq - update DE pipe interrupt 487 * @dev_priv: driver private 488 * @pipe: pipe whose interrupt to update 489 * @interrupt_mask: mask of interrupt bits to update 490 * @enabled_irq_mask: mask of interrupt bits to enable 491 */ 492 void bdw_update_pipe_irq(struct drm_i915_private *dev_priv, 493 enum i915_pipe pipe, 494 uint32_t interrupt_mask, 495 uint32_t enabled_irq_mask) 496 { 497 uint32_t new_val; 498 499 assert_spin_locked(&dev_priv->irq_lock); 500 501 WARN_ON(enabled_irq_mask & ~interrupt_mask); 502 503 if (WARN_ON(!intel_irqs_enabled(dev_priv))) 504 return; 505 506 new_val = dev_priv->de_irq_mask[pipe]; 507 new_val &= ~interrupt_mask; 508 new_val |= (~enabled_irq_mask & interrupt_mask); 509 510 if (new_val != dev_priv->de_irq_mask[pipe]) { 511 dev_priv->de_irq_mask[pipe] = new_val; 512 I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]); 513 POSTING_READ(GEN8_DE_PIPE_IMR(pipe)); 514 } 515 } 516 517 /** 518 * ibx_display_interrupt_update - update SDEIMR 519 * @dev_priv: driver private 520 * @interrupt_mask: mask of interrupt bits to update 521 * @enabled_irq_mask: mask of interrupt bits to enable 522 */ 523 void ibx_display_interrupt_update(struct drm_i915_private *dev_priv, 524 uint32_t interrupt_mask, 525 uint32_t enabled_irq_mask) 526 { 527 uint32_t sdeimr = I915_READ(SDEIMR); 528 sdeimr &= ~interrupt_mask; 529 sdeimr |= (~enabled_irq_mask & interrupt_mask); 530 531 WARN_ON(enabled_irq_mask & ~interrupt_mask); 532 533 assert_spin_locked(&dev_priv->irq_lock); 534 535 if (WARN_ON(!intel_irqs_enabled(dev_priv))) 536 return; 537 538 I915_WRITE(SDEIMR, sdeimr); 539 POSTING_READ(SDEIMR); 540 } 541 542 static void 543 __i915_enable_pipestat(struct drm_i915_private *dev_priv, enum i915_pipe pipe, 544 u32 enable_mask, u32 status_mask) 545 { 546 i915_reg_t reg = PIPESTAT(pipe); 547 u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK; 548 549 assert_spin_locked(&dev_priv->irq_lock); 550 WARN_ON(!intel_irqs_enabled(dev_priv)); 551 552 if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK || 553 status_mask & ~PIPESTAT_INT_STATUS_MASK, 554 "pipe %c: enable_mask=0x%x, status_mask=0x%x\n", 555 pipe_name(pipe), enable_mask, status_mask)) 556 return; 557 558 if ((pipestat & enable_mask) == enable_mask) 559 return; 560 561 dev_priv->pipestat_irq_mask[pipe] |= status_mask; 562 563 /* Enable the interrupt, clear any pending status */ 564 pipestat |= enable_mask | status_mask; 565 I915_WRITE(reg, pipestat); 566 POSTING_READ(reg); 567 } 568 569 static void 570 __i915_disable_pipestat(struct drm_i915_private *dev_priv, enum i915_pipe pipe, 571 u32 enable_mask, u32 status_mask) 572 { 573 i915_reg_t reg = PIPESTAT(pipe); 574 u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK; 575 576 assert_spin_locked(&dev_priv->irq_lock); 577 WARN_ON(!intel_irqs_enabled(dev_priv)); 578 579 if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK || 580 status_mask & ~PIPESTAT_INT_STATUS_MASK, 581 "pipe %c: enable_mask=0x%x, status_mask=0x%x\n", 582 pipe_name(pipe), enable_mask, status_mask)) 583 return; 584 585 if ((pipestat & enable_mask) == 0) 586 return; 587 588 dev_priv->pipestat_irq_mask[pipe] &= ~status_mask; 589 590 pipestat &= ~enable_mask; 591 I915_WRITE(reg, pipestat); 592 POSTING_READ(reg); 593 } 594 595 static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask) 596 { 597 u32 enable_mask = status_mask << 16; 598 599 /* 600 * On pipe A we don't support the PSR interrupt yet, 601 * on pipe B and C the same bit MBZ. 602 */ 603 if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV)) 604 return 0; 605 /* 606 * On pipe B and C we don't support the PSR interrupt yet, on pipe 607 * A the same bit is for perf counters which we don't use either. 608 */ 609 if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV)) 610 return 0; 611 612 enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS | 613 SPRITE0_FLIP_DONE_INT_EN_VLV | 614 SPRITE1_FLIP_DONE_INT_EN_VLV); 615 if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV) 616 enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV; 617 if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV) 618 enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV; 619 620 return enable_mask; 621 } 622 623 void 624 i915_enable_pipestat(struct drm_i915_private *dev_priv, enum i915_pipe pipe, 625 u32 status_mask) 626 { 627 u32 enable_mask; 628 629 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) 630 enable_mask = vlv_get_pipestat_enable_mask(&dev_priv->drm, 631 status_mask); 632 else 633 enable_mask = status_mask << 16; 634 __i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask); 635 } 636 637 void 638 i915_disable_pipestat(struct drm_i915_private *dev_priv, enum i915_pipe pipe, 639 u32 status_mask) 640 { 641 u32 enable_mask; 642 643 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) 644 enable_mask = vlv_get_pipestat_enable_mask(&dev_priv->drm, 645 status_mask); 646 else 647 enable_mask = status_mask << 16; 648 __i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask); 649 } 650 651 /** 652 * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion 653 * @dev_priv: i915 device private 654 */ 655 static void i915_enable_asle_pipestat(struct drm_i915_private *dev_priv) 656 { 657 if (!dev_priv->opregion.asle || !IS_MOBILE(dev_priv)) 658 return; 659 660 spin_lock_irq(&dev_priv->irq_lock); 661 662 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS); 663 if (INTEL_GEN(dev_priv) >= 4) 664 i915_enable_pipestat(dev_priv, PIPE_A, 665 PIPE_LEGACY_BLC_EVENT_STATUS); 666 667 spin_unlock_irq(&dev_priv->irq_lock); 668 } 669 670 /* 671 * This timing diagram depicts the video signal in and 672 * around the vertical blanking period. 673 * 674 * Assumptions about the fictitious mode used in this example: 675 * vblank_start >= 3 676 * vsync_start = vblank_start + 1 677 * vsync_end = vblank_start + 2 678 * vtotal = vblank_start + 3 679 * 680 * start of vblank: 681 * latch double buffered registers 682 * increment frame counter (ctg+) 683 * generate start of vblank interrupt (gen4+) 684 * | 685 * | frame start: 686 * | generate frame start interrupt (aka. vblank interrupt) (gmch) 687 * | may be shifted forward 1-3 extra lines via PIPECONF 688 * | | 689 * | | start of vsync: 690 * | | generate vsync interrupt 691 * | | | 692 * ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx 693 * . \hs/ . \hs/ \hs/ \hs/ . \hs/ 694 * ----va---> <-----------------vb--------------------> <--------va------------- 695 * | | <----vs-----> | 696 * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2) 697 * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+) 698 * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi) 699 * | | | 700 * last visible pixel first visible pixel 701 * | increment frame counter (gen3/4) 702 * pixel counter = vblank_start * htotal pixel counter = 0 (gen3/4) 703 * 704 * x = horizontal active 705 * _ = horizontal blanking 706 * hs = horizontal sync 707 * va = vertical active 708 * vb = vertical blanking 709 * vs = vertical sync 710 * vbs = vblank_start (number) 711 * 712 * Summary: 713 * - most events happen at the start of horizontal sync 714 * - frame start happens at the start of horizontal blank, 1-4 lines 715 * (depending on PIPECONF settings) after the start of vblank 716 * - gen3/4 pixel and frame counter are synchronized with the start 717 * of horizontal active on the first line of vertical active 718 */ 719 720 /* Called from drm generic code, passed a 'crtc', which 721 * we use as a pipe index 722 */ 723 static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe) 724 { 725 struct drm_i915_private *dev_priv = to_i915(dev); 726 i915_reg_t high_frame, low_frame; 727 u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal; 728 struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv, 729 pipe); 730 const struct drm_display_mode *mode = &intel_crtc->base.hwmode; 731 732 htotal = mode->crtc_htotal; 733 hsync_start = mode->crtc_hsync_start; 734 vbl_start = mode->crtc_vblank_start; 735 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 736 vbl_start = DIV_ROUND_UP(vbl_start, 2); 737 738 /* Convert to pixel count */ 739 vbl_start *= htotal; 740 741 /* Start of vblank event occurs at start of hsync */ 742 vbl_start -= htotal - hsync_start; 743 744 high_frame = PIPEFRAME(pipe); 745 low_frame = PIPEFRAMEPIXEL(pipe); 746 747 /* 748 * High & low register fields aren't synchronized, so make sure 749 * we get a low value that's stable across two reads of the high 750 * register. 751 */ 752 do { 753 high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK; 754 low = I915_READ(low_frame); 755 high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK; 756 } while (high1 != high2); 757 758 high1 >>= PIPE_FRAME_HIGH_SHIFT; 759 pixel = low & PIPE_PIXEL_MASK; 760 low >>= PIPE_FRAME_LOW_SHIFT; 761 762 /* 763 * The frame counter increments at beginning of active. 764 * Cook up a vblank counter by also checking the pixel 765 * counter against vblank start. 766 */ 767 return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff; 768 } 769 770 static u32 g4x_get_vblank_counter(struct drm_device *dev, unsigned int pipe) 771 { 772 struct drm_i915_private *dev_priv = to_i915(dev); 773 774 return I915_READ(PIPE_FRMCOUNT_G4X(pipe)); 775 } 776 777 /* I915_READ_FW, only for fast reads of display block, no need for forcewake etc. */ 778 static int __intel_get_crtc_scanline(struct intel_crtc *crtc) 779 { 780 struct drm_device *dev = crtc->base.dev; 781 struct drm_i915_private *dev_priv = to_i915(dev); 782 const struct drm_display_mode *mode = &crtc->base.hwmode; 783 enum i915_pipe pipe = crtc->pipe; 784 int position, vtotal; 785 786 vtotal = mode->crtc_vtotal; 787 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 788 vtotal /= 2; 789 790 if (IS_GEN2(dev_priv)) 791 position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN2; 792 else 793 position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN3; 794 795 /* 796 * On HSW, the DSL reg (0x70000) appears to return 0 if we 797 * read it just before the start of vblank. So try it again 798 * so we don't accidentally end up spanning a vblank frame 799 * increment, causing the pipe_update_end() code to squak at us. 800 * 801 * The nature of this problem means we can't simply check the ISR 802 * bit and return the vblank start value; nor can we use the scanline 803 * debug register in the transcoder as it appears to have the same 804 * problem. We may need to extend this to include other platforms, 805 * but so far testing only shows the problem on HSW. 806 */ 807 if (HAS_DDI(dev_priv) && !position) { 808 int i, temp; 809 810 for (i = 0; i < 100; i++) { 811 udelay(1); 812 temp = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & 813 DSL_LINEMASK_GEN3; 814 if (temp != position) { 815 position = temp; 816 break; 817 } 818 } 819 } 820 821 /* 822 * See update_scanline_offset() for the details on the 823 * scanline_offset adjustment. 824 */ 825 return (position + crtc->scanline_offset) % vtotal; 826 } 827 828 static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe, 829 unsigned int flags, int *vpos, int *hpos, 830 ktime_t *stime, ktime_t *etime, 831 const struct drm_display_mode *mode) 832 { 833 struct drm_i915_private *dev_priv = to_i915(dev); 834 struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv, 835 pipe); 836 int position; 837 int vbl_start, vbl_end, hsync_start, htotal, vtotal; 838 bool in_vbl = true; 839 int ret = 0; 840 unsigned long irqflags; 841 842 if (WARN_ON(!mode->crtc_clock)) { 843 DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled " 844 "pipe %c\n", pipe_name(pipe)); 845 return 0; 846 } 847 848 htotal = mode->crtc_htotal; 849 hsync_start = mode->crtc_hsync_start; 850 vtotal = mode->crtc_vtotal; 851 vbl_start = mode->crtc_vblank_start; 852 vbl_end = mode->crtc_vblank_end; 853 854 if (mode->flags & DRM_MODE_FLAG_INTERLACE) { 855 vbl_start = DIV_ROUND_UP(vbl_start, 2); 856 vbl_end /= 2; 857 vtotal /= 2; 858 } 859 860 ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE; 861 862 /* 863 * Lock uncore.lock, as we will do multiple timing critical raw 864 * register reads, potentially with preemption disabled, so the 865 * following code must not block on uncore.lock. 866 */ 867 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 868 869 /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */ 870 871 /* Get optional system timestamp before query. */ 872 if (stime) 873 *stime = ktime_get(); 874 875 if (IS_GEN2(dev_priv) || IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) { 876 /* No obvious pixelcount register. Only query vertical 877 * scanout position from Display scan line register. 878 */ 879 position = __intel_get_crtc_scanline(intel_crtc); 880 } else { 881 /* Have access to pixelcount since start of frame. 882 * We can split this into vertical and horizontal 883 * scanout position. 884 */ 885 position = (I915_READ_FW(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT; 886 887 /* convert to pixel counts */ 888 vbl_start *= htotal; 889 vbl_end *= htotal; 890 vtotal *= htotal; 891 892 /* 893 * In interlaced modes, the pixel counter counts all pixels, 894 * so one field will have htotal more pixels. In order to avoid 895 * the reported position from jumping backwards when the pixel 896 * counter is beyond the length of the shorter field, just 897 * clamp the position the length of the shorter field. This 898 * matches how the scanline counter based position works since 899 * the scanline counter doesn't count the two half lines. 900 */ 901 if (position >= vtotal) 902 position = vtotal - 1; 903 904 /* 905 * Start of vblank interrupt is triggered at start of hsync, 906 * just prior to the first active line of vblank. However we 907 * consider lines to start at the leading edge of horizontal 908 * active. So, should we get here before we've crossed into 909 * the horizontal active of the first line in vblank, we would 910 * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that, 911 * always add htotal-hsync_start to the current pixel position. 912 */ 913 position = (position + htotal - hsync_start) % vtotal; 914 } 915 916 /* Get optional system timestamp after query. */ 917 if (etime) 918 *etime = ktime_get(); 919 920 /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */ 921 922 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 923 924 in_vbl = position >= vbl_start && position < vbl_end; 925 926 /* 927 * While in vblank, position will be negative 928 * counting up towards 0 at vbl_end. And outside 929 * vblank, position will be positive counting 930 * up since vbl_end. 931 */ 932 if (position >= vbl_start) 933 position -= vbl_end; 934 else 935 position += vtotal - vbl_end; 936 937 if (IS_GEN2(dev_priv) || IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) { 938 *vpos = position; 939 *hpos = 0; 940 } else { 941 *vpos = position / htotal; 942 *hpos = position - (*vpos * htotal); 943 } 944 945 /* In vblank? */ 946 if (in_vbl) 947 ret |= DRM_SCANOUTPOS_IN_VBLANK; 948 949 return ret; 950 } 951 952 int intel_get_crtc_scanline(struct intel_crtc *crtc) 953 { 954 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 955 unsigned long irqflags; 956 int position; 957 958 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 959 position = __intel_get_crtc_scanline(crtc); 960 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 961 962 return position; 963 } 964 965 static int i915_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe, 966 int *max_error, 967 struct timeval *vblank_time, 968 unsigned flags) 969 { 970 struct drm_i915_private *dev_priv = to_i915(dev); 971 struct intel_crtc *crtc; 972 973 if (pipe >= INTEL_INFO(dev_priv)->num_pipes) { 974 DRM_ERROR("Invalid crtc %u\n", pipe); 975 return -EINVAL; 976 } 977 978 /* Get drm_crtc to timestamp: */ 979 crtc = intel_get_crtc_for_pipe(dev_priv, pipe); 980 if (crtc == NULL) { 981 DRM_ERROR("Invalid crtc %u\n", pipe); 982 return -EINVAL; 983 } 984 985 if (!crtc->base.hwmode.crtc_clock) { 986 DRM_DEBUG_KMS("crtc %u is disabled\n", pipe); 987 return -EBUSY; 988 } 989 990 /* Helper routine in DRM core does all the work: */ 991 return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error, 992 vblank_time, flags, 993 &crtc->base.hwmode); 994 } 995 996 static void ironlake_rps_change_irq_handler(struct drm_i915_private *dev_priv) 997 { 998 u32 busy_up, busy_down, max_avg, min_avg; 999 u8 new_delay; 1000 1001 lockmgr(&mchdev_lock, LK_EXCLUSIVE); 1002 1003 I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS)); 1004 1005 new_delay = dev_priv->ips.cur_delay; 1006 1007 I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG); 1008 busy_up = I915_READ(RCPREVBSYTUPAVG); 1009 busy_down = I915_READ(RCPREVBSYTDNAVG); 1010 max_avg = I915_READ(RCBMAXAVG); 1011 min_avg = I915_READ(RCBMINAVG); 1012 1013 /* Handle RCS change request from hw */ 1014 if (busy_up > max_avg) { 1015 if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay) 1016 new_delay = dev_priv->ips.cur_delay - 1; 1017 if (new_delay < dev_priv->ips.max_delay) 1018 new_delay = dev_priv->ips.max_delay; 1019 } else if (busy_down < min_avg) { 1020 if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay) 1021 new_delay = dev_priv->ips.cur_delay + 1; 1022 if (new_delay > dev_priv->ips.min_delay) 1023 new_delay = dev_priv->ips.min_delay; 1024 } 1025 1026 if (ironlake_set_drps(dev_priv, new_delay)) 1027 dev_priv->ips.cur_delay = new_delay; 1028 1029 lockmgr(&mchdev_lock, LK_RELEASE); 1030 1031 return; 1032 } 1033 1034 static void notify_ring(struct intel_engine_cs *engine) 1035 { 1036 smp_store_mb(engine->breadcrumbs.irq_posted, true); 1037 if (intel_engine_wakeup(engine)) 1038 trace_i915_gem_request_notify(engine); 1039 } 1040 1041 static void vlv_c0_read(struct drm_i915_private *dev_priv, 1042 struct intel_rps_ei *ei) 1043 { 1044 ei->cz_clock = vlv_punit_read(dev_priv, PUNIT_REG_CZ_TIMESTAMP); 1045 ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT); 1046 ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT); 1047 } 1048 1049 void gen6_rps_reset_ei(struct drm_i915_private *dev_priv) 1050 { 1051 memset(&dev_priv->rps.ei, 0, sizeof(dev_priv->rps.ei)); 1052 } 1053 1054 static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir) 1055 { 1056 const struct intel_rps_ei *prev = &dev_priv->rps.ei; 1057 struct intel_rps_ei now; 1058 u32 events = 0; 1059 1060 if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0) 1061 return 0; 1062 1063 vlv_c0_read(dev_priv, &now); 1064 if (now.cz_clock == 0) 1065 return 0; 1066 1067 if (prev->cz_clock) { 1068 u64 time, c0; 1069 unsigned int mul; 1070 1071 mul = VLV_CZ_CLOCK_TO_MILLI_SEC * 100; /* scale to threshold% */ 1072 if (I915_READ(VLV_COUNTER_CONTROL) & VLV_COUNT_RANGE_HIGH) 1073 mul <<= 8; 1074 1075 time = now.cz_clock - prev->cz_clock; 1076 time *= dev_priv->czclk_freq; 1077 1078 /* Workload can be split between render + media, 1079 * e.g. SwapBuffers being blitted in X after being rendered in 1080 * mesa. To account for this we need to combine both engines 1081 * into our activity counter. 1082 */ 1083 c0 = now.render_c0 - prev->render_c0; 1084 c0 += now.media_c0 - prev->media_c0; 1085 c0 *= mul; 1086 1087 if (c0 > time * dev_priv->rps.up_threshold) 1088 events = GEN6_PM_RP_UP_THRESHOLD; 1089 else if (c0 < time * dev_priv->rps.down_threshold) 1090 events = GEN6_PM_RP_DOWN_THRESHOLD; 1091 } 1092 1093 dev_priv->rps.ei = now; 1094 return events; 1095 } 1096 1097 static bool any_waiters(struct drm_i915_private *dev_priv) 1098 { 1099 struct intel_engine_cs *engine; 1100 enum intel_engine_id id; 1101 1102 for_each_engine(engine, dev_priv, id) 1103 if (intel_engine_has_waiter(engine)) 1104 return true; 1105 1106 return false; 1107 } 1108 1109 static void gen6_pm_rps_work(struct work_struct *work) 1110 { 1111 struct drm_i915_private *dev_priv = 1112 container_of(work, struct drm_i915_private, rps.work); 1113 bool client_boost; 1114 int new_delay, adj, min, max; 1115 u32 pm_iir; 1116 1117 spin_lock_irq(&dev_priv->irq_lock); 1118 /* Speed up work cancelation during disabling rps interrupts. */ 1119 if (!dev_priv->rps.interrupts_enabled) { 1120 spin_unlock_irq(&dev_priv->irq_lock); 1121 return; 1122 } 1123 1124 pm_iir = dev_priv->rps.pm_iir; 1125 dev_priv->rps.pm_iir = 0; 1126 /* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */ 1127 gen6_unmask_pm_irq(dev_priv, dev_priv->pm_rps_events); 1128 client_boost = dev_priv->rps.client_boost; 1129 dev_priv->rps.client_boost = false; 1130 spin_unlock_irq(&dev_priv->irq_lock); 1131 1132 /* Make sure we didn't queue anything we're not going to process. */ 1133 WARN_ON(pm_iir & ~dev_priv->pm_rps_events); 1134 1135 if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost) 1136 return; 1137 1138 mutex_lock(&dev_priv->rps.hw_lock); 1139 1140 pm_iir |= vlv_wa_c0_ei(dev_priv, pm_iir); 1141 1142 adj = dev_priv->rps.last_adj; 1143 new_delay = dev_priv->rps.cur_freq; 1144 min = dev_priv->rps.min_freq_softlimit; 1145 max = dev_priv->rps.max_freq_softlimit; 1146 if (client_boost || any_waiters(dev_priv)) 1147 max = dev_priv->rps.max_freq; 1148 if (client_boost && new_delay < dev_priv->rps.boost_freq) { 1149 new_delay = dev_priv->rps.boost_freq; 1150 adj = 0; 1151 } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) { 1152 if (adj > 0) 1153 adj *= 2; 1154 else /* CHV needs even encode values */ 1155 adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1; 1156 /* 1157 * For better performance, jump directly 1158 * to RPe if we're below it. 1159 */ 1160 if (new_delay < dev_priv->rps.efficient_freq - adj) { 1161 new_delay = dev_priv->rps.efficient_freq; 1162 adj = 0; 1163 } 1164 } else if (client_boost || any_waiters(dev_priv)) { 1165 adj = 0; 1166 } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) { 1167 if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq) 1168 new_delay = dev_priv->rps.efficient_freq; 1169 else 1170 new_delay = dev_priv->rps.min_freq_softlimit; 1171 adj = 0; 1172 } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) { 1173 if (adj < 0) 1174 adj *= 2; 1175 else /* CHV needs even encode values */ 1176 adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1; 1177 } else { /* unknown event */ 1178 adj = 0; 1179 } 1180 1181 dev_priv->rps.last_adj = adj; 1182 1183 /* sysfs frequency interfaces may have snuck in while servicing the 1184 * interrupt 1185 */ 1186 new_delay += adj; 1187 new_delay = clamp_t(int, new_delay, min, max); 1188 1189 intel_set_rps(dev_priv, new_delay); 1190 1191 mutex_unlock(&dev_priv->rps.hw_lock); 1192 } 1193 1194 1195 /** 1196 * ivybridge_parity_work - Workqueue called when a parity error interrupt 1197 * occurred. 1198 * @work: workqueue struct 1199 * 1200 * Doesn't actually do anything except notify userspace. As a consequence of 1201 * this event, userspace should try to remap the bad rows since statistically 1202 * it is likely the same row is more likely to go bad again. 1203 */ 1204 static void ivybridge_parity_work(struct work_struct *work) 1205 { 1206 struct drm_i915_private *dev_priv = 1207 container_of(work, struct drm_i915_private, l3_parity.error_work); 1208 u32 error_status, row, bank, subbank; 1209 char *parity_event[6]; 1210 uint32_t misccpctl; 1211 uint8_t slice = 0; 1212 1213 /* We must turn off DOP level clock gating to access the L3 registers. 1214 * In order to prevent a get/put style interface, acquire struct mutex 1215 * any time we access those registers. 1216 */ 1217 mutex_lock(&dev_priv->drm.struct_mutex); 1218 1219 /* If we've screwed up tracking, just let the interrupt fire again */ 1220 if (WARN_ON(!dev_priv->l3_parity.which_slice)) 1221 goto out; 1222 1223 misccpctl = I915_READ(GEN7_MISCCPCTL); 1224 I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE); 1225 POSTING_READ(GEN7_MISCCPCTL); 1226 1227 while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) { 1228 i915_reg_t reg; 1229 1230 slice--; 1231 if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv))) 1232 break; 1233 1234 dev_priv->l3_parity.which_slice &= ~(1<<slice); 1235 1236 reg = GEN7_L3CDERRST1(slice); 1237 1238 error_status = I915_READ(reg); 1239 row = GEN7_PARITY_ERROR_ROW(error_status); 1240 bank = GEN7_PARITY_ERROR_BANK(error_status); 1241 subbank = GEN7_PARITY_ERROR_SUBBANK(error_status); 1242 1243 I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE); 1244 POSTING_READ(reg); 1245 1246 parity_event[0] = I915_L3_PARITY_UEVENT "=1"; 1247 parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row); 1248 parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank); 1249 parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank); 1250 parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice); 1251 parity_event[5] = NULL; 1252 1253 kobject_uevent_env(&dev_priv->drm.primary->kdev->kobj, 1254 KOBJ_CHANGE, parity_event); 1255 1256 DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n", 1257 slice, row, bank, subbank); 1258 1259 kfree(parity_event[4]); 1260 kfree(parity_event[3]); 1261 kfree(parity_event[2]); 1262 kfree(parity_event[1]); 1263 } 1264 1265 I915_WRITE(GEN7_MISCCPCTL, misccpctl); 1266 1267 out: 1268 WARN_ON(dev_priv->l3_parity.which_slice); 1269 spin_lock_irq(&dev_priv->irq_lock); 1270 gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv)); 1271 spin_unlock_irq(&dev_priv->irq_lock); 1272 1273 mutex_unlock(&dev_priv->drm.struct_mutex); 1274 } 1275 1276 static void ivybridge_parity_error_irq_handler(struct drm_i915_private *dev_priv, 1277 u32 iir) 1278 { 1279 if (!HAS_L3_DPF(dev_priv)) 1280 return; 1281 1282 lockmgr(&dev_priv->irq_lock, LK_EXCLUSIVE); 1283 gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv)); 1284 lockmgr(&dev_priv->irq_lock, LK_RELEASE); 1285 1286 iir &= GT_PARITY_ERROR(dev_priv); 1287 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1) 1288 dev_priv->l3_parity.which_slice |= 1 << 1; 1289 1290 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT) 1291 dev_priv->l3_parity.which_slice |= 1 << 0; 1292 1293 queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work); 1294 } 1295 1296 static void ilk_gt_irq_handler(struct drm_i915_private *dev_priv, 1297 u32 gt_iir) 1298 { 1299 if (gt_iir & GT_RENDER_USER_INTERRUPT) 1300 notify_ring(dev_priv->engine[RCS]); 1301 if (gt_iir & ILK_BSD_USER_INTERRUPT) 1302 notify_ring(dev_priv->engine[VCS]); 1303 } 1304 1305 static void snb_gt_irq_handler(struct drm_i915_private *dev_priv, 1306 u32 gt_iir) 1307 { 1308 if (gt_iir & GT_RENDER_USER_INTERRUPT) 1309 notify_ring(dev_priv->engine[RCS]); 1310 if (gt_iir & GT_BSD_USER_INTERRUPT) 1311 notify_ring(dev_priv->engine[VCS]); 1312 if (gt_iir & GT_BLT_USER_INTERRUPT) 1313 notify_ring(dev_priv->engine[BCS]); 1314 1315 if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT | 1316 GT_BSD_CS_ERROR_INTERRUPT | 1317 GT_RENDER_CS_MASTER_ERROR_INTERRUPT)) 1318 DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir); 1319 1320 if (gt_iir & GT_PARITY_ERROR(dev_priv)) 1321 ivybridge_parity_error_irq_handler(dev_priv, gt_iir); 1322 } 1323 1324 static __always_inline void 1325 gen8_cs_irq_handler(struct intel_engine_cs *engine, u32 iir, int test_shift) 1326 { 1327 if (iir & (GT_RENDER_USER_INTERRUPT << test_shift)) 1328 notify_ring(engine); 1329 if (iir & (GT_CONTEXT_SWITCH_INTERRUPT << test_shift)) 1330 tasklet_schedule(&engine->irq_tasklet); 1331 } 1332 1333 static irqreturn_t gen8_gt_irq_ack(struct drm_i915_private *dev_priv, 1334 u32 master_ctl, 1335 u32 gt_iir[4]) 1336 { 1337 irqreturn_t ret = IRQ_NONE; 1338 1339 if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) { 1340 gt_iir[0] = I915_READ_FW(GEN8_GT_IIR(0)); 1341 if (gt_iir[0]) { 1342 I915_WRITE_FW(GEN8_GT_IIR(0), gt_iir[0]); 1343 ret = IRQ_HANDLED; 1344 } else 1345 DRM_ERROR("The master control interrupt lied (GT0)!\n"); 1346 } 1347 1348 if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) { 1349 gt_iir[1] = I915_READ_FW(GEN8_GT_IIR(1)); 1350 if (gt_iir[1]) { 1351 I915_WRITE_FW(GEN8_GT_IIR(1), gt_iir[1]); 1352 ret = IRQ_HANDLED; 1353 } else 1354 DRM_ERROR("The master control interrupt lied (GT1)!\n"); 1355 } 1356 1357 if (master_ctl & GEN8_GT_VECS_IRQ) { 1358 gt_iir[3] = I915_READ_FW(GEN8_GT_IIR(3)); 1359 if (gt_iir[3]) { 1360 I915_WRITE_FW(GEN8_GT_IIR(3), gt_iir[3]); 1361 ret = IRQ_HANDLED; 1362 } else 1363 DRM_ERROR("The master control interrupt lied (GT3)!\n"); 1364 } 1365 1366 if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) { 1367 gt_iir[2] = I915_READ_FW(GEN8_GT_IIR(2)); 1368 if (gt_iir[2] & (dev_priv->pm_rps_events | 1369 dev_priv->pm_guc_events)) { 1370 I915_WRITE_FW(GEN8_GT_IIR(2), 1371 gt_iir[2] & (dev_priv->pm_rps_events | 1372 dev_priv->pm_guc_events)); 1373 ret = IRQ_HANDLED; 1374 } else 1375 DRM_ERROR("The master control interrupt lied (PM)!\n"); 1376 } 1377 1378 return ret; 1379 } 1380 1381 static void gen8_gt_irq_handler(struct drm_i915_private *dev_priv, 1382 u32 gt_iir[4]) 1383 { 1384 if (gt_iir[0]) { 1385 gen8_cs_irq_handler(dev_priv->engine[RCS], 1386 gt_iir[0], GEN8_RCS_IRQ_SHIFT); 1387 gen8_cs_irq_handler(dev_priv->engine[BCS], 1388 gt_iir[0], GEN8_BCS_IRQ_SHIFT); 1389 } 1390 1391 if (gt_iir[1]) { 1392 gen8_cs_irq_handler(dev_priv->engine[VCS], 1393 gt_iir[1], GEN8_VCS1_IRQ_SHIFT); 1394 gen8_cs_irq_handler(dev_priv->engine[VCS2], 1395 gt_iir[1], GEN8_VCS2_IRQ_SHIFT); 1396 } 1397 1398 if (gt_iir[3]) 1399 gen8_cs_irq_handler(dev_priv->engine[VECS], 1400 gt_iir[3], GEN8_VECS_IRQ_SHIFT); 1401 1402 if (gt_iir[2] & dev_priv->pm_rps_events) 1403 gen6_rps_irq_handler(dev_priv, gt_iir[2]); 1404 1405 if (gt_iir[2] & dev_priv->pm_guc_events) 1406 gen9_guc_irq_handler(dev_priv, gt_iir[2]); 1407 } 1408 1409 static bool bxt_port_hotplug_long_detect(enum port port, u32 val) 1410 { 1411 switch (port) { 1412 case PORT_A: 1413 return val & PORTA_HOTPLUG_LONG_DETECT; 1414 case PORT_B: 1415 return val & PORTB_HOTPLUG_LONG_DETECT; 1416 case PORT_C: 1417 return val & PORTC_HOTPLUG_LONG_DETECT; 1418 default: 1419 return false; 1420 } 1421 } 1422 1423 static bool spt_port_hotplug2_long_detect(enum port port, u32 val) 1424 { 1425 switch (port) { 1426 case PORT_E: 1427 return val & PORTE_HOTPLUG_LONG_DETECT; 1428 default: 1429 return false; 1430 } 1431 } 1432 1433 static bool spt_port_hotplug_long_detect(enum port port, u32 val) 1434 { 1435 switch (port) { 1436 case PORT_A: 1437 return val & PORTA_HOTPLUG_LONG_DETECT; 1438 case PORT_B: 1439 return val & PORTB_HOTPLUG_LONG_DETECT; 1440 case PORT_C: 1441 return val & PORTC_HOTPLUG_LONG_DETECT; 1442 case PORT_D: 1443 return val & PORTD_HOTPLUG_LONG_DETECT; 1444 default: 1445 return false; 1446 } 1447 } 1448 1449 static bool ilk_port_hotplug_long_detect(enum port port, u32 val) 1450 { 1451 switch (port) { 1452 case PORT_A: 1453 return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT; 1454 default: 1455 return false; 1456 } 1457 } 1458 1459 static bool pch_port_hotplug_long_detect(enum port port, u32 val) 1460 { 1461 switch (port) { 1462 case PORT_B: 1463 return val & PORTB_HOTPLUG_LONG_DETECT; 1464 case PORT_C: 1465 return val & PORTC_HOTPLUG_LONG_DETECT; 1466 case PORT_D: 1467 return val & PORTD_HOTPLUG_LONG_DETECT; 1468 default: 1469 return false; 1470 } 1471 } 1472 1473 static bool i9xx_port_hotplug_long_detect(enum port port, u32 val) 1474 { 1475 switch (port) { 1476 case PORT_B: 1477 return val & PORTB_HOTPLUG_INT_LONG_PULSE; 1478 case PORT_C: 1479 return val & PORTC_HOTPLUG_INT_LONG_PULSE; 1480 case PORT_D: 1481 return val & PORTD_HOTPLUG_INT_LONG_PULSE; 1482 default: 1483 return false; 1484 } 1485 } 1486 1487 /* 1488 * Get a bit mask of pins that have triggered, and which ones may be long. 1489 * This can be called multiple times with the same masks to accumulate 1490 * hotplug detection results from several registers. 1491 * 1492 * Note that the caller is expected to zero out the masks initially. 1493 */ 1494 static void intel_get_hpd_pins(u32 *pin_mask, u32 *long_mask, 1495 u32 hotplug_trigger, u32 dig_hotplug_reg, 1496 const u32 hpd[HPD_NUM_PINS], 1497 bool long_pulse_detect(enum port port, u32 val)) 1498 { 1499 enum port port; 1500 int i; 1501 1502 for_each_hpd_pin(i) { 1503 if ((hpd[i] & hotplug_trigger) == 0) 1504 continue; 1505 1506 *pin_mask |= BIT(i); 1507 1508 if (!intel_hpd_pin_to_port(i, &port)) 1509 continue; 1510 1511 if (long_pulse_detect(port, dig_hotplug_reg)) 1512 *long_mask |= BIT(i); 1513 } 1514 1515 DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x\n", 1516 hotplug_trigger, dig_hotplug_reg, *pin_mask); 1517 1518 } 1519 1520 static void gmbus_irq_handler(struct drm_i915_private *dev_priv) 1521 { 1522 wake_up_all(&dev_priv->gmbus_wait_queue); 1523 } 1524 1525 static void dp_aux_irq_handler(struct drm_i915_private *dev_priv) 1526 { 1527 wake_up_all(&dev_priv->gmbus_wait_queue); 1528 } 1529 1530 #if defined(CONFIG_DEBUG_FS) 1531 static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv, 1532 enum i915_pipe pipe, 1533 uint32_t crc0, uint32_t crc1, 1534 uint32_t crc2, uint32_t crc3, 1535 uint32_t crc4) 1536 { 1537 struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe]; 1538 struct intel_pipe_crc_entry *entry; 1539 int head, tail; 1540 1541 lockmgr(&pipe_crc->lock, LK_EXCLUSIVE); 1542 1543 if (!pipe_crc->entries) { 1544 lockmgr(&pipe_crc->lock, LK_RELEASE); 1545 DRM_DEBUG_KMS("spurious interrupt\n"); 1546 return; 1547 } 1548 1549 head = pipe_crc->head; 1550 tail = pipe_crc->tail; 1551 1552 if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) { 1553 lockmgr(&pipe_crc->lock, LK_RELEASE); 1554 DRM_ERROR("CRC buffer overflowing\n"); 1555 return; 1556 } 1557 1558 entry = &pipe_crc->entries[head]; 1559 1560 entry->frame = dev_priv->drm.driver->get_vblank_counter(&dev_priv->drm, 1561 pipe); 1562 entry->crc[0] = crc0; 1563 entry->crc[1] = crc1; 1564 entry->crc[2] = crc2; 1565 entry->crc[3] = crc3; 1566 entry->crc[4] = crc4; 1567 1568 head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1); 1569 pipe_crc->head = head; 1570 1571 lockmgr(&pipe_crc->lock, LK_RELEASE); 1572 1573 wake_up_interruptible(&pipe_crc->wq); 1574 } 1575 #else 1576 static inline void 1577 display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv, 1578 enum i915_pipe pipe, 1579 uint32_t crc0, uint32_t crc1, 1580 uint32_t crc2, uint32_t crc3, 1581 uint32_t crc4) {} 1582 #endif 1583 1584 1585 static void hsw_pipe_crc_irq_handler(struct drm_i915_private *dev_priv, 1586 enum i915_pipe pipe) 1587 { 1588 display_pipe_crc_irq_handler(dev_priv, pipe, 1589 I915_READ(PIPE_CRC_RES_1_IVB(pipe)), 1590 0, 0, 0, 0); 1591 } 1592 1593 static void ivb_pipe_crc_irq_handler(struct drm_i915_private *dev_priv, 1594 enum i915_pipe pipe) 1595 { 1596 display_pipe_crc_irq_handler(dev_priv, pipe, 1597 I915_READ(PIPE_CRC_RES_1_IVB(pipe)), 1598 I915_READ(PIPE_CRC_RES_2_IVB(pipe)), 1599 I915_READ(PIPE_CRC_RES_3_IVB(pipe)), 1600 I915_READ(PIPE_CRC_RES_4_IVB(pipe)), 1601 I915_READ(PIPE_CRC_RES_5_IVB(pipe))); 1602 } 1603 1604 static void i9xx_pipe_crc_irq_handler(struct drm_i915_private *dev_priv, 1605 enum i915_pipe pipe) 1606 { 1607 uint32_t res1, res2; 1608 1609 if (INTEL_GEN(dev_priv) >= 3) 1610 res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe)); 1611 else 1612 res1 = 0; 1613 1614 if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv)) 1615 res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe)); 1616 else 1617 res2 = 0; 1618 1619 display_pipe_crc_irq_handler(dev_priv, pipe, 1620 I915_READ(PIPE_CRC_RES_RED(pipe)), 1621 I915_READ(PIPE_CRC_RES_GREEN(pipe)), 1622 I915_READ(PIPE_CRC_RES_BLUE(pipe)), 1623 res1, res2); 1624 } 1625 1626 /* The RPS events need forcewake, so we add them to a work queue and mask their 1627 * IMR bits until the work is done. Other interrupts can be processed without 1628 * the work queue. */ 1629 static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir) 1630 { 1631 if (pm_iir & dev_priv->pm_rps_events) { 1632 lockmgr(&dev_priv->irq_lock, LK_EXCLUSIVE); 1633 gen6_mask_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events); 1634 if (dev_priv->rps.interrupts_enabled) { 1635 dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events; 1636 schedule_work(&dev_priv->rps.work); 1637 } 1638 lockmgr(&dev_priv->irq_lock, LK_RELEASE); 1639 } 1640 1641 if (INTEL_INFO(dev_priv)->gen >= 8) 1642 return; 1643 1644 if (HAS_VEBOX(dev_priv)) { 1645 if (pm_iir & PM_VEBOX_USER_INTERRUPT) 1646 notify_ring(dev_priv->engine[VECS]); 1647 1648 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) 1649 DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir); 1650 } 1651 } 1652 1653 static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 gt_iir) 1654 { 1655 if (gt_iir & GEN9_GUC_TO_HOST_INT_EVENT) { 1656 /* Sample the log buffer flush related bits & clear them out now 1657 * itself from the message identity register to minimize the 1658 * probability of losing a flush interrupt, when there are back 1659 * to back flush interrupts. 1660 * There can be a new flush interrupt, for different log buffer 1661 * type (like for ISR), whilst Host is handling one (for DPC). 1662 * Since same bit is used in message register for ISR & DPC, it 1663 * could happen that GuC sets the bit for 2nd interrupt but Host 1664 * clears out the bit on handling the 1st interrupt. 1665 */ 1666 u32 msg, flush; 1667 1668 msg = I915_READ(SOFT_SCRATCH(15)); 1669 flush = msg & (GUC2HOST_MSG_CRASH_DUMP_POSTED | 1670 GUC2HOST_MSG_FLUSH_LOG_BUFFER); 1671 if (flush) { 1672 /* Clear the message bits that are handled */ 1673 I915_WRITE(SOFT_SCRATCH(15), msg & ~flush); 1674 1675 /* Handle flush interrupt in bottom half */ 1676 queue_work(dev_priv->guc.log.flush_wq, 1677 &dev_priv->guc.log.flush_work); 1678 1679 dev_priv->guc.log.flush_interrupt_count++; 1680 } else { 1681 /* Not clearing of unhandled event bits won't result in 1682 * re-triggering of the interrupt. 1683 */ 1684 } 1685 } 1686 } 1687 1688 static bool intel_pipe_handle_vblank(struct drm_i915_private *dev_priv, 1689 enum i915_pipe pipe) 1690 { 1691 bool ret; 1692 1693 ret = drm_handle_vblank(&dev_priv->drm, pipe); 1694 if (ret) 1695 intel_finish_page_flip_mmio(dev_priv, pipe); 1696 1697 return ret; 1698 } 1699 1700 static void valleyview_pipestat_irq_ack(struct drm_i915_private *dev_priv, 1701 u32 iir, u32 pipe_stats[I915_MAX_PIPES]) 1702 { 1703 int pipe; 1704 1705 lockmgr(&dev_priv->irq_lock, LK_EXCLUSIVE); 1706 1707 if (!dev_priv->display_irqs_enabled) { 1708 lockmgr(&dev_priv->irq_lock, LK_RELEASE); 1709 return; 1710 } 1711 1712 for_each_pipe(dev_priv, pipe) { 1713 i915_reg_t reg; 1714 u32 mask, iir_bit = 0; 1715 1716 /* 1717 * PIPESTAT bits get signalled even when the interrupt is 1718 * disabled with the mask bits, and some of the status bits do 1719 * not generate interrupts at all (like the underrun bit). Hence 1720 * we need to be careful that we only handle what we want to 1721 * handle. 1722 */ 1723 1724 /* fifo underruns are filterered in the underrun handler. */ 1725 mask = PIPE_FIFO_UNDERRUN_STATUS; 1726 1727 switch (pipe) { 1728 case PIPE_A: 1729 iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT; 1730 break; 1731 case PIPE_B: 1732 iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT; 1733 break; 1734 case PIPE_C: 1735 iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT; 1736 break; 1737 } 1738 if (iir & iir_bit) 1739 mask |= dev_priv->pipestat_irq_mask[pipe]; 1740 1741 if (!mask) 1742 continue; 1743 1744 reg = PIPESTAT(pipe); 1745 mask |= PIPESTAT_INT_ENABLE_MASK; 1746 pipe_stats[pipe] = I915_READ(reg) & mask; 1747 1748 /* 1749 * Clear the PIPE*STAT regs before the IIR 1750 */ 1751 if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS | 1752 PIPESTAT_INT_STATUS_MASK)) 1753 I915_WRITE(reg, pipe_stats[pipe]); 1754 } 1755 lockmgr(&dev_priv->irq_lock, LK_RELEASE); 1756 } 1757 1758 static void valleyview_pipestat_irq_handler(struct drm_i915_private *dev_priv, 1759 u32 pipe_stats[I915_MAX_PIPES]) 1760 { 1761 enum i915_pipe pipe; 1762 1763 for_each_pipe(dev_priv, pipe) { 1764 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS && 1765 intel_pipe_handle_vblank(dev_priv, pipe)) 1766 intel_check_page_flip(dev_priv, pipe); 1767 1768 if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) 1769 intel_finish_page_flip_cs(dev_priv, pipe); 1770 1771 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS) 1772 i9xx_pipe_crc_irq_handler(dev_priv, pipe); 1773 1774 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS) 1775 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe); 1776 } 1777 1778 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS) 1779 gmbus_irq_handler(dev_priv); 1780 } 1781 1782 static u32 i9xx_hpd_irq_ack(struct drm_i915_private *dev_priv) 1783 { 1784 u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT); 1785 1786 if (hotplug_status) 1787 I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status); 1788 1789 return hotplug_status; 1790 } 1791 1792 static void i9xx_hpd_irq_handler(struct drm_i915_private *dev_priv, 1793 u32 hotplug_status) 1794 { 1795 u32 pin_mask = 0, long_mask = 0; 1796 1797 if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) || 1798 IS_CHERRYVIEW(dev_priv)) { 1799 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X; 1800 1801 if (hotplug_trigger) { 1802 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger, 1803 hotplug_trigger, hpd_status_g4x, 1804 i9xx_port_hotplug_long_detect); 1805 1806 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); 1807 } 1808 1809 if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X) 1810 dp_aux_irq_handler(dev_priv); 1811 } else { 1812 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915; 1813 1814 if (hotplug_trigger) { 1815 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger, 1816 hotplug_trigger, hpd_status_i915, 1817 i9xx_port_hotplug_long_detect); 1818 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); 1819 } 1820 } 1821 } 1822 1823 static irqreturn_t valleyview_irq_handler(int irq, void *arg) 1824 { 1825 struct drm_device *dev = arg; 1826 struct drm_i915_private *dev_priv = to_i915(dev); 1827 irqreturn_t ret = IRQ_NONE; 1828 1829 if (!intel_irqs_enabled(dev_priv)) 1830 return IRQ_NONE; 1831 1832 /* IRQs are synced during runtime_suspend, we don't require a wakeref */ 1833 disable_rpm_wakeref_asserts(dev_priv); 1834 1835 do { 1836 u32 iir, gt_iir, pm_iir; 1837 u32 pipe_stats[I915_MAX_PIPES] = {}; 1838 u32 hotplug_status = 0; 1839 u32 ier = 0; 1840 1841 gt_iir = I915_READ(GTIIR); 1842 pm_iir = I915_READ(GEN6_PMIIR); 1843 iir = I915_READ(VLV_IIR); 1844 1845 if (gt_iir == 0 && pm_iir == 0 && iir == 0) 1846 break; 1847 1848 ret = IRQ_HANDLED; 1849 1850 /* 1851 * Theory on interrupt generation, based on empirical evidence: 1852 * 1853 * x = ((VLV_IIR & VLV_IER) || 1854 * (((GT_IIR & GT_IER) || (GEN6_PMIIR & GEN6_PMIER)) && 1855 * (VLV_MASTER_IER & MASTER_INTERRUPT_ENABLE))); 1856 * 1857 * A CPU interrupt will only be raised when 'x' has a 0->1 edge. 1858 * Hence we clear MASTER_INTERRUPT_ENABLE and VLV_IER to 1859 * guarantee the CPU interrupt will be raised again even if we 1860 * don't end up clearing all the VLV_IIR, GT_IIR, GEN6_PMIIR 1861 * bits this time around. 1862 */ 1863 I915_WRITE(VLV_MASTER_IER, 0); 1864 ier = I915_READ(VLV_IER); 1865 I915_WRITE(VLV_IER, 0); 1866 1867 if (gt_iir) 1868 I915_WRITE(GTIIR, gt_iir); 1869 if (pm_iir) 1870 I915_WRITE(GEN6_PMIIR, pm_iir); 1871 1872 if (iir & I915_DISPLAY_PORT_INTERRUPT) 1873 hotplug_status = i9xx_hpd_irq_ack(dev_priv); 1874 1875 /* Call regardless, as some status bits might not be 1876 * signalled in iir */ 1877 valleyview_pipestat_irq_ack(dev_priv, iir, pipe_stats); 1878 1879 /* 1880 * VLV_IIR is single buffered, and reflects the level 1881 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last. 1882 */ 1883 if (iir) 1884 I915_WRITE(VLV_IIR, iir); 1885 1886 I915_WRITE(VLV_IER, ier); 1887 I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE); 1888 POSTING_READ(VLV_MASTER_IER); 1889 1890 if (gt_iir) 1891 snb_gt_irq_handler(dev_priv, gt_iir); 1892 if (pm_iir) 1893 gen6_rps_irq_handler(dev_priv, pm_iir); 1894 1895 if (hotplug_status) 1896 i9xx_hpd_irq_handler(dev_priv, hotplug_status); 1897 1898 valleyview_pipestat_irq_handler(dev_priv, pipe_stats); 1899 } while (0); 1900 1901 enable_rpm_wakeref_asserts(dev_priv); 1902 1903 return ret; 1904 } 1905 1906 static irqreturn_t cherryview_irq_handler(int irq, void *arg) 1907 { 1908 struct drm_device *dev = arg; 1909 struct drm_i915_private *dev_priv = to_i915(dev); 1910 irqreturn_t ret = IRQ_NONE; 1911 1912 if (!intel_irqs_enabled(dev_priv)) 1913 return IRQ_NONE; 1914 1915 /* IRQs are synced during runtime_suspend, we don't require a wakeref */ 1916 disable_rpm_wakeref_asserts(dev_priv); 1917 1918 do { 1919 u32 master_ctl, iir; 1920 u32 gt_iir[4] = {}; 1921 u32 pipe_stats[I915_MAX_PIPES] = {}; 1922 u32 hotplug_status = 0; 1923 u32 ier = 0; 1924 1925 master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL; 1926 iir = I915_READ(VLV_IIR); 1927 1928 if (master_ctl == 0 && iir == 0) 1929 break; 1930 1931 ret = IRQ_HANDLED; 1932 1933 /* 1934 * Theory on interrupt generation, based on empirical evidence: 1935 * 1936 * x = ((VLV_IIR & VLV_IER) || 1937 * ((GEN8_MASTER_IRQ & ~GEN8_MASTER_IRQ_CONTROL) && 1938 * (GEN8_MASTER_IRQ & GEN8_MASTER_IRQ_CONTROL))); 1939 * 1940 * A CPU interrupt will only be raised when 'x' has a 0->1 edge. 1941 * Hence we clear GEN8_MASTER_IRQ_CONTROL and VLV_IER to 1942 * guarantee the CPU interrupt will be raised again even if we 1943 * don't end up clearing all the VLV_IIR and GEN8_MASTER_IRQ_CONTROL 1944 * bits this time around. 1945 */ 1946 I915_WRITE(GEN8_MASTER_IRQ, 0); 1947 ier = I915_READ(VLV_IER); 1948 I915_WRITE(VLV_IER, 0); 1949 1950 gen8_gt_irq_ack(dev_priv, master_ctl, gt_iir); 1951 1952 if (iir & I915_DISPLAY_PORT_INTERRUPT) 1953 hotplug_status = i9xx_hpd_irq_ack(dev_priv); 1954 1955 /* Call regardless, as some status bits might not be 1956 * signalled in iir */ 1957 valleyview_pipestat_irq_ack(dev_priv, iir, pipe_stats); 1958 1959 /* 1960 * VLV_IIR is single buffered, and reflects the level 1961 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last. 1962 */ 1963 if (iir) 1964 I915_WRITE(VLV_IIR, iir); 1965 1966 I915_WRITE(VLV_IER, ier); 1967 I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL); 1968 POSTING_READ(GEN8_MASTER_IRQ); 1969 1970 gen8_gt_irq_handler(dev_priv, gt_iir); 1971 1972 if (hotplug_status) 1973 i9xx_hpd_irq_handler(dev_priv, hotplug_status); 1974 1975 valleyview_pipestat_irq_handler(dev_priv, pipe_stats); 1976 } while (0); 1977 1978 enable_rpm_wakeref_asserts(dev_priv); 1979 1980 return ret; 1981 } 1982 1983 static void ibx_hpd_irq_handler(struct drm_i915_private *dev_priv, 1984 u32 hotplug_trigger, 1985 const u32 hpd[HPD_NUM_PINS]) 1986 { 1987 u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0; 1988 1989 /* 1990 * Somehow the PCH doesn't seem to really ack the interrupt to the CPU 1991 * unless we touch the hotplug register, even if hotplug_trigger is 1992 * zero. Not acking leads to "The master control interrupt lied (SDE)!" 1993 * errors. 1994 */ 1995 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG); 1996 if (!hotplug_trigger) { 1997 u32 mask = PORTA_HOTPLUG_STATUS_MASK | 1998 PORTD_HOTPLUG_STATUS_MASK | 1999 PORTC_HOTPLUG_STATUS_MASK | 2000 PORTB_HOTPLUG_STATUS_MASK; 2001 dig_hotplug_reg &= ~mask; 2002 } 2003 2004 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg); 2005 if (!hotplug_trigger) 2006 return; 2007 2008 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger, 2009 dig_hotplug_reg, hpd, 2010 pch_port_hotplug_long_detect); 2011 2012 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); 2013 } 2014 2015 static void ibx_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir) 2016 { 2017 int pipe; 2018 u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK; 2019 2020 ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ibx); 2021 2022 if (pch_iir & SDE_AUDIO_POWER_MASK) { 2023 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >> 2024 SDE_AUDIO_POWER_SHIFT); 2025 DRM_DEBUG_DRIVER("PCH audio power change on port %d\n", 2026 port_name(port)); 2027 } 2028 2029 if (pch_iir & SDE_AUX_MASK) 2030 dp_aux_irq_handler(dev_priv); 2031 2032 if (pch_iir & SDE_GMBUS) 2033 gmbus_irq_handler(dev_priv); 2034 2035 if (pch_iir & SDE_AUDIO_HDCP_MASK) 2036 DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n"); 2037 2038 if (pch_iir & SDE_AUDIO_TRANS_MASK) 2039 DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n"); 2040 2041 if (pch_iir & SDE_POISON) 2042 DRM_ERROR("PCH poison interrupt\n"); 2043 2044 if (pch_iir & SDE_FDI_MASK) 2045 for_each_pipe(dev_priv, pipe) 2046 DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n", 2047 pipe_name(pipe), 2048 I915_READ(FDI_RX_IIR(pipe))); 2049 2050 if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE)) 2051 DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n"); 2052 2053 if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR)) 2054 DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n"); 2055 2056 if (pch_iir & SDE_TRANSA_FIFO_UNDER) 2057 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A); 2058 2059 if (pch_iir & SDE_TRANSB_FIFO_UNDER) 2060 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B); 2061 } 2062 2063 static void ivb_err_int_handler(struct drm_i915_private *dev_priv) 2064 { 2065 u32 err_int = I915_READ(GEN7_ERR_INT); 2066 enum i915_pipe pipe; 2067 2068 if (err_int & ERR_INT_POISON) 2069 DRM_ERROR("Poison interrupt\n"); 2070 2071 for_each_pipe(dev_priv, pipe) { 2072 if (err_int & ERR_INT_FIFO_UNDERRUN(pipe)) 2073 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe); 2074 2075 if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) { 2076 if (IS_IVYBRIDGE(dev_priv)) 2077 ivb_pipe_crc_irq_handler(dev_priv, pipe); 2078 else 2079 hsw_pipe_crc_irq_handler(dev_priv, pipe); 2080 } 2081 } 2082 2083 I915_WRITE(GEN7_ERR_INT, err_int); 2084 } 2085 2086 static void cpt_serr_int_handler(struct drm_i915_private *dev_priv) 2087 { 2088 u32 serr_int = I915_READ(SERR_INT); 2089 2090 if (serr_int & SERR_INT_POISON) 2091 DRM_ERROR("PCH poison interrupt\n"); 2092 2093 if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN) 2094 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A); 2095 2096 if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN) 2097 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B); 2098 2099 if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN) 2100 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_C); 2101 2102 I915_WRITE(SERR_INT, serr_int); 2103 } 2104 2105 static void cpt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir) 2106 { 2107 int pipe; 2108 u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT; 2109 2110 ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_cpt); 2111 2112 if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) { 2113 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >> 2114 SDE_AUDIO_POWER_SHIFT_CPT); 2115 DRM_DEBUG_DRIVER("PCH audio power change on port %c\n", 2116 port_name(port)); 2117 } 2118 2119 if (pch_iir & SDE_AUX_MASK_CPT) 2120 dp_aux_irq_handler(dev_priv); 2121 2122 if (pch_iir & SDE_GMBUS_CPT) 2123 gmbus_irq_handler(dev_priv); 2124 2125 if (pch_iir & SDE_AUDIO_CP_REQ_CPT) 2126 DRM_DEBUG_DRIVER("Audio CP request interrupt\n"); 2127 2128 if (pch_iir & SDE_AUDIO_CP_CHG_CPT) 2129 DRM_DEBUG_DRIVER("Audio CP change interrupt\n"); 2130 2131 if (pch_iir & SDE_FDI_MASK_CPT) 2132 for_each_pipe(dev_priv, pipe) 2133 DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n", 2134 pipe_name(pipe), 2135 I915_READ(FDI_RX_IIR(pipe))); 2136 2137 if (pch_iir & SDE_ERROR_CPT) 2138 cpt_serr_int_handler(dev_priv); 2139 } 2140 2141 static void spt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir) 2142 { 2143 u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT & 2144 ~SDE_PORTE_HOTPLUG_SPT; 2145 u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT; 2146 u32 pin_mask = 0, long_mask = 0; 2147 2148 if (hotplug_trigger) { 2149 u32 dig_hotplug_reg; 2150 2151 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG); 2152 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg); 2153 2154 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger, 2155 dig_hotplug_reg, hpd_spt, 2156 spt_port_hotplug_long_detect); 2157 } 2158 2159 if (hotplug2_trigger) { 2160 u32 dig_hotplug_reg; 2161 2162 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2); 2163 I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg); 2164 2165 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug2_trigger, 2166 dig_hotplug_reg, hpd_spt, 2167 spt_port_hotplug2_long_detect); 2168 } 2169 2170 if (pin_mask) 2171 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); 2172 2173 if (pch_iir & SDE_GMBUS_CPT) 2174 gmbus_irq_handler(dev_priv); 2175 } 2176 2177 static void ilk_hpd_irq_handler(struct drm_i915_private *dev_priv, 2178 u32 hotplug_trigger, 2179 const u32 hpd[HPD_NUM_PINS]) 2180 { 2181 u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0; 2182 2183 dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL); 2184 I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg); 2185 2186 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger, 2187 dig_hotplug_reg, hpd, 2188 ilk_port_hotplug_long_detect); 2189 2190 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); 2191 } 2192 2193 static void ilk_display_irq_handler(struct drm_i915_private *dev_priv, 2194 u32 de_iir) 2195 { 2196 enum i915_pipe pipe; 2197 u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG; 2198 2199 if (hotplug_trigger) 2200 ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ilk); 2201 2202 if (de_iir & DE_AUX_CHANNEL_A) 2203 dp_aux_irq_handler(dev_priv); 2204 2205 if (de_iir & DE_GSE) 2206 intel_opregion_asle_intr(dev_priv); 2207 2208 if (de_iir & DE_POISON) 2209 DRM_ERROR("Poison interrupt\n"); 2210 2211 for_each_pipe(dev_priv, pipe) { 2212 if (de_iir & DE_PIPE_VBLANK(pipe) && 2213 intel_pipe_handle_vblank(dev_priv, pipe)) 2214 intel_check_page_flip(dev_priv, pipe); 2215 2216 if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe)) 2217 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe); 2218 2219 if (de_iir & DE_PIPE_CRC_DONE(pipe)) 2220 i9xx_pipe_crc_irq_handler(dev_priv, pipe); 2221 2222 /* plane/pipes map 1:1 on ilk+ */ 2223 if (de_iir & DE_PLANE_FLIP_DONE(pipe)) 2224 intel_finish_page_flip_cs(dev_priv, pipe); 2225 } 2226 2227 /* check event from PCH */ 2228 if (de_iir & DE_PCH_EVENT) { 2229 u32 pch_iir = I915_READ(SDEIIR); 2230 2231 if (HAS_PCH_CPT(dev_priv)) 2232 cpt_irq_handler(dev_priv, pch_iir); 2233 else 2234 ibx_irq_handler(dev_priv, pch_iir); 2235 2236 /* should clear PCH hotplug event before clear CPU irq */ 2237 I915_WRITE(SDEIIR, pch_iir); 2238 } 2239 2240 if (IS_GEN5(dev_priv) && de_iir & DE_PCU_EVENT) 2241 ironlake_rps_change_irq_handler(dev_priv); 2242 } 2243 2244 static void ivb_display_irq_handler(struct drm_i915_private *dev_priv, 2245 u32 de_iir) 2246 { 2247 enum i915_pipe pipe; 2248 u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB; 2249 2250 if (hotplug_trigger) 2251 ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ivb); 2252 2253 if (de_iir & DE_ERR_INT_IVB) 2254 ivb_err_int_handler(dev_priv); 2255 2256 if (de_iir & DE_AUX_CHANNEL_A_IVB) 2257 dp_aux_irq_handler(dev_priv); 2258 2259 if (de_iir & DE_GSE_IVB) 2260 intel_opregion_asle_intr(dev_priv); 2261 2262 for_each_pipe(dev_priv, pipe) { 2263 if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) && 2264 intel_pipe_handle_vblank(dev_priv, pipe)) 2265 intel_check_page_flip(dev_priv, pipe); 2266 2267 /* plane/pipes map 1:1 on ilk+ */ 2268 if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) 2269 intel_finish_page_flip_cs(dev_priv, pipe); 2270 } 2271 2272 /* check event from PCH */ 2273 if (!HAS_PCH_NOP(dev_priv) && (de_iir & DE_PCH_EVENT_IVB)) { 2274 u32 pch_iir = I915_READ(SDEIIR); 2275 2276 cpt_irq_handler(dev_priv, pch_iir); 2277 2278 /* clear PCH hotplug event before clear CPU irq */ 2279 I915_WRITE(SDEIIR, pch_iir); 2280 } 2281 } 2282 2283 /* 2284 * To handle irqs with the minimum potential races with fresh interrupts, we: 2285 * 1 - Disable Master Interrupt Control. 2286 * 2 - Find the source(s) of the interrupt. 2287 * 3 - Clear the Interrupt Identity bits (IIR). 2288 * 4 - Process the interrupt(s) that had bits set in the IIRs. 2289 * 5 - Re-enable Master Interrupt Control. 2290 */ 2291 static irqreturn_t ironlake_irq_handler(int irq, void *arg) 2292 { 2293 struct drm_device *dev = arg; 2294 struct drm_i915_private *dev_priv = to_i915(dev); 2295 u32 de_iir, gt_iir, de_ier, sde_ier = 0; 2296 irqreturn_t ret = IRQ_NONE; 2297 2298 if (!intel_irqs_enabled(dev_priv)) 2299 return IRQ_NONE; 2300 2301 /* IRQs are synced during runtime_suspend, we don't require a wakeref */ 2302 disable_rpm_wakeref_asserts(dev_priv); 2303 2304 /* disable master interrupt before clearing iir */ 2305 de_ier = I915_READ(DEIER); 2306 I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL); 2307 POSTING_READ(DEIER); 2308 2309 /* Disable south interrupts. We'll only write to SDEIIR once, so further 2310 * interrupts will will be stored on its back queue, and then we'll be 2311 * able to process them after we restore SDEIER (as soon as we restore 2312 * it, we'll get an interrupt if SDEIIR still has something to process 2313 * due to its back queue). */ 2314 if (!HAS_PCH_NOP(dev_priv)) { 2315 sde_ier = I915_READ(SDEIER); 2316 I915_WRITE(SDEIER, 0); 2317 POSTING_READ(SDEIER); 2318 } 2319 2320 /* Find, clear, then process each source of interrupt */ 2321 2322 gt_iir = I915_READ(GTIIR); 2323 if (gt_iir) { 2324 I915_WRITE(GTIIR, gt_iir); 2325 ret = IRQ_HANDLED; 2326 if (INTEL_GEN(dev_priv) >= 6) 2327 snb_gt_irq_handler(dev_priv, gt_iir); 2328 else 2329 ilk_gt_irq_handler(dev_priv, gt_iir); 2330 } 2331 2332 de_iir = I915_READ(DEIIR); 2333 if (de_iir) { 2334 I915_WRITE(DEIIR, de_iir); 2335 ret = IRQ_HANDLED; 2336 if (INTEL_GEN(dev_priv) >= 7) 2337 ivb_display_irq_handler(dev_priv, de_iir); 2338 else 2339 ilk_display_irq_handler(dev_priv, de_iir); 2340 } 2341 2342 if (INTEL_GEN(dev_priv) >= 6) { 2343 u32 pm_iir = I915_READ(GEN6_PMIIR); 2344 if (pm_iir) { 2345 I915_WRITE(GEN6_PMIIR, pm_iir); 2346 ret = IRQ_HANDLED; 2347 gen6_rps_irq_handler(dev_priv, pm_iir); 2348 } 2349 } 2350 2351 I915_WRITE(DEIER, de_ier); 2352 POSTING_READ(DEIER); 2353 if (!HAS_PCH_NOP(dev_priv)) { 2354 I915_WRITE(SDEIER, sde_ier); 2355 POSTING_READ(SDEIER); 2356 } 2357 2358 /* IRQs are synced during runtime_suspend, we don't require a wakeref */ 2359 enable_rpm_wakeref_asserts(dev_priv); 2360 2361 return ret; 2362 } 2363 2364 static void bxt_hpd_irq_handler(struct drm_i915_private *dev_priv, 2365 u32 hotplug_trigger, 2366 const u32 hpd[HPD_NUM_PINS]) 2367 { 2368 u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0; 2369 2370 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG); 2371 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg); 2372 2373 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger, 2374 dig_hotplug_reg, hpd, 2375 bxt_port_hotplug_long_detect); 2376 2377 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); 2378 } 2379 2380 static irqreturn_t 2381 gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl) 2382 { 2383 irqreturn_t ret = IRQ_NONE; 2384 u32 iir; 2385 enum i915_pipe pipe; 2386 2387 if (master_ctl & GEN8_DE_MISC_IRQ) { 2388 iir = I915_READ(GEN8_DE_MISC_IIR); 2389 if (iir) { 2390 I915_WRITE(GEN8_DE_MISC_IIR, iir); 2391 ret = IRQ_HANDLED; 2392 if (iir & GEN8_DE_MISC_GSE) 2393 intel_opregion_asle_intr(dev_priv); 2394 else 2395 DRM_ERROR("Unexpected DE Misc interrupt\n"); 2396 } 2397 else 2398 DRM_ERROR("The master control interrupt lied (DE MISC)!\n"); 2399 } 2400 2401 if (master_ctl & GEN8_DE_PORT_IRQ) { 2402 iir = I915_READ(GEN8_DE_PORT_IIR); 2403 if (iir) { 2404 u32 tmp_mask; 2405 bool found = false; 2406 2407 I915_WRITE(GEN8_DE_PORT_IIR, iir); 2408 ret = IRQ_HANDLED; 2409 2410 tmp_mask = GEN8_AUX_CHANNEL_A; 2411 if (INTEL_INFO(dev_priv)->gen >= 9) 2412 tmp_mask |= GEN9_AUX_CHANNEL_B | 2413 GEN9_AUX_CHANNEL_C | 2414 GEN9_AUX_CHANNEL_D; 2415 2416 if (iir & tmp_mask) { 2417 dp_aux_irq_handler(dev_priv); 2418 found = true; 2419 } 2420 2421 if (IS_BROXTON(dev_priv)) { 2422 tmp_mask = iir & BXT_DE_PORT_HOTPLUG_MASK; 2423 if (tmp_mask) { 2424 bxt_hpd_irq_handler(dev_priv, tmp_mask, 2425 hpd_bxt); 2426 found = true; 2427 } 2428 } else if (IS_BROADWELL(dev_priv)) { 2429 tmp_mask = iir & GEN8_PORT_DP_A_HOTPLUG; 2430 if (tmp_mask) { 2431 ilk_hpd_irq_handler(dev_priv, 2432 tmp_mask, hpd_bdw); 2433 found = true; 2434 } 2435 } 2436 2437 if (IS_BROXTON(dev_priv) && (iir & BXT_DE_PORT_GMBUS)) { 2438 gmbus_irq_handler(dev_priv); 2439 found = true; 2440 } 2441 2442 if (!found) 2443 DRM_ERROR("Unexpected DE Port interrupt\n"); 2444 } 2445 else 2446 DRM_ERROR("The master control interrupt lied (DE PORT)!\n"); 2447 } 2448 2449 for_each_pipe(dev_priv, pipe) { 2450 u32 flip_done, fault_errors; 2451 2452 if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe))) 2453 continue; 2454 2455 iir = I915_READ(GEN8_DE_PIPE_IIR(pipe)); 2456 if (!iir) { 2457 DRM_ERROR("The master control interrupt lied (DE PIPE)!\n"); 2458 continue; 2459 } 2460 2461 ret = IRQ_HANDLED; 2462 I915_WRITE(GEN8_DE_PIPE_IIR(pipe), iir); 2463 2464 if (iir & GEN8_PIPE_VBLANK && 2465 intel_pipe_handle_vblank(dev_priv, pipe)) 2466 intel_check_page_flip(dev_priv, pipe); 2467 2468 flip_done = iir; 2469 if (INTEL_INFO(dev_priv)->gen >= 9) 2470 flip_done &= GEN9_PIPE_PLANE1_FLIP_DONE; 2471 else 2472 flip_done &= GEN8_PIPE_PRIMARY_FLIP_DONE; 2473 2474 if (flip_done) 2475 intel_finish_page_flip_cs(dev_priv, pipe); 2476 2477 if (iir & GEN8_PIPE_CDCLK_CRC_DONE) 2478 hsw_pipe_crc_irq_handler(dev_priv, pipe); 2479 2480 if (iir & GEN8_PIPE_FIFO_UNDERRUN) 2481 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe); 2482 2483 fault_errors = iir; 2484 if (INTEL_INFO(dev_priv)->gen >= 9) 2485 fault_errors &= GEN9_DE_PIPE_IRQ_FAULT_ERRORS; 2486 else 2487 fault_errors &= GEN8_DE_PIPE_IRQ_FAULT_ERRORS; 2488 2489 if (fault_errors) 2490 DRM_ERROR("Fault errors on pipe %c: 0x%08x\n", 2491 pipe_name(pipe), 2492 fault_errors); 2493 } 2494 2495 if (HAS_PCH_SPLIT(dev_priv) && !HAS_PCH_NOP(dev_priv) && 2496 master_ctl & GEN8_DE_PCH_IRQ) { 2497 /* 2498 * FIXME(BDW): Assume for now that the new interrupt handling 2499 * scheme also closed the SDE interrupt handling race we've seen 2500 * on older pch-split platforms. But this needs testing. 2501 */ 2502 iir = I915_READ(SDEIIR); 2503 if (iir) { 2504 I915_WRITE(SDEIIR, iir); 2505 ret = IRQ_HANDLED; 2506 2507 if (HAS_PCH_SPT(dev_priv) || HAS_PCH_KBP(dev_priv)) 2508 spt_irq_handler(dev_priv, iir); 2509 else 2510 cpt_irq_handler(dev_priv, iir); 2511 } else { 2512 /* 2513 * Like on previous PCH there seems to be something 2514 * fishy going on with forwarding PCH interrupts. 2515 */ 2516 DRM_DEBUG_DRIVER("The master control interrupt lied (SDE)!\n"); 2517 } 2518 } 2519 2520 return ret; 2521 } 2522 2523 static irqreturn_t gen8_irq_handler(int irq, void *arg) 2524 { 2525 struct drm_device *dev = arg; 2526 struct drm_i915_private *dev_priv = to_i915(dev); 2527 u32 master_ctl; 2528 u32 gt_iir[4] = {}; 2529 irqreturn_t ret; 2530 2531 if (!intel_irqs_enabled(dev_priv)) 2532 return IRQ_NONE; 2533 2534 master_ctl = I915_READ_FW(GEN8_MASTER_IRQ); 2535 master_ctl &= ~GEN8_MASTER_IRQ_CONTROL; 2536 if (!master_ctl) 2537 return IRQ_NONE; 2538 2539 I915_WRITE_FW(GEN8_MASTER_IRQ, 0); 2540 2541 /* IRQs are synced during runtime_suspend, we don't require a wakeref */ 2542 disable_rpm_wakeref_asserts(dev_priv); 2543 2544 /* Find, clear, then process each source of interrupt */ 2545 ret = gen8_gt_irq_ack(dev_priv, master_ctl, gt_iir); 2546 gen8_gt_irq_handler(dev_priv, gt_iir); 2547 ret |= gen8_de_irq_handler(dev_priv, master_ctl); 2548 2549 I915_WRITE_FW(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL); 2550 POSTING_READ_FW(GEN8_MASTER_IRQ); 2551 2552 enable_rpm_wakeref_asserts(dev_priv); 2553 2554 return ret; 2555 } 2556 2557 static void i915_error_wake_up(struct drm_i915_private *dev_priv) 2558 { 2559 /* 2560 * Notify all waiters for GPU completion events that reset state has 2561 * been changed, and that they need to restart their wait after 2562 * checking for potential errors (and bail out to drop locks if there is 2563 * a gpu reset pending so that i915_error_work_func can acquire them). 2564 */ 2565 2566 /* Wake up __wait_seqno, potentially holding dev->struct_mutex. */ 2567 wake_up_all(&dev_priv->gpu_error.wait_queue); 2568 2569 /* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */ 2570 wake_up_all(&dev_priv->pending_flip_queue); 2571 } 2572 2573 /** 2574 * i915_reset_and_wakeup - do process context error handling work 2575 * @dev_priv: i915 device private 2576 * 2577 * Fire an error uevent so userspace can see that a hang or error 2578 * was detected. 2579 */ 2580 static void i915_reset_and_wakeup(struct drm_i915_private *dev_priv) 2581 { 2582 struct kobject *kobj = &dev_priv->drm.primary->kdev->kobj; 2583 char *error_event[] = { I915_ERROR_UEVENT "=1", NULL }; 2584 char *reset_event[] = { I915_RESET_UEVENT "=1", NULL }; 2585 char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL }; 2586 2587 kobject_uevent_env(kobj, KOBJ_CHANGE, error_event); 2588 2589 DRM_DEBUG_DRIVER("resetting chip\n"); 2590 kobject_uevent_env(kobj, KOBJ_CHANGE, reset_event); 2591 2592 /* 2593 * In most cases it's guaranteed that we get here with an RPM 2594 * reference held, for example because there is a pending GPU 2595 * request that won't finish until the reset is done. This 2596 * isn't the case at least when we get here by doing a 2597 * simulated reset via debugs, so get an RPM reference. 2598 */ 2599 intel_runtime_pm_get(dev_priv); 2600 intel_prepare_reset(dev_priv); 2601 2602 do { 2603 /* 2604 * All state reset _must_ be completed before we update the 2605 * reset counter, for otherwise waiters might miss the reset 2606 * pending state and not properly drop locks, resulting in 2607 * deadlocks with the reset work. 2608 */ 2609 if (mutex_trylock(&dev_priv->drm.struct_mutex)) { 2610 i915_reset(dev_priv); 2611 mutex_unlock(&dev_priv->drm.struct_mutex); 2612 } 2613 2614 /* We need to wait for anyone holding the lock to wakeup */ 2615 } while (wait_on_bit_timeout(&dev_priv->gpu_error.flags, 2616 I915_RESET_IN_PROGRESS, 2617 TASK_UNINTERRUPTIBLE, 2618 HZ)); 2619 2620 intel_finish_reset(dev_priv); 2621 intel_runtime_pm_put(dev_priv); 2622 2623 if (!test_bit(I915_WEDGED, &dev_priv->gpu_error.flags)) 2624 kobject_uevent_env(kobj, 2625 KOBJ_CHANGE, reset_done_event); 2626 2627 /* 2628 * Note: The wake_up also serves as a memory barrier so that 2629 * waiters see the updated value of the dev_priv->gpu_error. 2630 */ 2631 wake_up_all(&dev_priv->gpu_error.reset_queue); 2632 } 2633 2634 static inline void 2635 i915_err_print_instdone(struct drm_i915_private *dev_priv, 2636 struct intel_instdone *instdone) 2637 { 2638 int slice; 2639 int subslice; 2640 2641 pr_err(" INSTDONE: 0x%08x\n", instdone->instdone); 2642 2643 if (INTEL_GEN(dev_priv) <= 3) 2644 return; 2645 2646 pr_err(" SC_INSTDONE: 0x%08x\n", instdone->slice_common); 2647 2648 if (INTEL_GEN(dev_priv) <= 6) 2649 return; 2650 2651 for_each_instdone_slice_subslice(dev_priv, slice, subslice) 2652 pr_err(" SAMPLER_INSTDONE[%d][%d]: 0x%08x\n", 2653 slice, subslice, instdone->sampler[slice][subslice]); 2654 2655 for_each_instdone_slice_subslice(dev_priv, slice, subslice) 2656 pr_err(" ROW_INSTDONE[%d][%d]: 0x%08x\n", 2657 slice, subslice, instdone->row[slice][subslice]); 2658 } 2659 2660 static void i915_clear_error_registers(struct drm_i915_private *dev_priv) 2661 { 2662 u32 eir; 2663 2664 if (!IS_GEN2(dev_priv)) 2665 I915_WRITE(PGTBL_ER, I915_READ(PGTBL_ER)); 2666 2667 if (INTEL_GEN(dev_priv) < 4) 2668 I915_WRITE(IPEIR, I915_READ(IPEIR)); 2669 else 2670 I915_WRITE(IPEIR_I965, I915_READ(IPEIR_I965)); 2671 2672 I915_WRITE(EIR, I915_READ(EIR)); 2673 eir = I915_READ(EIR); 2674 if (eir) { 2675 /* 2676 * some errors might have become stuck, 2677 * mask them. 2678 */ 2679 DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masking\n", eir); 2680 I915_WRITE(EMR, I915_READ(EMR) | eir); 2681 I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT); 2682 } 2683 } 2684 2685 /** 2686 * i915_handle_error - handle a gpu error 2687 * @dev_priv: i915 device private 2688 * @engine_mask: mask representing engines that are hung 2689 * Do some basic checking of register state at error time and 2690 * dump it to the syslog. Also call i915_capture_error_state() to make 2691 * sure we get a record and make it available in debugfs. Fire a uevent 2692 * so userspace knows something bad happened (should trigger collection 2693 * of a ring dump etc.). 2694 * @fmt: Error message format string 2695 */ 2696 void i915_handle_error(struct drm_i915_private *dev_priv, 2697 u32 engine_mask, 2698 const char *fmt, ...) 2699 { 2700 va_list args; 2701 char error_msg[80]; 2702 2703 va_start(args, fmt); 2704 vscnprintf(error_msg, sizeof(error_msg), fmt, args); 2705 va_end(args); 2706 2707 i915_capture_error_state(dev_priv, engine_mask, error_msg); 2708 i915_clear_error_registers(dev_priv); 2709 2710 if (!engine_mask) 2711 return; 2712 2713 if (test_and_set_bit(I915_RESET_IN_PROGRESS, 2714 &dev_priv->gpu_error.flags)) 2715 return; 2716 2717 /* 2718 * Wakeup waiting processes so that the reset function 2719 * i915_reset_and_wakeup doesn't deadlock trying to grab 2720 * various locks. By bumping the reset counter first, the woken 2721 * processes will see a reset in progress and back off, 2722 * releasing their locks and then wait for the reset completion. 2723 * We must do this for _all_ gpu waiters that might hold locks 2724 * that the reset work needs to acquire. 2725 * 2726 * Note: The wake_up also provides a memory barrier to ensure that the 2727 * waiters see the updated value of the reset flags. 2728 */ 2729 i915_error_wake_up(dev_priv); 2730 2731 i915_reset_and_wakeup(dev_priv); 2732 } 2733 2734 /* Called from drm generic code, passed 'crtc' which 2735 * we use as a pipe index 2736 */ 2737 static int i8xx_enable_vblank(struct drm_device *dev, unsigned int pipe) 2738 { 2739 struct drm_i915_private *dev_priv = to_i915(dev); 2740 unsigned long irqflags; 2741 2742 spin_lock_irqsave(&dev_priv->irq_lock, irqflags); 2743 i915_enable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS); 2744 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); 2745 2746 return 0; 2747 } 2748 2749 static int i965_enable_vblank(struct drm_device *dev, unsigned int pipe) 2750 { 2751 struct drm_i915_private *dev_priv = to_i915(dev); 2752 unsigned long irqflags; 2753 2754 spin_lock_irqsave(&dev_priv->irq_lock, irqflags); 2755 i915_enable_pipestat(dev_priv, pipe, 2756 PIPE_START_VBLANK_INTERRUPT_STATUS); 2757 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); 2758 2759 return 0; 2760 } 2761 2762 static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe) 2763 { 2764 struct drm_i915_private *dev_priv = to_i915(dev); 2765 unsigned long irqflags; 2766 uint32_t bit = INTEL_GEN(dev_priv) >= 7 ? 2767 DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe); 2768 2769 spin_lock_irqsave(&dev_priv->irq_lock, irqflags); 2770 ilk_enable_display_irq(dev_priv, bit); 2771 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); 2772 2773 return 0; 2774 } 2775 2776 static int gen8_enable_vblank(struct drm_device *dev, unsigned int pipe) 2777 { 2778 struct drm_i915_private *dev_priv = to_i915(dev); 2779 unsigned long irqflags; 2780 2781 spin_lock_irqsave(&dev_priv->irq_lock, irqflags); 2782 bdw_enable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK); 2783 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); 2784 2785 return 0; 2786 } 2787 2788 /* Called from drm generic code, passed 'crtc' which 2789 * we use as a pipe index 2790 */ 2791 static void i8xx_disable_vblank(struct drm_device *dev, unsigned int pipe) 2792 { 2793 struct drm_i915_private *dev_priv = to_i915(dev); 2794 unsigned long irqflags; 2795 2796 spin_lock_irqsave(&dev_priv->irq_lock, irqflags); 2797 i915_disable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS); 2798 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); 2799 } 2800 2801 static void i965_disable_vblank(struct drm_device *dev, unsigned int pipe) 2802 { 2803 struct drm_i915_private *dev_priv = to_i915(dev); 2804 unsigned long irqflags; 2805 2806 spin_lock_irqsave(&dev_priv->irq_lock, irqflags); 2807 i915_disable_pipestat(dev_priv, pipe, 2808 PIPE_START_VBLANK_INTERRUPT_STATUS); 2809 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); 2810 } 2811 2812 static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe) 2813 { 2814 struct drm_i915_private *dev_priv = to_i915(dev); 2815 unsigned long irqflags; 2816 uint32_t bit = INTEL_GEN(dev_priv) >= 7 ? 2817 DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe); 2818 2819 spin_lock_irqsave(&dev_priv->irq_lock, irqflags); 2820 ilk_disable_display_irq(dev_priv, bit); 2821 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); 2822 } 2823 2824 static void gen8_disable_vblank(struct drm_device *dev, unsigned int pipe) 2825 { 2826 struct drm_i915_private *dev_priv = to_i915(dev); 2827 unsigned long irqflags; 2828 2829 spin_lock_irqsave(&dev_priv->irq_lock, irqflags); 2830 bdw_disable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK); 2831 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); 2832 } 2833 2834 static void ibx_irq_reset(struct drm_i915_private *dev_priv) 2835 { 2836 if (HAS_PCH_NOP(dev_priv)) 2837 return; 2838 2839 GEN5_IRQ_RESET(SDE); 2840 2841 if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv)) 2842 I915_WRITE(SERR_INT, 0xffffffff); 2843 } 2844 2845 /* 2846 * SDEIER is also touched by the interrupt handler to work around missed PCH 2847 * interrupts. Hence we can't update it after the interrupt handler is enabled - 2848 * instead we unconditionally enable all PCH interrupt sources here, but then 2849 * only unmask them as needed with SDEIMR. 2850 * 2851 * This function needs to be called before interrupts are enabled. 2852 */ 2853 static void ibx_irq_pre_postinstall(struct drm_device *dev) 2854 { 2855 struct drm_i915_private *dev_priv = to_i915(dev); 2856 2857 if (HAS_PCH_NOP(dev_priv)) 2858 return; 2859 2860 WARN_ON(I915_READ(SDEIER) != 0); 2861 I915_WRITE(SDEIER, 0xffffffff); 2862 POSTING_READ(SDEIER); 2863 } 2864 2865 static void gen5_gt_irq_reset(struct drm_i915_private *dev_priv) 2866 { 2867 GEN5_IRQ_RESET(GT); 2868 if (INTEL_GEN(dev_priv) >= 6) 2869 GEN5_IRQ_RESET(GEN6_PM); 2870 } 2871 2872 static void vlv_display_irq_reset(struct drm_i915_private *dev_priv) 2873 { 2874 enum i915_pipe pipe; 2875 2876 if (IS_CHERRYVIEW(dev_priv)) 2877 I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV); 2878 else 2879 I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK); 2880 2881 i915_hotplug_interrupt_update_locked(dev_priv, 0xffffffff, 0); 2882 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); 2883 2884 for_each_pipe(dev_priv, pipe) { 2885 I915_WRITE(PIPESTAT(pipe), 2886 PIPE_FIFO_UNDERRUN_STATUS | 2887 PIPESTAT_INT_STATUS_MASK); 2888 dev_priv->pipestat_irq_mask[pipe] = 0; 2889 } 2890 2891 GEN5_IRQ_RESET(VLV_); 2892 dev_priv->irq_mask = ~0; 2893 } 2894 2895 static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv) 2896 { 2897 u32 pipestat_mask; 2898 u32 enable_mask; 2899 enum i915_pipe pipe; 2900 2901 pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV | 2902 PIPE_CRC_DONE_INTERRUPT_STATUS; 2903 2904 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS); 2905 for_each_pipe(dev_priv, pipe) 2906 i915_enable_pipestat(dev_priv, pipe, pipestat_mask); 2907 2908 enable_mask = I915_DISPLAY_PORT_INTERRUPT | 2909 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | 2910 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT; 2911 if (IS_CHERRYVIEW(dev_priv)) 2912 enable_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT; 2913 2914 WARN_ON(dev_priv->irq_mask != ~0); 2915 2916 dev_priv->irq_mask = ~enable_mask; 2917 2918 GEN5_IRQ_INIT(VLV_, dev_priv->irq_mask, enable_mask); 2919 } 2920 2921 /* drm_dma.h hooks 2922 */ 2923 static void ironlake_irq_reset(struct drm_device *dev) 2924 { 2925 struct drm_i915_private *dev_priv = to_i915(dev); 2926 2927 I915_WRITE(HWSTAM, 0xffffffff); 2928 2929 GEN5_IRQ_RESET(DE); 2930 if (IS_GEN7(dev_priv)) 2931 I915_WRITE(GEN7_ERR_INT, 0xffffffff); 2932 2933 gen5_gt_irq_reset(dev_priv); 2934 2935 ibx_irq_reset(dev_priv); 2936 } 2937 2938 static void valleyview_irq_preinstall(struct drm_device *dev) 2939 { 2940 struct drm_i915_private *dev_priv = to_i915(dev); 2941 2942 I915_WRITE(VLV_MASTER_IER, 0); 2943 POSTING_READ(VLV_MASTER_IER); 2944 2945 gen5_gt_irq_reset(dev_priv); 2946 2947 spin_lock_irq(&dev_priv->irq_lock); 2948 if (dev_priv->display_irqs_enabled) 2949 vlv_display_irq_reset(dev_priv); 2950 spin_unlock_irq(&dev_priv->irq_lock); 2951 } 2952 2953 static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv) 2954 { 2955 GEN8_IRQ_RESET_NDX(GT, 0); 2956 GEN8_IRQ_RESET_NDX(GT, 1); 2957 GEN8_IRQ_RESET_NDX(GT, 2); 2958 GEN8_IRQ_RESET_NDX(GT, 3); 2959 } 2960 2961 static void gen8_irq_reset(struct drm_device *dev) 2962 { 2963 struct drm_i915_private *dev_priv = to_i915(dev); 2964 int pipe; 2965 2966 I915_WRITE(GEN8_MASTER_IRQ, 0); 2967 POSTING_READ(GEN8_MASTER_IRQ); 2968 2969 gen8_gt_irq_reset(dev_priv); 2970 2971 for_each_pipe(dev_priv, pipe) 2972 if (intel_display_power_is_enabled(dev_priv, 2973 POWER_DOMAIN_PIPE(pipe))) 2974 GEN8_IRQ_RESET_NDX(DE_PIPE, pipe); 2975 2976 GEN5_IRQ_RESET(GEN8_DE_PORT_); 2977 GEN5_IRQ_RESET(GEN8_DE_MISC_); 2978 GEN5_IRQ_RESET(GEN8_PCU_); 2979 2980 if (HAS_PCH_SPLIT(dev_priv)) 2981 ibx_irq_reset(dev_priv); 2982 } 2983 2984 void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv, 2985 unsigned int pipe_mask) 2986 { 2987 uint32_t extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN; 2988 enum i915_pipe pipe; 2989 2990 spin_lock_irq(&dev_priv->irq_lock); 2991 for_each_pipe_masked(dev_priv, pipe, pipe_mask) 2992 GEN8_IRQ_INIT_NDX(DE_PIPE, pipe, 2993 dev_priv->de_irq_mask[pipe], 2994 ~dev_priv->de_irq_mask[pipe] | extra_ier); 2995 spin_unlock_irq(&dev_priv->irq_lock); 2996 } 2997 2998 void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv, 2999 unsigned int pipe_mask) 3000 { 3001 enum i915_pipe pipe; 3002 3003 spin_lock_irq(&dev_priv->irq_lock); 3004 for_each_pipe_masked(dev_priv, pipe, pipe_mask) 3005 GEN8_IRQ_RESET_NDX(DE_PIPE, pipe); 3006 spin_unlock_irq(&dev_priv->irq_lock); 3007 3008 /* make sure we're done processing display irqs */ 3009 synchronize_irq(dev_priv->drm.irq); 3010 } 3011 3012 static void cherryview_irq_preinstall(struct drm_device *dev) 3013 { 3014 struct drm_i915_private *dev_priv = to_i915(dev); 3015 3016 I915_WRITE(GEN8_MASTER_IRQ, 0); 3017 POSTING_READ(GEN8_MASTER_IRQ); 3018 3019 gen8_gt_irq_reset(dev_priv); 3020 3021 GEN5_IRQ_RESET(GEN8_PCU_); 3022 3023 spin_lock_irq(&dev_priv->irq_lock); 3024 if (dev_priv->display_irqs_enabled) 3025 vlv_display_irq_reset(dev_priv); 3026 spin_unlock_irq(&dev_priv->irq_lock); 3027 } 3028 3029 static u32 intel_hpd_enabled_irqs(struct drm_i915_private *dev_priv, 3030 const u32 hpd[HPD_NUM_PINS]) 3031 { 3032 struct intel_encoder *encoder; 3033 u32 enabled_irqs = 0; 3034 3035 for_each_intel_encoder(&dev_priv->drm, encoder) 3036 if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED) 3037 enabled_irqs |= hpd[encoder->hpd_pin]; 3038 3039 return enabled_irqs; 3040 } 3041 3042 static void ibx_hpd_irq_setup(struct drm_i915_private *dev_priv) 3043 { 3044 u32 hotplug_irqs, hotplug, enabled_irqs; 3045 3046 if (HAS_PCH_IBX(dev_priv)) { 3047 hotplug_irqs = SDE_HOTPLUG_MASK; 3048 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ibx); 3049 } else { 3050 hotplug_irqs = SDE_HOTPLUG_MASK_CPT; 3051 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_cpt); 3052 } 3053 3054 ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs); 3055 3056 /* 3057 * Enable digital hotplug on the PCH, and configure the DP short pulse 3058 * duration to 2ms (which is the minimum in the Display Port spec). 3059 * The pulse duration bits are reserved on LPT+. 3060 */ 3061 hotplug = I915_READ(PCH_PORT_HOTPLUG); 3062 hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK); 3063 hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms; 3064 hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms; 3065 hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms; 3066 /* 3067 * When CPU and PCH are on the same package, port A 3068 * HPD must be enabled in both north and south. 3069 */ 3070 if (HAS_PCH_LPT_LP(dev_priv)) 3071 hotplug |= PORTA_HOTPLUG_ENABLE; 3072 I915_WRITE(PCH_PORT_HOTPLUG, hotplug); 3073 } 3074 3075 static void spt_hpd_detection_setup(struct drm_i915_private *dev_priv) 3076 { 3077 u32 hotplug; 3078 3079 /* Enable digital hotplug on the PCH */ 3080 hotplug = I915_READ(PCH_PORT_HOTPLUG); 3081 hotplug |= PORTA_HOTPLUG_ENABLE | 3082 PORTB_HOTPLUG_ENABLE | 3083 PORTC_HOTPLUG_ENABLE | 3084 PORTD_HOTPLUG_ENABLE; 3085 I915_WRITE(PCH_PORT_HOTPLUG, hotplug); 3086 3087 hotplug = I915_READ(PCH_PORT_HOTPLUG2); 3088 hotplug |= PORTE_HOTPLUG_ENABLE; 3089 I915_WRITE(PCH_PORT_HOTPLUG2, hotplug); 3090 } 3091 3092 static void spt_hpd_irq_setup(struct drm_i915_private *dev_priv) 3093 { 3094 u32 hotplug_irqs, enabled_irqs; 3095 3096 hotplug_irqs = SDE_HOTPLUG_MASK_SPT; 3097 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_spt); 3098 3099 ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs); 3100 3101 spt_hpd_detection_setup(dev_priv); 3102 } 3103 3104 static void ilk_hpd_irq_setup(struct drm_i915_private *dev_priv) 3105 { 3106 u32 hotplug_irqs, hotplug, enabled_irqs; 3107 3108 if (INTEL_GEN(dev_priv) >= 8) { 3109 hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG; 3110 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bdw); 3111 3112 bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs); 3113 } else if (INTEL_GEN(dev_priv) >= 7) { 3114 hotplug_irqs = DE_DP_A_HOTPLUG_IVB; 3115 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ivb); 3116 3117 ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs); 3118 } else { 3119 hotplug_irqs = DE_DP_A_HOTPLUG; 3120 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ilk); 3121 3122 ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs); 3123 } 3124 3125 /* 3126 * Enable digital hotplug on the CPU, and configure the DP short pulse 3127 * duration to 2ms (which is the minimum in the Display Port spec) 3128 * The pulse duration bits are reserved on HSW+. 3129 */ 3130 hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL); 3131 hotplug &= ~DIGITAL_PORTA_PULSE_DURATION_MASK; 3132 hotplug |= DIGITAL_PORTA_HOTPLUG_ENABLE | DIGITAL_PORTA_PULSE_DURATION_2ms; 3133 I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug); 3134 3135 ibx_hpd_irq_setup(dev_priv); 3136 } 3137 3138 static void __bxt_hpd_detection_setup(struct drm_i915_private *dev_priv, 3139 u32 enabled_irqs) 3140 { 3141 u32 hotplug; 3142 3143 hotplug = I915_READ(PCH_PORT_HOTPLUG); 3144 hotplug |= PORTA_HOTPLUG_ENABLE | 3145 PORTB_HOTPLUG_ENABLE | 3146 PORTC_HOTPLUG_ENABLE; 3147 3148 DRM_DEBUG_KMS("Invert bit setting: hp_ctl:%x hp_port:%x\n", 3149 hotplug, enabled_irqs); 3150 hotplug &= ~BXT_DDI_HPD_INVERT_MASK; 3151 3152 /* 3153 * For BXT invert bit has to be set based on AOB design 3154 * for HPD detection logic, update it based on VBT fields. 3155 */ 3156 if ((enabled_irqs & BXT_DE_PORT_HP_DDIA) && 3157 intel_bios_is_port_hpd_inverted(dev_priv, PORT_A)) 3158 hotplug |= BXT_DDIA_HPD_INVERT; 3159 if ((enabled_irqs & BXT_DE_PORT_HP_DDIB) && 3160 intel_bios_is_port_hpd_inverted(dev_priv, PORT_B)) 3161 hotplug |= BXT_DDIB_HPD_INVERT; 3162 if ((enabled_irqs & BXT_DE_PORT_HP_DDIC) && 3163 intel_bios_is_port_hpd_inverted(dev_priv, PORT_C)) 3164 hotplug |= BXT_DDIC_HPD_INVERT; 3165 3166 I915_WRITE(PCH_PORT_HOTPLUG, hotplug); 3167 } 3168 3169 static void bxt_hpd_detection_setup(struct drm_i915_private *dev_priv) 3170 { 3171 __bxt_hpd_detection_setup(dev_priv, BXT_DE_PORT_HOTPLUG_MASK); 3172 } 3173 3174 static void bxt_hpd_irq_setup(struct drm_i915_private *dev_priv) 3175 { 3176 u32 hotplug_irqs, enabled_irqs; 3177 3178 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bxt); 3179 hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK; 3180 3181 bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs); 3182 3183 __bxt_hpd_detection_setup(dev_priv, enabled_irqs); 3184 } 3185 3186 static void ibx_irq_postinstall(struct drm_device *dev) 3187 { 3188 struct drm_i915_private *dev_priv = to_i915(dev); 3189 u32 mask; 3190 3191 if (HAS_PCH_NOP(dev_priv)) 3192 return; 3193 3194 if (HAS_PCH_IBX(dev_priv)) 3195 mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON; 3196 else 3197 mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT; 3198 3199 gen5_assert_iir_is_zero(dev_priv, SDEIIR); 3200 I915_WRITE(SDEIMR, ~mask); 3201 3202 if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) || 3203 HAS_PCH_LPT(dev_priv)) 3204 ; /* TODO: Enable HPD detection on older PCH platforms too */ 3205 else 3206 spt_hpd_detection_setup(dev_priv); 3207 } 3208 3209 static void gen5_gt_irq_postinstall(struct drm_device *dev) 3210 { 3211 struct drm_i915_private *dev_priv = to_i915(dev); 3212 u32 pm_irqs, gt_irqs; 3213 3214 pm_irqs = gt_irqs = 0; 3215 3216 dev_priv->gt_irq_mask = ~0; 3217 if (HAS_L3_DPF(dev_priv)) { 3218 /* L3 parity interrupt is always unmasked. */ 3219 dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev_priv); 3220 gt_irqs |= GT_PARITY_ERROR(dev_priv); 3221 } 3222 3223 gt_irqs |= GT_RENDER_USER_INTERRUPT; 3224 if (IS_GEN5(dev_priv)) { 3225 gt_irqs |= ILK_BSD_USER_INTERRUPT; 3226 } else { 3227 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT; 3228 } 3229 3230 GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs); 3231 3232 if (INTEL_GEN(dev_priv) >= 6) { 3233 /* 3234 * RPS interrupts will get enabled/disabled on demand when RPS 3235 * itself is enabled/disabled. 3236 */ 3237 if (HAS_VEBOX(dev_priv)) { 3238 pm_irqs |= PM_VEBOX_USER_INTERRUPT; 3239 dev_priv->pm_ier |= PM_VEBOX_USER_INTERRUPT; 3240 } 3241 3242 dev_priv->pm_imr = 0xffffffff; 3243 GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_imr, pm_irqs); 3244 } 3245 } 3246 3247 static int ironlake_irq_postinstall(struct drm_device *dev) 3248 { 3249 struct drm_i915_private *dev_priv = to_i915(dev); 3250 u32 display_mask, extra_mask; 3251 3252 if (INTEL_GEN(dev_priv) >= 7) { 3253 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB | 3254 DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB | 3255 DE_PLANEB_FLIP_DONE_IVB | 3256 DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB); 3257 extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB | 3258 DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB | 3259 DE_DP_A_HOTPLUG_IVB); 3260 } else { 3261 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT | 3262 DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE | 3263 DE_AUX_CHANNEL_A | 3264 DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE | 3265 DE_POISON); 3266 extra_mask = (DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT | 3267 DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN | 3268 DE_DP_A_HOTPLUG); 3269 } 3270 3271 dev_priv->irq_mask = ~display_mask; 3272 3273 I915_WRITE(HWSTAM, 0xeffe); 3274 3275 ibx_irq_pre_postinstall(dev); 3276 3277 GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask); 3278 3279 gen5_gt_irq_postinstall(dev); 3280 3281 ibx_irq_postinstall(dev); 3282 3283 if (IS_IRONLAKE_M(dev_priv)) { 3284 /* Enable PCU event interrupts 3285 * 3286 * spinlocking not required here for correctness since interrupt 3287 * setup is guaranteed to run in single-threaded context. But we 3288 * need it to make the assert_spin_locked happy. */ 3289 spin_lock_irq(&dev_priv->irq_lock); 3290 ilk_enable_display_irq(dev_priv, DE_PCU_EVENT); 3291 spin_unlock_irq(&dev_priv->irq_lock); 3292 } 3293 3294 return 0; 3295 } 3296 3297 void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv) 3298 { 3299 assert_spin_locked(&dev_priv->irq_lock); 3300 3301 if (dev_priv->display_irqs_enabled) 3302 return; 3303 3304 dev_priv->display_irqs_enabled = true; 3305 3306 if (intel_irqs_enabled(dev_priv)) { 3307 vlv_display_irq_reset(dev_priv); 3308 vlv_display_irq_postinstall(dev_priv); 3309 } 3310 } 3311 3312 void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv) 3313 { 3314 assert_spin_locked(&dev_priv->irq_lock); 3315 3316 if (!dev_priv->display_irqs_enabled) 3317 return; 3318 3319 dev_priv->display_irqs_enabled = false; 3320 3321 if (intel_irqs_enabled(dev_priv)) 3322 vlv_display_irq_reset(dev_priv); 3323 } 3324 3325 3326 static int valleyview_irq_postinstall(struct drm_device *dev) 3327 { 3328 struct drm_i915_private *dev_priv = to_i915(dev); 3329 3330 gen5_gt_irq_postinstall(dev); 3331 3332 spin_lock_irq(&dev_priv->irq_lock); 3333 if (dev_priv->display_irqs_enabled) 3334 vlv_display_irq_postinstall(dev_priv); 3335 spin_unlock_irq(&dev_priv->irq_lock); 3336 3337 I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE); 3338 POSTING_READ(VLV_MASTER_IER); 3339 3340 return 0; 3341 } 3342 3343 static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv) 3344 { 3345 /* These are interrupts we'll toggle with the ring mask register */ 3346 uint32_t gt_interrupts[] = { 3347 GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT | 3348 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT | 3349 GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT | 3350 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT, 3351 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT | 3352 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT | 3353 GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT | 3354 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT, 3355 0, 3356 GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT | 3357 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT 3358 }; 3359 3360 if (HAS_L3_DPF(dev_priv)) 3361 gt_interrupts[0] |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT; 3362 3363 dev_priv->pm_ier = 0x0; 3364 dev_priv->pm_imr = ~dev_priv->pm_ier; 3365 GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]); 3366 GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]); 3367 /* 3368 * RPS interrupts will get enabled/disabled on demand when RPS itself 3369 * is enabled/disabled. Same wil be the case for GuC interrupts. 3370 */ 3371 GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_imr, dev_priv->pm_ier); 3372 GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]); 3373 } 3374 3375 static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv) 3376 { 3377 uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE; 3378 uint32_t de_pipe_enables; 3379 u32 de_port_masked = GEN8_AUX_CHANNEL_A; 3380 u32 de_port_enables; 3381 u32 de_misc_masked = GEN8_DE_MISC_GSE; 3382 enum i915_pipe pipe; 3383 3384 if (INTEL_INFO(dev_priv)->gen >= 9) { 3385 de_pipe_masked |= GEN9_PIPE_PLANE1_FLIP_DONE | 3386 GEN9_DE_PIPE_IRQ_FAULT_ERRORS; 3387 de_port_masked |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C | 3388 GEN9_AUX_CHANNEL_D; 3389 if (IS_BROXTON(dev_priv)) 3390 de_port_masked |= BXT_DE_PORT_GMBUS; 3391 } else { 3392 de_pipe_masked |= GEN8_PIPE_PRIMARY_FLIP_DONE | 3393 GEN8_DE_PIPE_IRQ_FAULT_ERRORS; 3394 } 3395 3396 de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK | 3397 GEN8_PIPE_FIFO_UNDERRUN; 3398 3399 de_port_enables = de_port_masked; 3400 if (IS_BROXTON(dev_priv)) 3401 de_port_enables |= BXT_DE_PORT_HOTPLUG_MASK; 3402 else if (IS_BROADWELL(dev_priv)) 3403 de_port_enables |= GEN8_PORT_DP_A_HOTPLUG; 3404 3405 dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked; 3406 dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked; 3407 dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked; 3408 3409 for_each_pipe(dev_priv, pipe) 3410 if (intel_display_power_is_enabled(dev_priv, 3411 POWER_DOMAIN_PIPE(pipe))) 3412 GEN8_IRQ_INIT_NDX(DE_PIPE, pipe, 3413 dev_priv->de_irq_mask[pipe], 3414 de_pipe_enables); 3415 3416 GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_masked, de_port_enables); 3417 GEN5_IRQ_INIT(GEN8_DE_MISC_, ~de_misc_masked, de_misc_masked); 3418 3419 if (IS_BROXTON(dev_priv)) 3420 bxt_hpd_detection_setup(dev_priv); 3421 } 3422 3423 static int gen8_irq_postinstall(struct drm_device *dev) 3424 { 3425 struct drm_i915_private *dev_priv = to_i915(dev); 3426 3427 if (HAS_PCH_SPLIT(dev_priv)) 3428 ibx_irq_pre_postinstall(dev); 3429 3430 gen8_gt_irq_postinstall(dev_priv); 3431 gen8_de_irq_postinstall(dev_priv); 3432 3433 if (HAS_PCH_SPLIT(dev_priv)) 3434 ibx_irq_postinstall(dev); 3435 3436 I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL); 3437 POSTING_READ(GEN8_MASTER_IRQ); 3438 3439 return 0; 3440 } 3441 3442 static int cherryview_irq_postinstall(struct drm_device *dev) 3443 { 3444 struct drm_i915_private *dev_priv = to_i915(dev); 3445 3446 gen8_gt_irq_postinstall(dev_priv); 3447 3448 spin_lock_irq(&dev_priv->irq_lock); 3449 if (dev_priv->display_irqs_enabled) 3450 vlv_display_irq_postinstall(dev_priv); 3451 spin_unlock_irq(&dev_priv->irq_lock); 3452 3453 I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL); 3454 POSTING_READ(GEN8_MASTER_IRQ); 3455 3456 return 0; 3457 } 3458 3459 static void gen8_irq_uninstall(struct drm_device *dev) 3460 { 3461 struct drm_i915_private *dev_priv = to_i915(dev); 3462 3463 if (!dev_priv) 3464 return; 3465 3466 gen8_irq_reset(dev); 3467 } 3468 3469 static void valleyview_irq_uninstall(struct drm_device *dev) 3470 { 3471 struct drm_i915_private *dev_priv = to_i915(dev); 3472 3473 if (!dev_priv) 3474 return; 3475 3476 I915_WRITE(VLV_MASTER_IER, 0); 3477 POSTING_READ(VLV_MASTER_IER); 3478 3479 gen5_gt_irq_reset(dev_priv); 3480 3481 I915_WRITE(HWSTAM, 0xffffffff); 3482 3483 spin_lock_irq(&dev_priv->irq_lock); 3484 if (dev_priv->display_irqs_enabled) 3485 vlv_display_irq_reset(dev_priv); 3486 spin_unlock_irq(&dev_priv->irq_lock); 3487 } 3488 3489 static void cherryview_irq_uninstall(struct drm_device *dev) 3490 { 3491 struct drm_i915_private *dev_priv = to_i915(dev); 3492 3493 if (!dev_priv) 3494 return; 3495 3496 I915_WRITE(GEN8_MASTER_IRQ, 0); 3497 POSTING_READ(GEN8_MASTER_IRQ); 3498 3499 gen8_gt_irq_reset(dev_priv); 3500 3501 GEN5_IRQ_RESET(GEN8_PCU_); 3502 3503 spin_lock_irq(&dev_priv->irq_lock); 3504 if (dev_priv->display_irqs_enabled) 3505 vlv_display_irq_reset(dev_priv); 3506 spin_unlock_irq(&dev_priv->irq_lock); 3507 } 3508 3509 static void ironlake_irq_uninstall(struct drm_device *dev) 3510 { 3511 struct drm_i915_private *dev_priv = to_i915(dev); 3512 3513 if (!dev_priv) 3514 return; 3515 3516 ironlake_irq_reset(dev); 3517 } 3518 3519 static void i8xx_irq_preinstall(struct drm_device * dev) 3520 { 3521 struct drm_i915_private *dev_priv = to_i915(dev); 3522 int pipe; 3523 3524 for_each_pipe(dev_priv, pipe) 3525 I915_WRITE(PIPESTAT(pipe), 0); 3526 I915_WRITE16(IMR, 0xffff); 3527 I915_WRITE16(IER, 0x0); 3528 POSTING_READ16(IER); 3529 } 3530 3531 static int i8xx_irq_postinstall(struct drm_device *dev) 3532 { 3533 struct drm_i915_private *dev_priv = to_i915(dev); 3534 3535 I915_WRITE16(EMR, 3536 ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH)); 3537 3538 /* Unmask the interrupts that we always want on. */ 3539 dev_priv->irq_mask = 3540 ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | 3541 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | 3542 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT | 3543 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT); 3544 I915_WRITE16(IMR, dev_priv->irq_mask); 3545 3546 I915_WRITE16(IER, 3547 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | 3548 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | 3549 I915_USER_INTERRUPT); 3550 POSTING_READ16(IER); 3551 3552 /* Interrupt setup is already guaranteed to be single-threaded, this is 3553 * just to make the assert_spin_locked check happy. */ 3554 spin_lock_irq(&dev_priv->irq_lock); 3555 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS); 3556 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS); 3557 spin_unlock_irq(&dev_priv->irq_lock); 3558 3559 return 0; 3560 } 3561 3562 /* 3563 * Returns true when a page flip has completed. 3564 */ 3565 static bool i8xx_handle_vblank(struct drm_i915_private *dev_priv, 3566 int plane, int pipe, u32 iir) 3567 { 3568 u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane); 3569 3570 if (!intel_pipe_handle_vblank(dev_priv, pipe)) 3571 return false; 3572 3573 if ((iir & flip_pending) == 0) 3574 goto check_page_flip; 3575 3576 /* We detect FlipDone by looking for the change in PendingFlip from '1' 3577 * to '0' on the following vblank, i.e. IIR has the Pendingflip 3578 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence 3579 * the flip is completed (no longer pending). Since this doesn't raise 3580 * an interrupt per se, we watch for the change at vblank. 3581 */ 3582 if (I915_READ16(ISR) & flip_pending) 3583 goto check_page_flip; 3584 3585 intel_finish_page_flip_cs(dev_priv, pipe); 3586 return true; 3587 3588 check_page_flip: 3589 intel_check_page_flip(dev_priv, pipe); 3590 return false; 3591 } 3592 3593 static irqreturn_t i8xx_irq_handler(int irq, void *arg) 3594 { 3595 struct drm_device *dev = arg; 3596 struct drm_i915_private *dev_priv = to_i915(dev); 3597 u16 iir, new_iir; 3598 u32 pipe_stats[2]; 3599 int pipe; 3600 u16 flip_mask = 3601 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT | 3602 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT; 3603 irqreturn_t ret; 3604 3605 if (!intel_irqs_enabled(dev_priv)) 3606 return IRQ_NONE; 3607 3608 /* IRQs are synced during runtime_suspend, we don't require a wakeref */ 3609 disable_rpm_wakeref_asserts(dev_priv); 3610 3611 ret = IRQ_NONE; 3612 iir = I915_READ16(IIR); 3613 if (iir == 0) 3614 goto out; 3615 3616 while (iir & ~flip_mask) { 3617 /* Can't rely on pipestat interrupt bit in iir as it might 3618 * have been cleared after the pipestat interrupt was received. 3619 * It doesn't set the bit in iir again, but it still produces 3620 * interrupts (for non-MSI). 3621 */ 3622 lockmgr(&dev_priv->irq_lock, LK_EXCLUSIVE); 3623 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT) 3624 DRM_DEBUG("Command parser error, iir 0x%08x\n", iir); 3625 3626 for_each_pipe(dev_priv, pipe) { 3627 i915_reg_t reg = PIPESTAT(pipe); 3628 pipe_stats[pipe] = I915_READ(reg); 3629 3630 /* 3631 * Clear the PIPE*STAT regs before the IIR 3632 */ 3633 if (pipe_stats[pipe] & 0x8000ffff) 3634 I915_WRITE(reg, pipe_stats[pipe]); 3635 } 3636 lockmgr(&dev_priv->irq_lock, LK_RELEASE); 3637 3638 I915_WRITE16(IIR, iir & ~flip_mask); 3639 new_iir = I915_READ16(IIR); /* Flush posted writes */ 3640 3641 if (iir & I915_USER_INTERRUPT) 3642 notify_ring(dev_priv->engine[RCS]); 3643 3644 for_each_pipe(dev_priv, pipe) { 3645 int plane = pipe; 3646 if (HAS_FBC(dev_priv)) 3647 plane = !plane; 3648 3649 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS && 3650 i8xx_handle_vblank(dev_priv, plane, pipe, iir)) 3651 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane); 3652 3653 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS) 3654 i9xx_pipe_crc_irq_handler(dev_priv, pipe); 3655 3656 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS) 3657 intel_cpu_fifo_underrun_irq_handler(dev_priv, 3658 pipe); 3659 } 3660 3661 iir = new_iir; 3662 } 3663 ret = IRQ_HANDLED; 3664 3665 out: 3666 enable_rpm_wakeref_asserts(dev_priv); 3667 3668 return ret; 3669 } 3670 3671 static void i8xx_irq_uninstall(struct drm_device * dev) 3672 { 3673 struct drm_i915_private *dev_priv = to_i915(dev); 3674 int pipe; 3675 3676 for_each_pipe(dev_priv, pipe) { 3677 /* Clear enable bits; then clear status bits */ 3678 I915_WRITE(PIPESTAT(pipe), 0); 3679 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe))); 3680 } 3681 I915_WRITE16(IMR, 0xffff); 3682 I915_WRITE16(IER, 0x0); 3683 I915_WRITE16(IIR, I915_READ16(IIR)); 3684 } 3685 3686 static void i915_irq_preinstall(struct drm_device * dev) 3687 { 3688 struct drm_i915_private *dev_priv = to_i915(dev); 3689 int pipe; 3690 3691 if (I915_HAS_HOTPLUG(dev_priv)) { 3692 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0); 3693 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); 3694 } 3695 3696 I915_WRITE16(HWSTAM, 0xeffe); 3697 for_each_pipe(dev_priv, pipe) 3698 I915_WRITE(PIPESTAT(pipe), 0); 3699 I915_WRITE(IMR, 0xffffffff); 3700 I915_WRITE(IER, 0x0); 3701 POSTING_READ(IER); 3702 } 3703 3704 static int i915_irq_postinstall(struct drm_device *dev) 3705 { 3706 struct drm_i915_private *dev_priv = to_i915(dev); 3707 u32 enable_mask; 3708 3709 I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH)); 3710 3711 /* Unmask the interrupts that we always want on. */ 3712 dev_priv->irq_mask = 3713 ~(I915_ASLE_INTERRUPT | 3714 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | 3715 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | 3716 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT | 3717 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT); 3718 3719 enable_mask = 3720 I915_ASLE_INTERRUPT | 3721 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | 3722 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | 3723 I915_USER_INTERRUPT; 3724 3725 if (I915_HAS_HOTPLUG(dev_priv)) { 3726 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0); 3727 POSTING_READ(PORT_HOTPLUG_EN); 3728 3729 /* Enable in IER... */ 3730 enable_mask |= I915_DISPLAY_PORT_INTERRUPT; 3731 /* and unmask in IMR */ 3732 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT; 3733 } 3734 3735 I915_WRITE(IMR, dev_priv->irq_mask); 3736 I915_WRITE(IER, enable_mask); 3737 POSTING_READ(IER); 3738 3739 i915_enable_asle_pipestat(dev_priv); 3740 3741 /* Interrupt setup is already guaranteed to be single-threaded, this is 3742 * just to make the assert_spin_locked check happy. */ 3743 spin_lock_irq(&dev_priv->irq_lock); 3744 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS); 3745 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS); 3746 spin_unlock_irq(&dev_priv->irq_lock); 3747 3748 return 0; 3749 } 3750 3751 /* 3752 * Returns true when a page flip has completed. 3753 */ 3754 static bool i915_handle_vblank(struct drm_i915_private *dev_priv, 3755 int plane, int pipe, u32 iir) 3756 { 3757 u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane); 3758 3759 if (!intel_pipe_handle_vblank(dev_priv, pipe)) 3760 return false; 3761 3762 if ((iir & flip_pending) == 0) 3763 goto check_page_flip; 3764 3765 /* We detect FlipDone by looking for the change in PendingFlip from '1' 3766 * to '0' on the following vblank, i.e. IIR has the Pendingflip 3767 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence 3768 * the flip is completed (no longer pending). Since this doesn't raise 3769 * an interrupt per se, we watch for the change at vblank. 3770 */ 3771 if (I915_READ(ISR) & flip_pending) 3772 goto check_page_flip; 3773 3774 intel_finish_page_flip_cs(dev_priv, pipe); 3775 return true; 3776 3777 check_page_flip: 3778 intel_check_page_flip(dev_priv, pipe); 3779 return false; 3780 } 3781 3782 static irqreturn_t i915_irq_handler(int irq, void *arg) 3783 { 3784 struct drm_device *dev = arg; 3785 struct drm_i915_private *dev_priv = to_i915(dev); 3786 u32 iir, new_iir, pipe_stats[I915_MAX_PIPES]; 3787 u32 flip_mask = 3788 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT | 3789 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT; 3790 int pipe, ret = IRQ_NONE; 3791 3792 if (!intel_irqs_enabled(dev_priv)) 3793 return IRQ_NONE; 3794 3795 /* IRQs are synced during runtime_suspend, we don't require a wakeref */ 3796 disable_rpm_wakeref_asserts(dev_priv); 3797 3798 iir = I915_READ(IIR); 3799 do { 3800 bool irq_received = (iir & ~flip_mask) != 0; 3801 bool blc_event = false; 3802 3803 /* Can't rely on pipestat interrupt bit in iir as it might 3804 * have been cleared after the pipestat interrupt was received. 3805 * It doesn't set the bit in iir again, but it still produces 3806 * interrupts (for non-MSI). 3807 */ 3808 lockmgr(&dev_priv->irq_lock, LK_EXCLUSIVE); 3809 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT) 3810 DRM_DEBUG("Command parser error, iir 0x%08x\n", iir); 3811 3812 for_each_pipe(dev_priv, pipe) { 3813 i915_reg_t reg = PIPESTAT(pipe); 3814 pipe_stats[pipe] = I915_READ(reg); 3815 3816 /* Clear the PIPE*STAT regs before the IIR */ 3817 if (pipe_stats[pipe] & 0x8000ffff) { 3818 I915_WRITE(reg, pipe_stats[pipe]); 3819 irq_received = true; 3820 } 3821 } 3822 lockmgr(&dev_priv->irq_lock, LK_RELEASE); 3823 3824 if (!irq_received) 3825 break; 3826 3827 /* Consume port. Then clear IIR or we'll miss events */ 3828 if (I915_HAS_HOTPLUG(dev_priv) && 3829 iir & I915_DISPLAY_PORT_INTERRUPT) { 3830 u32 hotplug_status = i9xx_hpd_irq_ack(dev_priv); 3831 if (hotplug_status) 3832 i9xx_hpd_irq_handler(dev_priv, hotplug_status); 3833 } 3834 3835 I915_WRITE(IIR, iir & ~flip_mask); 3836 new_iir = I915_READ(IIR); /* Flush posted writes */ 3837 3838 if (iir & I915_USER_INTERRUPT) 3839 notify_ring(dev_priv->engine[RCS]); 3840 3841 for_each_pipe(dev_priv, pipe) { 3842 int plane = pipe; 3843 if (HAS_FBC(dev_priv)) 3844 plane = !plane; 3845 3846 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS && 3847 i915_handle_vblank(dev_priv, plane, pipe, iir)) 3848 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane); 3849 3850 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS) 3851 blc_event = true; 3852 3853 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS) 3854 i9xx_pipe_crc_irq_handler(dev_priv, pipe); 3855 3856 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS) 3857 intel_cpu_fifo_underrun_irq_handler(dev_priv, 3858 pipe); 3859 } 3860 3861 if (blc_event || (iir & I915_ASLE_INTERRUPT)) 3862 intel_opregion_asle_intr(dev_priv); 3863 3864 /* With MSI, interrupts are only generated when iir 3865 * transitions from zero to nonzero. If another bit got 3866 * set while we were handling the existing iir bits, then 3867 * we would never get another interrupt. 3868 * 3869 * This is fine on non-MSI as well, as if we hit this path 3870 * we avoid exiting the interrupt handler only to generate 3871 * another one. 3872 * 3873 * Note that for MSI this could cause a stray interrupt report 3874 * if an interrupt landed in the time between writing IIR and 3875 * the posting read. This should be rare enough to never 3876 * trigger the 99% of 100,000 interrupts test for disabling 3877 * stray interrupts. 3878 */ 3879 ret = IRQ_HANDLED; 3880 iir = new_iir; 3881 } while (iir & ~flip_mask); 3882 3883 enable_rpm_wakeref_asserts(dev_priv); 3884 3885 return ret; 3886 } 3887 3888 static void i915_irq_uninstall(struct drm_device * dev) 3889 { 3890 struct drm_i915_private *dev_priv = to_i915(dev); 3891 int pipe; 3892 3893 if (I915_HAS_HOTPLUG(dev_priv)) { 3894 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0); 3895 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); 3896 } 3897 3898 I915_WRITE16(HWSTAM, 0xffff); 3899 for_each_pipe(dev_priv, pipe) { 3900 /* Clear enable bits; then clear status bits */ 3901 I915_WRITE(PIPESTAT(pipe), 0); 3902 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe))); 3903 } 3904 I915_WRITE(IMR, 0xffffffff); 3905 I915_WRITE(IER, 0x0); 3906 3907 I915_WRITE(IIR, I915_READ(IIR)); 3908 } 3909 3910 static void i965_irq_preinstall(struct drm_device * dev) 3911 { 3912 struct drm_i915_private *dev_priv = to_i915(dev); 3913 int pipe; 3914 3915 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0); 3916 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); 3917 3918 I915_WRITE(HWSTAM, 0xeffe); 3919 for_each_pipe(dev_priv, pipe) 3920 I915_WRITE(PIPESTAT(pipe), 0); 3921 I915_WRITE(IMR, 0xffffffff); 3922 I915_WRITE(IER, 0x0); 3923 POSTING_READ(IER); 3924 } 3925 3926 static int i965_irq_postinstall(struct drm_device *dev) 3927 { 3928 struct drm_i915_private *dev_priv = to_i915(dev); 3929 u32 enable_mask; 3930 u32 error_mask; 3931 3932 /* Unmask the interrupts that we always want on. */ 3933 dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT | 3934 I915_DISPLAY_PORT_INTERRUPT | 3935 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | 3936 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | 3937 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT | 3938 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT | 3939 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT); 3940 3941 enable_mask = ~dev_priv->irq_mask; 3942 enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT | 3943 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT); 3944 enable_mask |= I915_USER_INTERRUPT; 3945 3946 if (IS_G4X(dev_priv)) 3947 enable_mask |= I915_BSD_USER_INTERRUPT; 3948 3949 /* Interrupt setup is already guaranteed to be single-threaded, this is 3950 * just to make the assert_spin_locked check happy. */ 3951 spin_lock_irq(&dev_priv->irq_lock); 3952 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS); 3953 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS); 3954 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS); 3955 spin_unlock_irq(&dev_priv->irq_lock); 3956 3957 /* 3958 * Enable some error detection, note the instruction error mask 3959 * bit is reserved, so we leave it masked. 3960 */ 3961 if (IS_G4X(dev_priv)) { 3962 error_mask = ~(GM45_ERROR_PAGE_TABLE | 3963 GM45_ERROR_MEM_PRIV | 3964 GM45_ERROR_CP_PRIV | 3965 I915_ERROR_MEMORY_REFRESH); 3966 } else { 3967 error_mask = ~(I915_ERROR_PAGE_TABLE | 3968 I915_ERROR_MEMORY_REFRESH); 3969 } 3970 I915_WRITE(EMR, error_mask); 3971 3972 I915_WRITE(IMR, dev_priv->irq_mask); 3973 I915_WRITE(IER, enable_mask); 3974 POSTING_READ(IER); 3975 3976 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0); 3977 POSTING_READ(PORT_HOTPLUG_EN); 3978 3979 i915_enable_asle_pipestat(dev_priv); 3980 3981 return 0; 3982 } 3983 3984 static void i915_hpd_irq_setup(struct drm_i915_private *dev_priv) 3985 { 3986 u32 hotplug_en; 3987 3988 assert_spin_locked(&dev_priv->irq_lock); 3989 3990 /* Note HDMI and DP share hotplug bits */ 3991 /* enable bits are the same for all generations */ 3992 hotplug_en = intel_hpd_enabled_irqs(dev_priv, hpd_mask_i915); 3993 /* Programming the CRT detection parameters tends 3994 to generate a spurious hotplug event about three 3995 seconds later. So just do it once. 3996 */ 3997 if (IS_G4X(dev_priv)) 3998 hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64; 3999 hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50; 4000 4001 /* Ignore TV since it's buggy */ 4002 i915_hotplug_interrupt_update_locked(dev_priv, 4003 HOTPLUG_INT_EN_MASK | 4004 CRT_HOTPLUG_VOLTAGE_COMPARE_MASK | 4005 CRT_HOTPLUG_ACTIVATION_PERIOD_64, 4006 hotplug_en); 4007 } 4008 4009 static irqreturn_t i965_irq_handler(int irq, void *arg) 4010 { 4011 struct drm_device *dev = arg; 4012 struct drm_i915_private *dev_priv = to_i915(dev); 4013 u32 iir, new_iir; 4014 u32 pipe_stats[I915_MAX_PIPES]; 4015 int ret = IRQ_NONE, pipe; 4016 u32 flip_mask = 4017 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT | 4018 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT; 4019 4020 if (!intel_irqs_enabled(dev_priv)) 4021 return IRQ_NONE; 4022 4023 /* IRQs are synced during runtime_suspend, we don't require a wakeref */ 4024 disable_rpm_wakeref_asserts(dev_priv); 4025 4026 iir = I915_READ(IIR); 4027 4028 for (;;) { 4029 bool irq_received = (iir & ~flip_mask) != 0; 4030 bool blc_event = false; 4031 4032 /* Can't rely on pipestat interrupt bit in iir as it might 4033 * have been cleared after the pipestat interrupt was received. 4034 * It doesn't set the bit in iir again, but it still produces 4035 * interrupts (for non-MSI). 4036 */ 4037 lockmgr(&dev_priv->irq_lock, LK_EXCLUSIVE); 4038 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT) 4039 DRM_DEBUG("Command parser error, iir 0x%08x\n", iir); 4040 4041 for_each_pipe(dev_priv, pipe) { 4042 i915_reg_t reg = PIPESTAT(pipe); 4043 pipe_stats[pipe] = I915_READ(reg); 4044 4045 /* 4046 * Clear the PIPE*STAT regs before the IIR 4047 */ 4048 if (pipe_stats[pipe] & 0x8000ffff) { 4049 I915_WRITE(reg, pipe_stats[pipe]); 4050 irq_received = true; 4051 } 4052 } 4053 lockmgr(&dev_priv->irq_lock, LK_RELEASE); 4054 4055 if (!irq_received) 4056 break; 4057 4058 ret = IRQ_HANDLED; 4059 4060 /* Consume port. Then clear IIR or we'll miss events */ 4061 if (iir & I915_DISPLAY_PORT_INTERRUPT) { 4062 u32 hotplug_status = i9xx_hpd_irq_ack(dev_priv); 4063 if (hotplug_status) 4064 i9xx_hpd_irq_handler(dev_priv, hotplug_status); 4065 } 4066 4067 I915_WRITE(IIR, iir & ~flip_mask); 4068 new_iir = I915_READ(IIR); /* Flush posted writes */ 4069 4070 if (iir & I915_USER_INTERRUPT) 4071 notify_ring(dev_priv->engine[RCS]); 4072 if (iir & I915_BSD_USER_INTERRUPT) 4073 notify_ring(dev_priv->engine[VCS]); 4074 4075 for_each_pipe(dev_priv, pipe) { 4076 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS && 4077 i915_handle_vblank(dev_priv, pipe, pipe, iir)) 4078 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe); 4079 4080 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS) 4081 blc_event = true; 4082 4083 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS) 4084 i9xx_pipe_crc_irq_handler(dev_priv, pipe); 4085 4086 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS) 4087 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe); 4088 } 4089 4090 if (blc_event || (iir & I915_ASLE_INTERRUPT)) 4091 intel_opregion_asle_intr(dev_priv); 4092 4093 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS) 4094 gmbus_irq_handler(dev_priv); 4095 4096 /* With MSI, interrupts are only generated when iir 4097 * transitions from zero to nonzero. If another bit got 4098 * set while we were handling the existing iir bits, then 4099 * we would never get another interrupt. 4100 * 4101 * This is fine on non-MSI as well, as if we hit this path 4102 * we avoid exiting the interrupt handler only to generate 4103 * another one. 4104 * 4105 * Note that for MSI this could cause a stray interrupt report 4106 * if an interrupt landed in the time between writing IIR and 4107 * the posting read. This should be rare enough to never 4108 * trigger the 99% of 100,000 interrupts test for disabling 4109 * stray interrupts. 4110 */ 4111 iir = new_iir; 4112 } 4113 4114 enable_rpm_wakeref_asserts(dev_priv); 4115 4116 return ret; 4117 } 4118 4119 static void i965_irq_uninstall(struct drm_device * dev) 4120 { 4121 struct drm_i915_private *dev_priv = to_i915(dev); 4122 int pipe; 4123 4124 if (!dev_priv) 4125 return; 4126 4127 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0); 4128 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); 4129 4130 I915_WRITE(HWSTAM, 0xffffffff); 4131 for_each_pipe(dev_priv, pipe) 4132 I915_WRITE(PIPESTAT(pipe), 0); 4133 I915_WRITE(IMR, 0xffffffff); 4134 I915_WRITE(IER, 0x0); 4135 4136 for_each_pipe(dev_priv, pipe) 4137 I915_WRITE(PIPESTAT(pipe), 4138 I915_READ(PIPESTAT(pipe)) & 0x8000ffff); 4139 I915_WRITE(IIR, I915_READ(IIR)); 4140 } 4141 4142 /** 4143 * intel_irq_init - initializes irq support 4144 * @dev_priv: i915 device instance 4145 * 4146 * This function initializes all the irq support including work items, timers 4147 * and all the vtables. It does not setup the interrupt itself though. 4148 */ 4149 void intel_irq_init(struct drm_i915_private *dev_priv) 4150 { 4151 struct drm_device *dev = &dev_priv->drm; 4152 4153 intel_hpd_init_work(dev_priv); 4154 4155 INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work); 4156 INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work); 4157 4158 if (HAS_GUC_SCHED(dev_priv)) 4159 dev_priv->pm_guc_events = GEN9_GUC_TO_HOST_INT_EVENT; 4160 4161 /* Let's track the enabled rps events */ 4162 if (IS_VALLEYVIEW(dev_priv)) 4163 /* WaGsvRC0ResidencyMethod:vlv */ 4164 dev_priv->pm_rps_events = GEN6_PM_RP_UP_EI_EXPIRED; 4165 else 4166 dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS; 4167 4168 dev_priv->rps.pm_intr_keep = 0; 4169 4170 /* 4171 * SNB,IVB can while VLV,CHV may hard hang on looping batchbuffer 4172 * if GEN6_PM_UP_EI_EXPIRED is masked. 4173 * 4174 * TODO: verify if this can be reproduced on VLV,CHV. 4175 */ 4176 if (INTEL_INFO(dev_priv)->gen <= 7 && !IS_HASWELL(dev_priv)) 4177 dev_priv->rps.pm_intr_keep |= GEN6_PM_RP_UP_EI_EXPIRED; 4178 4179 if (INTEL_INFO(dev_priv)->gen >= 8) 4180 dev_priv->rps.pm_intr_keep |= GEN8_PMINTR_REDIRECT_TO_GUC; 4181 4182 if (IS_GEN2(dev_priv)) { 4183 /* Gen2 doesn't have a hardware frame counter */ 4184 dev->max_vblank_count = 0; 4185 dev->driver->get_vblank_counter = drm_vblank_no_hw_counter; 4186 } else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) { 4187 dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */ 4188 dev->driver->get_vblank_counter = g4x_get_vblank_counter; 4189 } else { 4190 dev->driver->get_vblank_counter = i915_get_vblank_counter; 4191 dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */ 4192 } 4193 4194 /* 4195 * Opt out of the vblank disable timer on everything except gen2. 4196 * Gen2 doesn't have a hardware frame counter and so depends on 4197 * vblank interrupts to produce sane vblank seuquence numbers. 4198 */ 4199 if (!IS_GEN2(dev_priv)) 4200 dev->vblank_disable_immediate = true; 4201 4202 /* Most platforms treat the display irq block as an always-on 4203 * power domain. vlv/chv can disable it at runtime and need 4204 * special care to avoid writing any of the display block registers 4205 * outside of the power domain. We defer setting up the display irqs 4206 * in this case to the runtime pm. 4207 */ 4208 dev_priv->display_irqs_enabled = true; 4209 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) 4210 dev_priv->display_irqs_enabled = false; 4211 4212 dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp; 4213 dev->driver->get_scanout_position = i915_get_crtc_scanoutpos; 4214 4215 if (IS_CHERRYVIEW(dev_priv)) { 4216 dev->driver->irq_handler = cherryview_irq_handler; 4217 dev->driver->irq_preinstall = cherryview_irq_preinstall; 4218 dev->driver->irq_postinstall = cherryview_irq_postinstall; 4219 dev->driver->irq_uninstall = cherryview_irq_uninstall; 4220 dev->driver->enable_vblank = i965_enable_vblank; 4221 dev->driver->disable_vblank = i965_disable_vblank; 4222 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup; 4223 } else if (IS_VALLEYVIEW(dev_priv)) { 4224 dev->driver->irq_handler = valleyview_irq_handler; 4225 dev->driver->irq_preinstall = valleyview_irq_preinstall; 4226 dev->driver->irq_postinstall = valleyview_irq_postinstall; 4227 dev->driver->irq_uninstall = valleyview_irq_uninstall; 4228 dev->driver->enable_vblank = i965_enable_vblank; 4229 dev->driver->disable_vblank = i965_disable_vblank; 4230 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup; 4231 } else if (INTEL_INFO(dev_priv)->gen >= 8) { 4232 dev->driver->irq_handler = gen8_irq_handler; 4233 dev->driver->irq_preinstall = gen8_irq_reset; 4234 dev->driver->irq_postinstall = gen8_irq_postinstall; 4235 dev->driver->irq_uninstall = gen8_irq_uninstall; 4236 dev->driver->enable_vblank = gen8_enable_vblank; 4237 dev->driver->disable_vblank = gen8_disable_vblank; 4238 if (IS_BROXTON(dev_priv)) 4239 dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup; 4240 else if (HAS_PCH_SPT(dev_priv) || HAS_PCH_KBP(dev_priv)) 4241 dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup; 4242 else 4243 dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup; 4244 } else if (HAS_PCH_SPLIT(dev_priv)) { 4245 dev->driver->irq_handler = ironlake_irq_handler; 4246 dev->driver->irq_preinstall = ironlake_irq_reset; 4247 dev->driver->irq_postinstall = ironlake_irq_postinstall; 4248 dev->driver->irq_uninstall = ironlake_irq_uninstall; 4249 dev->driver->enable_vblank = ironlake_enable_vblank; 4250 dev->driver->disable_vblank = ironlake_disable_vblank; 4251 dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup; 4252 } else { 4253 if (IS_GEN2(dev_priv)) { 4254 dev->driver->irq_preinstall = i8xx_irq_preinstall; 4255 dev->driver->irq_postinstall = i8xx_irq_postinstall; 4256 dev->driver->irq_handler = i8xx_irq_handler; 4257 dev->driver->irq_uninstall = i8xx_irq_uninstall; 4258 dev->driver->enable_vblank = i8xx_enable_vblank; 4259 dev->driver->disable_vblank = i8xx_disable_vblank; 4260 } else if (IS_GEN3(dev_priv)) { 4261 dev->driver->irq_preinstall = i915_irq_preinstall; 4262 dev->driver->irq_postinstall = i915_irq_postinstall; 4263 dev->driver->irq_uninstall = i915_irq_uninstall; 4264 dev->driver->irq_handler = i915_irq_handler; 4265 dev->driver->enable_vblank = i8xx_enable_vblank; 4266 dev->driver->disable_vblank = i8xx_disable_vblank; 4267 } else { 4268 dev->driver->irq_preinstall = i965_irq_preinstall; 4269 dev->driver->irq_postinstall = i965_irq_postinstall; 4270 dev->driver->irq_uninstall = i965_irq_uninstall; 4271 dev->driver->irq_handler = i965_irq_handler; 4272 dev->driver->enable_vblank = i965_enable_vblank; 4273 dev->driver->disable_vblank = i965_disable_vblank; 4274 } 4275 if (I915_HAS_HOTPLUG(dev_priv)) 4276 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup; 4277 } 4278 } 4279 4280 /** 4281 * intel_irq_install - enables the hardware interrupt 4282 * @dev_priv: i915 device instance 4283 * 4284 * This function enables the hardware interrupt handling, but leaves the hotplug 4285 * handling still disabled. It is called after intel_irq_init(). 4286 * 4287 * In the driver load and resume code we need working interrupts in a few places 4288 * but don't want to deal with the hassle of concurrent probe and hotplug 4289 * workers. Hence the split into this two-stage approach. 4290 */ 4291 int intel_irq_install(struct drm_i915_private *dev_priv) 4292 { 4293 /* 4294 * We enable some interrupt sources in our postinstall hooks, so mark 4295 * interrupts as enabled _before_ actually enabling them to avoid 4296 * special cases in our ordering checks. 4297 */ 4298 dev_priv->pm.irqs_enabled = true; 4299 4300 return drm_irq_install(&dev_priv->drm, dev_priv->drm.pdev->irq); 4301 } 4302 4303 /** 4304 * intel_irq_uninstall - finilizes all irq handling 4305 * @dev_priv: i915 device instance 4306 * 4307 * This stops interrupt and hotplug handling and unregisters and frees all 4308 * resources acquired in the init functions. 4309 */ 4310 void intel_irq_uninstall(struct drm_i915_private *dev_priv) 4311 { 4312 drm_irq_uninstall(&dev_priv->drm); 4313 intel_hpd_cancel_work(dev_priv); 4314 dev_priv->pm.irqs_enabled = false; 4315 } 4316 4317 /** 4318 * intel_runtime_pm_disable_interrupts - runtime interrupt disabling 4319 * @dev_priv: i915 device instance 4320 * 4321 * This function is used to disable interrupts at runtime, both in the runtime 4322 * pm and the system suspend/resume code. 4323 */ 4324 void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv) 4325 { 4326 dev_priv->drm.driver->irq_uninstall(&dev_priv->drm); 4327 dev_priv->pm.irqs_enabled = false; 4328 synchronize_irq(dev_priv->drm.irq); 4329 } 4330 4331 /** 4332 * intel_runtime_pm_enable_interrupts - runtime interrupt enabling 4333 * @dev_priv: i915 device instance 4334 * 4335 * This function is used to enable interrupts at runtime, both in the runtime 4336 * pm and the system suspend/resume code. 4337 */ 4338 void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv) 4339 { 4340 dev_priv->pm.irqs_enabled = true; 4341 dev_priv->drm.driver->irq_preinstall(&dev_priv->drm); 4342 dev_priv->drm.driver->irq_postinstall(&dev_priv->drm); 4343 } 4344