1 /* 2 * Copyright © 2012 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eugeni Dodonov <eugeni.dodonov@intel.com> 25 * 26 */ 27 28 #include "i915_drv.h" 29 #include "intel_drv.h" 30 #include <linux/module.h> 31 #include <machine/clock.h> 32 #include <drm/i915_powerwell.h> 33 34 /** 35 * RC6 is a special power stage which allows the GPU to enter an very 36 * low-voltage mode when idle, using down to 0V while at this stage. This 37 * stage is entered automatically when the GPU is idle when RC6 support is 38 * enabled, and as soon as new workload arises GPU wakes up automatically as well. 39 * 40 * There are different RC6 modes available in Intel GPU, which differentiate 41 * among each other with the latency required to enter and leave RC6 and 42 * voltage consumed by the GPU in different states. 43 * 44 * The combination of the following flags define which states GPU is allowed 45 * to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and 46 * RC6pp is deepest RC6. Their support by hardware varies according to the 47 * GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one 48 * which brings the most power savings; deeper states save more power, but 49 * require higher latency to switch to and wake up. 50 */ 51 #define INTEL_RC6_ENABLE (1<<0) 52 #define INTEL_RC6p_ENABLE (1<<1) 53 #define INTEL_RC6pp_ENABLE (1<<2) 54 55 /* FBC, or Frame Buffer Compression, is a technique employed to compress the 56 * framebuffer contents in-memory, aiming at reducing the required bandwidth 57 * during in-memory transfers and, therefore, reduce the power packet. 58 * 59 * The benefits of FBC are mostly visible with solid backgrounds and 60 * variation-less patterns. 61 * 62 * FBC-related functionality can be enabled by the means of the 63 * i915.i915_enable_fbc parameter 64 */ 65 66 static void i8xx_disable_fbc(struct drm_device *dev) 67 { 68 struct drm_i915_private *dev_priv = dev->dev_private; 69 u32 fbc_ctl; 70 71 /* Disable compression */ 72 fbc_ctl = I915_READ(FBC_CONTROL); 73 if ((fbc_ctl & FBC_CTL_EN) == 0) 74 return; 75 76 fbc_ctl &= ~FBC_CTL_EN; 77 I915_WRITE(FBC_CONTROL, fbc_ctl); 78 79 /* Wait for compressing bit to clear */ 80 if (wait_for((I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING) == 0, 10)) { 81 DRM_DEBUG_KMS("FBC idle timed out\n"); 82 return; 83 } 84 85 DRM_DEBUG_KMS("disabled FBC\n"); 86 } 87 88 static void i8xx_enable_fbc(struct drm_crtc *crtc) 89 { 90 struct drm_device *dev = crtc->dev; 91 struct drm_i915_private *dev_priv = dev->dev_private; 92 struct drm_framebuffer *fb = crtc->primary->fb; 93 struct drm_i915_gem_object *obj = intel_fb_obj(fb); 94 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 95 int cfb_pitch; 96 int i; 97 u32 fbc_ctl; 98 99 cfb_pitch = dev_priv->fbc.size / FBC_LL_SIZE; 100 if (fb->pitches[0] < cfb_pitch) 101 cfb_pitch = fb->pitches[0]; 102 103 /* FBC_CTL wants 32B or 64B units */ 104 if (IS_GEN2(dev)) 105 cfb_pitch = (cfb_pitch / 32) - 1; 106 else 107 cfb_pitch = (cfb_pitch / 64) - 1; 108 109 /* Clear old tags */ 110 for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++) 111 I915_WRITE(FBC_TAG + (i * 4), 0); 112 113 if (IS_GEN4(dev)) { 114 u32 fbc_ctl2; 115 116 /* Set it up... */ 117 fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | FBC_CTL_CPU_FENCE; 118 fbc_ctl2 |= FBC_CTL_PLANE(intel_crtc->plane); 119 I915_WRITE(FBC_CONTROL2, fbc_ctl2); 120 I915_WRITE(FBC_FENCE_OFF, crtc->y); 121 } 122 123 /* enable it... */ 124 fbc_ctl = I915_READ(FBC_CONTROL); 125 fbc_ctl &= 0x3fff << FBC_CTL_INTERVAL_SHIFT; 126 fbc_ctl |= FBC_CTL_EN | FBC_CTL_PERIODIC; 127 if (IS_I945GM(dev)) 128 fbc_ctl |= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */ 129 fbc_ctl |= (cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT; 130 fbc_ctl |= obj->fence_reg; 131 I915_WRITE(FBC_CONTROL, fbc_ctl); 132 133 DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %c\n", 134 cfb_pitch, crtc->y, plane_name(intel_crtc->plane)); 135 } 136 137 static bool i8xx_fbc_enabled(struct drm_device *dev) 138 { 139 struct drm_i915_private *dev_priv = dev->dev_private; 140 141 return I915_READ(FBC_CONTROL) & FBC_CTL_EN; 142 } 143 144 static void g4x_enable_fbc(struct drm_crtc *crtc) 145 { 146 struct drm_device *dev = crtc->dev; 147 struct drm_i915_private *dev_priv = dev->dev_private; 148 struct drm_framebuffer *fb = crtc->primary->fb; 149 struct drm_i915_gem_object *obj = intel_fb_obj(fb); 150 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 151 u32 dpfc_ctl; 152 153 dpfc_ctl = DPFC_CTL_PLANE(intel_crtc->plane) | DPFC_SR_EN; 154 if (drm_format_plane_cpp(fb->pixel_format, 0) == 2) 155 dpfc_ctl |= DPFC_CTL_LIMIT_2X; 156 else 157 dpfc_ctl |= DPFC_CTL_LIMIT_1X; 158 dpfc_ctl |= DPFC_CTL_FENCE_EN | obj->fence_reg; 159 160 I915_WRITE(DPFC_FENCE_YOFF, crtc->y); 161 162 /* enable it... */ 163 I915_WRITE(DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN); 164 165 DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane)); 166 } 167 168 static void g4x_disable_fbc(struct drm_device *dev) 169 { 170 struct drm_i915_private *dev_priv = dev->dev_private; 171 u32 dpfc_ctl; 172 173 /* Disable compression */ 174 dpfc_ctl = I915_READ(DPFC_CONTROL); 175 if (dpfc_ctl & DPFC_CTL_EN) { 176 dpfc_ctl &= ~DPFC_CTL_EN; 177 I915_WRITE(DPFC_CONTROL, dpfc_ctl); 178 179 DRM_DEBUG_KMS("disabled FBC\n"); 180 } 181 } 182 183 static bool g4x_fbc_enabled(struct drm_device *dev) 184 { 185 struct drm_i915_private *dev_priv = dev->dev_private; 186 187 return I915_READ(DPFC_CONTROL) & DPFC_CTL_EN; 188 } 189 190 static void sandybridge_blit_fbc_update(struct drm_device *dev) 191 { 192 struct drm_i915_private *dev_priv = dev->dev_private; 193 u32 blt_ecoskpd; 194 195 /* Make sure blitter notifies FBC of writes */ 196 197 /* Blitter is part of Media powerwell on VLV. No impact of 198 * his param in other platforms for now */ 199 gen6_gt_force_wake_get(dev_priv, FORCEWAKE_MEDIA); 200 201 blt_ecoskpd = I915_READ(GEN6_BLITTER_ECOSKPD); 202 blt_ecoskpd |= GEN6_BLITTER_FBC_NOTIFY << 203 GEN6_BLITTER_LOCK_SHIFT; 204 I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd); 205 blt_ecoskpd |= GEN6_BLITTER_FBC_NOTIFY; 206 I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd); 207 blt_ecoskpd &= ~(GEN6_BLITTER_FBC_NOTIFY << 208 GEN6_BLITTER_LOCK_SHIFT); 209 I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd); 210 POSTING_READ(GEN6_BLITTER_ECOSKPD); 211 212 gen6_gt_force_wake_put(dev_priv, FORCEWAKE_MEDIA); 213 } 214 215 static void ironlake_enable_fbc(struct drm_crtc *crtc) 216 { 217 struct drm_device *dev = crtc->dev; 218 struct drm_i915_private *dev_priv = dev->dev_private; 219 struct drm_framebuffer *fb = crtc->primary->fb; 220 struct drm_i915_gem_object *obj = intel_fb_obj(fb); 221 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 222 u32 dpfc_ctl; 223 224 dpfc_ctl = DPFC_CTL_PLANE(intel_crtc->plane); 225 if (drm_format_plane_cpp(fb->pixel_format, 0) == 2) 226 dev_priv->fbc.threshold++; 227 228 switch (dev_priv->fbc.threshold) { 229 case 4: 230 case 3: 231 dpfc_ctl |= DPFC_CTL_LIMIT_4X; 232 break; 233 case 2: 234 dpfc_ctl |= DPFC_CTL_LIMIT_2X; 235 break; 236 case 1: 237 dpfc_ctl |= DPFC_CTL_LIMIT_1X; 238 break; 239 } 240 dpfc_ctl |= DPFC_CTL_FENCE_EN; 241 if (IS_GEN5(dev)) 242 dpfc_ctl |= obj->fence_reg; 243 244 I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y); 245 I915_WRITE(ILK_FBC_RT_BASE, i915_gem_obj_ggtt_offset(obj) | ILK_FBC_RT_VALID); 246 /* enable it... */ 247 I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN); 248 249 if (IS_GEN6(dev)) { 250 I915_WRITE(SNB_DPFC_CTL_SA, 251 SNB_CPU_FENCE_ENABLE | obj->fence_reg); 252 I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y); 253 sandybridge_blit_fbc_update(dev); 254 } 255 256 DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane)); 257 } 258 259 static void ironlake_disable_fbc(struct drm_device *dev) 260 { 261 struct drm_i915_private *dev_priv = dev->dev_private; 262 u32 dpfc_ctl; 263 264 /* Disable compression */ 265 dpfc_ctl = I915_READ(ILK_DPFC_CONTROL); 266 if (dpfc_ctl & DPFC_CTL_EN) { 267 dpfc_ctl &= ~DPFC_CTL_EN; 268 I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl); 269 270 DRM_DEBUG_KMS("disabled FBC\n"); 271 } 272 } 273 274 static bool ironlake_fbc_enabled(struct drm_device *dev) 275 { 276 struct drm_i915_private *dev_priv = dev->dev_private; 277 278 return I915_READ(ILK_DPFC_CONTROL) & DPFC_CTL_EN; 279 } 280 281 static void gen7_enable_fbc(struct drm_crtc *crtc) 282 { 283 struct drm_device *dev = crtc->dev; 284 struct drm_i915_private *dev_priv = dev->dev_private; 285 struct drm_framebuffer *fb = crtc->primary->fb; 286 struct drm_i915_gem_object *obj = intel_fb_obj(fb); 287 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 288 u32 dpfc_ctl; 289 290 dpfc_ctl = IVB_DPFC_CTL_PLANE(intel_crtc->plane); 291 if (drm_format_plane_cpp(fb->pixel_format, 0) == 2) 292 dev_priv->fbc.threshold++; 293 294 switch (dev_priv->fbc.threshold) { 295 case 4: 296 case 3: 297 dpfc_ctl |= DPFC_CTL_LIMIT_4X; 298 break; 299 case 2: 300 dpfc_ctl |= DPFC_CTL_LIMIT_2X; 301 break; 302 case 1: 303 dpfc_ctl |= DPFC_CTL_LIMIT_1X; 304 break; 305 } 306 307 dpfc_ctl |= IVB_DPFC_CTL_FENCE_EN; 308 309 if (dev_priv->fbc.false_color) 310 dpfc_ctl |= FBC_CTL_FALSE_COLOR; 311 312 I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN); 313 314 if (IS_IVYBRIDGE(dev)) { 315 /* WaFbcAsynchFlipDisableFbcQueue:ivb */ 316 I915_WRITE(ILK_DISPLAY_CHICKEN1, 317 I915_READ(ILK_DISPLAY_CHICKEN1) | 318 ILK_FBCQ_DIS); 319 } else { 320 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */ 321 I915_WRITE(CHICKEN_PIPESL_1(intel_crtc->pipe), 322 I915_READ(CHICKEN_PIPESL_1(intel_crtc->pipe)) | 323 HSW_FBCQ_DIS); 324 } 325 326 I915_WRITE(SNB_DPFC_CTL_SA, 327 SNB_CPU_FENCE_ENABLE | obj->fence_reg); 328 I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y); 329 330 sandybridge_blit_fbc_update(dev); 331 332 DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane)); 333 } 334 335 bool intel_fbc_enabled(struct drm_device *dev) 336 { 337 struct drm_i915_private *dev_priv = dev->dev_private; 338 339 if (!dev_priv->display.fbc_enabled) 340 return false; 341 342 return dev_priv->display.fbc_enabled(dev); 343 } 344 345 void gen8_fbc_sw_flush(struct drm_device *dev, u32 value) 346 { 347 struct drm_i915_private *dev_priv = dev->dev_private; 348 349 if (!IS_GEN8(dev)) 350 return; 351 352 I915_WRITE(MSG_FBC_REND_STATE, value); 353 } 354 355 static void intel_fbc_work_fn(struct work_struct *__work) 356 { 357 struct intel_fbc_work *work = 358 container_of(to_delayed_work(__work), 359 struct intel_fbc_work, work); 360 struct drm_device *dev = work->crtc->dev; 361 struct drm_i915_private *dev_priv = dev->dev_private; 362 363 mutex_lock(&dev->struct_mutex); 364 if (work == dev_priv->fbc.fbc_work) { 365 /* Double check that we haven't switched fb without cancelling 366 * the prior work. 367 */ 368 if (work->crtc->primary->fb == work->fb) { 369 dev_priv->display.enable_fbc(work->crtc); 370 371 dev_priv->fbc.plane = to_intel_crtc(work->crtc)->plane; 372 dev_priv->fbc.fb_id = work->crtc->primary->fb->base.id; 373 dev_priv->fbc.y = work->crtc->y; 374 } 375 376 dev_priv->fbc.fbc_work = NULL; 377 } 378 mutex_unlock(&dev->struct_mutex); 379 380 kfree(work); 381 } 382 383 static void intel_cancel_fbc_work(struct drm_i915_private *dev_priv) 384 { 385 if (dev_priv->fbc.fbc_work == NULL) 386 return; 387 388 DRM_DEBUG_KMS("cancelling pending FBC enable\n"); 389 390 /* Synchronisation is provided by struct_mutex and checking of 391 * dev_priv->fbc.fbc_work, so we can perform the cancellation 392 * entirely asynchronously. 393 */ 394 if (cancel_delayed_work(&dev_priv->fbc.fbc_work->work)) 395 /* tasklet was killed before being run, clean up */ 396 kfree(dev_priv->fbc.fbc_work); 397 398 /* Mark the work as no longer wanted so that if it does 399 * wake-up (because the work was already running and waiting 400 * for our mutex), it will discover that is no longer 401 * necessary to run. 402 */ 403 dev_priv->fbc.fbc_work = NULL; 404 } 405 406 static void intel_enable_fbc(struct drm_crtc *crtc) 407 { 408 struct intel_fbc_work *work; 409 struct drm_device *dev = crtc->dev; 410 struct drm_i915_private *dev_priv = dev->dev_private; 411 412 if (!dev_priv->display.enable_fbc) 413 return; 414 415 intel_cancel_fbc_work(dev_priv); 416 417 work = kzalloc(sizeof(*work), GFP_KERNEL); 418 if (work == NULL) { 419 DRM_ERROR("Failed to allocate FBC work structure\n"); 420 dev_priv->display.enable_fbc(crtc); 421 return; 422 } 423 424 work->crtc = crtc; 425 work->fb = crtc->primary->fb; 426 INIT_DELAYED_WORK(&work->work, intel_fbc_work_fn); 427 428 dev_priv->fbc.fbc_work = work; 429 430 /* Delay the actual enabling to let pageflipping cease and the 431 * display to settle before starting the compression. Note that 432 * this delay also serves a second purpose: it allows for a 433 * vblank to pass after disabling the FBC before we attempt 434 * to modify the control registers. 435 * 436 * A more complicated solution would involve tracking vblanks 437 * following the termination of the page-flipping sequence 438 * and indeed performing the enable as a co-routine and not 439 * waiting synchronously upon the vblank. 440 * 441 * WaFbcWaitForVBlankBeforeEnable:ilk,snb 442 */ 443 schedule_delayed_work(&work->work, msecs_to_jiffies(50)); 444 } 445 446 void intel_disable_fbc(struct drm_device *dev) 447 { 448 struct drm_i915_private *dev_priv = dev->dev_private; 449 450 intel_cancel_fbc_work(dev_priv); 451 452 if (!dev_priv->display.disable_fbc) 453 return; 454 455 dev_priv->display.disable_fbc(dev); 456 dev_priv->fbc.plane = -1; 457 } 458 459 static bool set_no_fbc_reason(struct drm_i915_private *dev_priv, 460 enum no_fbc_reason reason) 461 { 462 if (dev_priv->fbc.no_fbc_reason == reason) 463 return false; 464 465 dev_priv->fbc.no_fbc_reason = reason; 466 return true; 467 } 468 469 /** 470 * intel_update_fbc - enable/disable FBC as needed 471 * @dev: the drm_device 472 * 473 * Set up the framebuffer compression hardware at mode set time. We 474 * enable it if possible: 475 * - plane A only (on pre-965) 476 * - no pixel mulitply/line duplication 477 * - no alpha buffer discard 478 * - no dual wide 479 * - framebuffer <= max_hdisplay in width, max_vdisplay in height 480 * 481 * We can't assume that any compression will take place (worst case), 482 * so the compressed buffer has to be the same size as the uncompressed 483 * one. It also must reside (along with the line length buffer) in 484 * stolen memory. 485 * 486 * We need to enable/disable FBC on a global basis. 487 */ 488 void intel_update_fbc(struct drm_device *dev) 489 { 490 struct drm_i915_private *dev_priv = dev->dev_private; 491 struct drm_crtc *crtc = NULL, *tmp_crtc; 492 struct intel_crtc *intel_crtc; 493 struct drm_framebuffer *fb; 494 struct drm_i915_gem_object *obj; 495 const struct drm_display_mode *adjusted_mode; 496 unsigned int max_width, max_height; 497 498 if (!HAS_FBC(dev)) { 499 set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED); 500 return; 501 } 502 503 if (!i915.powersave) { 504 if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM)) 505 DRM_DEBUG_KMS("fbc disabled per module param\n"); 506 return; 507 } 508 509 /* 510 * If FBC is already on, we just have to verify that we can 511 * keep it that way... 512 * Need to disable if: 513 * - more than one pipe is active 514 * - changing FBC params (stride, fence, mode) 515 * - new fb is too large to fit in compressed buffer 516 * - going to an unsupported config (interlace, pixel multiply, etc.) 517 */ 518 for_each_crtc(dev, tmp_crtc) { 519 if (intel_crtc_active(tmp_crtc) && 520 to_intel_crtc(tmp_crtc)->primary_enabled) { 521 if (crtc) { 522 if (set_no_fbc_reason(dev_priv, FBC_MULTIPLE_PIPES)) 523 DRM_DEBUG_KMS("more than one pipe active, disabling compression\n"); 524 goto out_disable; 525 } 526 crtc = tmp_crtc; 527 } 528 } 529 530 if (!crtc || crtc->primary->fb == NULL) { 531 if (set_no_fbc_reason(dev_priv, FBC_NO_OUTPUT)) 532 DRM_DEBUG_KMS("no output, disabling\n"); 533 goto out_disable; 534 } 535 536 intel_crtc = to_intel_crtc(crtc); 537 fb = crtc->primary->fb; 538 obj = intel_fb_obj(fb); 539 adjusted_mode = &intel_crtc->config.adjusted_mode; 540 541 if (i915.enable_fbc < 0) { 542 if (set_no_fbc_reason(dev_priv, FBC_CHIP_DEFAULT)) 543 DRM_DEBUG_KMS("disabled per chip default\n"); 544 goto out_disable; 545 } 546 if (!i915.enable_fbc) { 547 if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM)) 548 DRM_DEBUG_KMS("fbc disabled per module param\n"); 549 goto out_disable; 550 } 551 if ((adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) || 552 (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN)) { 553 if (set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED_MODE)) 554 DRM_DEBUG_KMS("mode incompatible with compression, " 555 "disabling\n"); 556 goto out_disable; 557 } 558 559 if (INTEL_INFO(dev)->gen >= 8 || IS_HASWELL(dev)) { 560 max_width = 4096; 561 max_height = 4096; 562 } else if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) { 563 max_width = 4096; 564 max_height = 2048; 565 } else { 566 max_width = 2048; 567 max_height = 1536; 568 } 569 if (intel_crtc->config.pipe_src_w > max_width || 570 intel_crtc->config.pipe_src_h > max_height) { 571 if (set_no_fbc_reason(dev_priv, FBC_MODE_TOO_LARGE)) 572 DRM_DEBUG_KMS("mode too large for compression, disabling\n"); 573 goto out_disable; 574 } 575 if ((INTEL_INFO(dev)->gen < 4 || HAS_DDI(dev)) && 576 intel_crtc->plane != PLANE_A) { 577 if (set_no_fbc_reason(dev_priv, FBC_BAD_PLANE)) 578 DRM_DEBUG_KMS("plane not A, disabling compression\n"); 579 goto out_disable; 580 } 581 582 /* The use of a CPU fence is mandatory in order to detect writes 583 * by the CPU to the scanout and trigger updates to the FBC. 584 */ 585 if (obj->tiling_mode != I915_TILING_X || 586 obj->fence_reg == I915_FENCE_REG_NONE) { 587 if (set_no_fbc_reason(dev_priv, FBC_NOT_TILED)) 588 DRM_DEBUG_KMS("framebuffer not tiled or fenced, disabling compression\n"); 589 goto out_disable; 590 } 591 if (INTEL_INFO(dev)->gen <= 4 && !IS_G4X(dev) && 592 to_intel_plane(crtc->primary)->rotation != BIT(DRM_ROTATE_0)) { 593 if (set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED_MODE)) 594 DRM_DEBUG_KMS("Rotation unsupported, disabling\n"); 595 goto out_disable; 596 } 597 598 /* If the kernel debugger is active, always disable compression */ 599 #ifdef DDB 600 if (in_dbg_master()) 601 goto out_disable; 602 #endif 603 604 if (i915_gem_stolen_setup_compression(dev, obj->base.size, 605 drm_format_plane_cpp(fb->pixel_format, 0))) { 606 if (set_no_fbc_reason(dev_priv, FBC_STOLEN_TOO_SMALL)) 607 DRM_DEBUG_KMS("framebuffer too large, disabling compression\n"); 608 goto out_disable; 609 } 610 611 /* If the scanout has not changed, don't modify the FBC settings. 612 * Note that we make the fundamental assumption that the fb->obj 613 * cannot be unpinned (and have its GTT offset and fence revoked) 614 * without first being decoupled from the scanout and FBC disabled. 615 */ 616 if (dev_priv->fbc.plane == intel_crtc->plane && 617 dev_priv->fbc.fb_id == fb->base.id && 618 dev_priv->fbc.y == crtc->y) 619 return; 620 621 if (intel_fbc_enabled(dev)) { 622 /* We update FBC along two paths, after changing fb/crtc 623 * configuration (modeswitching) and after page-flipping 624 * finishes. For the latter, we know that not only did 625 * we disable the FBC at the start of the page-flip 626 * sequence, but also more than one vblank has passed. 627 * 628 * For the former case of modeswitching, it is possible 629 * to switch between two FBC valid configurations 630 * instantaneously so we do need to disable the FBC 631 * before we can modify its control registers. We also 632 * have to wait for the next vblank for that to take 633 * effect. However, since we delay enabling FBC we can 634 * assume that a vblank has passed since disabling and 635 * that we can safely alter the registers in the deferred 636 * callback. 637 * 638 * In the scenario that we go from a valid to invalid 639 * and then back to valid FBC configuration we have 640 * no strict enforcement that a vblank occurred since 641 * disabling the FBC. However, along all current pipe 642 * disabling paths we do need to wait for a vblank at 643 * some point. And we wait before enabling FBC anyway. 644 */ 645 DRM_DEBUG_KMS("disabling active FBC for update\n"); 646 intel_disable_fbc(dev); 647 } 648 649 intel_enable_fbc(crtc); 650 dev_priv->fbc.no_fbc_reason = FBC_OK; 651 return; 652 653 out_disable: 654 /* Multiple disables should be harmless */ 655 if (intel_fbc_enabled(dev)) { 656 DRM_DEBUG_KMS("unsupported config, disabling FBC\n"); 657 intel_disable_fbc(dev); 658 } 659 i915_gem_stolen_cleanup_compression(dev); 660 } 661 662 static void i915_pineview_get_mem_freq(struct drm_device *dev) 663 { 664 struct drm_i915_private *dev_priv = dev->dev_private; 665 u32 tmp; 666 667 tmp = I915_READ(CLKCFG); 668 669 switch (tmp & CLKCFG_FSB_MASK) { 670 case CLKCFG_FSB_533: 671 dev_priv->fsb_freq = 533; /* 133*4 */ 672 break; 673 case CLKCFG_FSB_800: 674 dev_priv->fsb_freq = 800; /* 200*4 */ 675 break; 676 case CLKCFG_FSB_667: 677 dev_priv->fsb_freq = 667; /* 167*4 */ 678 break; 679 case CLKCFG_FSB_400: 680 dev_priv->fsb_freq = 400; /* 100*4 */ 681 break; 682 } 683 684 switch (tmp & CLKCFG_MEM_MASK) { 685 case CLKCFG_MEM_533: 686 dev_priv->mem_freq = 533; 687 break; 688 case CLKCFG_MEM_667: 689 dev_priv->mem_freq = 667; 690 break; 691 case CLKCFG_MEM_800: 692 dev_priv->mem_freq = 800; 693 break; 694 } 695 696 /* detect pineview DDR3 setting */ 697 tmp = I915_READ(CSHRDDR3CTL); 698 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0; 699 } 700 701 static void i915_ironlake_get_mem_freq(struct drm_device *dev) 702 { 703 struct drm_i915_private *dev_priv = dev->dev_private; 704 u16 ddrpll, csipll; 705 706 ddrpll = I915_READ16(DDRMPLL1); 707 csipll = I915_READ16(CSIPLL0); 708 709 switch (ddrpll & 0xff) { 710 case 0xc: 711 dev_priv->mem_freq = 800; 712 break; 713 case 0x10: 714 dev_priv->mem_freq = 1066; 715 break; 716 case 0x14: 717 dev_priv->mem_freq = 1333; 718 break; 719 case 0x18: 720 dev_priv->mem_freq = 1600; 721 break; 722 default: 723 DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n", 724 ddrpll & 0xff); 725 dev_priv->mem_freq = 0; 726 break; 727 } 728 729 dev_priv->ips.r_t = dev_priv->mem_freq; 730 731 switch (csipll & 0x3ff) { 732 case 0x00c: 733 dev_priv->fsb_freq = 3200; 734 break; 735 case 0x00e: 736 dev_priv->fsb_freq = 3733; 737 break; 738 case 0x010: 739 dev_priv->fsb_freq = 4266; 740 break; 741 case 0x012: 742 dev_priv->fsb_freq = 4800; 743 break; 744 case 0x014: 745 dev_priv->fsb_freq = 5333; 746 break; 747 case 0x016: 748 dev_priv->fsb_freq = 5866; 749 break; 750 case 0x018: 751 dev_priv->fsb_freq = 6400; 752 break; 753 default: 754 DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n", 755 csipll & 0x3ff); 756 dev_priv->fsb_freq = 0; 757 break; 758 } 759 760 if (dev_priv->fsb_freq == 3200) { 761 dev_priv->ips.c_m = 0; 762 } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) { 763 dev_priv->ips.c_m = 1; 764 } else { 765 dev_priv->ips.c_m = 2; 766 } 767 } 768 769 static const struct cxsr_latency cxsr_latency_table[] = { 770 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */ 771 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */ 772 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */ 773 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */ 774 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */ 775 776 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */ 777 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */ 778 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */ 779 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */ 780 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */ 781 782 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */ 783 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */ 784 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */ 785 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */ 786 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */ 787 788 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */ 789 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */ 790 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */ 791 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */ 792 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */ 793 794 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */ 795 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */ 796 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */ 797 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */ 798 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */ 799 800 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */ 801 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */ 802 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */ 803 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */ 804 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */ 805 }; 806 807 static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop, 808 int is_ddr3, 809 int fsb, 810 int mem) 811 { 812 const struct cxsr_latency *latency; 813 int i; 814 815 if (fsb == 0 || mem == 0) 816 return NULL; 817 818 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) { 819 latency = &cxsr_latency_table[i]; 820 if (is_desktop == latency->is_desktop && 821 is_ddr3 == latency->is_ddr3 && 822 fsb == latency->fsb_freq && mem == latency->mem_freq) 823 return latency; 824 } 825 826 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n"); 827 828 return NULL; 829 } 830 831 void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable) 832 { 833 struct drm_device *dev = dev_priv->dev; 834 u32 val; 835 836 if (IS_VALLEYVIEW(dev)) { 837 I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0); 838 } else if (IS_G4X(dev) || IS_CRESTLINE(dev)) { 839 I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0); 840 } else if (IS_PINEVIEW(dev)) { 841 val = I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN; 842 val |= enable ? PINEVIEW_SELF_REFRESH_EN : 0; 843 I915_WRITE(DSPFW3, val); 844 } else if (IS_I945G(dev) || IS_I945GM(dev)) { 845 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) : 846 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN); 847 I915_WRITE(FW_BLC_SELF, val); 848 } else if (IS_I915GM(dev)) { 849 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) : 850 _MASKED_BIT_DISABLE(INSTPM_SELF_EN); 851 I915_WRITE(INSTPM, val); 852 } else { 853 return; 854 } 855 856 DRM_DEBUG_KMS("memory self-refresh is %s\n", 857 enable ? "enabled" : "disabled"); 858 } 859 860 /* 861 * Latency for FIFO fetches is dependent on several factors: 862 * - memory configuration (speed, channels) 863 * - chipset 864 * - current MCH state 865 * It can be fairly high in some situations, so here we assume a fairly 866 * pessimal value. It's a tradeoff between extra memory fetches (if we 867 * set this value too high, the FIFO will fetch frequently to stay full) 868 * and power consumption (set it too low to save power and we might see 869 * FIFO underruns and display "flicker"). 870 * 871 * A value of 5us seems to be a good balance; safe for very low end 872 * platforms but not overly aggressive on lower latency configs. 873 */ 874 static const int pessimal_latency_ns = 5000; 875 876 static int i9xx_get_fifo_size(struct drm_device *dev, int plane) 877 { 878 struct drm_i915_private *dev_priv = dev->dev_private; 879 uint32_t dsparb = I915_READ(DSPARB); 880 int size; 881 882 size = dsparb & 0x7f; 883 if (plane) 884 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size; 885 886 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb, 887 plane ? "B" : "A", size); 888 889 return size; 890 } 891 892 static int i830_get_fifo_size(struct drm_device *dev, int plane) 893 { 894 struct drm_i915_private *dev_priv = dev->dev_private; 895 uint32_t dsparb = I915_READ(DSPARB); 896 int size; 897 898 size = dsparb & 0x1ff; 899 if (plane) 900 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size; 901 size >>= 1; /* Convert to cachelines */ 902 903 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb, 904 plane ? "B" : "A", size); 905 906 return size; 907 } 908 909 static int i845_get_fifo_size(struct drm_device *dev, int plane) 910 { 911 struct drm_i915_private *dev_priv = dev->dev_private; 912 uint32_t dsparb = I915_READ(DSPARB); 913 int size; 914 915 size = dsparb & 0x7f; 916 size >>= 2; /* Convert to cachelines */ 917 918 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb, 919 plane ? "B" : "A", 920 size); 921 922 return size; 923 } 924 925 /* Pineview has different values for various configs */ 926 static const struct intel_watermark_params pineview_display_wm = { 927 .fifo_size = PINEVIEW_DISPLAY_FIFO, 928 .max_wm = PINEVIEW_MAX_WM, 929 .default_wm = PINEVIEW_DFT_WM, 930 .guard_size = PINEVIEW_GUARD_WM, 931 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE, 932 }; 933 static const struct intel_watermark_params pineview_display_hplloff_wm = { 934 .fifo_size = PINEVIEW_DISPLAY_FIFO, 935 .max_wm = PINEVIEW_MAX_WM, 936 .default_wm = PINEVIEW_DFT_HPLLOFF_WM, 937 .guard_size = PINEVIEW_GUARD_WM, 938 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE, 939 }; 940 static const struct intel_watermark_params pineview_cursor_wm = { 941 .fifo_size = PINEVIEW_CURSOR_FIFO, 942 .max_wm = PINEVIEW_CURSOR_MAX_WM, 943 .default_wm = PINEVIEW_CURSOR_DFT_WM, 944 .guard_size = PINEVIEW_CURSOR_GUARD_WM, 945 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE, 946 }; 947 static const struct intel_watermark_params pineview_cursor_hplloff_wm = { 948 .fifo_size = PINEVIEW_CURSOR_FIFO, 949 .max_wm = PINEVIEW_CURSOR_MAX_WM, 950 .default_wm = PINEVIEW_CURSOR_DFT_WM, 951 .guard_size = PINEVIEW_CURSOR_GUARD_WM, 952 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE, 953 }; 954 static const struct intel_watermark_params g4x_wm_info = { 955 .fifo_size = G4X_FIFO_SIZE, 956 .max_wm = G4X_MAX_WM, 957 .default_wm = G4X_MAX_WM, 958 .guard_size = 2, 959 .cacheline_size = G4X_FIFO_LINE_SIZE, 960 }; 961 static const struct intel_watermark_params g4x_cursor_wm_info = { 962 .fifo_size = I965_CURSOR_FIFO, 963 .max_wm = I965_CURSOR_MAX_WM, 964 .default_wm = I965_CURSOR_DFT_WM, 965 .guard_size = 2, 966 .cacheline_size = G4X_FIFO_LINE_SIZE, 967 }; 968 static const struct intel_watermark_params valleyview_wm_info = { 969 .fifo_size = VALLEYVIEW_FIFO_SIZE, 970 .max_wm = VALLEYVIEW_MAX_WM, 971 .default_wm = VALLEYVIEW_MAX_WM, 972 .guard_size = 2, 973 .cacheline_size = G4X_FIFO_LINE_SIZE, 974 }; 975 static const struct intel_watermark_params valleyview_cursor_wm_info = { 976 .fifo_size = I965_CURSOR_FIFO, 977 .max_wm = VALLEYVIEW_CURSOR_MAX_WM, 978 .default_wm = I965_CURSOR_DFT_WM, 979 .guard_size = 2, 980 .cacheline_size = G4X_FIFO_LINE_SIZE, 981 }; 982 static const struct intel_watermark_params i965_cursor_wm_info = { 983 .fifo_size = I965_CURSOR_FIFO, 984 .max_wm = I965_CURSOR_MAX_WM, 985 .default_wm = I965_CURSOR_DFT_WM, 986 .guard_size = 2, 987 .cacheline_size = I915_FIFO_LINE_SIZE, 988 }; 989 static const struct intel_watermark_params i945_wm_info = { 990 .fifo_size = I945_FIFO_SIZE, 991 .max_wm = I915_MAX_WM, 992 .default_wm = 1, 993 .guard_size = 2, 994 .cacheline_size = I915_FIFO_LINE_SIZE, 995 }; 996 static const struct intel_watermark_params i915_wm_info = { 997 .fifo_size = I915_FIFO_SIZE, 998 .max_wm = I915_MAX_WM, 999 .default_wm = 1, 1000 .guard_size = 2, 1001 .cacheline_size = I915_FIFO_LINE_SIZE, 1002 }; 1003 static const struct intel_watermark_params i830_a_wm_info = { 1004 .fifo_size = I855GM_FIFO_SIZE, 1005 .max_wm = I915_MAX_WM, 1006 .default_wm = 1, 1007 .guard_size = 2, 1008 .cacheline_size = I830_FIFO_LINE_SIZE, 1009 }; 1010 static const struct intel_watermark_params i830_bc_wm_info = { 1011 .fifo_size = I855GM_FIFO_SIZE, 1012 .max_wm = I915_MAX_WM/2, 1013 .default_wm = 1, 1014 .guard_size = 2, 1015 .cacheline_size = I830_FIFO_LINE_SIZE, 1016 }; 1017 static const struct intel_watermark_params i845_wm_info = { 1018 .fifo_size = I830_FIFO_SIZE, 1019 .max_wm = I915_MAX_WM, 1020 .default_wm = 1, 1021 .guard_size = 2, 1022 .cacheline_size = I830_FIFO_LINE_SIZE, 1023 }; 1024 1025 /** 1026 * intel_calculate_wm - calculate watermark level 1027 * @clock_in_khz: pixel clock 1028 * @wm: chip FIFO params 1029 * @pixel_size: display pixel size 1030 * @latency_ns: memory latency for the platform 1031 * 1032 * Calculate the watermark level (the level at which the display plane will 1033 * start fetching from memory again). Each chip has a different display 1034 * FIFO size and allocation, so the caller needs to figure that out and pass 1035 * in the correct intel_watermark_params structure. 1036 * 1037 * As the pixel clock runs, the FIFO will be drained at a rate that depends 1038 * on the pixel size. When it reaches the watermark level, it'll start 1039 * fetching FIFO line sized based chunks from memory until the FIFO fills 1040 * past the watermark point. If the FIFO drains completely, a FIFO underrun 1041 * will occur, and a display engine hang could result. 1042 */ 1043 static unsigned long intel_calculate_wm(unsigned long clock_in_khz, 1044 const struct intel_watermark_params *wm, 1045 int fifo_size, 1046 int pixel_size, 1047 unsigned long latency_ns) 1048 { 1049 long entries_required, wm_size; 1050 1051 /* 1052 * Note: we need to make sure we don't overflow for various clock & 1053 * latency values. 1054 * clocks go from a few thousand to several hundred thousand. 1055 * latency is usually a few thousand 1056 */ 1057 entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) / 1058 1000; 1059 entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size); 1060 1061 DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required); 1062 1063 wm_size = fifo_size - (entries_required + wm->guard_size); 1064 1065 DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size); 1066 1067 /* Don't promote wm_size to unsigned... */ 1068 if (wm_size > (long)wm->max_wm) 1069 wm_size = wm->max_wm; 1070 if (wm_size <= 0) 1071 wm_size = wm->default_wm; 1072 1073 /* 1074 * Bspec seems to indicate that the value shouldn't be lower than 1075 * 'burst size + 1'. Certainly 830 is quite unhappy with low values. 1076 * Lets go for 8 which is the burst size since certain platforms 1077 * already use a hardcoded 8 (which is what the spec says should be 1078 * done). 1079 */ 1080 if (wm_size <= 8) 1081 wm_size = 8; 1082 1083 return wm_size; 1084 } 1085 1086 static struct drm_crtc *single_enabled_crtc(struct drm_device *dev) 1087 { 1088 struct drm_crtc *crtc, *enabled = NULL; 1089 1090 for_each_crtc(dev, crtc) { 1091 if (intel_crtc_active(crtc)) { 1092 if (enabled) 1093 return NULL; 1094 enabled = crtc; 1095 } 1096 } 1097 1098 return enabled; 1099 } 1100 1101 static void pineview_update_wm(struct drm_crtc *unused_crtc) 1102 { 1103 struct drm_device *dev = unused_crtc->dev; 1104 struct drm_i915_private *dev_priv = dev->dev_private; 1105 struct drm_crtc *crtc; 1106 const struct cxsr_latency *latency; 1107 u32 reg; 1108 unsigned long wm; 1109 1110 latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3, 1111 dev_priv->fsb_freq, dev_priv->mem_freq); 1112 if (!latency) { 1113 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n"); 1114 intel_set_memory_cxsr(dev_priv, false); 1115 return; 1116 } 1117 1118 crtc = single_enabled_crtc(dev); 1119 if (crtc) { 1120 const struct drm_display_mode *adjusted_mode; 1121 int pixel_size = crtc->primary->fb->bits_per_pixel / 8; 1122 int clock; 1123 1124 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode; 1125 clock = adjusted_mode->crtc_clock; 1126 1127 /* Display SR */ 1128 wm = intel_calculate_wm(clock, &pineview_display_wm, 1129 pineview_display_wm.fifo_size, 1130 pixel_size, latency->display_sr); 1131 reg = I915_READ(DSPFW1); 1132 reg &= ~DSPFW_SR_MASK; 1133 reg |= wm << DSPFW_SR_SHIFT; 1134 I915_WRITE(DSPFW1, reg); 1135 DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg); 1136 1137 /* cursor SR */ 1138 wm = intel_calculate_wm(clock, &pineview_cursor_wm, 1139 pineview_display_wm.fifo_size, 1140 pixel_size, latency->cursor_sr); 1141 reg = I915_READ(DSPFW3); 1142 reg &= ~DSPFW_CURSOR_SR_MASK; 1143 reg |= (wm & 0x3f) << DSPFW_CURSOR_SR_SHIFT; 1144 I915_WRITE(DSPFW3, reg); 1145 1146 /* Display HPLL off SR */ 1147 wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm, 1148 pineview_display_hplloff_wm.fifo_size, 1149 pixel_size, latency->display_hpll_disable); 1150 reg = I915_READ(DSPFW3); 1151 reg &= ~DSPFW_HPLL_SR_MASK; 1152 reg |= wm & DSPFW_HPLL_SR_MASK; 1153 I915_WRITE(DSPFW3, reg); 1154 1155 /* cursor HPLL off SR */ 1156 wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm, 1157 pineview_display_hplloff_wm.fifo_size, 1158 pixel_size, latency->cursor_hpll_disable); 1159 reg = I915_READ(DSPFW3); 1160 reg &= ~DSPFW_HPLL_CURSOR_MASK; 1161 reg |= (wm & 0x3f) << DSPFW_HPLL_CURSOR_SHIFT; 1162 I915_WRITE(DSPFW3, reg); 1163 DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg); 1164 1165 intel_set_memory_cxsr(dev_priv, true); 1166 } else { 1167 intel_set_memory_cxsr(dev_priv, false); 1168 } 1169 } 1170 1171 static bool g4x_compute_wm0(struct drm_device *dev, 1172 int plane, 1173 const struct intel_watermark_params *display, 1174 int display_latency_ns, 1175 const struct intel_watermark_params *cursor, 1176 int cursor_latency_ns, 1177 int *plane_wm, 1178 int *cursor_wm) 1179 { 1180 struct drm_crtc *crtc; 1181 const struct drm_display_mode *adjusted_mode; 1182 int htotal, hdisplay, clock, pixel_size; 1183 int line_time_us, line_count; 1184 int entries, tlb_miss; 1185 1186 crtc = intel_get_crtc_for_plane(dev, plane); 1187 if (!intel_crtc_active(crtc)) { 1188 *cursor_wm = cursor->guard_size; 1189 *plane_wm = display->guard_size; 1190 return false; 1191 } 1192 1193 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode; 1194 clock = adjusted_mode->crtc_clock; 1195 htotal = adjusted_mode->crtc_htotal; 1196 hdisplay = to_intel_crtc(crtc)->config.pipe_src_w; 1197 pixel_size = crtc->primary->fb->bits_per_pixel / 8; 1198 1199 /* Use the small buffer method to calculate plane watermark */ 1200 entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000; 1201 tlb_miss = display->fifo_size*display->cacheline_size - hdisplay * 8; 1202 if (tlb_miss > 0) 1203 entries += tlb_miss; 1204 entries = DIV_ROUND_UP(entries, display->cacheline_size); 1205 *plane_wm = entries + display->guard_size; 1206 if (*plane_wm > (int)display->max_wm) 1207 *plane_wm = display->max_wm; 1208 1209 /* Use the large buffer method to calculate cursor watermark */ 1210 line_time_us = max(htotal * 1000 / clock, 1); 1211 line_count = (cursor_latency_ns / line_time_us + 1000) / 1000; 1212 entries = line_count * to_intel_crtc(crtc)->cursor_width * pixel_size; 1213 tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8; 1214 if (tlb_miss > 0) 1215 entries += tlb_miss; 1216 entries = DIV_ROUND_UP(entries, cursor->cacheline_size); 1217 *cursor_wm = entries + cursor->guard_size; 1218 if (*cursor_wm > (int)cursor->max_wm) 1219 *cursor_wm = (int)cursor->max_wm; 1220 1221 return true; 1222 } 1223 1224 /* 1225 * Check the wm result. 1226 * 1227 * If any calculated watermark values is larger than the maximum value that 1228 * can be programmed into the associated watermark register, that watermark 1229 * must be disabled. 1230 */ 1231 static bool g4x_check_srwm(struct drm_device *dev, 1232 int display_wm, int cursor_wm, 1233 const struct intel_watermark_params *display, 1234 const struct intel_watermark_params *cursor) 1235 { 1236 DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n", 1237 display_wm, cursor_wm); 1238 1239 if (display_wm > display->max_wm) { 1240 DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n", 1241 display_wm, display->max_wm); 1242 return false; 1243 } 1244 1245 if (cursor_wm > cursor->max_wm) { 1246 DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n", 1247 cursor_wm, cursor->max_wm); 1248 return false; 1249 } 1250 1251 if (!(display_wm || cursor_wm)) { 1252 DRM_DEBUG_KMS("SR latency is 0, disabling\n"); 1253 return false; 1254 } 1255 1256 return true; 1257 } 1258 1259 static bool g4x_compute_srwm(struct drm_device *dev, 1260 int plane, 1261 int latency_ns, 1262 const struct intel_watermark_params *display, 1263 const struct intel_watermark_params *cursor, 1264 int *display_wm, int *cursor_wm) 1265 { 1266 struct drm_crtc *crtc; 1267 const struct drm_display_mode *adjusted_mode; 1268 int hdisplay, htotal, pixel_size, clock; 1269 unsigned long line_time_us; 1270 int line_count, line_size; 1271 int small, large; 1272 int entries; 1273 1274 if (!latency_ns) { 1275 *display_wm = *cursor_wm = 0; 1276 return false; 1277 } 1278 1279 crtc = intel_get_crtc_for_plane(dev, plane); 1280 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode; 1281 clock = adjusted_mode->crtc_clock; 1282 htotal = adjusted_mode->crtc_htotal; 1283 hdisplay = to_intel_crtc(crtc)->config.pipe_src_w; 1284 pixel_size = crtc->primary->fb->bits_per_pixel / 8; 1285 1286 line_time_us = max(htotal * 1000 / clock, 1); 1287 line_count = (latency_ns / line_time_us + 1000) / 1000; 1288 line_size = hdisplay * pixel_size; 1289 1290 /* Use the minimum of the small and large buffer method for primary */ 1291 small = ((clock * pixel_size / 1000) * latency_ns) / 1000; 1292 large = line_count * line_size; 1293 1294 entries = DIV_ROUND_UP(min(small, large), display->cacheline_size); 1295 *display_wm = entries + display->guard_size; 1296 1297 /* calculate the self-refresh watermark for display cursor */ 1298 entries = line_count * pixel_size * to_intel_crtc(crtc)->cursor_width; 1299 entries = DIV_ROUND_UP(entries, cursor->cacheline_size); 1300 *cursor_wm = entries + cursor->guard_size; 1301 1302 return g4x_check_srwm(dev, 1303 *display_wm, *cursor_wm, 1304 display, cursor); 1305 } 1306 1307 static bool vlv_compute_drain_latency(struct drm_crtc *crtc, 1308 int pixel_size, 1309 int *prec_mult, 1310 int *drain_latency) 1311 { 1312 int entries; 1313 int clock = to_intel_crtc(crtc)->config.adjusted_mode.crtc_clock; 1314 1315 if (WARN(clock == 0, "Pixel clock is zero!\n")) 1316 return false; 1317 1318 if (WARN(pixel_size == 0, "Pixel size is zero!\n")) 1319 return false; 1320 1321 entries = DIV_ROUND_UP(clock, 1000) * pixel_size; 1322 *prec_mult = (entries > 128) ? DRAIN_LATENCY_PRECISION_64 : 1323 DRAIN_LATENCY_PRECISION_32; 1324 *drain_latency = (64 * (*prec_mult) * 4) / entries; 1325 1326 if (*drain_latency > DRAIN_LATENCY_MASK) 1327 *drain_latency = DRAIN_LATENCY_MASK; 1328 1329 return true; 1330 } 1331 1332 /* 1333 * Update drain latency registers of memory arbiter 1334 * 1335 * Valleyview SoC has a new memory arbiter and needs drain latency registers 1336 * to be programmed. Each plane has a drain latency multiplier and a drain 1337 * latency value. 1338 */ 1339 1340 static void vlv_update_drain_latency(struct drm_crtc *crtc) 1341 { 1342 struct drm_i915_private *dev_priv = crtc->dev->dev_private; 1343 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 1344 int pixel_size; 1345 int drain_latency; 1346 enum i915_pipe pipe = intel_crtc->pipe; 1347 int plane_prec, prec_mult, plane_dl; 1348 1349 plane_dl = I915_READ(VLV_DDL(pipe)) & ~(DDL_PLANE_PRECISION_64 | 1350 DRAIN_LATENCY_MASK | DDL_CURSOR_PRECISION_64 | 1351 (DRAIN_LATENCY_MASK << DDL_CURSOR_SHIFT)); 1352 1353 if (!intel_crtc_active(crtc)) { 1354 I915_WRITE(VLV_DDL(pipe), plane_dl); 1355 return; 1356 } 1357 1358 /* Primary plane Drain Latency */ 1359 pixel_size = crtc->primary->fb->bits_per_pixel / 8; /* BPP */ 1360 if (vlv_compute_drain_latency(crtc, pixel_size, &prec_mult, &drain_latency)) { 1361 plane_prec = (prec_mult == DRAIN_LATENCY_PRECISION_64) ? 1362 DDL_PLANE_PRECISION_64 : 1363 DDL_PLANE_PRECISION_32; 1364 plane_dl |= plane_prec | drain_latency; 1365 } 1366 1367 /* Cursor Drain Latency 1368 * BPP is always 4 for cursor 1369 */ 1370 pixel_size = 4; 1371 1372 /* Program cursor DL only if it is enabled */ 1373 if (intel_crtc->cursor_base && 1374 vlv_compute_drain_latency(crtc, pixel_size, &prec_mult, &drain_latency)) { 1375 plane_prec = (prec_mult == DRAIN_LATENCY_PRECISION_64) ? 1376 DDL_CURSOR_PRECISION_64 : 1377 DDL_CURSOR_PRECISION_32; 1378 plane_dl |= plane_prec | (drain_latency << DDL_CURSOR_SHIFT); 1379 } 1380 1381 I915_WRITE(VLV_DDL(pipe), plane_dl); 1382 } 1383 1384 #define single_plane_enabled(mask) is_power_of_2(mask) 1385 1386 static void valleyview_update_wm(struct drm_crtc *crtc) 1387 { 1388 struct drm_device *dev = crtc->dev; 1389 static const int sr_latency_ns = 12000; 1390 struct drm_i915_private *dev_priv = dev->dev_private; 1391 int planea_wm, planeb_wm, cursora_wm, cursorb_wm; 1392 int plane_sr, cursor_sr; 1393 int ignore_plane_sr, ignore_cursor_sr; 1394 unsigned int enabled = 0; 1395 bool cxsr_enabled; 1396 1397 vlv_update_drain_latency(crtc); 1398 1399 if (g4x_compute_wm0(dev, PIPE_A, 1400 &valleyview_wm_info, pessimal_latency_ns, 1401 &valleyview_cursor_wm_info, pessimal_latency_ns, 1402 &planea_wm, &cursora_wm)) 1403 enabled |= 1 << PIPE_A; 1404 1405 if (g4x_compute_wm0(dev, PIPE_B, 1406 &valleyview_wm_info, pessimal_latency_ns, 1407 &valleyview_cursor_wm_info, pessimal_latency_ns, 1408 &planeb_wm, &cursorb_wm)) 1409 enabled |= 1 << PIPE_B; 1410 1411 if (single_plane_enabled(enabled) && 1412 g4x_compute_srwm(dev, ffs(enabled) - 1, 1413 sr_latency_ns, 1414 &valleyview_wm_info, 1415 &valleyview_cursor_wm_info, 1416 &plane_sr, &ignore_cursor_sr) && 1417 g4x_compute_srwm(dev, ffs(enabled) - 1, 1418 2*sr_latency_ns, 1419 &valleyview_wm_info, 1420 &valleyview_cursor_wm_info, 1421 &ignore_plane_sr, &cursor_sr)) { 1422 cxsr_enabled = true; 1423 } else { 1424 cxsr_enabled = false; 1425 intel_set_memory_cxsr(dev_priv, false); 1426 plane_sr = cursor_sr = 0; 1427 } 1428 1429 DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, " 1430 "B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n", 1431 planea_wm, cursora_wm, 1432 planeb_wm, cursorb_wm, 1433 plane_sr, cursor_sr); 1434 1435 I915_WRITE(DSPFW1, 1436 (plane_sr << DSPFW_SR_SHIFT) | 1437 (cursorb_wm << DSPFW_CURSORB_SHIFT) | 1438 (planeb_wm << DSPFW_PLANEB_SHIFT) | 1439 (planea_wm << DSPFW_PLANEA_SHIFT)); 1440 I915_WRITE(DSPFW2, 1441 (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) | 1442 (cursora_wm << DSPFW_CURSORA_SHIFT)); 1443 I915_WRITE(DSPFW3, 1444 (I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) | 1445 (cursor_sr << DSPFW_CURSOR_SR_SHIFT)); 1446 1447 if (cxsr_enabled) 1448 intel_set_memory_cxsr(dev_priv, true); 1449 } 1450 1451 static void cherryview_update_wm(struct drm_crtc *crtc) 1452 { 1453 struct drm_device *dev = crtc->dev; 1454 static const int sr_latency_ns = 12000; 1455 struct drm_i915_private *dev_priv = dev->dev_private; 1456 int planea_wm, planeb_wm, planec_wm; 1457 int cursora_wm, cursorb_wm, cursorc_wm; 1458 int plane_sr, cursor_sr; 1459 int ignore_plane_sr, ignore_cursor_sr; 1460 unsigned int enabled = 0; 1461 bool cxsr_enabled; 1462 1463 vlv_update_drain_latency(crtc); 1464 1465 if (g4x_compute_wm0(dev, PIPE_A, 1466 &valleyview_wm_info, pessimal_latency_ns, 1467 &valleyview_cursor_wm_info, pessimal_latency_ns, 1468 &planea_wm, &cursora_wm)) 1469 enabled |= 1 << PIPE_A; 1470 1471 if (g4x_compute_wm0(dev, PIPE_B, 1472 &valleyview_wm_info, pessimal_latency_ns, 1473 &valleyview_cursor_wm_info, pessimal_latency_ns, 1474 &planeb_wm, &cursorb_wm)) 1475 enabled |= 1 << PIPE_B; 1476 1477 if (g4x_compute_wm0(dev, PIPE_C, 1478 &valleyview_wm_info, pessimal_latency_ns, 1479 &valleyview_cursor_wm_info, pessimal_latency_ns, 1480 &planec_wm, &cursorc_wm)) 1481 enabled |= 1 << PIPE_C; 1482 1483 if (single_plane_enabled(enabled) && 1484 g4x_compute_srwm(dev, ffs(enabled) - 1, 1485 sr_latency_ns, 1486 &valleyview_wm_info, 1487 &valleyview_cursor_wm_info, 1488 &plane_sr, &ignore_cursor_sr) && 1489 g4x_compute_srwm(dev, ffs(enabled) - 1, 1490 2*sr_latency_ns, 1491 &valleyview_wm_info, 1492 &valleyview_cursor_wm_info, 1493 &ignore_plane_sr, &cursor_sr)) { 1494 cxsr_enabled = true; 1495 } else { 1496 cxsr_enabled = false; 1497 intel_set_memory_cxsr(dev_priv, false); 1498 plane_sr = cursor_sr = 0; 1499 } 1500 1501 DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, " 1502 "B: plane=%d, cursor=%d, C: plane=%d, cursor=%d, " 1503 "SR: plane=%d, cursor=%d\n", 1504 planea_wm, cursora_wm, 1505 planeb_wm, cursorb_wm, 1506 planec_wm, cursorc_wm, 1507 plane_sr, cursor_sr); 1508 1509 I915_WRITE(DSPFW1, 1510 (plane_sr << DSPFW_SR_SHIFT) | 1511 (cursorb_wm << DSPFW_CURSORB_SHIFT) | 1512 (planeb_wm << DSPFW_PLANEB_SHIFT) | 1513 (planea_wm << DSPFW_PLANEA_SHIFT)); 1514 I915_WRITE(DSPFW2, 1515 (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) | 1516 (cursora_wm << DSPFW_CURSORA_SHIFT)); 1517 I915_WRITE(DSPFW3, 1518 (I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) | 1519 (cursor_sr << DSPFW_CURSOR_SR_SHIFT)); 1520 I915_WRITE(DSPFW9_CHV, 1521 (I915_READ(DSPFW9_CHV) & ~(DSPFW_PLANEC_MASK | 1522 DSPFW_CURSORC_MASK)) | 1523 (planec_wm << DSPFW_PLANEC_SHIFT) | 1524 (cursorc_wm << DSPFW_CURSORC_SHIFT)); 1525 1526 if (cxsr_enabled) 1527 intel_set_memory_cxsr(dev_priv, true); 1528 } 1529 1530 static void valleyview_update_sprite_wm(struct drm_plane *plane, 1531 struct drm_crtc *crtc, 1532 uint32_t sprite_width, 1533 uint32_t sprite_height, 1534 int pixel_size, 1535 bool enabled, bool scaled) 1536 { 1537 struct drm_device *dev = crtc->dev; 1538 struct drm_i915_private *dev_priv = dev->dev_private; 1539 int pipe = to_intel_plane(plane)->pipe; 1540 int sprite = to_intel_plane(plane)->plane; 1541 int drain_latency; 1542 int plane_prec; 1543 int sprite_dl; 1544 int prec_mult; 1545 1546 sprite_dl = I915_READ(VLV_DDL(pipe)) & ~(DDL_SPRITE_PRECISION_64(sprite) | 1547 (DRAIN_LATENCY_MASK << DDL_SPRITE_SHIFT(sprite))); 1548 1549 if (enabled && vlv_compute_drain_latency(crtc, pixel_size, &prec_mult, 1550 &drain_latency)) { 1551 plane_prec = (prec_mult == DRAIN_LATENCY_PRECISION_64) ? 1552 DDL_SPRITE_PRECISION_64(sprite) : 1553 DDL_SPRITE_PRECISION_32(sprite); 1554 sprite_dl |= plane_prec | 1555 (drain_latency << DDL_SPRITE_SHIFT(sprite)); 1556 } 1557 1558 I915_WRITE(VLV_DDL(pipe), sprite_dl); 1559 } 1560 1561 static void g4x_update_wm(struct drm_crtc *crtc) 1562 { 1563 struct drm_device *dev = crtc->dev; 1564 static const int sr_latency_ns = 12000; 1565 struct drm_i915_private *dev_priv = dev->dev_private; 1566 int planea_wm, planeb_wm, cursora_wm, cursorb_wm; 1567 int plane_sr, cursor_sr; 1568 unsigned int enabled = 0; 1569 bool cxsr_enabled; 1570 1571 if (g4x_compute_wm0(dev, PIPE_A, 1572 &g4x_wm_info, pessimal_latency_ns, 1573 &g4x_cursor_wm_info, pessimal_latency_ns, 1574 &planea_wm, &cursora_wm)) 1575 enabled |= 1 << PIPE_A; 1576 1577 if (g4x_compute_wm0(dev, PIPE_B, 1578 &g4x_wm_info, pessimal_latency_ns, 1579 &g4x_cursor_wm_info, pessimal_latency_ns, 1580 &planeb_wm, &cursorb_wm)) 1581 enabled |= 1 << PIPE_B; 1582 1583 if (single_plane_enabled(enabled) && 1584 g4x_compute_srwm(dev, ffs(enabled) - 1, 1585 sr_latency_ns, 1586 &g4x_wm_info, 1587 &g4x_cursor_wm_info, 1588 &plane_sr, &cursor_sr)) { 1589 cxsr_enabled = true; 1590 } else { 1591 cxsr_enabled = false; 1592 intel_set_memory_cxsr(dev_priv, false); 1593 plane_sr = cursor_sr = 0; 1594 } 1595 1596 DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, " 1597 "B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n", 1598 planea_wm, cursora_wm, 1599 planeb_wm, cursorb_wm, 1600 plane_sr, cursor_sr); 1601 1602 I915_WRITE(DSPFW1, 1603 (plane_sr << DSPFW_SR_SHIFT) | 1604 (cursorb_wm << DSPFW_CURSORB_SHIFT) | 1605 (planeb_wm << DSPFW_PLANEB_SHIFT) | 1606 (planea_wm << DSPFW_PLANEA_SHIFT)); 1607 I915_WRITE(DSPFW2, 1608 (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) | 1609 (cursora_wm << DSPFW_CURSORA_SHIFT)); 1610 /* HPLL off in SR has some issues on G4x... disable it */ 1611 I915_WRITE(DSPFW3, 1612 (I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN | DSPFW_CURSOR_SR_MASK)) | 1613 (cursor_sr << DSPFW_CURSOR_SR_SHIFT)); 1614 1615 if (cxsr_enabled) 1616 intel_set_memory_cxsr(dev_priv, true); 1617 } 1618 1619 static void i965_update_wm(struct drm_crtc *unused_crtc) 1620 { 1621 struct drm_device *dev = unused_crtc->dev; 1622 struct drm_i915_private *dev_priv = dev->dev_private; 1623 struct drm_crtc *crtc; 1624 int srwm = 1; 1625 int cursor_sr = 16; 1626 bool cxsr_enabled; 1627 1628 /* Calc sr entries for one plane configs */ 1629 crtc = single_enabled_crtc(dev); 1630 if (crtc) { 1631 /* self-refresh has much higher latency */ 1632 static const int sr_latency_ns = 12000; 1633 const struct drm_display_mode *adjusted_mode = 1634 &to_intel_crtc(crtc)->config.adjusted_mode; 1635 int clock = adjusted_mode->crtc_clock; 1636 int htotal = adjusted_mode->crtc_htotal; 1637 int hdisplay = to_intel_crtc(crtc)->config.pipe_src_w; 1638 int pixel_size = crtc->primary->fb->bits_per_pixel / 8; 1639 unsigned long line_time_us; 1640 int entries; 1641 1642 line_time_us = max(htotal * 1000 / clock, 1); 1643 1644 /* Use ns/us then divide to preserve precision */ 1645 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) * 1646 pixel_size * hdisplay; 1647 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE); 1648 srwm = I965_FIFO_SIZE - entries; 1649 if (srwm < 0) 1650 srwm = 1; 1651 srwm &= 0x1ff; 1652 DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n", 1653 entries, srwm); 1654 1655 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) * 1656 pixel_size * to_intel_crtc(crtc)->cursor_width; 1657 entries = DIV_ROUND_UP(entries, 1658 i965_cursor_wm_info.cacheline_size); 1659 cursor_sr = i965_cursor_wm_info.fifo_size - 1660 (entries + i965_cursor_wm_info.guard_size); 1661 1662 if (cursor_sr > i965_cursor_wm_info.max_wm) 1663 cursor_sr = i965_cursor_wm_info.max_wm; 1664 1665 DRM_DEBUG_KMS("self-refresh watermark: display plane %d " 1666 "cursor %d\n", srwm, cursor_sr); 1667 1668 cxsr_enabled = true; 1669 } else { 1670 cxsr_enabled = false; 1671 /* Turn off self refresh if both pipes are enabled */ 1672 intel_set_memory_cxsr(dev_priv, false); 1673 } 1674 1675 DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n", 1676 srwm); 1677 1678 /* 965 has limitations... */ 1679 I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) | 1680 (8 << DSPFW_CURSORB_SHIFT) | 1681 (8 << DSPFW_PLANEB_SHIFT) | 1682 (8 << DSPFW_PLANEA_SHIFT)); 1683 I915_WRITE(DSPFW2, (8 << DSPFW_CURSORA_SHIFT) | 1684 (8 << DSPFW_PLANEC_SHIFT_OLD)); 1685 /* update cursor SR watermark */ 1686 I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT)); 1687 1688 if (cxsr_enabled) 1689 intel_set_memory_cxsr(dev_priv, true); 1690 } 1691 1692 static void i9xx_update_wm(struct drm_crtc *unused_crtc) 1693 { 1694 struct drm_device *dev = unused_crtc->dev; 1695 struct drm_i915_private *dev_priv = dev->dev_private; 1696 const struct intel_watermark_params *wm_info; 1697 uint32_t fwater_lo; 1698 uint32_t fwater_hi; 1699 int cwm, srwm = 1; 1700 int fifo_size; 1701 int planea_wm, planeb_wm; 1702 struct drm_crtc *crtc, *enabled = NULL; 1703 1704 if (IS_I945GM(dev)) 1705 wm_info = &i945_wm_info; 1706 else if (!IS_GEN2(dev)) 1707 wm_info = &i915_wm_info; 1708 else 1709 wm_info = &i830_a_wm_info; 1710 1711 fifo_size = dev_priv->display.get_fifo_size(dev, 0); 1712 crtc = intel_get_crtc_for_plane(dev, 0); 1713 if (intel_crtc_active(crtc)) { 1714 const struct drm_display_mode *adjusted_mode; 1715 int cpp = crtc->primary->fb->bits_per_pixel / 8; 1716 if (IS_GEN2(dev)) 1717 cpp = 4; 1718 1719 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode; 1720 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock, 1721 wm_info, fifo_size, cpp, 1722 pessimal_latency_ns); 1723 enabled = crtc; 1724 } else { 1725 planea_wm = fifo_size - wm_info->guard_size; 1726 if (planea_wm > (long)wm_info->max_wm) 1727 planea_wm = wm_info->max_wm; 1728 } 1729 1730 if (IS_GEN2(dev)) 1731 wm_info = &i830_bc_wm_info; 1732 1733 fifo_size = dev_priv->display.get_fifo_size(dev, 1); 1734 crtc = intel_get_crtc_for_plane(dev, 1); 1735 if (intel_crtc_active(crtc)) { 1736 const struct drm_display_mode *adjusted_mode; 1737 int cpp = crtc->primary->fb->bits_per_pixel / 8; 1738 if (IS_GEN2(dev)) 1739 cpp = 4; 1740 1741 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode; 1742 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock, 1743 wm_info, fifo_size, cpp, 1744 pessimal_latency_ns); 1745 if (enabled == NULL) 1746 enabled = crtc; 1747 else 1748 enabled = NULL; 1749 } else { 1750 planeb_wm = fifo_size - wm_info->guard_size; 1751 if (planeb_wm > (long)wm_info->max_wm) 1752 planeb_wm = wm_info->max_wm; 1753 } 1754 1755 DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm); 1756 1757 if (IS_I915GM(dev) && enabled) { 1758 struct drm_i915_gem_object *obj; 1759 1760 obj = intel_fb_obj(enabled->primary->fb); 1761 1762 /* self-refresh seems busted with untiled */ 1763 if (obj->tiling_mode == I915_TILING_NONE) 1764 enabled = NULL; 1765 } 1766 1767 /* 1768 * Overlay gets an aggressive default since video jitter is bad. 1769 */ 1770 cwm = 2; 1771 1772 /* Play safe and disable self-refresh before adjusting watermarks. */ 1773 intel_set_memory_cxsr(dev_priv, false); 1774 1775 /* Calc sr entries for one plane configs */ 1776 if (HAS_FW_BLC(dev) && enabled) { 1777 /* self-refresh has much higher latency */ 1778 static const int sr_latency_ns = 6000; 1779 const struct drm_display_mode *adjusted_mode = 1780 &to_intel_crtc(enabled)->config.adjusted_mode; 1781 int clock = adjusted_mode->crtc_clock; 1782 int htotal = adjusted_mode->crtc_htotal; 1783 int hdisplay = to_intel_crtc(enabled)->config.pipe_src_w; 1784 int pixel_size = enabled->primary->fb->bits_per_pixel / 8; 1785 unsigned long line_time_us; 1786 int entries; 1787 1788 line_time_us = max(htotal * 1000 / clock, 1); 1789 1790 /* Use ns/us then divide to preserve precision */ 1791 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) * 1792 pixel_size * hdisplay; 1793 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size); 1794 DRM_DEBUG_KMS("self-refresh entries: %d\n", entries); 1795 srwm = wm_info->fifo_size - entries; 1796 if (srwm < 0) 1797 srwm = 1; 1798 1799 if (IS_I945G(dev) || IS_I945GM(dev)) 1800 I915_WRITE(FW_BLC_SELF, 1801 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff)); 1802 else if (IS_I915GM(dev)) 1803 I915_WRITE(FW_BLC_SELF, srwm & 0x3f); 1804 } 1805 1806 DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n", 1807 planea_wm, planeb_wm, cwm, srwm); 1808 1809 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f); 1810 fwater_hi = (cwm & 0x1f); 1811 1812 /* Set request length to 8 cachelines per fetch */ 1813 fwater_lo = fwater_lo | (1 << 24) | (1 << 8); 1814 fwater_hi = fwater_hi | (1 << 8); 1815 1816 I915_WRITE(FW_BLC, fwater_lo); 1817 I915_WRITE(FW_BLC2, fwater_hi); 1818 1819 if (enabled) 1820 intel_set_memory_cxsr(dev_priv, true); 1821 } 1822 1823 static void i845_update_wm(struct drm_crtc *unused_crtc) 1824 { 1825 struct drm_device *dev = unused_crtc->dev; 1826 struct drm_i915_private *dev_priv = dev->dev_private; 1827 struct drm_crtc *crtc; 1828 const struct drm_display_mode *adjusted_mode; 1829 uint32_t fwater_lo; 1830 int planea_wm; 1831 1832 crtc = single_enabled_crtc(dev); 1833 if (crtc == NULL) 1834 return; 1835 1836 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode; 1837 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock, 1838 &i845_wm_info, 1839 dev_priv->display.get_fifo_size(dev, 0), 1840 4, pessimal_latency_ns); 1841 fwater_lo = I915_READ(FW_BLC) & ~0xfff; 1842 fwater_lo |= (3<<8) | planea_wm; 1843 1844 DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm); 1845 1846 I915_WRITE(FW_BLC, fwater_lo); 1847 } 1848 1849 static uint32_t ilk_pipe_pixel_rate(struct drm_device *dev, 1850 struct drm_crtc *crtc) 1851 { 1852 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 1853 uint32_t pixel_rate; 1854 1855 pixel_rate = intel_crtc->config.adjusted_mode.crtc_clock; 1856 1857 /* We only use IF-ID interlacing. If we ever use PF-ID we'll need to 1858 * adjust the pixel_rate here. */ 1859 1860 if (intel_crtc->config.pch_pfit.enabled) { 1861 uint64_t pipe_w, pipe_h, pfit_w, pfit_h; 1862 uint32_t pfit_size = intel_crtc->config.pch_pfit.size; 1863 1864 pipe_w = intel_crtc->config.pipe_src_w; 1865 pipe_h = intel_crtc->config.pipe_src_h; 1866 pfit_w = (pfit_size >> 16) & 0xFFFF; 1867 pfit_h = pfit_size & 0xFFFF; 1868 if (pipe_w < pfit_w) 1869 pipe_w = pfit_w; 1870 if (pipe_h < pfit_h) 1871 pipe_h = pfit_h; 1872 1873 pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h, 1874 pfit_w * pfit_h); 1875 } 1876 1877 return pixel_rate; 1878 } 1879 1880 /* latency must be in 0.1us units. */ 1881 static uint32_t ilk_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel, 1882 uint32_t latency) 1883 { 1884 uint64_t ret; 1885 1886 if (WARN(latency == 0, "Latency value missing\n")) 1887 return UINT_MAX; 1888 1889 ret = (uint64_t) pixel_rate * bytes_per_pixel * latency; 1890 ret = DIV_ROUND_UP_ULL(ret, 64 * 10000) + 2; 1891 1892 return ret; 1893 } 1894 1895 /* latency must be in 0.1us units. */ 1896 static uint32_t ilk_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal, 1897 uint32_t horiz_pixels, uint8_t bytes_per_pixel, 1898 uint32_t latency) 1899 { 1900 uint32_t ret; 1901 1902 if (WARN(latency == 0, "Latency value missing\n")) 1903 return UINT_MAX; 1904 1905 ret = (latency * pixel_rate) / (pipe_htotal * 10000); 1906 ret = (ret + 1) * horiz_pixels * bytes_per_pixel; 1907 ret = DIV_ROUND_UP(ret, 64) + 2; 1908 return ret; 1909 } 1910 1911 static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels, 1912 uint8_t bytes_per_pixel) 1913 { 1914 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * bytes_per_pixel) + 2; 1915 } 1916 1917 struct ilk_pipe_wm_parameters { 1918 bool active; 1919 uint32_t pipe_htotal; 1920 uint32_t pixel_rate; 1921 struct intel_plane_wm_parameters pri; 1922 struct intel_plane_wm_parameters spr; 1923 struct intel_plane_wm_parameters cur; 1924 }; 1925 1926 struct ilk_wm_maximums { 1927 uint16_t pri; 1928 uint16_t spr; 1929 uint16_t cur; 1930 uint16_t fbc; 1931 }; 1932 1933 /* used in computing the new watermarks state */ 1934 struct intel_wm_config { 1935 unsigned int num_pipes_active; 1936 bool sprites_enabled; 1937 bool sprites_scaled; 1938 }; 1939 1940 /* 1941 * For both WM_PIPE and WM_LP. 1942 * mem_value must be in 0.1us units. 1943 */ 1944 static uint32_t ilk_compute_pri_wm(const struct ilk_pipe_wm_parameters *params, 1945 uint32_t mem_value, 1946 bool is_lp) 1947 { 1948 uint32_t method1, method2; 1949 1950 if (!params->active || !params->pri.enabled) 1951 return 0; 1952 1953 method1 = ilk_wm_method1(params->pixel_rate, 1954 params->pri.bytes_per_pixel, 1955 mem_value); 1956 1957 if (!is_lp) 1958 return method1; 1959 1960 method2 = ilk_wm_method2(params->pixel_rate, 1961 params->pipe_htotal, 1962 params->pri.horiz_pixels, 1963 params->pri.bytes_per_pixel, 1964 mem_value); 1965 1966 return min(method1, method2); 1967 } 1968 1969 /* 1970 * For both WM_PIPE and WM_LP. 1971 * mem_value must be in 0.1us units. 1972 */ 1973 static uint32_t ilk_compute_spr_wm(const struct ilk_pipe_wm_parameters *params, 1974 uint32_t mem_value) 1975 { 1976 uint32_t method1, method2; 1977 1978 if (!params->active || !params->spr.enabled) 1979 return 0; 1980 1981 method1 = ilk_wm_method1(params->pixel_rate, 1982 params->spr.bytes_per_pixel, 1983 mem_value); 1984 method2 = ilk_wm_method2(params->pixel_rate, 1985 params->pipe_htotal, 1986 params->spr.horiz_pixels, 1987 params->spr.bytes_per_pixel, 1988 mem_value); 1989 return min(method1, method2); 1990 } 1991 1992 /* 1993 * For both WM_PIPE and WM_LP. 1994 * mem_value must be in 0.1us units. 1995 */ 1996 static uint32_t ilk_compute_cur_wm(const struct ilk_pipe_wm_parameters *params, 1997 uint32_t mem_value) 1998 { 1999 if (!params->active || !params->cur.enabled) 2000 return 0; 2001 2002 return ilk_wm_method2(params->pixel_rate, 2003 params->pipe_htotal, 2004 params->cur.horiz_pixels, 2005 params->cur.bytes_per_pixel, 2006 mem_value); 2007 } 2008 2009 /* Only for WM_LP. */ 2010 static uint32_t ilk_compute_fbc_wm(const struct ilk_pipe_wm_parameters *params, 2011 uint32_t pri_val) 2012 { 2013 if (!params->active || !params->pri.enabled) 2014 return 0; 2015 2016 return ilk_wm_fbc(pri_val, 2017 params->pri.horiz_pixels, 2018 params->pri.bytes_per_pixel); 2019 } 2020 2021 static unsigned int ilk_display_fifo_size(const struct drm_device *dev) 2022 { 2023 if (INTEL_INFO(dev)->gen >= 8) 2024 return 3072; 2025 else if (INTEL_INFO(dev)->gen >= 7) 2026 return 768; 2027 else 2028 return 512; 2029 } 2030 2031 static unsigned int ilk_plane_wm_reg_max(const struct drm_device *dev, 2032 int level, bool is_sprite) 2033 { 2034 if (INTEL_INFO(dev)->gen >= 8) 2035 /* BDW primary/sprite plane watermarks */ 2036 return level == 0 ? 255 : 2047; 2037 else if (INTEL_INFO(dev)->gen >= 7) 2038 /* IVB/HSW primary/sprite plane watermarks */ 2039 return level == 0 ? 127 : 1023; 2040 else if (!is_sprite) 2041 /* ILK/SNB primary plane watermarks */ 2042 return level == 0 ? 127 : 511; 2043 else 2044 /* ILK/SNB sprite plane watermarks */ 2045 return level == 0 ? 63 : 255; 2046 } 2047 2048 static unsigned int ilk_cursor_wm_reg_max(const struct drm_device *dev, 2049 int level) 2050 { 2051 if (INTEL_INFO(dev)->gen >= 7) 2052 return level == 0 ? 63 : 255; 2053 else 2054 return level == 0 ? 31 : 63; 2055 } 2056 2057 static unsigned int ilk_fbc_wm_reg_max(const struct drm_device *dev) 2058 { 2059 if (INTEL_INFO(dev)->gen >= 8) 2060 return 31; 2061 else 2062 return 15; 2063 } 2064 2065 /* Calculate the maximum primary/sprite plane watermark */ 2066 static unsigned int ilk_plane_wm_max(const struct drm_device *dev, 2067 int level, 2068 const struct intel_wm_config *config, 2069 enum intel_ddb_partitioning ddb_partitioning, 2070 bool is_sprite) 2071 { 2072 unsigned int fifo_size = ilk_display_fifo_size(dev); 2073 2074 /* if sprites aren't enabled, sprites get nothing */ 2075 if (is_sprite && !config->sprites_enabled) 2076 return 0; 2077 2078 /* HSW allows LP1+ watermarks even with multiple pipes */ 2079 if (level == 0 || config->num_pipes_active > 1) { 2080 fifo_size /= INTEL_INFO(dev)->num_pipes; 2081 2082 /* 2083 * For some reason the non self refresh 2084 * FIFO size is only half of the self 2085 * refresh FIFO size on ILK/SNB. 2086 */ 2087 if (INTEL_INFO(dev)->gen <= 6) 2088 fifo_size /= 2; 2089 } 2090 2091 if (config->sprites_enabled) { 2092 /* level 0 is always calculated with 1:1 split */ 2093 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) { 2094 if (is_sprite) 2095 fifo_size *= 5; 2096 fifo_size /= 6; 2097 } else { 2098 fifo_size /= 2; 2099 } 2100 } 2101 2102 /* clamp to max that the registers can hold */ 2103 return min(fifo_size, ilk_plane_wm_reg_max(dev, level, is_sprite)); 2104 } 2105 2106 /* Calculate the maximum cursor plane watermark */ 2107 static unsigned int ilk_cursor_wm_max(const struct drm_device *dev, 2108 int level, 2109 const struct intel_wm_config *config) 2110 { 2111 /* HSW LP1+ watermarks w/ multiple pipes */ 2112 if (level > 0 && config->num_pipes_active > 1) 2113 return 64; 2114 2115 /* otherwise just report max that registers can hold */ 2116 return ilk_cursor_wm_reg_max(dev, level); 2117 } 2118 2119 static void ilk_compute_wm_maximums(const struct drm_device *dev, 2120 int level, 2121 const struct intel_wm_config *config, 2122 enum intel_ddb_partitioning ddb_partitioning, 2123 struct ilk_wm_maximums *max) 2124 { 2125 max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false); 2126 max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true); 2127 max->cur = ilk_cursor_wm_max(dev, level, config); 2128 max->fbc = ilk_fbc_wm_reg_max(dev); 2129 } 2130 2131 static void ilk_compute_wm_reg_maximums(struct drm_device *dev, 2132 int level, 2133 struct ilk_wm_maximums *max) 2134 { 2135 max->pri = ilk_plane_wm_reg_max(dev, level, false); 2136 max->spr = ilk_plane_wm_reg_max(dev, level, true); 2137 max->cur = ilk_cursor_wm_reg_max(dev, level); 2138 max->fbc = ilk_fbc_wm_reg_max(dev); 2139 } 2140 2141 static bool ilk_validate_wm_level(int level, 2142 const struct ilk_wm_maximums *max, 2143 struct intel_wm_level *result) 2144 { 2145 bool ret; 2146 2147 /* already determined to be invalid? */ 2148 if (!result->enable) 2149 return false; 2150 2151 result->enable = result->pri_val <= max->pri && 2152 result->spr_val <= max->spr && 2153 result->cur_val <= max->cur; 2154 2155 ret = result->enable; 2156 2157 /* 2158 * HACK until we can pre-compute everything, 2159 * and thus fail gracefully if LP0 watermarks 2160 * are exceeded... 2161 */ 2162 if (level == 0 && !result->enable) { 2163 if (result->pri_val > max->pri) 2164 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n", 2165 level, result->pri_val, max->pri); 2166 if (result->spr_val > max->spr) 2167 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n", 2168 level, result->spr_val, max->spr); 2169 if (result->cur_val > max->cur) 2170 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n", 2171 level, result->cur_val, max->cur); 2172 2173 result->pri_val = min_t(uint32_t, result->pri_val, max->pri); 2174 result->spr_val = min_t(uint32_t, result->spr_val, max->spr); 2175 result->cur_val = min_t(uint32_t, result->cur_val, max->cur); 2176 result->enable = true; 2177 } 2178 2179 return ret; 2180 } 2181 2182 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv, 2183 int level, 2184 const struct ilk_pipe_wm_parameters *p, 2185 struct intel_wm_level *result) 2186 { 2187 uint16_t pri_latency = dev_priv->wm.pri_latency[level]; 2188 uint16_t spr_latency = dev_priv->wm.spr_latency[level]; 2189 uint16_t cur_latency = dev_priv->wm.cur_latency[level]; 2190 2191 /* WM1+ latency values stored in 0.5us units */ 2192 if (level > 0) { 2193 pri_latency *= 5; 2194 spr_latency *= 5; 2195 cur_latency *= 5; 2196 } 2197 2198 result->pri_val = ilk_compute_pri_wm(p, pri_latency, level); 2199 result->spr_val = ilk_compute_spr_wm(p, spr_latency); 2200 result->cur_val = ilk_compute_cur_wm(p, cur_latency); 2201 result->fbc_val = ilk_compute_fbc_wm(p, result->pri_val); 2202 result->enable = true; 2203 } 2204 2205 static uint32_t 2206 hsw_compute_linetime_wm(struct drm_device *dev, struct drm_crtc *crtc) 2207 { 2208 struct drm_i915_private *dev_priv = dev->dev_private; 2209 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 2210 struct drm_display_mode *mode = &intel_crtc->config.adjusted_mode; 2211 u32 linetime, ips_linetime; 2212 2213 if (!intel_crtc_active(crtc)) 2214 return 0; 2215 2216 /* The WM are computed with base on how long it takes to fill a single 2217 * row at the given clock rate, multiplied by 8. 2218 * */ 2219 linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8, 2220 mode->crtc_clock); 2221 ips_linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8, 2222 intel_ddi_get_cdclk_freq(dev_priv)); 2223 2224 return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) | 2225 PIPE_WM_LINETIME_TIME(linetime); 2226 } 2227 2228 static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[5]) 2229 { 2230 struct drm_i915_private *dev_priv = dev->dev_private; 2231 2232 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) { 2233 uint64_t sskpd = I915_READ64(MCH_SSKPD); 2234 2235 wm[0] = (sskpd >> 56) & 0xFF; 2236 if (wm[0] == 0) 2237 wm[0] = sskpd & 0xF; 2238 wm[1] = (sskpd >> 4) & 0xFF; 2239 wm[2] = (sskpd >> 12) & 0xFF; 2240 wm[3] = (sskpd >> 20) & 0x1FF; 2241 wm[4] = (sskpd >> 32) & 0x1FF; 2242 } else if (INTEL_INFO(dev)->gen >= 6) { 2243 uint32_t sskpd = I915_READ(MCH_SSKPD); 2244 2245 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK; 2246 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK; 2247 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK; 2248 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK; 2249 } else if (INTEL_INFO(dev)->gen >= 5) { 2250 uint32_t mltr = I915_READ(MLTR_ILK); 2251 2252 /* ILK primary LP0 latency is 700 ns */ 2253 wm[0] = 7; 2254 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK; 2255 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK; 2256 } 2257 } 2258 2259 static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5]) 2260 { 2261 /* ILK sprite LP0 latency is 1300 ns */ 2262 if (INTEL_INFO(dev)->gen == 5) 2263 wm[0] = 13; 2264 } 2265 2266 static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5]) 2267 { 2268 /* ILK cursor LP0 latency is 1300 ns */ 2269 if (INTEL_INFO(dev)->gen == 5) 2270 wm[0] = 13; 2271 2272 /* WaDoubleCursorLP3Latency:ivb */ 2273 if (IS_IVYBRIDGE(dev)) 2274 wm[3] *= 2; 2275 } 2276 2277 int ilk_wm_max_level(const struct drm_device *dev) 2278 { 2279 /* how many WM levels are we expecting */ 2280 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) 2281 return 4; 2282 else if (INTEL_INFO(dev)->gen >= 6) 2283 return 3; 2284 else 2285 return 2; 2286 } 2287 2288 static void intel_print_wm_latency(struct drm_device *dev, 2289 const char *name, 2290 const uint16_t wm[5]) 2291 { 2292 int level, max_level = ilk_wm_max_level(dev); 2293 2294 for (level = 0; level <= max_level; level++) { 2295 unsigned int latency = wm[level]; 2296 2297 if (latency == 0) { 2298 DRM_ERROR("%s WM%d latency not provided\n", 2299 name, level); 2300 continue; 2301 } 2302 2303 /* WM1+ latency values in 0.5us units */ 2304 if (level > 0) 2305 latency *= 5; 2306 2307 DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n", 2308 name, level, wm[level], 2309 latency / 10, latency % 10); 2310 } 2311 } 2312 2313 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv, 2314 uint16_t wm[5], uint16_t min) 2315 { 2316 int level, max_level = ilk_wm_max_level(dev_priv->dev); 2317 2318 if (wm[0] >= min) 2319 return false; 2320 2321 wm[0] = max(wm[0], min); 2322 for (level = 1; level <= max_level; level++) 2323 wm[level] = max_t(uint16_t, wm[level], DIV_ROUND_UP(min, 5)); 2324 2325 return true; 2326 } 2327 2328 static void snb_wm_latency_quirk(struct drm_device *dev) 2329 { 2330 struct drm_i915_private *dev_priv = dev->dev_private; 2331 bool changed; 2332 2333 /* 2334 * The BIOS provided WM memory latency values are often 2335 * inadequate for high resolution displays. Adjust them. 2336 */ 2337 changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) | 2338 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) | 2339 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12); 2340 2341 if (!changed) 2342 return; 2343 2344 DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n"); 2345 intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency); 2346 intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency); 2347 intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency); 2348 } 2349 2350 static void ilk_setup_wm_latency(struct drm_device *dev) 2351 { 2352 struct drm_i915_private *dev_priv = dev->dev_private; 2353 2354 intel_read_wm_latency(dev, dev_priv->wm.pri_latency); 2355 2356 memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency, 2357 sizeof(dev_priv->wm.pri_latency)); 2358 memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency, 2359 sizeof(dev_priv->wm.pri_latency)); 2360 2361 intel_fixup_spr_wm_latency(dev, dev_priv->wm.spr_latency); 2362 intel_fixup_cur_wm_latency(dev, dev_priv->wm.cur_latency); 2363 2364 intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency); 2365 intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency); 2366 intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency); 2367 2368 if (IS_GEN6(dev)) 2369 snb_wm_latency_quirk(dev); 2370 } 2371 2372 static void ilk_compute_wm_parameters(struct drm_crtc *crtc, 2373 struct ilk_pipe_wm_parameters *p) 2374 { 2375 struct drm_device *dev = crtc->dev; 2376 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 2377 enum i915_pipe pipe = intel_crtc->pipe; 2378 struct drm_plane *plane; 2379 2380 if (!intel_crtc_active(crtc)) 2381 return; 2382 2383 p->active = true; 2384 p->pipe_htotal = intel_crtc->config.adjusted_mode.crtc_htotal; 2385 p->pixel_rate = ilk_pipe_pixel_rate(dev, crtc); 2386 p->pri.bytes_per_pixel = crtc->primary->fb->bits_per_pixel / 8; 2387 p->cur.bytes_per_pixel = 4; 2388 p->pri.horiz_pixels = intel_crtc->config.pipe_src_w; 2389 p->cur.horiz_pixels = intel_crtc->cursor_width; 2390 /* TODO: for now, assume primary and cursor planes are always enabled. */ 2391 p->pri.enabled = true; 2392 p->cur.enabled = true; 2393 2394 drm_for_each_legacy_plane(plane, &dev->mode_config.plane_list) { 2395 struct intel_plane *intel_plane = to_intel_plane(plane); 2396 2397 if (intel_plane->pipe == pipe) { 2398 p->spr = intel_plane->wm; 2399 break; 2400 } 2401 } 2402 } 2403 2404 static void ilk_compute_wm_config(struct drm_device *dev, 2405 struct intel_wm_config *config) 2406 { 2407 struct intel_crtc *intel_crtc; 2408 2409 /* Compute the currently _active_ config */ 2410 for_each_intel_crtc(dev, intel_crtc) { 2411 const struct intel_pipe_wm *wm = &intel_crtc->wm.active; 2412 2413 if (!wm->pipe_enabled) 2414 continue; 2415 2416 config->sprites_enabled |= wm->sprites_enabled; 2417 config->sprites_scaled |= wm->sprites_scaled; 2418 config->num_pipes_active++; 2419 } 2420 } 2421 2422 /* Compute new watermarks for the pipe */ 2423 static bool intel_compute_pipe_wm(struct drm_crtc *crtc, 2424 const struct ilk_pipe_wm_parameters *params, 2425 struct intel_pipe_wm *pipe_wm) 2426 { 2427 struct drm_device *dev = crtc->dev; 2428 const struct drm_i915_private *dev_priv = dev->dev_private; 2429 int level, max_level = ilk_wm_max_level(dev); 2430 /* LP0 watermark maximums depend on this pipe alone */ 2431 struct intel_wm_config config = { 2432 .num_pipes_active = 1, 2433 .sprites_enabled = params->spr.enabled, 2434 .sprites_scaled = params->spr.scaled, 2435 }; 2436 struct ilk_wm_maximums max; 2437 2438 pipe_wm->pipe_enabled = params->active; 2439 pipe_wm->sprites_enabled = params->spr.enabled; 2440 pipe_wm->sprites_scaled = params->spr.scaled; 2441 2442 /* ILK/SNB: LP2+ watermarks only w/o sprites */ 2443 if (INTEL_INFO(dev)->gen <= 6 && params->spr.enabled) 2444 max_level = 1; 2445 2446 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */ 2447 if (params->spr.scaled) 2448 max_level = 0; 2449 2450 ilk_compute_wm_level(dev_priv, 0, params, &pipe_wm->wm[0]); 2451 2452 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) 2453 pipe_wm->linetime = hsw_compute_linetime_wm(dev, crtc); 2454 2455 /* LP0 watermarks always use 1/2 DDB partitioning */ 2456 ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max); 2457 2458 /* At least LP0 must be valid */ 2459 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) 2460 return false; 2461 2462 ilk_compute_wm_reg_maximums(dev, 1, &max); 2463 2464 for (level = 1; level <= max_level; level++) { 2465 struct intel_wm_level wm = {}; 2466 2467 ilk_compute_wm_level(dev_priv, level, params, &wm); 2468 2469 /* 2470 * Disable any watermark level that exceeds the 2471 * register maximums since such watermarks are 2472 * always invalid. 2473 */ 2474 if (!ilk_validate_wm_level(level, &max, &wm)) 2475 break; 2476 2477 pipe_wm->wm[level] = wm; 2478 } 2479 2480 return true; 2481 } 2482 2483 /* 2484 * Merge the watermarks from all active pipes for a specific level. 2485 */ 2486 static void ilk_merge_wm_level(struct drm_device *dev, 2487 int level, 2488 struct intel_wm_level *ret_wm) 2489 { 2490 struct intel_crtc *intel_crtc; 2491 2492 ret_wm->enable = true; 2493 2494 for_each_intel_crtc(dev, intel_crtc) { 2495 const struct intel_pipe_wm *active = &intel_crtc->wm.active; 2496 const struct intel_wm_level *wm = &active->wm[level]; 2497 2498 if (!active->pipe_enabled) 2499 continue; 2500 2501 /* 2502 * The watermark values may have been used in the past, 2503 * so we must maintain them in the registers for some 2504 * time even if the level is now disabled. 2505 */ 2506 if (!wm->enable) 2507 ret_wm->enable = false; 2508 2509 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val); 2510 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val); 2511 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val); 2512 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val); 2513 } 2514 } 2515 2516 /* 2517 * Merge all low power watermarks for all active pipes. 2518 */ 2519 static void ilk_wm_merge(struct drm_device *dev, 2520 const struct intel_wm_config *config, 2521 const struct ilk_wm_maximums *max, 2522 struct intel_pipe_wm *merged) 2523 { 2524 int level, max_level = ilk_wm_max_level(dev); 2525 int last_enabled_level = max_level; 2526 2527 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */ 2528 if ((INTEL_INFO(dev)->gen <= 6 || IS_IVYBRIDGE(dev)) && 2529 config->num_pipes_active > 1) 2530 return; 2531 2532 /* ILK: FBC WM must be disabled always */ 2533 merged->fbc_wm_enabled = INTEL_INFO(dev)->gen >= 6; 2534 2535 /* merge each WM1+ level */ 2536 for (level = 1; level <= max_level; level++) { 2537 struct intel_wm_level *wm = &merged->wm[level]; 2538 2539 ilk_merge_wm_level(dev, level, wm); 2540 2541 if (level > last_enabled_level) 2542 wm->enable = false; 2543 else if (!ilk_validate_wm_level(level, max, wm)) 2544 /* make sure all following levels get disabled */ 2545 last_enabled_level = level - 1; 2546 2547 /* 2548 * The spec says it is preferred to disable 2549 * FBC WMs instead of disabling a WM level. 2550 */ 2551 if (wm->fbc_val > max->fbc) { 2552 if (wm->enable) 2553 merged->fbc_wm_enabled = false; 2554 wm->fbc_val = 0; 2555 } 2556 } 2557 2558 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */ 2559 /* 2560 * FIXME this is racy. FBC might get enabled later. 2561 * What we should check here is whether FBC can be 2562 * enabled sometime later. 2563 */ 2564 if (IS_GEN5(dev) && !merged->fbc_wm_enabled && intel_fbc_enabled(dev)) { 2565 for (level = 2; level <= max_level; level++) { 2566 struct intel_wm_level *wm = &merged->wm[level]; 2567 2568 wm->enable = false; 2569 } 2570 } 2571 } 2572 2573 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm) 2574 { 2575 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */ 2576 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable); 2577 } 2578 2579 /* The value we need to program into the WM_LPx latency field */ 2580 static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level) 2581 { 2582 struct drm_i915_private *dev_priv = dev->dev_private; 2583 2584 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) 2585 return 2 * level; 2586 else 2587 return dev_priv->wm.pri_latency[level]; 2588 } 2589 2590 static void ilk_compute_wm_results(struct drm_device *dev, 2591 const struct intel_pipe_wm *merged, 2592 enum intel_ddb_partitioning partitioning, 2593 struct ilk_wm_values *results) 2594 { 2595 struct intel_crtc *intel_crtc; 2596 int level, wm_lp; 2597 2598 results->enable_fbc_wm = merged->fbc_wm_enabled; 2599 results->partitioning = partitioning; 2600 2601 /* LP1+ register values */ 2602 for (wm_lp = 1; wm_lp <= 3; wm_lp++) { 2603 const struct intel_wm_level *r; 2604 2605 level = ilk_wm_lp_to_level(wm_lp, merged); 2606 2607 r = &merged->wm[level]; 2608 2609 /* 2610 * Maintain the watermark values even if the level is 2611 * disabled. Doing otherwise could cause underruns. 2612 */ 2613 results->wm_lp[wm_lp - 1] = 2614 (ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) | 2615 (r->pri_val << WM1_LP_SR_SHIFT) | 2616 r->cur_val; 2617 2618 if (r->enable) 2619 results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN; 2620 2621 if (INTEL_INFO(dev)->gen >= 8) 2622 results->wm_lp[wm_lp - 1] |= 2623 r->fbc_val << WM1_LP_FBC_SHIFT_BDW; 2624 else 2625 results->wm_lp[wm_lp - 1] |= 2626 r->fbc_val << WM1_LP_FBC_SHIFT; 2627 2628 /* 2629 * Always set WM1S_LP_EN when spr_val != 0, even if the 2630 * level is disabled. Doing otherwise could cause underruns. 2631 */ 2632 if (INTEL_INFO(dev)->gen <= 6 && r->spr_val) { 2633 WARN_ON(wm_lp != 1); 2634 results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val; 2635 } else 2636 results->wm_lp_spr[wm_lp - 1] = r->spr_val; 2637 } 2638 2639 /* LP0 register values */ 2640 for_each_intel_crtc(dev, intel_crtc) { 2641 enum i915_pipe pipe = intel_crtc->pipe; 2642 const struct intel_wm_level *r = 2643 &intel_crtc->wm.active.wm[0]; 2644 2645 if (WARN_ON(!r->enable)) 2646 continue; 2647 2648 results->wm_linetime[pipe] = intel_crtc->wm.active.linetime; 2649 2650 results->wm_pipe[pipe] = 2651 (r->pri_val << WM0_PIPE_PLANE_SHIFT) | 2652 (r->spr_val << WM0_PIPE_SPRITE_SHIFT) | 2653 r->cur_val; 2654 } 2655 } 2656 2657 /* Find the result with the highest level enabled. Check for enable_fbc_wm in 2658 * case both are at the same level. Prefer r1 in case they're the same. */ 2659 static struct intel_pipe_wm *ilk_find_best_result(struct drm_device *dev, 2660 struct intel_pipe_wm *r1, 2661 struct intel_pipe_wm *r2) 2662 { 2663 int level, max_level = ilk_wm_max_level(dev); 2664 int level1 = 0, level2 = 0; 2665 2666 for (level = 1; level <= max_level; level++) { 2667 if (r1->wm[level].enable) 2668 level1 = level; 2669 if (r2->wm[level].enable) 2670 level2 = level; 2671 } 2672 2673 if (level1 == level2) { 2674 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled) 2675 return r2; 2676 else 2677 return r1; 2678 } else if (level1 > level2) { 2679 return r1; 2680 } else { 2681 return r2; 2682 } 2683 } 2684 2685 /* dirty bits used to track which watermarks need changes */ 2686 #define WM_DIRTY_PIPE(pipe) (1 << (pipe)) 2687 #define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe))) 2688 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp))) 2689 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3)) 2690 #define WM_DIRTY_FBC (1 << 24) 2691 #define WM_DIRTY_DDB (1 << 25) 2692 2693 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv, 2694 const struct ilk_wm_values *old, 2695 const struct ilk_wm_values *new) 2696 { 2697 unsigned int dirty = 0; 2698 enum i915_pipe pipe; 2699 int wm_lp; 2700 2701 for_each_pipe(dev_priv, pipe) { 2702 if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) { 2703 dirty |= WM_DIRTY_LINETIME(pipe); 2704 /* Must disable LP1+ watermarks too */ 2705 dirty |= WM_DIRTY_LP_ALL; 2706 } 2707 2708 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) { 2709 dirty |= WM_DIRTY_PIPE(pipe); 2710 /* Must disable LP1+ watermarks too */ 2711 dirty |= WM_DIRTY_LP_ALL; 2712 } 2713 } 2714 2715 if (old->enable_fbc_wm != new->enable_fbc_wm) { 2716 dirty |= WM_DIRTY_FBC; 2717 /* Must disable LP1+ watermarks too */ 2718 dirty |= WM_DIRTY_LP_ALL; 2719 } 2720 2721 if (old->partitioning != new->partitioning) { 2722 dirty |= WM_DIRTY_DDB; 2723 /* Must disable LP1+ watermarks too */ 2724 dirty |= WM_DIRTY_LP_ALL; 2725 } 2726 2727 /* LP1+ watermarks already deemed dirty, no need to continue */ 2728 if (dirty & WM_DIRTY_LP_ALL) 2729 return dirty; 2730 2731 /* Find the lowest numbered LP1+ watermark in need of an update... */ 2732 for (wm_lp = 1; wm_lp <= 3; wm_lp++) { 2733 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] || 2734 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1]) 2735 break; 2736 } 2737 2738 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */ 2739 for (; wm_lp <= 3; wm_lp++) 2740 dirty |= WM_DIRTY_LP(wm_lp); 2741 2742 return dirty; 2743 } 2744 2745 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv, 2746 unsigned int dirty) 2747 { 2748 struct ilk_wm_values *previous = &dev_priv->wm.hw; 2749 bool changed = false; 2750 2751 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) { 2752 previous->wm_lp[2] &= ~WM1_LP_SR_EN; 2753 I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]); 2754 changed = true; 2755 } 2756 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) { 2757 previous->wm_lp[1] &= ~WM1_LP_SR_EN; 2758 I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]); 2759 changed = true; 2760 } 2761 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) { 2762 previous->wm_lp[0] &= ~WM1_LP_SR_EN; 2763 I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]); 2764 changed = true; 2765 } 2766 2767 /* 2768 * Don't touch WM1S_LP_EN here. 2769 * Doing so could cause underruns. 2770 */ 2771 2772 return changed; 2773 } 2774 2775 /* 2776 * The spec says we shouldn't write when we don't need, because every write 2777 * causes WMs to be re-evaluated, expending some power. 2778 */ 2779 static void ilk_write_wm_values(struct drm_i915_private *dev_priv, 2780 struct ilk_wm_values *results) 2781 { 2782 struct drm_device *dev = dev_priv->dev; 2783 struct ilk_wm_values *previous = &dev_priv->wm.hw; 2784 unsigned int dirty; 2785 uint32_t val; 2786 2787 dirty = ilk_compute_wm_dirty(dev_priv, previous, results); 2788 if (!dirty) 2789 return; 2790 2791 _ilk_disable_lp_wm(dev_priv, dirty); 2792 2793 if (dirty & WM_DIRTY_PIPE(PIPE_A)) 2794 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]); 2795 if (dirty & WM_DIRTY_PIPE(PIPE_B)) 2796 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]); 2797 if (dirty & WM_DIRTY_PIPE(PIPE_C)) 2798 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]); 2799 2800 if (dirty & WM_DIRTY_LINETIME(PIPE_A)) 2801 I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]); 2802 if (dirty & WM_DIRTY_LINETIME(PIPE_B)) 2803 I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]); 2804 if (dirty & WM_DIRTY_LINETIME(PIPE_C)) 2805 I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]); 2806 2807 if (dirty & WM_DIRTY_DDB) { 2808 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) { 2809 val = I915_READ(WM_MISC); 2810 if (results->partitioning == INTEL_DDB_PART_1_2) 2811 val &= ~WM_MISC_DATA_PARTITION_5_6; 2812 else 2813 val |= WM_MISC_DATA_PARTITION_5_6; 2814 I915_WRITE(WM_MISC, val); 2815 } else { 2816 val = I915_READ(DISP_ARB_CTL2); 2817 if (results->partitioning == INTEL_DDB_PART_1_2) 2818 val &= ~DISP_DATA_PARTITION_5_6; 2819 else 2820 val |= DISP_DATA_PARTITION_5_6; 2821 I915_WRITE(DISP_ARB_CTL2, val); 2822 } 2823 } 2824 2825 if (dirty & WM_DIRTY_FBC) { 2826 val = I915_READ(DISP_ARB_CTL); 2827 if (results->enable_fbc_wm) 2828 val &= ~DISP_FBC_WM_DIS; 2829 else 2830 val |= DISP_FBC_WM_DIS; 2831 I915_WRITE(DISP_ARB_CTL, val); 2832 } 2833 2834 if (dirty & WM_DIRTY_LP(1) && 2835 previous->wm_lp_spr[0] != results->wm_lp_spr[0]) 2836 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]); 2837 2838 if (INTEL_INFO(dev)->gen >= 7) { 2839 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1]) 2840 I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]); 2841 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2]) 2842 I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]); 2843 } 2844 2845 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0]) 2846 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]); 2847 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1]) 2848 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]); 2849 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2]) 2850 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]); 2851 2852 dev_priv->wm.hw = *results; 2853 } 2854 2855 static bool ilk_disable_lp_wm(struct drm_device *dev) 2856 { 2857 struct drm_i915_private *dev_priv = dev->dev_private; 2858 2859 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL); 2860 } 2861 2862 static void ilk_update_wm(struct drm_crtc *crtc) 2863 { 2864 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 2865 struct drm_device *dev = crtc->dev; 2866 struct drm_i915_private *dev_priv = dev->dev_private; 2867 struct ilk_wm_maximums max; 2868 struct ilk_pipe_wm_parameters params = {}; 2869 struct ilk_wm_values results = {}; 2870 enum intel_ddb_partitioning partitioning; 2871 struct intel_pipe_wm pipe_wm = {}; 2872 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm; 2873 struct intel_wm_config config = {}; 2874 2875 ilk_compute_wm_parameters(crtc, ¶ms); 2876 2877 intel_compute_pipe_wm(crtc, ¶ms, &pipe_wm); 2878 2879 if (!memcmp(&intel_crtc->wm.active, &pipe_wm, sizeof(pipe_wm))) 2880 return; 2881 2882 intel_crtc->wm.active = pipe_wm; 2883 2884 ilk_compute_wm_config(dev, &config); 2885 2886 ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max); 2887 ilk_wm_merge(dev, &config, &max, &lp_wm_1_2); 2888 2889 /* 5/6 split only in single pipe config on IVB+ */ 2890 if (INTEL_INFO(dev)->gen >= 7 && 2891 config.num_pipes_active == 1 && config.sprites_enabled) { 2892 ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max); 2893 ilk_wm_merge(dev, &config, &max, &lp_wm_5_6); 2894 2895 best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6); 2896 } else { 2897 best_lp_wm = &lp_wm_1_2; 2898 } 2899 2900 partitioning = (best_lp_wm == &lp_wm_1_2) ? 2901 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6; 2902 2903 ilk_compute_wm_results(dev, best_lp_wm, partitioning, &results); 2904 2905 ilk_write_wm_values(dev_priv, &results); 2906 } 2907 2908 static void 2909 ilk_update_sprite_wm(struct drm_plane *plane, 2910 struct drm_crtc *crtc, 2911 uint32_t sprite_width, uint32_t sprite_height, 2912 int pixel_size, bool enabled, bool scaled) 2913 { 2914 struct drm_device *dev = plane->dev; 2915 struct intel_plane *intel_plane = to_intel_plane(plane); 2916 2917 intel_plane->wm.enabled = enabled; 2918 intel_plane->wm.scaled = scaled; 2919 intel_plane->wm.horiz_pixels = sprite_width; 2920 intel_plane->wm.vert_pixels = sprite_width; 2921 intel_plane->wm.bytes_per_pixel = pixel_size; 2922 2923 /* 2924 * IVB workaround: must disable low power watermarks for at least 2925 * one frame before enabling scaling. LP watermarks can be re-enabled 2926 * when scaling is disabled. 2927 * 2928 * WaCxSRDisabledForSpriteScaling:ivb 2929 */ 2930 if (IS_IVYBRIDGE(dev) && scaled && ilk_disable_lp_wm(dev)) 2931 intel_wait_for_vblank(dev, intel_plane->pipe); 2932 2933 ilk_update_wm(crtc); 2934 } 2935 2936 static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc) 2937 { 2938 struct drm_device *dev = crtc->dev; 2939 struct drm_i915_private *dev_priv = dev->dev_private; 2940 struct ilk_wm_values *hw = &dev_priv->wm.hw; 2941 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 2942 struct intel_pipe_wm *active = &intel_crtc->wm.active; 2943 enum i915_pipe pipe = intel_crtc->pipe; 2944 static const unsigned int wm0_pipe_reg[] = { 2945 [PIPE_A] = WM0_PIPEA_ILK, 2946 [PIPE_B] = WM0_PIPEB_ILK, 2947 [PIPE_C] = WM0_PIPEC_IVB, 2948 }; 2949 2950 hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]); 2951 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) 2952 hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe)); 2953 2954 active->pipe_enabled = intel_crtc_active(crtc); 2955 2956 if (active->pipe_enabled) { 2957 u32 tmp = hw->wm_pipe[pipe]; 2958 2959 /* 2960 * For active pipes LP0 watermark is marked as 2961 * enabled, and LP1+ watermaks as disabled since 2962 * we can't really reverse compute them in case 2963 * multiple pipes are active. 2964 */ 2965 active->wm[0].enable = true; 2966 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT; 2967 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT; 2968 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK; 2969 active->linetime = hw->wm_linetime[pipe]; 2970 } else { 2971 int level, max_level = ilk_wm_max_level(dev); 2972 2973 /* 2974 * For inactive pipes, all watermark levels 2975 * should be marked as enabled but zeroed, 2976 * which is what we'd compute them to. 2977 */ 2978 for (level = 0; level <= max_level; level++) 2979 active->wm[level].enable = true; 2980 } 2981 } 2982 2983 void ilk_wm_get_hw_state(struct drm_device *dev) 2984 { 2985 struct drm_i915_private *dev_priv = dev->dev_private; 2986 struct ilk_wm_values *hw = &dev_priv->wm.hw; 2987 struct drm_crtc *crtc; 2988 2989 for_each_crtc(dev, crtc) 2990 ilk_pipe_wm_get_hw_state(crtc); 2991 2992 hw->wm_lp[0] = I915_READ(WM1_LP_ILK); 2993 hw->wm_lp[1] = I915_READ(WM2_LP_ILK); 2994 hw->wm_lp[2] = I915_READ(WM3_LP_ILK); 2995 2996 hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK); 2997 if (INTEL_INFO(dev)->gen >= 7) { 2998 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB); 2999 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB); 3000 } 3001 3002 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) 3003 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ? 3004 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2; 3005 else if (IS_IVYBRIDGE(dev)) 3006 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ? 3007 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2; 3008 3009 hw->enable_fbc_wm = 3010 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS); 3011 } 3012 3013 /** 3014 * intel_update_watermarks - update FIFO watermark values based on current modes 3015 * 3016 * Calculate watermark values for the various WM regs based on current mode 3017 * and plane configuration. 3018 * 3019 * There are several cases to deal with here: 3020 * - normal (i.e. non-self-refresh) 3021 * - self-refresh (SR) mode 3022 * - lines are large relative to FIFO size (buffer can hold up to 2) 3023 * - lines are small relative to FIFO size (buffer can hold more than 2 3024 * lines), so need to account for TLB latency 3025 * 3026 * The normal calculation is: 3027 * watermark = dotclock * bytes per pixel * latency 3028 * where latency is platform & configuration dependent (we assume pessimal 3029 * values here). 3030 * 3031 * The SR calculation is: 3032 * watermark = (trunc(latency/line time)+1) * surface width * 3033 * bytes per pixel 3034 * where 3035 * line time = htotal / dotclock 3036 * surface width = hdisplay for normal plane and 64 for cursor 3037 * and latency is assumed to be high, as above. 3038 * 3039 * The final value programmed to the register should always be rounded up, 3040 * and include an extra 2 entries to account for clock crossings. 3041 * 3042 * We don't use the sprite, so we can ignore that. And on Crestline we have 3043 * to set the non-SR watermarks to 8. 3044 */ 3045 void intel_update_watermarks(struct drm_crtc *crtc) 3046 { 3047 struct drm_i915_private *dev_priv = crtc->dev->dev_private; 3048 3049 if (dev_priv->display.update_wm) 3050 dev_priv->display.update_wm(crtc); 3051 } 3052 3053 void intel_update_sprite_watermarks(struct drm_plane *plane, 3054 struct drm_crtc *crtc, 3055 uint32_t sprite_width, 3056 uint32_t sprite_height, 3057 int pixel_size, 3058 bool enabled, bool scaled) 3059 { 3060 struct drm_i915_private *dev_priv = plane->dev->dev_private; 3061 3062 if (dev_priv->display.update_sprite_wm) 3063 dev_priv->display.update_sprite_wm(plane, crtc, 3064 sprite_width, sprite_height, 3065 pixel_size, enabled, scaled); 3066 } 3067 3068 static struct drm_i915_gem_object * 3069 intel_alloc_context_page(struct drm_device *dev) 3070 { 3071 struct drm_i915_gem_object *ctx; 3072 int ret; 3073 3074 WARN_ON(!mutex_is_locked(&dev->struct_mutex)); 3075 3076 ctx = i915_gem_alloc_object(dev, 4096); 3077 if (!ctx) { 3078 DRM_DEBUG("failed to alloc power context, RC6 disabled\n"); 3079 return NULL; 3080 } 3081 3082 ret = i915_gem_obj_ggtt_pin(ctx, 4096, 0); 3083 if (ret) { 3084 DRM_ERROR("failed to pin power context: %d\n", ret); 3085 goto err_unref; 3086 } 3087 3088 ret = i915_gem_object_set_to_gtt_domain(ctx, 1); 3089 if (ret) { 3090 DRM_ERROR("failed to set-domain on power context: %d\n", ret); 3091 goto err_unpin; 3092 } 3093 3094 return ctx; 3095 3096 err_unpin: 3097 i915_gem_object_ggtt_unpin(ctx); 3098 err_unref: 3099 drm_gem_object_unreference(&ctx->base); 3100 return NULL; 3101 } 3102 3103 /** 3104 * Lock protecting IPS related data structures 3105 */ 3106 struct lock mchdev_lock; 3107 LOCK_SYSINIT(mchdev, &mchdev_lock, "mchdev", LK_CANRECURSE); 3108 3109 /* Global for IPS driver to get at the current i915 device. Protected by 3110 * mchdev_lock. */ 3111 static struct drm_i915_private *i915_mch_dev; 3112 3113 bool ironlake_set_drps(struct drm_device *dev, u8 val) 3114 { 3115 struct drm_i915_private *dev_priv = dev->dev_private; 3116 u16 rgvswctl; 3117 3118 assert_spin_locked(&mchdev_lock); 3119 3120 rgvswctl = I915_READ16(MEMSWCTL); 3121 if (rgvswctl & MEMCTL_CMD_STS) { 3122 DRM_DEBUG("gpu busy, RCS change rejected\n"); 3123 return false; /* still busy with another command */ 3124 } 3125 3126 rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) | 3127 (val << MEMCTL_FREQ_SHIFT) | MEMCTL_SFCAVM; 3128 I915_WRITE16(MEMSWCTL, rgvswctl); 3129 POSTING_READ16(MEMSWCTL); 3130 3131 rgvswctl |= MEMCTL_CMD_STS; 3132 I915_WRITE16(MEMSWCTL, rgvswctl); 3133 3134 return true; 3135 } 3136 3137 static void ironlake_enable_drps(struct drm_device *dev) 3138 { 3139 struct drm_i915_private *dev_priv = dev->dev_private; 3140 u32 rgvmodectl = I915_READ(MEMMODECTL); 3141 u8 fmax, fmin, fstart, vstart; 3142 3143 lockmgr(&mchdev_lock, LK_EXCLUSIVE); 3144 3145 /* Enable temp reporting */ 3146 I915_WRITE16(PMMISC, I915_READ(PMMISC) | MCPPCE_EN); 3147 I915_WRITE16(TSC1, I915_READ(TSC1) | TSE); 3148 3149 /* 100ms RC evaluation intervals */ 3150 I915_WRITE(RCUPEI, 100000); 3151 I915_WRITE(RCDNEI, 100000); 3152 3153 /* Set max/min thresholds to 90ms and 80ms respectively */ 3154 I915_WRITE(RCBMAXAVG, 90000); 3155 I915_WRITE(RCBMINAVG, 80000); 3156 3157 I915_WRITE(MEMIHYST, 1); 3158 3159 /* Set up min, max, and cur for interrupt handling */ 3160 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT; 3161 fmin = (rgvmodectl & MEMMODE_FMIN_MASK); 3162 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> 3163 MEMMODE_FSTART_SHIFT; 3164 3165 vstart = (I915_READ(PXVFREQ_BASE + (fstart * 4)) & PXVFREQ_PX_MASK) >> 3166 PXVFREQ_PX_SHIFT; 3167 3168 dev_priv->ips.fmax = fmax; /* IPS callback will increase this */ 3169 dev_priv->ips.fstart = fstart; 3170 3171 dev_priv->ips.max_delay = fstart; 3172 dev_priv->ips.min_delay = fmin; 3173 dev_priv->ips.cur_delay = fstart; 3174 3175 DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n", 3176 fmax, fmin, fstart); 3177 3178 I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN); 3179 3180 /* 3181 * Interrupts will be enabled in ironlake_irq_postinstall 3182 */ 3183 3184 I915_WRITE(VIDSTART, vstart); 3185 POSTING_READ(VIDSTART); 3186 3187 rgvmodectl |= MEMMODE_SWMODE_EN; 3188 I915_WRITE(MEMMODECTL, rgvmodectl); 3189 3190 if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10)) 3191 DRM_ERROR("stuck trying to change perf mode\n"); 3192 mdelay(1); 3193 3194 ironlake_set_drps(dev, fstart); 3195 3196 dev_priv->ips.last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) + 3197 I915_READ(0x112e0); 3198 dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies); 3199 dev_priv->ips.last_count2 = I915_READ(0x112f4); 3200 getrawmonotonic(&dev_priv->ips.last_time2); 3201 3202 lockmgr(&mchdev_lock, LK_RELEASE); 3203 } 3204 3205 static void ironlake_disable_drps(struct drm_device *dev) 3206 { 3207 struct drm_i915_private *dev_priv = dev->dev_private; 3208 u16 rgvswctl; 3209 3210 lockmgr(&mchdev_lock, LK_EXCLUSIVE); 3211 3212 rgvswctl = I915_READ16(MEMSWCTL); 3213 3214 /* Ack interrupts, disable EFC interrupt */ 3215 I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN); 3216 I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG); 3217 I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT); 3218 I915_WRITE(DEIIR, DE_PCU_EVENT); 3219 I915_WRITE(DEIMR, I915_READ(DEIMR) | DE_PCU_EVENT); 3220 3221 /* Go back to the starting frequency */ 3222 ironlake_set_drps(dev, dev_priv->ips.fstart); 3223 mdelay(1); 3224 rgvswctl |= MEMCTL_CMD_STS; 3225 I915_WRITE(MEMSWCTL, rgvswctl); 3226 mdelay(1); 3227 3228 lockmgr(&mchdev_lock, LK_RELEASE); 3229 } 3230 3231 /* There's a funny hw issue where the hw returns all 0 when reading from 3232 * GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value 3233 * ourselves, instead of doing a rmw cycle (which might result in us clearing 3234 * all limits and the gpu stuck at whatever frequency it is at atm). 3235 */ 3236 static u32 gen6_rps_limits(struct drm_i915_private *dev_priv, u8 val) 3237 { 3238 u32 limits; 3239 3240 /* Only set the down limit when we've reached the lowest level to avoid 3241 * getting more interrupts, otherwise leave this clear. This prevents a 3242 * race in the hw when coming out of rc6: There's a tiny window where 3243 * the hw runs at the minimal clock before selecting the desired 3244 * frequency, if the down threshold expires in that window we will not 3245 * receive a down interrupt. */ 3246 limits = dev_priv->rps.max_freq_softlimit << 24; 3247 if (val <= dev_priv->rps.min_freq_softlimit) 3248 limits |= dev_priv->rps.min_freq_softlimit << 16; 3249 3250 return limits; 3251 } 3252 3253 static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val) 3254 { 3255 int new_power; 3256 3257 new_power = dev_priv->rps.power; 3258 switch (dev_priv->rps.power) { 3259 case LOW_POWER: 3260 if (val > dev_priv->rps.efficient_freq + 1 && val > dev_priv->rps.cur_freq) 3261 new_power = BETWEEN; 3262 break; 3263 3264 case BETWEEN: 3265 if (val <= dev_priv->rps.efficient_freq && val < dev_priv->rps.cur_freq) 3266 new_power = LOW_POWER; 3267 else if (val >= dev_priv->rps.rp0_freq && val > dev_priv->rps.cur_freq) 3268 new_power = HIGH_POWER; 3269 break; 3270 3271 case HIGH_POWER: 3272 if (val < (dev_priv->rps.rp1_freq + dev_priv->rps.rp0_freq) >> 1 && val < dev_priv->rps.cur_freq) 3273 new_power = BETWEEN; 3274 break; 3275 } 3276 /* Max/min bins are special */ 3277 if (val == dev_priv->rps.min_freq_softlimit) 3278 new_power = LOW_POWER; 3279 if (val == dev_priv->rps.max_freq_softlimit) 3280 new_power = HIGH_POWER; 3281 if (new_power == dev_priv->rps.power) 3282 return; 3283 3284 /* Note the units here are not exactly 1us, but 1280ns. */ 3285 switch (new_power) { 3286 case LOW_POWER: 3287 /* Upclock if more than 95% busy over 16ms */ 3288 I915_WRITE(GEN6_RP_UP_EI, 12500); 3289 I915_WRITE(GEN6_RP_UP_THRESHOLD, 11800); 3290 3291 /* Downclock if less than 85% busy over 32ms */ 3292 I915_WRITE(GEN6_RP_DOWN_EI, 25000); 3293 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 21250); 3294 3295 I915_WRITE(GEN6_RP_CONTROL, 3296 GEN6_RP_MEDIA_TURBO | 3297 GEN6_RP_MEDIA_HW_NORMAL_MODE | 3298 GEN6_RP_MEDIA_IS_GFX | 3299 GEN6_RP_ENABLE | 3300 GEN6_RP_UP_BUSY_AVG | 3301 GEN6_RP_DOWN_IDLE_AVG); 3302 break; 3303 3304 case BETWEEN: 3305 /* Upclock if more than 90% busy over 13ms */ 3306 I915_WRITE(GEN6_RP_UP_EI, 10250); 3307 I915_WRITE(GEN6_RP_UP_THRESHOLD, 9225); 3308 3309 /* Downclock if less than 75% busy over 32ms */ 3310 I915_WRITE(GEN6_RP_DOWN_EI, 25000); 3311 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 18750); 3312 3313 I915_WRITE(GEN6_RP_CONTROL, 3314 GEN6_RP_MEDIA_TURBO | 3315 GEN6_RP_MEDIA_HW_NORMAL_MODE | 3316 GEN6_RP_MEDIA_IS_GFX | 3317 GEN6_RP_ENABLE | 3318 GEN6_RP_UP_BUSY_AVG | 3319 GEN6_RP_DOWN_IDLE_AVG); 3320 break; 3321 3322 case HIGH_POWER: 3323 /* Upclock if more than 85% busy over 10ms */ 3324 I915_WRITE(GEN6_RP_UP_EI, 8000); 3325 I915_WRITE(GEN6_RP_UP_THRESHOLD, 6800); 3326 3327 /* Downclock if less than 60% busy over 32ms */ 3328 I915_WRITE(GEN6_RP_DOWN_EI, 25000); 3329 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 15000); 3330 3331 I915_WRITE(GEN6_RP_CONTROL, 3332 GEN6_RP_MEDIA_TURBO | 3333 GEN6_RP_MEDIA_HW_NORMAL_MODE | 3334 GEN6_RP_MEDIA_IS_GFX | 3335 GEN6_RP_ENABLE | 3336 GEN6_RP_UP_BUSY_AVG | 3337 GEN6_RP_DOWN_IDLE_AVG); 3338 break; 3339 } 3340 3341 dev_priv->rps.power = new_power; 3342 dev_priv->rps.last_adj = 0; 3343 } 3344 3345 static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val) 3346 { 3347 u32 mask = 0; 3348 3349 if (val > dev_priv->rps.min_freq_softlimit) 3350 mask |= GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT; 3351 if (val < dev_priv->rps.max_freq_softlimit) 3352 mask |= GEN6_PM_RP_UP_THRESHOLD; 3353 3354 mask |= dev_priv->pm_rps_events & (GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED); 3355 mask &= dev_priv->pm_rps_events; 3356 3357 /* IVB and SNB hard hangs on looping batchbuffer 3358 * if GEN6_PM_UP_EI_EXPIRED is masked. 3359 */ 3360 if (INTEL_INFO(dev_priv->dev)->gen <= 7 && !IS_HASWELL(dev_priv->dev)) 3361 mask |= GEN6_PM_RP_UP_EI_EXPIRED; 3362 3363 if (IS_GEN8(dev_priv->dev)) 3364 mask |= GEN8_PMINTR_REDIRECT_TO_NON_DISP; 3365 3366 return ~mask; 3367 } 3368 3369 /* gen6_set_rps is called to update the frequency request, but should also be 3370 * called when the range (min_delay and max_delay) is modified so that we can 3371 * update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */ 3372 void gen6_set_rps(struct drm_device *dev, u8 val) 3373 { 3374 struct drm_i915_private *dev_priv = dev->dev_private; 3375 3376 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); 3377 WARN_ON(val > dev_priv->rps.max_freq_softlimit); 3378 WARN_ON(val < dev_priv->rps.min_freq_softlimit); 3379 3380 /* min/max delay may still have been modified so be sure to 3381 * write the limits value. 3382 */ 3383 if (val != dev_priv->rps.cur_freq) { 3384 gen6_set_rps_thresholds(dev_priv, val); 3385 3386 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) 3387 I915_WRITE(GEN6_RPNSWREQ, 3388 HSW_FREQUENCY(val)); 3389 else 3390 I915_WRITE(GEN6_RPNSWREQ, 3391 GEN6_FREQUENCY(val) | 3392 GEN6_OFFSET(0) | 3393 GEN6_AGGRESSIVE_TURBO); 3394 } 3395 3396 /* Make sure we continue to get interrupts 3397 * until we hit the minimum or maximum frequencies. 3398 */ 3399 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, gen6_rps_limits(dev_priv, val)); 3400 I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val)); 3401 3402 POSTING_READ(GEN6_RPNSWREQ); 3403 3404 dev_priv->rps.cur_freq = val; 3405 trace_intel_gpu_freq_change(val * 50); 3406 } 3407 3408 /* vlv_set_rps_idle: Set the frequency to Rpn if Gfx clocks are down 3409 * 3410 * * If Gfx is Idle, then 3411 * 1. Mask Turbo interrupts 3412 * 2. Bring up Gfx clock 3413 * 3. Change the freq to Rpn and wait till P-Unit updates freq 3414 * 4. Clear the Force GFX CLK ON bit so that Gfx can down 3415 * 5. Unmask Turbo interrupts 3416 */ 3417 static void vlv_set_rps_idle(struct drm_i915_private *dev_priv) 3418 { 3419 int revision; 3420 3421 struct drm_device *dev = dev_priv->dev; 3422 3423 /* Latest VLV doesn't need to force the gfx clock */ 3424 revision = pci_read_config(dev->dev, PCIR_REVID, 1); 3425 if (revision >= 0xd) { 3426 valleyview_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit); 3427 return; 3428 } 3429 3430 /* 3431 * When we are idle. Drop to min voltage state. 3432 */ 3433 3434 if (dev_priv->rps.cur_freq <= dev_priv->rps.min_freq_softlimit) 3435 return; 3436 3437 /* Mask turbo interrupt so that they will not come in between */ 3438 I915_WRITE(GEN6_PMINTRMSK, 0xffffffff); 3439 3440 vlv_force_gfx_clock(dev_priv, true); 3441 3442 dev_priv->rps.cur_freq = dev_priv->rps.min_freq_softlimit; 3443 3444 vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, 3445 dev_priv->rps.min_freq_softlimit); 3446 3447 if (wait_for(((vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS)) 3448 & GENFREQSTATUS) == 0, 5)) 3449 DRM_ERROR("timed out waiting for Punit\n"); 3450 3451 vlv_force_gfx_clock(dev_priv, false); 3452 3453 I915_WRITE(GEN6_PMINTRMSK, 3454 gen6_rps_pm_mask(dev_priv, dev_priv->rps.cur_freq)); 3455 } 3456 3457 void gen6_rps_idle(struct drm_i915_private *dev_priv) 3458 { 3459 struct drm_device *dev = dev_priv->dev; 3460 3461 mutex_lock(&dev_priv->rps.hw_lock); 3462 if (dev_priv->rps.enabled) { 3463 if (IS_CHERRYVIEW(dev)) 3464 valleyview_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit); 3465 else if (IS_VALLEYVIEW(dev)) 3466 vlv_set_rps_idle(dev_priv); 3467 else 3468 gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit); 3469 dev_priv->rps.last_adj = 0; 3470 } 3471 mutex_unlock(&dev_priv->rps.hw_lock); 3472 } 3473 3474 void gen6_rps_boost(struct drm_i915_private *dev_priv) 3475 { 3476 struct drm_device *dev = dev_priv->dev; 3477 3478 mutex_lock(&dev_priv->rps.hw_lock); 3479 if (dev_priv->rps.enabled) { 3480 if (IS_VALLEYVIEW(dev)) 3481 valleyview_set_rps(dev_priv->dev, dev_priv->rps.max_freq_softlimit); 3482 else 3483 gen6_set_rps(dev_priv->dev, dev_priv->rps.max_freq_softlimit); 3484 dev_priv->rps.last_adj = 0; 3485 } 3486 mutex_unlock(&dev_priv->rps.hw_lock); 3487 } 3488 3489 void valleyview_set_rps(struct drm_device *dev, u8 val) 3490 { 3491 struct drm_i915_private *dev_priv = dev->dev_private; 3492 3493 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); 3494 WARN_ON(val > dev_priv->rps.max_freq_softlimit); 3495 WARN_ON(val < dev_priv->rps.min_freq_softlimit); 3496 3497 if (WARN_ONCE(IS_CHERRYVIEW(dev) && (val & 1), 3498 "Odd GPU freq value\n")) 3499 val &= ~1; 3500 3501 if (val != dev_priv->rps.cur_freq) { 3502 DRM_DEBUG_DRIVER("GPU freq request from %d MHz (%u) to %d MHz (%u)\n", 3503 vlv_gpu_freq(dev_priv, dev_priv->rps.cur_freq), 3504 dev_priv->rps.cur_freq, 3505 vlv_gpu_freq(dev_priv, val), val); 3506 3507 vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val); 3508 } 3509 3510 I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val)); 3511 3512 dev_priv->rps.cur_freq = val; 3513 trace_intel_gpu_freq_change(vlv_gpu_freq(dev_priv, val)); 3514 } 3515 3516 static void gen8_disable_rps_interrupts(struct drm_device *dev) 3517 { 3518 struct drm_i915_private *dev_priv = dev->dev_private; 3519 3520 I915_WRITE(GEN6_PMINTRMSK, ~GEN8_PMINTR_REDIRECT_TO_NON_DISP); 3521 I915_WRITE(GEN8_GT_IER(2), I915_READ(GEN8_GT_IER(2)) & 3522 ~dev_priv->pm_rps_events); 3523 /* Complete PM interrupt masking here doesn't race with the rps work 3524 * item again unmasking PM interrupts because that is using a different 3525 * register (GEN8_GT_IMR(2)) to mask PM interrupts. The only risk is in 3526 * leaving stale bits in GEN8_GT_IIR(2) and GEN8_GT_IMR(2) which 3527 * gen8_enable_rps will clean up. */ 3528 3529 lockmgr(&dev_priv->irq_lock, LK_EXCLUSIVE); 3530 dev_priv->rps.pm_iir = 0; 3531 lockmgr(&dev_priv->irq_lock, LK_RELEASE); 3532 3533 I915_WRITE(GEN8_GT_IIR(2), dev_priv->pm_rps_events); 3534 } 3535 3536 static void gen6_disable_rps_interrupts(struct drm_device *dev) 3537 { 3538 struct drm_i915_private *dev_priv = dev->dev_private; 3539 3540 I915_WRITE(GEN6_PMINTRMSK, 0xffffffff); 3541 I915_WRITE(GEN6_PMIER, I915_READ(GEN6_PMIER) & 3542 ~dev_priv->pm_rps_events); 3543 /* Complete PM interrupt masking here doesn't race with the rps work 3544 * item again unmasking PM interrupts because that is using a different 3545 * register (PMIMR) to mask PM interrupts. The only risk is in leaving 3546 * stale bits in PMIIR and PMIMR which gen6_enable_rps will clean up. */ 3547 3548 lockmgr(&dev_priv->irq_lock, LK_EXCLUSIVE); 3549 dev_priv->rps.pm_iir = 0; 3550 lockmgr(&dev_priv->irq_lock, LK_RELEASE); 3551 3552 I915_WRITE(GEN6_PMIIR, dev_priv->pm_rps_events); 3553 } 3554 3555 static void gen6_disable_rps(struct drm_device *dev) 3556 { 3557 struct drm_i915_private *dev_priv = dev->dev_private; 3558 3559 I915_WRITE(GEN6_RC_CONTROL, 0); 3560 I915_WRITE(GEN6_RPNSWREQ, 1 << 31); 3561 3562 if (IS_BROADWELL(dev)) 3563 gen8_disable_rps_interrupts(dev); 3564 else 3565 gen6_disable_rps_interrupts(dev); 3566 } 3567 3568 static void cherryview_disable_rps(struct drm_device *dev) 3569 { 3570 struct drm_i915_private *dev_priv = dev->dev_private; 3571 3572 I915_WRITE(GEN6_RC_CONTROL, 0); 3573 3574 gen8_disable_rps_interrupts(dev); 3575 } 3576 3577 static void valleyview_disable_rps(struct drm_device *dev) 3578 { 3579 struct drm_i915_private *dev_priv = dev->dev_private; 3580 3581 /* we're doing forcewake before Disabling RC6, 3582 * This what the BIOS expects when going into suspend */ 3583 gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL); 3584 3585 I915_WRITE(GEN6_RC_CONTROL, 0); 3586 3587 gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL); 3588 3589 gen6_disable_rps_interrupts(dev); 3590 } 3591 3592 static void intel_print_rc6_info(struct drm_device *dev, u32 mode) 3593 { 3594 if (IS_VALLEYVIEW(dev)) { 3595 if (mode & (GEN7_RC_CTL_TO_MODE | GEN6_RC_CTL_EI_MODE(1))) 3596 mode = GEN6_RC_CTL_RC6_ENABLE; 3597 else 3598 mode = 0; 3599 } 3600 DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s, RC6p %s, RC6pp %s\n", 3601 (mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off", 3602 (mode & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off", 3603 (mode & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off"); 3604 } 3605 3606 static int sanitize_rc6_option(const struct drm_device *dev, int enable_rc6) 3607 { 3608 /* No RC6 before Ironlake */ 3609 if (INTEL_INFO(dev)->gen < 5) 3610 return 0; 3611 3612 /* RC6 is only on Ironlake mobile not on desktop */ 3613 if (INTEL_INFO(dev)->gen == 5 && !IS_IRONLAKE_M(dev)) 3614 return 0; 3615 3616 /* Respect the kernel parameter if it is set */ 3617 if (enable_rc6 >= 0) { 3618 int mask; 3619 3620 if (INTEL_INFO(dev)->gen == 6 || IS_IVYBRIDGE(dev)) 3621 mask = INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE | 3622 INTEL_RC6pp_ENABLE; 3623 else 3624 mask = INTEL_RC6_ENABLE; 3625 3626 if ((enable_rc6 & mask) != enable_rc6) 3627 DRM_DEBUG_KMS("Adjusting RC6 mask to %d (requested %d, valid %d)\n", 3628 enable_rc6 & mask, enable_rc6, mask); 3629 3630 return enable_rc6 & mask; 3631 } 3632 3633 /* Disable RC6 on Ironlake */ 3634 if (INTEL_INFO(dev)->gen == 5) 3635 return 0; 3636 3637 if (IS_IVYBRIDGE(dev)) 3638 return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE); 3639 3640 return INTEL_RC6_ENABLE; 3641 } 3642 3643 int intel_enable_rc6(const struct drm_device *dev) 3644 { 3645 return i915.enable_rc6; 3646 } 3647 3648 static void gen8_enable_rps_interrupts(struct drm_device *dev) 3649 { 3650 struct drm_i915_private *dev_priv = dev->dev_private; 3651 3652 lockmgr(&dev_priv->irq_lock, LK_EXCLUSIVE); 3653 WARN_ON(dev_priv->rps.pm_iir); 3654 gen8_enable_pm_irq(dev_priv, dev_priv->pm_rps_events); 3655 I915_WRITE(GEN8_GT_IIR(2), dev_priv->pm_rps_events); 3656 lockmgr(&dev_priv->irq_lock, LK_RELEASE); 3657 } 3658 3659 static void gen6_enable_rps_interrupts(struct drm_device *dev) 3660 { 3661 struct drm_i915_private *dev_priv = dev->dev_private; 3662 3663 lockmgr(&dev_priv->irq_lock, LK_EXCLUSIVE); 3664 WARN_ON(dev_priv->rps.pm_iir); 3665 gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events); 3666 I915_WRITE(GEN6_PMIIR, dev_priv->pm_rps_events); 3667 lockmgr(&dev_priv->irq_lock, LK_RELEASE); 3668 } 3669 3670 static void parse_rp_state_cap(struct drm_i915_private *dev_priv, u32 rp_state_cap) 3671 { 3672 /* All of these values are in units of 50MHz */ 3673 dev_priv->rps.cur_freq = 0; 3674 /* static values from HW: RP0 < RPe < RP1 < RPn (min_freq) */ 3675 dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff; 3676 dev_priv->rps.rp0_freq = (rp_state_cap >> 0) & 0xff; 3677 dev_priv->rps.min_freq = (rp_state_cap >> 16) & 0xff; 3678 /* XXX: only BYT has a special efficient freq */ 3679 dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq; 3680 /* hw_max = RP0 until we check for overclocking */ 3681 dev_priv->rps.max_freq = dev_priv->rps.rp0_freq; 3682 3683 /* Preserve min/max settings in case of re-init */ 3684 if (dev_priv->rps.max_freq_softlimit == 0) 3685 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq; 3686 3687 if (dev_priv->rps.min_freq_softlimit == 0) 3688 dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq; 3689 } 3690 3691 static void gen8_enable_rps(struct drm_device *dev) 3692 { 3693 struct drm_i915_private *dev_priv = dev->dev_private; 3694 struct intel_engine_cs *ring; 3695 uint32_t rc6_mask = 0, rp_state_cap; 3696 int unused; 3697 3698 /* 1a: Software RC state - RC0 */ 3699 I915_WRITE(GEN6_RC_STATE, 0); 3700 3701 /* 1c & 1d: Get forcewake during program sequence. Although the driver 3702 * hasn't enabled a state yet where we need forcewake, BIOS may have.*/ 3703 gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL); 3704 3705 /* 2a: Disable RC states. */ 3706 I915_WRITE(GEN6_RC_CONTROL, 0); 3707 3708 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP); 3709 parse_rp_state_cap(dev_priv, rp_state_cap); 3710 3711 /* 2b: Program RC6 thresholds.*/ 3712 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16); 3713 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ 3714 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ 3715 for_each_ring(ring, dev_priv, unused) 3716 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10); 3717 I915_WRITE(GEN6_RC_SLEEP, 0); 3718 if (IS_BROADWELL(dev)) 3719 I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */ 3720 else 3721 I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */ 3722 3723 /* 3: Enable RC6 */ 3724 if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE) 3725 rc6_mask = GEN6_RC_CTL_RC6_ENABLE; 3726 intel_print_rc6_info(dev, rc6_mask); 3727 if (IS_BROADWELL(dev)) 3728 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE | 3729 GEN7_RC_CTL_TO_MODE | 3730 rc6_mask); 3731 else 3732 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE | 3733 GEN6_RC_CTL_EI_MODE(1) | 3734 rc6_mask); 3735 3736 /* 4 Program defaults and thresholds for RPS*/ 3737 I915_WRITE(GEN6_RPNSWREQ, 3738 HSW_FREQUENCY(dev_priv->rps.rp1_freq)); 3739 I915_WRITE(GEN6_RC_VIDEO_FREQ, 3740 HSW_FREQUENCY(dev_priv->rps.rp1_freq)); 3741 /* NB: Docs say 1s, and 1000000 - which aren't equivalent */ 3742 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */ 3743 3744 /* Docs recommend 900MHz, and 300 MHz respectively */ 3745 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, 3746 dev_priv->rps.max_freq_softlimit << 24 | 3747 dev_priv->rps.min_freq_softlimit << 16); 3748 3749 I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */ 3750 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/ 3751 I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */ 3752 I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */ 3753 3754 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10); 3755 3756 /* 5: Enable RPS */ 3757 I915_WRITE(GEN6_RP_CONTROL, 3758 GEN6_RP_MEDIA_TURBO | 3759 GEN6_RP_MEDIA_HW_NORMAL_MODE | 3760 GEN6_RP_MEDIA_IS_GFX | 3761 GEN6_RP_ENABLE | 3762 GEN6_RP_UP_BUSY_AVG | 3763 GEN6_RP_DOWN_IDLE_AVG); 3764 3765 /* 6: Ring frequency + overclocking (our driver does this later */ 3766 3767 gen6_set_rps(dev, (I915_READ(GEN6_GT_PERF_STATUS) & 0xff00) >> 8); 3768 3769 gen8_enable_rps_interrupts(dev); 3770 3771 gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL); 3772 } 3773 3774 static void gen6_enable_rps(struct drm_device *dev) 3775 { 3776 struct drm_i915_private *dev_priv = dev->dev_private; 3777 struct intel_engine_cs *ring; 3778 u32 rp_state_cap; 3779 u32 rc6vids, pcu_mbox = 0, rc6_mask = 0; 3780 u32 gtfifodbg; 3781 int rc6_mode; 3782 int i, ret; 3783 3784 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); 3785 3786 /* Here begins a magic sequence of register writes to enable 3787 * auto-downclocking. 3788 * 3789 * Perhaps there might be some value in exposing these to 3790 * userspace... 3791 */ 3792 I915_WRITE(GEN6_RC_STATE, 0); 3793 3794 /* Clear the DBG now so we don't confuse earlier errors */ 3795 if ((gtfifodbg = I915_READ(GTFIFODBG))) { 3796 DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg); 3797 I915_WRITE(GTFIFODBG, gtfifodbg); 3798 } 3799 3800 gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL); 3801 3802 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP); 3803 3804 parse_rp_state_cap(dev_priv, rp_state_cap); 3805 3806 /* disable the counters and set deterministic thresholds */ 3807 I915_WRITE(GEN6_RC_CONTROL, 0); 3808 3809 I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16); 3810 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30); 3811 I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30); 3812 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); 3813 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); 3814 3815 for_each_ring(ring, dev_priv, i) 3816 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10); 3817 3818 I915_WRITE(GEN6_RC_SLEEP, 0); 3819 I915_WRITE(GEN6_RC1e_THRESHOLD, 1000); 3820 if (IS_IVYBRIDGE(dev)) 3821 I915_WRITE(GEN6_RC6_THRESHOLD, 125000); 3822 else 3823 I915_WRITE(GEN6_RC6_THRESHOLD, 50000); 3824 I915_WRITE(GEN6_RC6p_THRESHOLD, 150000); 3825 I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */ 3826 3827 /* Check if we are enabling RC6 */ 3828 rc6_mode = intel_enable_rc6(dev_priv->dev); 3829 if (rc6_mode & INTEL_RC6_ENABLE) 3830 rc6_mask |= GEN6_RC_CTL_RC6_ENABLE; 3831 3832 /* We don't use those on Haswell */ 3833 if (!IS_HASWELL(dev)) { 3834 if (rc6_mode & INTEL_RC6p_ENABLE) 3835 rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE; 3836 3837 if (rc6_mode & INTEL_RC6pp_ENABLE) 3838 rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE; 3839 } 3840 3841 intel_print_rc6_info(dev, rc6_mask); 3842 3843 I915_WRITE(GEN6_RC_CONTROL, 3844 rc6_mask | 3845 GEN6_RC_CTL_EI_MODE(1) | 3846 GEN6_RC_CTL_HW_ENABLE); 3847 3848 /* Power down if completely idle for over 50ms */ 3849 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000); 3850 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10); 3851 3852 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0); 3853 if (ret) 3854 DRM_DEBUG_DRIVER("Failed to set the min frequency\n"); 3855 3856 ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox); 3857 if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */ 3858 DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n", 3859 (dev_priv->rps.max_freq_softlimit & 0xff) * 50, 3860 (pcu_mbox & 0xff) * 50); 3861 dev_priv->rps.max_freq = pcu_mbox & 0xff; 3862 } 3863 3864 dev_priv->rps.power = HIGH_POWER; /* force a reset */ 3865 gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit); 3866 3867 gen6_enable_rps_interrupts(dev); 3868 3869 rc6vids = 0; 3870 ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids); 3871 if (IS_GEN6(dev) && ret) { 3872 DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n"); 3873 } else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) { 3874 DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n", 3875 GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450); 3876 rc6vids &= 0xffff00; 3877 rc6vids |= GEN6_ENCODE_RC6_VID(450); 3878 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids); 3879 if (ret) 3880 DRM_ERROR("Couldn't fix incorrect rc6 voltage\n"); 3881 } 3882 3883 gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL); 3884 } 3885 3886 static void __gen6_update_ring_freq(struct drm_device *dev) 3887 { 3888 struct drm_i915_private *dev_priv = dev->dev_private; 3889 int min_freq = 15; 3890 unsigned int gpu_freq; 3891 unsigned int max_ia_freq, min_ring_freq; 3892 int scaling_factor = 180; 3893 3894 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); 3895 3896 #if 0 3897 policy = cpufreq_cpu_get(0); 3898 if (policy) { 3899 max_ia_freq = policy->cpuinfo.max_freq; 3900 cpufreq_cpu_put(policy); 3901 } else { 3902 /* 3903 * Default to measured freq if none found, PCU will ensure we 3904 * don't go over 3905 */ 3906 max_ia_freq = tsc_khz; 3907 } 3908 #else 3909 max_ia_freq = tsc_frequency / 1000; 3910 #endif 3911 3912 /* Convert from kHz to MHz */ 3913 max_ia_freq /= 1000; 3914 3915 min_ring_freq = I915_READ(DCLK) & 0xf; 3916 /* convert DDR frequency from units of 266.6MHz to bandwidth */ 3917 min_ring_freq = mult_frac(min_ring_freq, 8, 3); 3918 3919 /* 3920 * For each potential GPU frequency, load a ring frequency we'd like 3921 * to use for memory access. We do this by specifying the IA frequency 3922 * the PCU should use as a reference to determine the ring frequency. 3923 */ 3924 for (gpu_freq = dev_priv->rps.max_freq_softlimit; gpu_freq >= dev_priv->rps.min_freq_softlimit; 3925 gpu_freq--) { 3926 int diff = dev_priv->rps.max_freq_softlimit - gpu_freq; 3927 unsigned int ia_freq = 0, ring_freq = 0; 3928 3929 if (INTEL_INFO(dev)->gen >= 8) { 3930 /* max(2 * GT, DDR). NB: GT is 50MHz units */ 3931 ring_freq = max(min_ring_freq, gpu_freq); 3932 } else if (IS_HASWELL(dev)) { 3933 ring_freq = mult_frac(gpu_freq, 5, 4); 3934 ring_freq = max(min_ring_freq, ring_freq); 3935 /* leave ia_freq as the default, chosen by cpufreq */ 3936 } else { 3937 /* On older processors, there is no separate ring 3938 * clock domain, so in order to boost the bandwidth 3939 * of the ring, we need to upclock the CPU (ia_freq). 3940 * 3941 * For GPU frequencies less than 750MHz, 3942 * just use the lowest ring freq. 3943 */ 3944 if (gpu_freq < min_freq) 3945 ia_freq = 800; 3946 else 3947 ia_freq = max_ia_freq - ((diff * scaling_factor) / 2); 3948 ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100); 3949 } 3950 3951 sandybridge_pcode_write(dev_priv, 3952 GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 3953 ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT | 3954 ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT | 3955 gpu_freq); 3956 } 3957 } 3958 3959 void gen6_update_ring_freq(struct drm_device *dev) 3960 { 3961 struct drm_i915_private *dev_priv = dev->dev_private; 3962 3963 if (INTEL_INFO(dev)->gen < 6 || IS_VALLEYVIEW(dev)) 3964 return; 3965 3966 mutex_lock(&dev_priv->rps.hw_lock); 3967 __gen6_update_ring_freq(dev); 3968 mutex_unlock(&dev_priv->rps.hw_lock); 3969 } 3970 3971 static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv) 3972 { 3973 u32 val, rp0; 3974 3975 val = vlv_punit_read(dev_priv, PUNIT_GPU_STATUS_REG); 3976 rp0 = (val >> PUNIT_GPU_STATUS_MAX_FREQ_SHIFT) & PUNIT_GPU_STATUS_MAX_FREQ_MASK; 3977 3978 return rp0; 3979 } 3980 3981 static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv) 3982 { 3983 u32 val, rpe; 3984 3985 val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG); 3986 rpe = (val >> PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT) & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK; 3987 3988 return rpe; 3989 } 3990 3991 static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv) 3992 { 3993 u32 val, rp1; 3994 3995 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); 3996 rp1 = (val >> PUNIT_GPU_STATUS_MAX_FREQ_SHIFT) & PUNIT_GPU_STATUS_MAX_FREQ_MASK; 3997 3998 return rp1; 3999 } 4000 4001 static int cherryview_rps_min_freq(struct drm_i915_private *dev_priv) 4002 { 4003 u32 val, rpn; 4004 4005 val = vlv_punit_read(dev_priv, PUNIT_GPU_STATUS_REG); 4006 rpn = (val >> PUNIT_GPU_STATIS_GFX_MIN_FREQ_SHIFT) & PUNIT_GPU_STATUS_GFX_MIN_FREQ_MASK; 4007 return rpn; 4008 } 4009 4010 static int valleyview_rps_guar_freq(struct drm_i915_private *dev_priv) 4011 { 4012 u32 val, rp1; 4013 4014 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE); 4015 4016 rp1 = (val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK) >> FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT; 4017 4018 return rp1; 4019 } 4020 4021 static int valleyview_rps_max_freq(struct drm_i915_private *dev_priv) 4022 { 4023 u32 val, rp0; 4024 4025 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE); 4026 4027 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT; 4028 /* Clamp to max */ 4029 rp0 = min_t(u32, rp0, 0xea); 4030 4031 return rp0; 4032 } 4033 4034 static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv) 4035 { 4036 u32 val, rpe; 4037 4038 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO); 4039 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT; 4040 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI); 4041 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5; 4042 4043 return rpe; 4044 } 4045 4046 static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv) 4047 { 4048 return vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff; 4049 } 4050 4051 /* Check that the pctx buffer wasn't move under us. */ 4052 static void valleyview_check_pctx(struct drm_i915_private *dev_priv) 4053 { 4054 unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095; 4055 4056 /* DragonFly - if EDID fails vlv_pctx can wind up NULL */ 4057 if (WARN_ON(!dev_priv->vlv_pctx)) 4058 return; 4059 4060 WARN_ON(pctx_addr != dev_priv->mm.stolen_base + 4061 dev_priv->vlv_pctx->stolen->start); 4062 } 4063 4064 4065 /* Check that the pcbr address is not empty. */ 4066 static void cherryview_check_pctx(struct drm_i915_private *dev_priv) 4067 { 4068 unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095; 4069 4070 WARN_ON((pctx_addr >> VLV_PCBR_ADDR_SHIFT) == 0); 4071 } 4072 4073 static void cherryview_setup_pctx(struct drm_device *dev) 4074 { 4075 struct drm_i915_private *dev_priv = dev->dev_private; 4076 unsigned long pctx_paddr, paddr; 4077 struct i915_gtt *gtt = &dev_priv->gtt; 4078 u32 pcbr; 4079 int pctx_size = 32*1024; 4080 4081 WARN_ON(!mutex_is_locked(&dev->struct_mutex)); 4082 4083 pcbr = I915_READ(VLV_PCBR); 4084 if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) { 4085 paddr = (dev_priv->mm.stolen_base + 4086 (gtt->stolen_size - pctx_size)); 4087 4088 pctx_paddr = (paddr & (~4095)); 4089 I915_WRITE(VLV_PCBR, pctx_paddr); 4090 } 4091 } 4092 4093 static void valleyview_setup_pctx(struct drm_device *dev) 4094 { 4095 struct drm_i915_private *dev_priv = dev->dev_private; 4096 struct drm_i915_gem_object *pctx; 4097 unsigned long pctx_paddr; 4098 u32 pcbr; 4099 int pctx_size = 24*1024; 4100 4101 WARN_ON(!mutex_is_locked(&dev->struct_mutex)); 4102 4103 pcbr = I915_READ(VLV_PCBR); 4104 if (pcbr) { 4105 /* BIOS set it up already, grab the pre-alloc'd space */ 4106 int pcbr_offset; 4107 4108 pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base; 4109 pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv->dev, 4110 pcbr_offset, 4111 I915_GTT_OFFSET_NONE, 4112 pctx_size); 4113 goto out; 4114 } 4115 4116 /* 4117 * From the Gunit register HAS: 4118 * The Gfx driver is expected to program this register and ensure 4119 * proper allocation within Gfx stolen memory. For example, this 4120 * register should be programmed such than the PCBR range does not 4121 * overlap with other ranges, such as the frame buffer, protected 4122 * memory, or any other relevant ranges. 4123 */ 4124 pctx = i915_gem_object_create_stolen(dev, pctx_size); 4125 if (!pctx) { 4126 DRM_DEBUG("not enough stolen space for PCTX, disabling\n"); 4127 return; 4128 } 4129 4130 pctx_paddr = dev_priv->mm.stolen_base + pctx->stolen->start; 4131 I915_WRITE(VLV_PCBR, pctx_paddr); 4132 4133 out: 4134 dev_priv->vlv_pctx = pctx; 4135 } 4136 4137 static void valleyview_cleanup_pctx(struct drm_device *dev) 4138 { 4139 struct drm_i915_private *dev_priv = dev->dev_private; 4140 4141 if (WARN_ON(!dev_priv->vlv_pctx)) 4142 return; 4143 4144 drm_gem_object_unreference(&dev_priv->vlv_pctx->base); 4145 dev_priv->vlv_pctx = NULL; 4146 } 4147 4148 static void valleyview_init_gt_powersave(struct drm_device *dev) 4149 { 4150 struct drm_i915_private *dev_priv = dev->dev_private; 4151 u32 val; 4152 4153 valleyview_setup_pctx(dev); 4154 4155 mutex_lock(&dev_priv->rps.hw_lock); 4156 4157 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); 4158 switch ((val >> 6) & 3) { 4159 case 0: 4160 case 1: 4161 dev_priv->mem_freq = 800; 4162 break; 4163 case 2: 4164 dev_priv->mem_freq = 1066; 4165 break; 4166 case 3: 4167 dev_priv->mem_freq = 1333; 4168 break; 4169 } 4170 DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq); 4171 4172 dev_priv->rps.max_freq = valleyview_rps_max_freq(dev_priv); 4173 dev_priv->rps.rp0_freq = dev_priv->rps.max_freq; 4174 DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n", 4175 vlv_gpu_freq(dev_priv, dev_priv->rps.max_freq), 4176 dev_priv->rps.max_freq); 4177 4178 dev_priv->rps.efficient_freq = valleyview_rps_rpe_freq(dev_priv); 4179 DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n", 4180 vlv_gpu_freq(dev_priv, dev_priv->rps.efficient_freq), 4181 dev_priv->rps.efficient_freq); 4182 4183 dev_priv->rps.rp1_freq = valleyview_rps_guar_freq(dev_priv); 4184 DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n", 4185 vlv_gpu_freq(dev_priv, dev_priv->rps.rp1_freq), 4186 dev_priv->rps.rp1_freq); 4187 4188 dev_priv->rps.min_freq = valleyview_rps_min_freq(dev_priv); 4189 DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n", 4190 vlv_gpu_freq(dev_priv, dev_priv->rps.min_freq), 4191 dev_priv->rps.min_freq); 4192 4193 /* Preserve min/max settings in case of re-init */ 4194 if (dev_priv->rps.max_freq_softlimit == 0) 4195 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq; 4196 4197 if (dev_priv->rps.min_freq_softlimit == 0) 4198 dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq; 4199 4200 mutex_unlock(&dev_priv->rps.hw_lock); 4201 } 4202 4203 static void cherryview_init_gt_powersave(struct drm_device *dev) 4204 { 4205 struct drm_i915_private *dev_priv = dev->dev_private; 4206 u32 val; 4207 4208 cherryview_setup_pctx(dev); 4209 4210 mutex_lock(&dev_priv->rps.hw_lock); 4211 4212 val = vlv_punit_read(dev_priv, CCK_FUSE_REG); 4213 switch ((val >> 2) & 0x7) { 4214 case 0: 4215 case 1: 4216 dev_priv->rps.cz_freq = 200; 4217 dev_priv->mem_freq = 1600; 4218 break; 4219 case 2: 4220 dev_priv->rps.cz_freq = 267; 4221 dev_priv->mem_freq = 1600; 4222 break; 4223 case 3: 4224 dev_priv->rps.cz_freq = 333; 4225 dev_priv->mem_freq = 2000; 4226 break; 4227 case 4: 4228 dev_priv->rps.cz_freq = 320; 4229 dev_priv->mem_freq = 1600; 4230 break; 4231 case 5: 4232 dev_priv->rps.cz_freq = 400; 4233 dev_priv->mem_freq = 1600; 4234 break; 4235 } 4236 DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq); 4237 4238 dev_priv->rps.max_freq = cherryview_rps_max_freq(dev_priv); 4239 dev_priv->rps.rp0_freq = dev_priv->rps.max_freq; 4240 DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n", 4241 vlv_gpu_freq(dev_priv, dev_priv->rps.max_freq), 4242 dev_priv->rps.max_freq); 4243 4244 dev_priv->rps.efficient_freq = cherryview_rps_rpe_freq(dev_priv); 4245 DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n", 4246 vlv_gpu_freq(dev_priv, dev_priv->rps.efficient_freq), 4247 dev_priv->rps.efficient_freq); 4248 4249 dev_priv->rps.rp1_freq = cherryview_rps_guar_freq(dev_priv); 4250 DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n", 4251 vlv_gpu_freq(dev_priv, dev_priv->rps.rp1_freq), 4252 dev_priv->rps.rp1_freq); 4253 4254 dev_priv->rps.min_freq = cherryview_rps_min_freq(dev_priv); 4255 DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n", 4256 vlv_gpu_freq(dev_priv, dev_priv->rps.min_freq), 4257 dev_priv->rps.min_freq); 4258 4259 WARN_ONCE((dev_priv->rps.max_freq | 4260 dev_priv->rps.efficient_freq | 4261 dev_priv->rps.rp1_freq | 4262 dev_priv->rps.min_freq) & 1, 4263 "Odd GPU freq values\n"); 4264 4265 /* Preserve min/max settings in case of re-init */ 4266 if (dev_priv->rps.max_freq_softlimit == 0) 4267 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq; 4268 4269 if (dev_priv->rps.min_freq_softlimit == 0) 4270 dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq; 4271 4272 mutex_unlock(&dev_priv->rps.hw_lock); 4273 } 4274 4275 static void valleyview_cleanup_gt_powersave(struct drm_device *dev) 4276 { 4277 valleyview_cleanup_pctx(dev); 4278 } 4279 4280 static void cherryview_enable_rps(struct drm_device *dev) 4281 { 4282 struct drm_i915_private *dev_priv = dev->dev_private; 4283 struct intel_engine_cs *ring; 4284 u32 gtfifodbg, val, rc6_mode = 0, pcbr; 4285 int i; 4286 4287 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); 4288 4289 gtfifodbg = I915_READ(GTFIFODBG); 4290 if (gtfifodbg) { 4291 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n", 4292 gtfifodbg); 4293 I915_WRITE(GTFIFODBG, gtfifodbg); 4294 } 4295 4296 cherryview_check_pctx(dev_priv); 4297 4298 /* 1a & 1b: Get forcewake during program sequence. Although the driver 4299 * hasn't enabled a state yet where we need forcewake, BIOS may have.*/ 4300 gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL); 4301 4302 /* 2a: Program RC6 thresholds.*/ 4303 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16); 4304 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ 4305 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ 4306 4307 for_each_ring(ring, dev_priv, i) 4308 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10); 4309 I915_WRITE(GEN6_RC_SLEEP, 0); 4310 4311 I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */ 4312 4313 /* allows RC6 residency counter to work */ 4314 I915_WRITE(VLV_COUNTER_CONTROL, 4315 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH | 4316 VLV_MEDIA_RC6_COUNT_EN | 4317 VLV_RENDER_RC6_COUNT_EN)); 4318 4319 /* For now we assume BIOS is allocating and populating the PCBR */ 4320 pcbr = I915_READ(VLV_PCBR); 4321 4322 DRM_DEBUG_DRIVER("PCBR offset : 0x%x\n", pcbr); 4323 4324 /* 3: Enable RC6 */ 4325 if ((intel_enable_rc6(dev) & INTEL_RC6_ENABLE) && 4326 (pcbr >> VLV_PCBR_ADDR_SHIFT)) 4327 rc6_mode = GEN6_RC_CTL_EI_MODE(1); 4328 4329 I915_WRITE(GEN6_RC_CONTROL, rc6_mode); 4330 4331 /* 4 Program defaults and thresholds for RPS*/ 4332 I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400); 4333 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000); 4334 I915_WRITE(GEN6_RP_UP_EI, 66000); 4335 I915_WRITE(GEN6_RP_DOWN_EI, 350000); 4336 4337 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10); 4338 4339 /* WaDisablePwrmtrEvent:chv (pre-production hw) */ 4340 I915_WRITE(0xA80C, I915_READ(0xA80C) & 0x00ffffff); 4341 I915_WRITE(0xA810, I915_READ(0xA810) & 0xffffff00); 4342 4343 /* 5: Enable RPS */ 4344 I915_WRITE(GEN6_RP_CONTROL, 4345 GEN6_RP_MEDIA_HW_NORMAL_MODE | 4346 GEN6_RP_MEDIA_IS_GFX | /* WaSetMaskForGfxBusyness:chv (pre-production hw ?) */ 4347 GEN6_RP_ENABLE | 4348 GEN6_RP_UP_BUSY_AVG | 4349 GEN6_RP_DOWN_IDLE_AVG); 4350 4351 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); 4352 4353 DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & 0x10 ? "yes" : "no"); 4354 DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val); 4355 4356 dev_priv->rps.cur_freq = (val >> 8) & 0xff; 4357 DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n", 4358 vlv_gpu_freq(dev_priv, dev_priv->rps.cur_freq), 4359 dev_priv->rps.cur_freq); 4360 4361 DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n", 4362 vlv_gpu_freq(dev_priv, dev_priv->rps.efficient_freq), 4363 dev_priv->rps.efficient_freq); 4364 4365 valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq); 4366 4367 gen8_enable_rps_interrupts(dev); 4368 4369 gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL); 4370 } 4371 4372 static void valleyview_enable_rps(struct drm_device *dev) 4373 { 4374 struct drm_i915_private *dev_priv = dev->dev_private; 4375 struct intel_engine_cs *ring; 4376 u32 gtfifodbg, val, rc6_mode = 0; 4377 int i; 4378 4379 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); 4380 4381 valleyview_check_pctx(dev_priv); 4382 4383 if ((gtfifodbg = I915_READ(GTFIFODBG))) { 4384 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n", 4385 gtfifodbg); 4386 I915_WRITE(GTFIFODBG, gtfifodbg); 4387 } 4388 4389 /* If VLV, Forcewake all wells, else re-direct to regular path */ 4390 gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL); 4391 4392 I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400); 4393 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000); 4394 I915_WRITE(GEN6_RP_UP_EI, 66000); 4395 I915_WRITE(GEN6_RP_DOWN_EI, 350000); 4396 4397 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10); 4398 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 0xf4240); 4399 4400 I915_WRITE(GEN6_RP_CONTROL, 4401 GEN6_RP_MEDIA_TURBO | 4402 GEN6_RP_MEDIA_HW_NORMAL_MODE | 4403 GEN6_RP_MEDIA_IS_GFX | 4404 GEN6_RP_ENABLE | 4405 GEN6_RP_UP_BUSY_AVG | 4406 GEN6_RP_DOWN_IDLE_CONT); 4407 4408 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000); 4409 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); 4410 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); 4411 4412 for_each_ring(ring, dev_priv, i) 4413 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10); 4414 4415 I915_WRITE(GEN6_RC6_THRESHOLD, 0x557); 4416 4417 /* allows RC6 residency counter to work */ 4418 I915_WRITE(VLV_COUNTER_CONTROL, 4419 _MASKED_BIT_ENABLE(VLV_MEDIA_RC0_COUNT_EN | 4420 VLV_RENDER_RC0_COUNT_EN | 4421 VLV_MEDIA_RC6_COUNT_EN | 4422 VLV_RENDER_RC6_COUNT_EN)); 4423 4424 if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE) 4425 rc6_mode = GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL; 4426 4427 intel_print_rc6_info(dev, rc6_mode); 4428 4429 I915_WRITE(GEN6_RC_CONTROL, rc6_mode); 4430 4431 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); 4432 4433 DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & 0x10 ? "yes" : "no"); 4434 DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val); 4435 4436 dev_priv->rps.cur_freq = (val >> 8) & 0xff; 4437 DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n", 4438 vlv_gpu_freq(dev_priv, dev_priv->rps.cur_freq), 4439 dev_priv->rps.cur_freq); 4440 4441 DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n", 4442 vlv_gpu_freq(dev_priv, dev_priv->rps.efficient_freq), 4443 dev_priv->rps.efficient_freq); 4444 4445 valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq); 4446 4447 gen6_enable_rps_interrupts(dev); 4448 4449 gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL); 4450 } 4451 4452 void ironlake_teardown_rc6(struct drm_device *dev) 4453 { 4454 struct drm_i915_private *dev_priv = dev->dev_private; 4455 4456 if (dev_priv->ips.renderctx) { 4457 i915_gem_object_ggtt_unpin(dev_priv->ips.renderctx); 4458 drm_gem_object_unreference(&dev_priv->ips.renderctx->base); 4459 dev_priv->ips.renderctx = NULL; 4460 } 4461 4462 if (dev_priv->ips.pwrctx) { 4463 i915_gem_object_ggtt_unpin(dev_priv->ips.pwrctx); 4464 drm_gem_object_unreference(&dev_priv->ips.pwrctx->base); 4465 dev_priv->ips.pwrctx = NULL; 4466 } 4467 } 4468 4469 static void ironlake_disable_rc6(struct drm_device *dev) 4470 { 4471 struct drm_i915_private *dev_priv = dev->dev_private; 4472 4473 if (I915_READ(PWRCTXA)) { 4474 /* Wake the GPU, prevent RC6, then restore RSTDBYCTL */ 4475 I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) | RCX_SW_EXIT); 4476 wait_for(((I915_READ(RSTDBYCTL) & RSX_STATUS_MASK) == RSX_STATUS_ON), 4477 50); 4478 4479 I915_WRITE(PWRCTXA, 0); 4480 POSTING_READ(PWRCTXA); 4481 4482 I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT); 4483 POSTING_READ(RSTDBYCTL); 4484 } 4485 } 4486 4487 static int ironlake_setup_rc6(struct drm_device *dev) 4488 { 4489 struct drm_i915_private *dev_priv = dev->dev_private; 4490 4491 if (dev_priv->ips.renderctx == NULL) 4492 dev_priv->ips.renderctx = intel_alloc_context_page(dev); 4493 if (!dev_priv->ips.renderctx) 4494 return -ENOMEM; 4495 4496 if (dev_priv->ips.pwrctx == NULL) 4497 dev_priv->ips.pwrctx = intel_alloc_context_page(dev); 4498 if (!dev_priv->ips.pwrctx) { 4499 ironlake_teardown_rc6(dev); 4500 return -ENOMEM; 4501 } 4502 4503 return 0; 4504 } 4505 4506 static void ironlake_enable_rc6(struct drm_device *dev) 4507 { 4508 struct drm_i915_private *dev_priv = dev->dev_private; 4509 struct intel_engine_cs *ring = &dev_priv->ring[RCS]; 4510 bool was_interruptible; 4511 int ret; 4512 4513 /* rc6 disabled by default due to repeated reports of hanging during 4514 * boot and resume. 4515 */ 4516 if (!intel_enable_rc6(dev)) 4517 return; 4518 4519 WARN_ON(!mutex_is_locked(&dev->struct_mutex)); 4520 4521 ret = ironlake_setup_rc6(dev); 4522 if (ret) 4523 return; 4524 4525 was_interruptible = dev_priv->mm.interruptible; 4526 dev_priv->mm.interruptible = false; 4527 4528 /* 4529 * GPU can automatically power down the render unit if given a page 4530 * to save state. 4531 */ 4532 ret = intel_ring_begin(ring, 6); 4533 if (ret) { 4534 ironlake_teardown_rc6(dev); 4535 dev_priv->mm.interruptible = was_interruptible; 4536 return; 4537 } 4538 4539 intel_ring_emit(ring, MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN); 4540 intel_ring_emit(ring, MI_SET_CONTEXT); 4541 intel_ring_emit(ring, i915_gem_obj_ggtt_offset(dev_priv->ips.renderctx) | 4542 MI_MM_SPACE_GTT | 4543 MI_SAVE_EXT_STATE_EN | 4544 MI_RESTORE_EXT_STATE_EN | 4545 MI_RESTORE_INHIBIT); 4546 intel_ring_emit(ring, MI_SUSPEND_FLUSH); 4547 intel_ring_emit(ring, MI_NOOP); 4548 intel_ring_emit(ring, MI_FLUSH); 4549 intel_ring_advance(ring); 4550 4551 /* 4552 * Wait for the command parser to advance past MI_SET_CONTEXT. The HW 4553 * does an implicit flush, combined with MI_FLUSH above, it should be 4554 * safe to assume that renderctx is valid 4555 */ 4556 ret = intel_ring_idle(ring); 4557 dev_priv->mm.interruptible = was_interruptible; 4558 if (ret) { 4559 DRM_ERROR("failed to enable ironlake power savings\n"); 4560 ironlake_teardown_rc6(dev); 4561 return; 4562 } 4563 4564 I915_WRITE(PWRCTXA, i915_gem_obj_ggtt_offset(dev_priv->ips.pwrctx) | PWRCTX_EN); 4565 I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT); 4566 4567 intel_print_rc6_info(dev, GEN6_RC_CTL_RC6_ENABLE); 4568 } 4569 4570 static unsigned long intel_pxfreq(u32 vidfreq) 4571 { 4572 unsigned long freq; 4573 int div = (vidfreq & 0x3f0000) >> 16; 4574 int post = (vidfreq & 0x3000) >> 12; 4575 int pre = (vidfreq & 0x7); 4576 4577 if (!pre) 4578 return 0; 4579 4580 freq = ((div * 133333) / ((1<<post) * pre)); 4581 4582 return freq; 4583 } 4584 4585 static const struct cparams { 4586 u16 i; 4587 u16 t; 4588 u16 m; 4589 u16 c; 4590 } cparams[] = { 4591 { 1, 1333, 301, 28664 }, 4592 { 1, 1066, 294, 24460 }, 4593 { 1, 800, 294, 25192 }, 4594 { 0, 1333, 276, 27605 }, 4595 { 0, 1066, 276, 27605 }, 4596 { 0, 800, 231, 23784 }, 4597 }; 4598 4599 static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv) 4600 { 4601 u64 total_count, diff, ret; 4602 u32 count1, count2, count3, m = 0, c = 0; 4603 unsigned long now = jiffies_to_msecs(jiffies), diff1; 4604 int i; 4605 4606 assert_spin_locked(&mchdev_lock); 4607 4608 diff1 = now - dev_priv->ips.last_time1; 4609 4610 /* Prevent division-by-zero if we are asking too fast. 4611 * Also, we don't get interesting results if we are polling 4612 * faster than once in 10ms, so just return the saved value 4613 * in such cases. 4614 */ 4615 if (diff1 <= 10) 4616 return dev_priv->ips.chipset_power; 4617 4618 count1 = I915_READ(DMIEC); 4619 count2 = I915_READ(DDREC); 4620 count3 = I915_READ(CSIEC); 4621 4622 total_count = count1 + count2 + count3; 4623 4624 /* FIXME: handle per-counter overflow */ 4625 if (total_count < dev_priv->ips.last_count1) { 4626 diff = ~0UL - dev_priv->ips.last_count1; 4627 diff += total_count; 4628 } else { 4629 diff = total_count - dev_priv->ips.last_count1; 4630 } 4631 4632 for (i = 0; i < ARRAY_SIZE(cparams); i++) { 4633 if (cparams[i].i == dev_priv->ips.c_m && 4634 cparams[i].t == dev_priv->ips.r_t) { 4635 m = cparams[i].m; 4636 c = cparams[i].c; 4637 break; 4638 } 4639 } 4640 4641 diff = div_u64(diff, diff1); 4642 ret = ((m * diff) + c); 4643 ret = div_u64(ret, 10); 4644 4645 dev_priv->ips.last_count1 = total_count; 4646 dev_priv->ips.last_time1 = now; 4647 4648 dev_priv->ips.chipset_power = ret; 4649 4650 return ret; 4651 } 4652 4653 unsigned long i915_chipset_val(struct drm_i915_private *dev_priv) 4654 { 4655 struct drm_device *dev = dev_priv->dev; 4656 unsigned long val; 4657 4658 if (INTEL_INFO(dev)->gen != 5) 4659 return 0; 4660 4661 lockmgr(&mchdev_lock, LK_EXCLUSIVE); 4662 4663 val = __i915_chipset_val(dev_priv); 4664 4665 lockmgr(&mchdev_lock, LK_RELEASE); 4666 4667 return val; 4668 } 4669 4670 unsigned long i915_mch_val(struct drm_i915_private *dev_priv) 4671 { 4672 unsigned long m, x, b; 4673 u32 tsfs; 4674 4675 tsfs = I915_READ(TSFS); 4676 4677 m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT); 4678 x = I915_READ8(TR1); 4679 4680 b = tsfs & TSFS_INTR_MASK; 4681 4682 return ((m * x) / 127) - b; 4683 } 4684 4685 static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid) 4686 { 4687 struct drm_device *dev = dev_priv->dev; 4688 static const struct v_table { 4689 u16 vd; /* in .1 mil */ 4690 u16 vm; /* in .1 mil */ 4691 } v_table[] = { 4692 { 0, 0, }, 4693 { 375, 0, }, 4694 { 500, 0, }, 4695 { 625, 0, }, 4696 { 750, 0, }, 4697 { 875, 0, }, 4698 { 1000, 0, }, 4699 { 1125, 0, }, 4700 { 4125, 3000, }, 4701 { 4125, 3000, }, 4702 { 4125, 3000, }, 4703 { 4125, 3000, }, 4704 { 4125, 3000, }, 4705 { 4125, 3000, }, 4706 { 4125, 3000, }, 4707 { 4125, 3000, }, 4708 { 4125, 3000, }, 4709 { 4125, 3000, }, 4710 { 4125, 3000, }, 4711 { 4125, 3000, }, 4712 { 4125, 3000, }, 4713 { 4125, 3000, }, 4714 { 4125, 3000, }, 4715 { 4125, 3000, }, 4716 { 4125, 3000, }, 4717 { 4125, 3000, }, 4718 { 4125, 3000, }, 4719 { 4125, 3000, }, 4720 { 4125, 3000, }, 4721 { 4125, 3000, }, 4722 { 4125, 3000, }, 4723 { 4125, 3000, }, 4724 { 4250, 3125, }, 4725 { 4375, 3250, }, 4726 { 4500, 3375, }, 4727 { 4625, 3500, }, 4728 { 4750, 3625, }, 4729 { 4875, 3750, }, 4730 { 5000, 3875, }, 4731 { 5125, 4000, }, 4732 { 5250, 4125, }, 4733 { 5375, 4250, }, 4734 { 5500, 4375, }, 4735 { 5625, 4500, }, 4736 { 5750, 4625, }, 4737 { 5875, 4750, }, 4738 { 6000, 4875, }, 4739 { 6125, 5000, }, 4740 { 6250, 5125, }, 4741 { 6375, 5250, }, 4742 { 6500, 5375, }, 4743 { 6625, 5500, }, 4744 { 6750, 5625, }, 4745 { 6875, 5750, }, 4746 { 7000, 5875, }, 4747 { 7125, 6000, }, 4748 { 7250, 6125, }, 4749 { 7375, 6250, }, 4750 { 7500, 6375, }, 4751 { 7625, 6500, }, 4752 { 7750, 6625, }, 4753 { 7875, 6750, }, 4754 { 8000, 6875, }, 4755 { 8125, 7000, }, 4756 { 8250, 7125, }, 4757 { 8375, 7250, }, 4758 { 8500, 7375, }, 4759 { 8625, 7500, }, 4760 { 8750, 7625, }, 4761 { 8875, 7750, }, 4762 { 9000, 7875, }, 4763 { 9125, 8000, }, 4764 { 9250, 8125, }, 4765 { 9375, 8250, }, 4766 { 9500, 8375, }, 4767 { 9625, 8500, }, 4768 { 9750, 8625, }, 4769 { 9875, 8750, }, 4770 { 10000, 8875, }, 4771 { 10125, 9000, }, 4772 { 10250, 9125, }, 4773 { 10375, 9250, }, 4774 { 10500, 9375, }, 4775 { 10625, 9500, }, 4776 { 10750, 9625, }, 4777 { 10875, 9750, }, 4778 { 11000, 9875, }, 4779 { 11125, 10000, }, 4780 { 11250, 10125, }, 4781 { 11375, 10250, }, 4782 { 11500, 10375, }, 4783 { 11625, 10500, }, 4784 { 11750, 10625, }, 4785 { 11875, 10750, }, 4786 { 12000, 10875, }, 4787 { 12125, 11000, }, 4788 { 12250, 11125, }, 4789 { 12375, 11250, }, 4790 { 12500, 11375, }, 4791 { 12625, 11500, }, 4792 { 12750, 11625, }, 4793 { 12875, 11750, }, 4794 { 13000, 11875, }, 4795 { 13125, 12000, }, 4796 { 13250, 12125, }, 4797 { 13375, 12250, }, 4798 { 13500, 12375, }, 4799 { 13625, 12500, }, 4800 { 13750, 12625, }, 4801 { 13875, 12750, }, 4802 { 14000, 12875, }, 4803 { 14125, 13000, }, 4804 { 14250, 13125, }, 4805 { 14375, 13250, }, 4806 { 14500, 13375, }, 4807 { 14625, 13500, }, 4808 { 14750, 13625, }, 4809 { 14875, 13750, }, 4810 { 15000, 13875, }, 4811 { 15125, 14000, }, 4812 { 15250, 14125, }, 4813 { 15375, 14250, }, 4814 { 15500, 14375, }, 4815 { 15625, 14500, }, 4816 { 15750, 14625, }, 4817 { 15875, 14750, }, 4818 { 16000, 14875, }, 4819 { 16125, 15000, }, 4820 }; 4821 if (INTEL_INFO(dev)->is_mobile) 4822 return v_table[pxvid].vm; 4823 else 4824 return v_table[pxvid].vd; 4825 } 4826 4827 static void __i915_update_gfx_val(struct drm_i915_private *dev_priv) 4828 { 4829 struct timespec now, diff1; 4830 u64 diff; 4831 unsigned long diffms; 4832 u32 count; 4833 4834 assert_spin_locked(&mchdev_lock); 4835 4836 getrawmonotonic(&now); 4837 diff1 = timespec_sub(now, dev_priv->ips.last_time2); 4838 4839 /* Don't divide by 0 */ 4840 diffms = diff1.tv_sec * 1000 + diff1.tv_nsec / 1000000; 4841 if (!diffms) 4842 return; 4843 4844 count = I915_READ(GFXEC); 4845 4846 if (count < dev_priv->ips.last_count2) { 4847 diff = ~0UL - dev_priv->ips.last_count2; 4848 diff += count; 4849 } else { 4850 diff = count - dev_priv->ips.last_count2; 4851 } 4852 4853 dev_priv->ips.last_count2 = count; 4854 dev_priv->ips.last_time2 = now; 4855 4856 /* More magic constants... */ 4857 diff = diff * 1181; 4858 diff = div_u64(diff, diffms * 10); 4859 dev_priv->ips.gfx_power = diff; 4860 } 4861 4862 void i915_update_gfx_val(struct drm_i915_private *dev_priv) 4863 { 4864 struct drm_device *dev = dev_priv->dev; 4865 4866 if (INTEL_INFO(dev)->gen != 5) 4867 return; 4868 4869 lockmgr(&mchdev_lock, LK_EXCLUSIVE); 4870 4871 __i915_update_gfx_val(dev_priv); 4872 4873 lockmgr(&mchdev_lock, LK_RELEASE); 4874 } 4875 4876 static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv) 4877 { 4878 unsigned long t, corr, state1, corr2, state2; 4879 u32 pxvid, ext_v; 4880 4881 assert_spin_locked(&mchdev_lock); 4882 4883 pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_freq * 4)); 4884 pxvid = (pxvid >> 24) & 0x7f; 4885 ext_v = pvid_to_extvid(dev_priv, pxvid); 4886 4887 state1 = ext_v; 4888 4889 t = i915_mch_val(dev_priv); 4890 4891 /* Revel in the empirically derived constants */ 4892 4893 /* Correction factor in 1/100000 units */ 4894 if (t > 80) 4895 corr = ((t * 2349) + 135940); 4896 else if (t >= 50) 4897 corr = ((t * 964) + 29317); 4898 else /* < 50 */ 4899 corr = ((t * 301) + 1004); 4900 4901 corr = corr * ((150142 * state1) / 10000 - 78642); 4902 corr /= 100000; 4903 corr2 = (corr * dev_priv->ips.corr); 4904 4905 state2 = (corr2 * state1) / 10000; 4906 state2 /= 100; /* convert to mW */ 4907 4908 __i915_update_gfx_val(dev_priv); 4909 4910 return dev_priv->ips.gfx_power + state2; 4911 } 4912 4913 unsigned long i915_gfx_val(struct drm_i915_private *dev_priv) 4914 { 4915 struct drm_device *dev = dev_priv->dev; 4916 unsigned long val; 4917 4918 if (INTEL_INFO(dev)->gen != 5) 4919 return 0; 4920 4921 lockmgr(&mchdev_lock, LK_EXCLUSIVE); 4922 4923 val = __i915_gfx_val(dev_priv); 4924 4925 lockmgr(&mchdev_lock, LK_RELEASE); 4926 4927 return val; 4928 } 4929 4930 /** 4931 * i915_read_mch_val - return value for IPS use 4932 * 4933 * Calculate and return a value for the IPS driver to use when deciding whether 4934 * we have thermal and power headroom to increase CPU or GPU power budget. 4935 */ 4936 unsigned long i915_read_mch_val(void) 4937 { 4938 struct drm_i915_private *dev_priv; 4939 unsigned long chipset_val, graphics_val, ret = 0; 4940 4941 lockmgr(&mchdev_lock, LK_EXCLUSIVE); 4942 if (!i915_mch_dev) 4943 goto out_unlock; 4944 dev_priv = i915_mch_dev; 4945 4946 chipset_val = __i915_chipset_val(dev_priv); 4947 graphics_val = __i915_gfx_val(dev_priv); 4948 4949 ret = chipset_val + graphics_val; 4950 4951 out_unlock: 4952 lockmgr(&mchdev_lock, LK_RELEASE); 4953 4954 return ret; 4955 } 4956 4957 /** 4958 * i915_gpu_raise - raise GPU frequency limit 4959 * 4960 * Raise the limit; IPS indicates we have thermal headroom. 4961 */ 4962 bool i915_gpu_raise(void) 4963 { 4964 struct drm_i915_private *dev_priv; 4965 bool ret = true; 4966 4967 lockmgr(&mchdev_lock, LK_EXCLUSIVE); 4968 if (!i915_mch_dev) { 4969 ret = false; 4970 goto out_unlock; 4971 } 4972 dev_priv = i915_mch_dev; 4973 4974 if (dev_priv->ips.max_delay > dev_priv->ips.fmax) 4975 dev_priv->ips.max_delay--; 4976 4977 out_unlock: 4978 lockmgr(&mchdev_lock, LK_RELEASE); 4979 4980 return ret; 4981 } 4982 4983 /** 4984 * i915_gpu_lower - lower GPU frequency limit 4985 * 4986 * IPS indicates we're close to a thermal limit, so throttle back the GPU 4987 * frequency maximum. 4988 */ 4989 bool i915_gpu_lower(void) 4990 { 4991 struct drm_i915_private *dev_priv; 4992 bool ret = true; 4993 4994 lockmgr(&mchdev_lock, LK_EXCLUSIVE); 4995 if (!i915_mch_dev) { 4996 ret = false; 4997 goto out_unlock; 4998 } 4999 dev_priv = i915_mch_dev; 5000 5001 if (dev_priv->ips.max_delay < dev_priv->ips.min_delay) 5002 dev_priv->ips.max_delay++; 5003 5004 out_unlock: 5005 lockmgr(&mchdev_lock, LK_RELEASE); 5006 5007 return ret; 5008 } 5009 5010 /** 5011 * i915_gpu_busy - indicate GPU business to IPS 5012 * 5013 * Tell the IPS driver whether or not the GPU is busy. 5014 */ 5015 bool i915_gpu_busy(void) 5016 { 5017 struct drm_i915_private *dev_priv; 5018 struct intel_engine_cs *ring; 5019 bool ret = false; 5020 int i; 5021 5022 lockmgr(&mchdev_lock, LK_EXCLUSIVE); 5023 if (!i915_mch_dev) 5024 goto out_unlock; 5025 dev_priv = i915_mch_dev; 5026 5027 for_each_ring(ring, dev_priv, i) 5028 ret |= !list_empty(&ring->request_list); 5029 5030 out_unlock: 5031 lockmgr(&mchdev_lock, LK_RELEASE); 5032 5033 return ret; 5034 } 5035 5036 /** 5037 * i915_gpu_turbo_disable - disable graphics turbo 5038 * 5039 * Disable graphics turbo by resetting the max frequency and setting the 5040 * current frequency to the default. 5041 */ 5042 bool i915_gpu_turbo_disable(void) 5043 { 5044 struct drm_i915_private *dev_priv; 5045 bool ret = true; 5046 5047 lockmgr(&mchdev_lock, LK_EXCLUSIVE); 5048 if (!i915_mch_dev) { 5049 ret = false; 5050 goto out_unlock; 5051 } 5052 dev_priv = i915_mch_dev; 5053 5054 dev_priv->ips.max_delay = dev_priv->ips.fstart; 5055 5056 if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart)) 5057 ret = false; 5058 5059 out_unlock: 5060 lockmgr(&mchdev_lock, LK_RELEASE); 5061 5062 return ret; 5063 } 5064 5065 #if 0 5066 /** 5067 * Tells the intel_ips driver that the i915 driver is now loaded, if 5068 * IPS got loaded first. 5069 * 5070 * This awkward dance is so that neither module has to depend on the 5071 * other in order for IPS to do the appropriate communication of 5072 * GPU turbo limits to i915. 5073 */ 5074 static void 5075 ips_ping_for_i915_load(void) 5076 { 5077 void (*link)(void); 5078 5079 link = symbol_get(ips_link_to_i915_driver); 5080 if (link) { 5081 link(); 5082 symbol_put(ips_link_to_i915_driver); 5083 } 5084 } 5085 #endif 5086 5087 void intel_gpu_ips_init(struct drm_i915_private *dev_priv) 5088 { 5089 /* We only register the i915 ips part with intel-ips once everything is 5090 * set up, to avoid intel-ips sneaking in and reading bogus values. */ 5091 lockmgr(&mchdev_lock, LK_EXCLUSIVE); 5092 i915_mch_dev = dev_priv; 5093 lockmgr(&mchdev_lock, LK_RELEASE); 5094 } 5095 5096 void intel_gpu_ips_teardown(void) 5097 { 5098 lockmgr(&mchdev_lock, LK_EXCLUSIVE); 5099 i915_mch_dev = NULL; 5100 lockmgr(&mchdev_lock, LK_RELEASE); 5101 } 5102 5103 static void intel_init_emon(struct drm_device *dev) 5104 { 5105 struct drm_i915_private *dev_priv = dev->dev_private; 5106 u32 lcfuse; 5107 u8 pxw[16]; 5108 int i; 5109 5110 /* Disable to program */ 5111 I915_WRITE(ECR, 0); 5112 POSTING_READ(ECR); 5113 5114 /* Program energy weights for various events */ 5115 I915_WRITE(SDEW, 0x15040d00); 5116 I915_WRITE(CSIEW0, 0x007f0000); 5117 I915_WRITE(CSIEW1, 0x1e220004); 5118 I915_WRITE(CSIEW2, 0x04000004); 5119 5120 for (i = 0; i < 5; i++) 5121 I915_WRITE(PEW + (i * 4), 0); 5122 for (i = 0; i < 3; i++) 5123 I915_WRITE(DEW + (i * 4), 0); 5124 5125 /* Program P-state weights to account for frequency power adjustment */ 5126 for (i = 0; i < 16; i++) { 5127 u32 pxvidfreq = I915_READ(PXVFREQ_BASE + (i * 4)); 5128 unsigned long freq = intel_pxfreq(pxvidfreq); 5129 unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >> 5130 PXVFREQ_PX_SHIFT; 5131 unsigned long val; 5132 5133 val = vid * vid; 5134 val *= (freq / 1000); 5135 val *= 255; 5136 val /= (127*127*900); 5137 if (val > 0xff) 5138 DRM_ERROR("bad pxval: %ld\n", val); 5139 pxw[i] = val; 5140 } 5141 /* Render standby states get 0 weight */ 5142 pxw[14] = 0; 5143 pxw[15] = 0; 5144 5145 for (i = 0; i < 4; i++) { 5146 u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) | 5147 (pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]); 5148 I915_WRITE(PXW + (i * 4), val); 5149 } 5150 5151 /* Adjust magic regs to magic values (more experimental results) */ 5152 I915_WRITE(OGW0, 0); 5153 I915_WRITE(OGW1, 0); 5154 I915_WRITE(EG0, 0x00007f00); 5155 I915_WRITE(EG1, 0x0000000e); 5156 I915_WRITE(EG2, 0x000e0000); 5157 I915_WRITE(EG3, 0x68000300); 5158 I915_WRITE(EG4, 0x42000000); 5159 I915_WRITE(EG5, 0x00140031); 5160 I915_WRITE(EG6, 0); 5161 I915_WRITE(EG7, 0); 5162 5163 for (i = 0; i < 8; i++) 5164 I915_WRITE(PXWL + (i * 4), 0); 5165 5166 /* Enable PMON + select events */ 5167 I915_WRITE(ECR, 0x80000019); 5168 5169 lcfuse = I915_READ(LCFUSE02); 5170 5171 dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK); 5172 } 5173 5174 void intel_init_gt_powersave(struct drm_device *dev) 5175 { 5176 i915.enable_rc6 = sanitize_rc6_option(dev, i915.enable_rc6); 5177 5178 if (IS_CHERRYVIEW(dev)) 5179 cherryview_init_gt_powersave(dev); 5180 else if (IS_VALLEYVIEW(dev)) 5181 valleyview_init_gt_powersave(dev); 5182 } 5183 5184 void intel_cleanup_gt_powersave(struct drm_device *dev) 5185 { 5186 if (IS_CHERRYVIEW(dev)) 5187 return; 5188 else if (IS_VALLEYVIEW(dev)) 5189 valleyview_cleanup_gt_powersave(dev); 5190 } 5191 5192 /** 5193 * intel_suspend_gt_powersave - suspend PM work and helper threads 5194 * @dev: drm device 5195 * 5196 * We don't want to disable RC6 or other features here, we just want 5197 * to make sure any work we've queued has finished and won't bother 5198 * us while we're suspended. 5199 */ 5200 void intel_suspend_gt_powersave(struct drm_device *dev) 5201 { 5202 struct drm_i915_private *dev_priv = dev->dev_private; 5203 5204 /* Interrupts should be disabled already to avoid re-arming. */ 5205 WARN_ON(intel_irqs_enabled(dev_priv)); 5206 5207 #if 0 5208 flush_delayed_work(&dev_priv->rps.delayed_resume_work); 5209 #endif 5210 5211 cancel_work_sync(&dev_priv->rps.work); 5212 5213 /* Force GPU to min freq during suspend */ 5214 gen6_rps_idle(dev_priv); 5215 } 5216 5217 void intel_disable_gt_powersave(struct drm_device *dev) 5218 { 5219 struct drm_i915_private *dev_priv = dev->dev_private; 5220 5221 /* Interrupts should be disabled already to avoid re-arming. */ 5222 WARN_ON(intel_irqs_enabled(dev_priv)); 5223 5224 if (IS_IRONLAKE_M(dev)) { 5225 ironlake_disable_drps(dev); 5226 ironlake_disable_rc6(dev); 5227 } else if (INTEL_INFO(dev)->gen >= 6) { 5228 intel_suspend_gt_powersave(dev); 5229 5230 mutex_lock(&dev_priv->rps.hw_lock); 5231 if (IS_CHERRYVIEW(dev)) 5232 cherryview_disable_rps(dev); 5233 else if (IS_VALLEYVIEW(dev)) 5234 valleyview_disable_rps(dev); 5235 else 5236 gen6_disable_rps(dev); 5237 dev_priv->rps.enabled = false; 5238 mutex_unlock(&dev_priv->rps.hw_lock); 5239 } 5240 } 5241 5242 static void intel_gen6_powersave_work(struct work_struct *work) 5243 { 5244 struct drm_i915_private *dev_priv = 5245 container_of(work, struct drm_i915_private, 5246 rps.delayed_resume_work.work); 5247 struct drm_device *dev = dev_priv->dev; 5248 5249 mutex_lock(&dev_priv->rps.hw_lock); 5250 5251 if (IS_CHERRYVIEW(dev)) { 5252 cherryview_enable_rps(dev); 5253 } else if (IS_VALLEYVIEW(dev)) { 5254 valleyview_enable_rps(dev); 5255 } else if (IS_BROADWELL(dev)) { 5256 gen8_enable_rps(dev); 5257 __gen6_update_ring_freq(dev); 5258 } else { 5259 gen6_enable_rps(dev); 5260 __gen6_update_ring_freq(dev); 5261 } 5262 dev_priv->rps.enabled = true; 5263 mutex_unlock(&dev_priv->rps.hw_lock); 5264 5265 intel_runtime_pm_put(dev_priv); 5266 } 5267 5268 void intel_enable_gt_powersave(struct drm_device *dev) 5269 { 5270 struct drm_i915_private *dev_priv = dev->dev_private; 5271 5272 if (IS_IRONLAKE_M(dev)) { 5273 mutex_lock(&dev->struct_mutex); 5274 ironlake_enable_drps(dev); 5275 ironlake_enable_rc6(dev); 5276 intel_init_emon(dev); 5277 mutex_unlock(&dev->struct_mutex); 5278 } else if (INTEL_INFO(dev)->gen >= 6) { 5279 /* 5280 * PCU communication is slow and this doesn't need to be 5281 * done at any specific time, so do this out of our fast path 5282 * to make resume and init faster. 5283 * 5284 * We depend on the HW RC6 power context save/restore 5285 * mechanism when entering D3 through runtime PM suspend. So 5286 * disable RPM until RPS/RC6 is properly setup. We can only 5287 * get here via the driver load/system resume/runtime resume 5288 * paths, so the _noresume version is enough (and in case of 5289 * runtime resume it's necessary). 5290 */ 5291 if (schedule_delayed_work(&dev_priv->rps.delayed_resume_work, 5292 round_jiffies_up_relative(HZ))) 5293 intel_runtime_pm_get_noresume(dev_priv); 5294 } 5295 } 5296 5297 void intel_reset_gt_powersave(struct drm_device *dev) 5298 { 5299 struct drm_i915_private *dev_priv = dev->dev_private; 5300 5301 dev_priv->rps.enabled = false; 5302 intel_enable_gt_powersave(dev); 5303 } 5304 5305 static void ibx_init_clock_gating(struct drm_device *dev) 5306 { 5307 struct drm_i915_private *dev_priv = dev->dev_private; 5308 5309 /* 5310 * On Ibex Peak and Cougar Point, we need to disable clock 5311 * gating for the panel power sequencer or it will fail to 5312 * start up when no ports are active. 5313 */ 5314 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE); 5315 } 5316 5317 static void g4x_disable_trickle_feed(struct drm_device *dev) 5318 { 5319 struct drm_i915_private *dev_priv = dev->dev_private; 5320 int pipe; 5321 5322 for_each_pipe(dev_priv, pipe) { 5323 I915_WRITE(DSPCNTR(pipe), 5324 I915_READ(DSPCNTR(pipe)) | 5325 DISPPLANE_TRICKLE_FEED_DISABLE); 5326 intel_flush_primary_plane(dev_priv, pipe); 5327 } 5328 } 5329 5330 static void ilk_init_lp_watermarks(struct drm_device *dev) 5331 { 5332 struct drm_i915_private *dev_priv = dev->dev_private; 5333 5334 I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN); 5335 I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN); 5336 I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN); 5337 5338 /* 5339 * Don't touch WM1S_LP_EN here. 5340 * Doing so could cause underruns. 5341 */ 5342 } 5343 5344 static void ironlake_init_clock_gating(struct drm_device *dev) 5345 { 5346 struct drm_i915_private *dev_priv = dev->dev_private; 5347 uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE; 5348 5349 /* 5350 * Required for FBC 5351 * WaFbcDisableDpfcClockGating:ilk 5352 */ 5353 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE | 5354 ILK_DPFCUNIT_CLOCK_GATE_DISABLE | 5355 ILK_DPFDUNIT_CLOCK_GATE_ENABLE; 5356 5357 I915_WRITE(PCH_3DCGDIS0, 5358 MARIUNIT_CLOCK_GATE_DISABLE | 5359 SVSMUNIT_CLOCK_GATE_DISABLE); 5360 I915_WRITE(PCH_3DCGDIS1, 5361 VFMUNIT_CLOCK_GATE_DISABLE); 5362 5363 /* 5364 * According to the spec the following bits should be set in 5365 * order to enable memory self-refresh 5366 * The bit 22/21 of 0x42004 5367 * The bit 5 of 0x42020 5368 * The bit 15 of 0x45000 5369 */ 5370 I915_WRITE(ILK_DISPLAY_CHICKEN2, 5371 (I915_READ(ILK_DISPLAY_CHICKEN2) | 5372 ILK_DPARB_GATE | ILK_VSDPFD_FULL)); 5373 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE; 5374 I915_WRITE(DISP_ARB_CTL, 5375 (I915_READ(DISP_ARB_CTL) | 5376 DISP_FBC_WM_DIS)); 5377 5378 ilk_init_lp_watermarks(dev); 5379 5380 /* 5381 * Based on the document from hardware guys the following bits 5382 * should be set unconditionally in order to enable FBC. 5383 * The bit 22 of 0x42000 5384 * The bit 22 of 0x42004 5385 * The bit 7,8,9 of 0x42020. 5386 */ 5387 if (IS_IRONLAKE_M(dev)) { 5388 /* WaFbcAsynchFlipDisableFbcQueue:ilk */ 5389 I915_WRITE(ILK_DISPLAY_CHICKEN1, 5390 I915_READ(ILK_DISPLAY_CHICKEN1) | 5391 ILK_FBCQ_DIS); 5392 I915_WRITE(ILK_DISPLAY_CHICKEN2, 5393 I915_READ(ILK_DISPLAY_CHICKEN2) | 5394 ILK_DPARB_GATE); 5395 } 5396 5397 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate); 5398 5399 I915_WRITE(ILK_DISPLAY_CHICKEN2, 5400 I915_READ(ILK_DISPLAY_CHICKEN2) | 5401 ILK_ELPIN_409_SELECT); 5402 I915_WRITE(_3D_CHICKEN2, 5403 _3D_CHICKEN2_WM_READ_PIPELINED << 16 | 5404 _3D_CHICKEN2_WM_READ_PIPELINED); 5405 5406 /* WaDisableRenderCachePipelinedFlush:ilk */ 5407 I915_WRITE(CACHE_MODE_0, 5408 _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE)); 5409 5410 /* WaDisable_RenderCache_OperationalFlush:ilk */ 5411 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); 5412 5413 g4x_disable_trickle_feed(dev); 5414 5415 ibx_init_clock_gating(dev); 5416 } 5417 5418 static void cpt_init_clock_gating(struct drm_device *dev) 5419 { 5420 struct drm_i915_private *dev_priv = dev->dev_private; 5421 int pipe; 5422 uint32_t val; 5423 5424 /* 5425 * On Ibex Peak and Cougar Point, we need to disable clock 5426 * gating for the panel power sequencer or it will fail to 5427 * start up when no ports are active. 5428 */ 5429 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE | 5430 PCH_DPLUNIT_CLOCK_GATE_DISABLE | 5431 PCH_CPUNIT_CLOCK_GATE_DISABLE); 5432 I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) | 5433 DPLS_EDP_PPS_FIX_DIS); 5434 /* The below fixes the weird display corruption, a few pixels shifted 5435 * downward, on (only) LVDS of some HP laptops with IVY. 5436 */ 5437 for_each_pipe(dev_priv, pipe) { 5438 val = I915_READ(TRANS_CHICKEN2(pipe)); 5439 val |= TRANS_CHICKEN2_TIMING_OVERRIDE; 5440 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED; 5441 if (dev_priv->vbt.fdi_rx_polarity_inverted) 5442 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED; 5443 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK; 5444 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER; 5445 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH; 5446 I915_WRITE(TRANS_CHICKEN2(pipe), val); 5447 } 5448 /* WADP0ClockGatingDisable */ 5449 for_each_pipe(dev_priv, pipe) { 5450 I915_WRITE(TRANS_CHICKEN1(pipe), 5451 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE); 5452 } 5453 } 5454 5455 static void gen6_check_mch_setup(struct drm_device *dev) 5456 { 5457 struct drm_i915_private *dev_priv = dev->dev_private; 5458 uint32_t tmp; 5459 5460 tmp = I915_READ(MCH_SSKPD); 5461 if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL) 5462 DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n", 5463 tmp); 5464 } 5465 5466 static void gen6_init_clock_gating(struct drm_device *dev) 5467 { 5468 struct drm_i915_private *dev_priv = dev->dev_private; 5469 uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE; 5470 5471 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate); 5472 5473 I915_WRITE(ILK_DISPLAY_CHICKEN2, 5474 I915_READ(ILK_DISPLAY_CHICKEN2) | 5475 ILK_ELPIN_409_SELECT); 5476 5477 /* WaDisableHiZPlanesWhenMSAAEnabled:snb */ 5478 I915_WRITE(_3D_CHICKEN, 5479 _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB)); 5480 5481 /* WaDisable_RenderCache_OperationalFlush:snb */ 5482 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); 5483 5484 /* 5485 * BSpec recoomends 8x4 when MSAA is used, 5486 * however in practice 16x4 seems fastest. 5487 * 5488 * Note that PS/WM thread counts depend on the WIZ hashing 5489 * disable bit, which we don't touch here, but it's good 5490 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). 5491 */ 5492 I915_WRITE(GEN6_GT_MODE, 5493 GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4); 5494 5495 ilk_init_lp_watermarks(dev); 5496 5497 I915_WRITE(CACHE_MODE_0, 5498 _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB)); 5499 5500 I915_WRITE(GEN6_UCGCTL1, 5501 I915_READ(GEN6_UCGCTL1) | 5502 GEN6_BLBUNIT_CLOCK_GATE_DISABLE | 5503 GEN6_CSUNIT_CLOCK_GATE_DISABLE); 5504 5505 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock 5506 * gating disable must be set. Failure to set it results in 5507 * flickering pixels due to Z write ordering failures after 5508 * some amount of runtime in the Mesa "fire" demo, and Unigine 5509 * Sanctuary and Tropics, and apparently anything else with 5510 * alpha test or pixel discard. 5511 * 5512 * According to the spec, bit 11 (RCCUNIT) must also be set, 5513 * but we didn't debug actual testcases to find it out. 5514 * 5515 * WaDisableRCCUnitClockGating:snb 5516 * WaDisableRCPBUnitClockGating:snb 5517 */ 5518 I915_WRITE(GEN6_UCGCTL2, 5519 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE | 5520 GEN6_RCCUNIT_CLOCK_GATE_DISABLE); 5521 5522 /* WaStripsFansDisableFastClipPerformanceFix:snb */ 5523 I915_WRITE(_3D_CHICKEN3, 5524 _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL)); 5525 5526 /* 5527 * Bspec says: 5528 * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and 5529 * 3DSTATE_SF number of SF output attributes is more than 16." 5530 */ 5531 I915_WRITE(_3D_CHICKEN3, 5532 _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH)); 5533 5534 /* 5535 * According to the spec the following bits should be 5536 * set in order to enable memory self-refresh and fbc: 5537 * The bit21 and bit22 of 0x42000 5538 * The bit21 and bit22 of 0x42004 5539 * The bit5 and bit7 of 0x42020 5540 * The bit14 of 0x70180 5541 * The bit14 of 0x71180 5542 * 5543 * WaFbcAsynchFlipDisableFbcQueue:snb 5544 */ 5545 I915_WRITE(ILK_DISPLAY_CHICKEN1, 5546 I915_READ(ILK_DISPLAY_CHICKEN1) | 5547 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS); 5548 I915_WRITE(ILK_DISPLAY_CHICKEN2, 5549 I915_READ(ILK_DISPLAY_CHICKEN2) | 5550 ILK_DPARB_GATE | ILK_VSDPFD_FULL); 5551 I915_WRITE(ILK_DSPCLK_GATE_D, 5552 I915_READ(ILK_DSPCLK_GATE_D) | 5553 ILK_DPARBUNIT_CLOCK_GATE_ENABLE | 5554 ILK_DPFDUNIT_CLOCK_GATE_ENABLE); 5555 5556 g4x_disable_trickle_feed(dev); 5557 5558 cpt_init_clock_gating(dev); 5559 5560 gen6_check_mch_setup(dev); 5561 } 5562 5563 static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv) 5564 { 5565 uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE); 5566 5567 /* 5568 * WaVSThreadDispatchOverride:ivb,vlv 5569 * 5570 * This actually overrides the dispatch 5571 * mode for all thread types. 5572 */ 5573 reg &= ~GEN7_FF_SCHED_MASK; 5574 reg |= GEN7_FF_TS_SCHED_HW; 5575 reg |= GEN7_FF_VS_SCHED_HW; 5576 reg |= GEN7_FF_DS_SCHED_HW; 5577 5578 I915_WRITE(GEN7_FF_THREAD_MODE, reg); 5579 } 5580 5581 static void lpt_init_clock_gating(struct drm_device *dev) 5582 { 5583 struct drm_i915_private *dev_priv = dev->dev_private; 5584 5585 /* 5586 * TODO: this bit should only be enabled when really needed, then 5587 * disabled when not needed anymore in order to save power. 5588 */ 5589 if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) 5590 I915_WRITE(SOUTH_DSPCLK_GATE_D, 5591 I915_READ(SOUTH_DSPCLK_GATE_D) | 5592 PCH_LP_PARTITION_LEVEL_DISABLE); 5593 5594 /* WADPOClockGatingDisable:hsw */ 5595 I915_WRITE(_TRANSA_CHICKEN1, 5596 I915_READ(_TRANSA_CHICKEN1) | 5597 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE); 5598 } 5599 5600 static void lpt_suspend_hw(struct drm_device *dev) 5601 { 5602 struct drm_i915_private *dev_priv = dev->dev_private; 5603 5604 if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) { 5605 uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D); 5606 5607 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE; 5608 I915_WRITE(SOUTH_DSPCLK_GATE_D, val); 5609 } 5610 } 5611 5612 static void broadwell_init_clock_gating(struct drm_device *dev) 5613 { 5614 struct drm_i915_private *dev_priv = dev->dev_private; 5615 enum i915_pipe pipe; 5616 5617 I915_WRITE(WM3_LP_ILK, 0); 5618 I915_WRITE(WM2_LP_ILK, 0); 5619 I915_WRITE(WM1_LP_ILK, 0); 5620 5621 /* FIXME(BDW): Check all the w/a, some might only apply to 5622 * pre-production hw. */ 5623 5624 5625 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_BWGTLB_DISABLE)); 5626 5627 I915_WRITE(_3D_CHICKEN3, 5628 _MASKED_BIT_ENABLE(_3D_CHICKEN_SDE_LIMIT_FIFO_POLY_DEPTH(2))); 5629 5630 5631 /* WaSwitchSolVfFArbitrationPriority:bdw */ 5632 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL); 5633 5634 /* WaPsrDPAMaskVBlankInSRD:bdw */ 5635 I915_WRITE(CHICKEN_PAR1_1, 5636 I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD); 5637 5638 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */ 5639 for_each_pipe(dev_priv, pipe) { 5640 I915_WRITE(CHICKEN_PIPESL_1(pipe), 5641 I915_READ(CHICKEN_PIPESL_1(pipe)) | 5642 BDW_DPRS_MASK_VBLANK_SRD); 5643 } 5644 5645 /* WaVSRefCountFullforceMissDisable:bdw */ 5646 /* WaDSRefCountFullforceMissDisable:bdw */ 5647 I915_WRITE(GEN7_FF_THREAD_MODE, 5648 I915_READ(GEN7_FF_THREAD_MODE) & 5649 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME)); 5650 5651 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL, 5652 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE)); 5653 5654 /* WaDisableSDEUnitClockGating:bdw */ 5655 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | 5656 GEN8_SDEUNIT_CLOCK_GATE_DISABLE); 5657 5658 lpt_init_clock_gating(dev); 5659 } 5660 5661 static void haswell_init_clock_gating(struct drm_device *dev) 5662 { 5663 struct drm_i915_private *dev_priv = dev->dev_private; 5664 5665 ilk_init_lp_watermarks(dev); 5666 5667 /* L3 caching of data atomics doesn't work -- disable it. */ 5668 I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE); 5669 I915_WRITE(HSW_ROW_CHICKEN3, 5670 _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE)); 5671 5672 /* This is required by WaCatErrorRejectionIssue:hsw */ 5673 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG, 5674 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) | 5675 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB); 5676 5677 /* WaVSRefCountFullforceMissDisable:hsw */ 5678 I915_WRITE(GEN7_FF_THREAD_MODE, 5679 I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME); 5680 5681 /* WaDisable_RenderCache_OperationalFlush:hsw */ 5682 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); 5683 5684 /* enable HiZ Raw Stall Optimization */ 5685 I915_WRITE(CACHE_MODE_0_GEN7, 5686 _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE)); 5687 5688 /* WaDisable4x2SubspanOptimization:hsw */ 5689 I915_WRITE(CACHE_MODE_1, 5690 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE)); 5691 5692 /* 5693 * BSpec recommends 8x4 when MSAA is used, 5694 * however in practice 16x4 seems fastest. 5695 * 5696 * Note that PS/WM thread counts depend on the WIZ hashing 5697 * disable bit, which we don't touch here, but it's good 5698 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). 5699 */ 5700 I915_WRITE(GEN7_GT_MODE, 5701 GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4); 5702 5703 /* WaSwitchSolVfFArbitrationPriority:hsw */ 5704 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL); 5705 5706 /* WaRsPkgCStateDisplayPMReq:hsw */ 5707 I915_WRITE(CHICKEN_PAR1_1, 5708 I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES); 5709 5710 lpt_init_clock_gating(dev); 5711 } 5712 5713 static void ivybridge_init_clock_gating(struct drm_device *dev) 5714 { 5715 struct drm_i915_private *dev_priv = dev->dev_private; 5716 uint32_t snpcr; 5717 5718 ilk_init_lp_watermarks(dev); 5719 5720 I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE); 5721 5722 /* WaDisableEarlyCull:ivb */ 5723 I915_WRITE(_3D_CHICKEN3, 5724 _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL)); 5725 5726 /* WaDisableBackToBackFlipFix:ivb */ 5727 I915_WRITE(IVB_CHICKEN3, 5728 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE | 5729 CHICKEN3_DGMG_DONE_FIX_DISABLE); 5730 5731 /* WaDisablePSDDualDispatchEnable:ivb */ 5732 if (IS_IVB_GT1(dev)) 5733 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1, 5734 _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE)); 5735 5736 /* WaDisable_RenderCache_OperationalFlush:ivb */ 5737 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); 5738 5739 /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */ 5740 I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1, 5741 GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC); 5742 5743 /* WaApplyL3ControlAndL3ChickenMode:ivb */ 5744 I915_WRITE(GEN7_L3CNTLREG1, 5745 GEN7_WA_FOR_GEN7_L3_CONTROL); 5746 I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER, 5747 GEN7_WA_L3_CHICKEN_MODE); 5748 if (IS_IVB_GT1(dev)) 5749 I915_WRITE(GEN7_ROW_CHICKEN2, 5750 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); 5751 else { 5752 /* must write both registers */ 5753 I915_WRITE(GEN7_ROW_CHICKEN2, 5754 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); 5755 I915_WRITE(GEN7_ROW_CHICKEN2_GT2, 5756 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); 5757 } 5758 5759 /* WaForceL3Serialization:ivb */ 5760 I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) & 5761 ~L3SQ_URB_READ_CAM_MATCH_DISABLE); 5762 5763 /* 5764 * According to the spec, bit 13 (RCZUNIT) must be set on IVB. 5765 * This implements the WaDisableRCZUnitClockGating:ivb workaround. 5766 */ 5767 I915_WRITE(GEN6_UCGCTL2, 5768 GEN6_RCZUNIT_CLOCK_GATE_DISABLE); 5769 5770 /* This is required by WaCatErrorRejectionIssue:ivb */ 5771 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG, 5772 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) | 5773 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB); 5774 5775 g4x_disable_trickle_feed(dev); 5776 5777 gen7_setup_fixed_func_scheduler(dev_priv); 5778 5779 if (0) { /* causes HiZ corruption on ivb:gt1 */ 5780 /* enable HiZ Raw Stall Optimization */ 5781 I915_WRITE(CACHE_MODE_0_GEN7, 5782 _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE)); 5783 } 5784 5785 /* WaDisable4x2SubspanOptimization:ivb */ 5786 I915_WRITE(CACHE_MODE_1, 5787 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE)); 5788 5789 /* 5790 * BSpec recommends 8x4 when MSAA is used, 5791 * however in practice 16x4 seems fastest. 5792 * 5793 * Note that PS/WM thread counts depend on the WIZ hashing 5794 * disable bit, which we don't touch here, but it's good 5795 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). 5796 */ 5797 I915_WRITE(GEN7_GT_MODE, 5798 GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4); 5799 5800 snpcr = I915_READ(GEN6_MBCUNIT_SNPCR); 5801 snpcr &= ~GEN6_MBC_SNPCR_MASK; 5802 snpcr |= GEN6_MBC_SNPCR_MED; 5803 I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr); 5804 5805 if (!HAS_PCH_NOP(dev)) 5806 cpt_init_clock_gating(dev); 5807 5808 gen6_check_mch_setup(dev); 5809 } 5810 5811 static void valleyview_init_clock_gating(struct drm_device *dev) 5812 { 5813 struct drm_i915_private *dev_priv = dev->dev_private; 5814 5815 I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE); 5816 5817 /* WaDisableEarlyCull:vlv */ 5818 I915_WRITE(_3D_CHICKEN3, 5819 _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL)); 5820 5821 /* WaDisableBackToBackFlipFix:vlv */ 5822 I915_WRITE(IVB_CHICKEN3, 5823 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE | 5824 CHICKEN3_DGMG_DONE_FIX_DISABLE); 5825 5826 /* WaPsdDispatchEnable:vlv */ 5827 /* WaDisablePSDDualDispatchEnable:vlv */ 5828 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1, 5829 _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP | 5830 GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE)); 5831 5832 /* WaDisable_RenderCache_OperationalFlush:vlv */ 5833 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); 5834 5835 /* WaForceL3Serialization:vlv */ 5836 I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) & 5837 ~L3SQ_URB_READ_CAM_MATCH_DISABLE); 5838 5839 /* WaDisableDopClockGating:vlv */ 5840 I915_WRITE(GEN7_ROW_CHICKEN2, 5841 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); 5842 5843 /* This is required by WaCatErrorRejectionIssue:vlv */ 5844 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG, 5845 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) | 5846 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB); 5847 5848 gen7_setup_fixed_func_scheduler(dev_priv); 5849 5850 /* 5851 * According to the spec, bit 13 (RCZUNIT) must be set on IVB. 5852 * This implements the WaDisableRCZUnitClockGating:vlv workaround. 5853 */ 5854 I915_WRITE(GEN6_UCGCTL2, 5855 GEN6_RCZUNIT_CLOCK_GATE_DISABLE); 5856 5857 /* WaDisableL3Bank2xClockGate:vlv 5858 * Disabling L3 clock gating- MMIO 940c[25] = 1 5859 * Set bit 25, to disable L3_BANK_2x_CLK_GATING */ 5860 I915_WRITE(GEN7_UCGCTL4, 5861 I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE); 5862 5863 I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE); 5864 5865 /* 5866 * BSpec says this must be set, even though 5867 * WaDisable4x2SubspanOptimization isn't listed for VLV. 5868 */ 5869 I915_WRITE(CACHE_MODE_1, 5870 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE)); 5871 5872 /* 5873 * WaIncreaseL3CreditsForVLVB0:vlv 5874 * This is the hardware default actually. 5875 */ 5876 I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE); 5877 5878 /* 5879 * WaDisableVLVClockGating_VBIIssue:vlv 5880 * Disable clock gating on th GCFG unit to prevent a delay 5881 * in the reporting of vblank events. 5882 */ 5883 I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS); 5884 } 5885 5886 static void cherryview_init_clock_gating(struct drm_device *dev) 5887 { 5888 struct drm_i915_private *dev_priv = dev->dev_private; 5889 5890 I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE); 5891 5892 I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE); 5893 5894 /* WaVSRefCountFullforceMissDisable:chv */ 5895 /* WaDSRefCountFullforceMissDisable:chv */ 5896 I915_WRITE(GEN7_FF_THREAD_MODE, 5897 I915_READ(GEN7_FF_THREAD_MODE) & 5898 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME)); 5899 5900 /* WaDisableSemaphoreAndSyncFlipWait:chv */ 5901 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL, 5902 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE)); 5903 5904 /* WaDisableCSUnitClockGating:chv */ 5905 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) | 5906 GEN6_CSUNIT_CLOCK_GATE_DISABLE); 5907 5908 /* WaDisableSDEUnitClockGating:chv */ 5909 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | 5910 GEN8_SDEUNIT_CLOCK_GATE_DISABLE); 5911 5912 /* WaDisableGunitClockGating:chv (pre-production hw) */ 5913 I915_WRITE(VLV_GUNIT_CLOCK_GATE, I915_READ(VLV_GUNIT_CLOCK_GATE) | 5914 GINT_DIS); 5915 5916 /* WaDisableFfDopClockGating:chv (pre-production hw) */ 5917 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL, 5918 _MASKED_BIT_ENABLE(GEN8_FF_DOP_CLOCK_GATE_DISABLE)); 5919 5920 /* WaDisableDopClockGating:chv (pre-production hw) */ 5921 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) | 5922 GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE); 5923 } 5924 5925 static void g4x_init_clock_gating(struct drm_device *dev) 5926 { 5927 struct drm_i915_private *dev_priv = dev->dev_private; 5928 uint32_t dspclk_gate; 5929 5930 I915_WRITE(RENCLK_GATE_D1, 0); 5931 I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE | 5932 GS_UNIT_CLOCK_GATE_DISABLE | 5933 CL_UNIT_CLOCK_GATE_DISABLE); 5934 I915_WRITE(RAMCLK_GATE_D, 0); 5935 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE | 5936 OVRUNIT_CLOCK_GATE_DISABLE | 5937 OVCUNIT_CLOCK_GATE_DISABLE; 5938 if (IS_GM45(dev)) 5939 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE; 5940 I915_WRITE(DSPCLK_GATE_D, dspclk_gate); 5941 5942 /* WaDisableRenderCachePipelinedFlush */ 5943 I915_WRITE(CACHE_MODE_0, 5944 _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE)); 5945 5946 /* WaDisable_RenderCache_OperationalFlush:g4x */ 5947 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); 5948 5949 g4x_disable_trickle_feed(dev); 5950 } 5951 5952 static void crestline_init_clock_gating(struct drm_device *dev) 5953 { 5954 struct drm_i915_private *dev_priv = dev->dev_private; 5955 5956 I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE); 5957 I915_WRITE(RENCLK_GATE_D2, 0); 5958 I915_WRITE(DSPCLK_GATE_D, 0); 5959 I915_WRITE(RAMCLK_GATE_D, 0); 5960 I915_WRITE16(DEUC, 0); 5961 I915_WRITE(MI_ARB_STATE, 5962 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE)); 5963 5964 /* WaDisable_RenderCache_OperationalFlush:gen4 */ 5965 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); 5966 } 5967 5968 static void broadwater_init_clock_gating(struct drm_device *dev) 5969 { 5970 struct drm_i915_private *dev_priv = dev->dev_private; 5971 5972 I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE | 5973 I965_RCC_CLOCK_GATE_DISABLE | 5974 I965_RCPB_CLOCK_GATE_DISABLE | 5975 I965_ISC_CLOCK_GATE_DISABLE | 5976 I965_FBC_CLOCK_GATE_DISABLE); 5977 I915_WRITE(RENCLK_GATE_D2, 0); 5978 I915_WRITE(MI_ARB_STATE, 5979 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE)); 5980 5981 /* WaDisable_RenderCache_OperationalFlush:gen4 */ 5982 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); 5983 } 5984 5985 static void gen3_init_clock_gating(struct drm_device *dev) 5986 { 5987 struct drm_i915_private *dev_priv = dev->dev_private; 5988 u32 dstate = I915_READ(D_STATE); 5989 5990 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING | 5991 DSTATE_DOT_CLOCK_GATING; 5992 I915_WRITE(D_STATE, dstate); 5993 5994 if (IS_PINEVIEW(dev)) 5995 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY)); 5996 5997 /* IIR "flip pending" means done if this bit is set */ 5998 I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE)); 5999 6000 /* interrupts should cause a wake up from C3 */ 6001 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN)); 6002 6003 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ 6004 I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE)); 6005 6006 I915_WRITE(MI_ARB_STATE, 6007 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE)); 6008 } 6009 6010 static void i85x_init_clock_gating(struct drm_device *dev) 6011 { 6012 struct drm_i915_private *dev_priv = dev->dev_private; 6013 6014 I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE); 6015 6016 /* interrupts should cause a wake up from C3 */ 6017 I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) | 6018 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE)); 6019 6020 I915_WRITE(MEM_MODE, 6021 _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE)); 6022 } 6023 6024 static void i830_init_clock_gating(struct drm_device *dev) 6025 { 6026 struct drm_i915_private *dev_priv = dev->dev_private; 6027 6028 I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE); 6029 6030 I915_WRITE(MEM_MODE, 6031 _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) | 6032 _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE)); 6033 } 6034 6035 void intel_init_clock_gating(struct drm_device *dev) 6036 { 6037 struct drm_i915_private *dev_priv = dev->dev_private; 6038 6039 dev_priv->display.init_clock_gating(dev); 6040 } 6041 6042 void intel_suspend_hw(struct drm_device *dev) 6043 { 6044 if (HAS_PCH_LPT(dev)) 6045 lpt_suspend_hw(dev); 6046 } 6047 6048 #define for_each_power_well(i, power_well, domain_mask, power_domains) \ 6049 for (i = 0; \ 6050 i < (power_domains)->power_well_count && \ 6051 ((power_well) = &(power_domains)->power_wells[i]); \ 6052 i++) \ 6053 if ((power_well)->domains & (domain_mask)) 6054 6055 #define for_each_power_well_rev(i, power_well, domain_mask, power_domains) \ 6056 for (i = (power_domains)->power_well_count - 1; \ 6057 i >= 0 && ((power_well) = &(power_domains)->power_wells[i]);\ 6058 i--) \ 6059 if ((power_well)->domains & (domain_mask)) 6060 6061 /** 6062 * We should only use the power well if we explicitly asked the hardware to 6063 * enable it, so check if it's enabled and also check if we've requested it to 6064 * be enabled. 6065 */ 6066 static bool hsw_power_well_enabled(struct drm_i915_private *dev_priv, 6067 struct i915_power_well *power_well) 6068 { 6069 return I915_READ(HSW_PWR_WELL_DRIVER) == 6070 (HSW_PWR_WELL_ENABLE_REQUEST | HSW_PWR_WELL_STATE_ENABLED); 6071 } 6072 6073 bool intel_display_power_enabled_unlocked(struct drm_i915_private *dev_priv, 6074 enum intel_display_power_domain domain) 6075 { 6076 struct i915_power_domains *power_domains; 6077 struct i915_power_well *power_well; 6078 bool is_enabled; 6079 int i; 6080 6081 if (dev_priv->pm.suspended) 6082 return false; 6083 6084 power_domains = &dev_priv->power_domains; 6085 6086 is_enabled = true; 6087 6088 for_each_power_well_rev(i, power_well, BIT(domain), power_domains) { 6089 if (power_well->always_on) 6090 continue; 6091 6092 if (!power_well->hw_enabled) { 6093 is_enabled = false; 6094 break; 6095 } 6096 } 6097 6098 return is_enabled; 6099 } 6100 6101 bool intel_display_power_enabled(struct drm_i915_private *dev_priv, 6102 enum intel_display_power_domain domain) 6103 { 6104 struct i915_power_domains *power_domains; 6105 bool ret; 6106 6107 power_domains = &dev_priv->power_domains; 6108 6109 mutex_lock(&power_domains->lock); 6110 ret = intel_display_power_enabled_unlocked(dev_priv, domain); 6111 mutex_unlock(&power_domains->lock); 6112 6113 return ret; 6114 } 6115 6116 /* 6117 * Starting with Haswell, we have a "Power Down Well" that can be turned off 6118 * when not needed anymore. We have 4 registers that can request the power well 6119 * to be enabled, and it will only be disabled if none of the registers is 6120 * requesting it to be enabled. 6121 */ 6122 static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv) 6123 { 6124 struct drm_device *dev = dev_priv->dev; 6125 6126 /* 6127 * After we re-enable the power well, if we touch VGA register 0x3d5 6128 * we'll get unclaimed register interrupts. This stops after we write 6129 * anything to the VGA MSR register. The vgacon module uses this 6130 * register all the time, so if we unbind our driver and, as a 6131 * consequence, bind vgacon, we'll get stuck in an infinite loop at 6132 * console_unlock(). So make here we touch the VGA MSR register, making 6133 * sure vgacon can keep working normally without triggering interrupts 6134 * and error messages. 6135 */ 6136 #if 0 6137 vga_get_uninterruptible(dev->pdev, VGA_RSRC_LEGACY_IO); 6138 outb(inb(VGA_MSR_READ), VGA_MSR_WRITE); 6139 vga_put(dev->pdev, VGA_RSRC_LEGACY_IO); 6140 #endif 6141 6142 if (IS_BROADWELL(dev)) 6143 gen8_irq_power_well_post_enable(dev_priv); 6144 } 6145 6146 static void hsw_set_power_well(struct drm_i915_private *dev_priv, 6147 struct i915_power_well *power_well, bool enable) 6148 { 6149 bool is_enabled, enable_requested; 6150 uint32_t tmp; 6151 6152 tmp = I915_READ(HSW_PWR_WELL_DRIVER); 6153 is_enabled = tmp & HSW_PWR_WELL_STATE_ENABLED; 6154 enable_requested = tmp & HSW_PWR_WELL_ENABLE_REQUEST; 6155 6156 if (enable) { 6157 if (!enable_requested) 6158 I915_WRITE(HSW_PWR_WELL_DRIVER, 6159 HSW_PWR_WELL_ENABLE_REQUEST); 6160 6161 if (!is_enabled) { 6162 DRM_DEBUG_KMS("Enabling power well\n"); 6163 if (wait_for((I915_READ(HSW_PWR_WELL_DRIVER) & 6164 HSW_PWR_WELL_STATE_ENABLED), 20)) 6165 DRM_ERROR("Timeout enabling power well\n"); 6166 } 6167 6168 hsw_power_well_post_enable(dev_priv); 6169 } else { 6170 if (enable_requested) { 6171 I915_WRITE(HSW_PWR_WELL_DRIVER, 0); 6172 POSTING_READ(HSW_PWR_WELL_DRIVER); 6173 DRM_DEBUG_KMS("Requesting to disable the power well\n"); 6174 } 6175 } 6176 } 6177 6178 static void hsw_power_well_sync_hw(struct drm_i915_private *dev_priv, 6179 struct i915_power_well *power_well) 6180 { 6181 hsw_set_power_well(dev_priv, power_well, power_well->count > 0); 6182 6183 /* 6184 * We're taking over the BIOS, so clear any requests made by it since 6185 * the driver is in charge now. 6186 */ 6187 if (I915_READ(HSW_PWR_WELL_BIOS) & HSW_PWR_WELL_ENABLE_REQUEST) 6188 I915_WRITE(HSW_PWR_WELL_BIOS, 0); 6189 } 6190 6191 static void hsw_power_well_enable(struct drm_i915_private *dev_priv, 6192 struct i915_power_well *power_well) 6193 { 6194 hsw_set_power_well(dev_priv, power_well, true); 6195 } 6196 6197 static void hsw_power_well_disable(struct drm_i915_private *dev_priv, 6198 struct i915_power_well *power_well) 6199 { 6200 hsw_set_power_well(dev_priv, power_well, false); 6201 } 6202 6203 static void i9xx_always_on_power_well_noop(struct drm_i915_private *dev_priv, 6204 struct i915_power_well *power_well) 6205 { 6206 } 6207 6208 static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv, 6209 struct i915_power_well *power_well) 6210 { 6211 return true; 6212 } 6213 6214 static void vlv_set_power_well(struct drm_i915_private *dev_priv, 6215 struct i915_power_well *power_well, bool enable) 6216 { 6217 enum punit_power_well power_well_id = power_well->data; 6218 u32 mask; 6219 u32 state; 6220 u32 ctrl; 6221 6222 mask = PUNIT_PWRGT_MASK(power_well_id); 6223 state = enable ? PUNIT_PWRGT_PWR_ON(power_well_id) : 6224 PUNIT_PWRGT_PWR_GATE(power_well_id); 6225 6226 mutex_lock(&dev_priv->rps.hw_lock); 6227 6228 #define COND \ 6229 ((vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask) == state) 6230 6231 if (COND) 6232 goto out; 6233 6234 ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL); 6235 ctrl &= ~mask; 6236 ctrl |= state; 6237 vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, ctrl); 6238 6239 if (wait_for(COND, 100)) 6240 DRM_ERROR("timout setting power well state %08x (%08x)\n", 6241 state, 6242 vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL)); 6243 6244 #undef COND 6245 6246 out: 6247 mutex_unlock(&dev_priv->rps.hw_lock); 6248 } 6249 6250 static void vlv_power_well_sync_hw(struct drm_i915_private *dev_priv, 6251 struct i915_power_well *power_well) 6252 { 6253 vlv_set_power_well(dev_priv, power_well, power_well->count > 0); 6254 } 6255 6256 static void vlv_power_well_enable(struct drm_i915_private *dev_priv, 6257 struct i915_power_well *power_well) 6258 { 6259 vlv_set_power_well(dev_priv, power_well, true); 6260 } 6261 6262 static void vlv_power_well_disable(struct drm_i915_private *dev_priv, 6263 struct i915_power_well *power_well) 6264 { 6265 vlv_set_power_well(dev_priv, power_well, false); 6266 } 6267 6268 static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv, 6269 struct i915_power_well *power_well) 6270 { 6271 int power_well_id = power_well->data; 6272 bool enabled = false; 6273 u32 mask; 6274 u32 state; 6275 u32 ctrl; 6276 6277 mask = PUNIT_PWRGT_MASK(power_well_id); 6278 ctrl = PUNIT_PWRGT_PWR_ON(power_well_id); 6279 6280 mutex_lock(&dev_priv->rps.hw_lock); 6281 6282 state = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask; 6283 /* 6284 * We only ever set the power-on and power-gate states, anything 6285 * else is unexpected. 6286 */ 6287 WARN_ON(state != PUNIT_PWRGT_PWR_ON(power_well_id) && 6288 state != PUNIT_PWRGT_PWR_GATE(power_well_id)); 6289 if (state == ctrl) 6290 enabled = true; 6291 6292 /* 6293 * A transient state at this point would mean some unexpected party 6294 * is poking at the power controls too. 6295 */ 6296 ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL) & mask; 6297 WARN_ON(ctrl != state); 6298 6299 mutex_unlock(&dev_priv->rps.hw_lock); 6300 6301 return enabled; 6302 } 6303 6304 static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv, 6305 struct i915_power_well *power_well) 6306 { 6307 WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D); 6308 6309 vlv_set_power_well(dev_priv, power_well, true); 6310 6311 lockmgr(&dev_priv->irq_lock, LK_EXCLUSIVE); 6312 valleyview_enable_display_irqs(dev_priv); 6313 lockmgr(&dev_priv->irq_lock, LK_RELEASE); 6314 6315 /* 6316 * During driver initialization/resume we can avoid restoring the 6317 * part of the HW/SW state that will be inited anyway explicitly. 6318 */ 6319 if (dev_priv->power_domains.initializing) 6320 return; 6321 6322 intel_hpd_init(dev_priv->dev); 6323 6324 i915_redisable_vga_power_on(dev_priv->dev); 6325 } 6326 6327 static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv, 6328 struct i915_power_well *power_well) 6329 { 6330 WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D); 6331 6332 lockmgr(&dev_priv->irq_lock, LK_EXCLUSIVE); 6333 valleyview_disable_display_irqs(dev_priv); 6334 lockmgr(&dev_priv->irq_lock, LK_RELEASE); 6335 6336 vlv_set_power_well(dev_priv, power_well, false); 6337 6338 vlv_power_sequencer_reset(dev_priv); 6339 } 6340 6341 static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv, 6342 struct i915_power_well *power_well) 6343 { 6344 WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC); 6345 6346 /* 6347 * Enable the CRI clock source so we can get at the 6348 * display and the reference clock for VGA 6349 * hotplug / manual detection. 6350 */ 6351 I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) | 6352 DPLL_REFA_CLK_ENABLE_VLV | DPLL_INTEGRATED_CRI_CLK_VLV); 6353 udelay(1); /* >10ns for cmnreset, >0ns for sidereset */ 6354 6355 vlv_set_power_well(dev_priv, power_well, true); 6356 6357 /* 6358 * From VLV2A0_DP_eDP_DPIO_driver_vbios_notes_10.docx - 6359 * 6. De-assert cmn_reset/side_reset. Same as VLV X0. 6360 * a. GUnit 0x2110 bit[0] set to 1 (def 0) 6361 * b. The other bits such as sfr settings / modesel may all 6362 * be set to 0. 6363 * 6364 * This should only be done on init and resume from S3 with 6365 * both PLLs disabled, or we risk losing DPIO and PLL 6366 * synchronization. 6367 */ 6368 I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) | DPIO_CMNRST); 6369 } 6370 6371 static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv, 6372 struct i915_power_well *power_well) 6373 { 6374 enum i915_pipe pipe; 6375 6376 WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC); 6377 6378 for_each_pipe(dev_priv, pipe) 6379 assert_pll_disabled(dev_priv, pipe); 6380 6381 /* Assert common reset */ 6382 I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) & ~DPIO_CMNRST); 6383 6384 vlv_set_power_well(dev_priv, power_well, false); 6385 } 6386 6387 static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv, 6388 struct i915_power_well *power_well) 6389 { 6390 enum dpio_phy phy; 6391 6392 WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC && 6393 power_well->data != PUNIT_POWER_WELL_DPIO_CMN_D); 6394 6395 /* 6396 * Enable the CRI clock source so we can get at the 6397 * display and the reference clock for VGA 6398 * hotplug / manual detection. 6399 */ 6400 if (power_well->data == PUNIT_POWER_WELL_DPIO_CMN_BC) { 6401 phy = DPIO_PHY0; 6402 I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) | 6403 DPLL_REFA_CLK_ENABLE_VLV); 6404 I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) | 6405 DPLL_REFA_CLK_ENABLE_VLV | DPLL_INTEGRATED_CRI_CLK_VLV); 6406 } else { 6407 phy = DPIO_PHY1; 6408 I915_WRITE(DPLL(PIPE_C), I915_READ(DPLL(PIPE_C)) | 6409 DPLL_REFA_CLK_ENABLE_VLV | DPLL_INTEGRATED_CRI_CLK_VLV); 6410 } 6411 udelay(1); /* >10ns for cmnreset, >0ns for sidereset */ 6412 vlv_set_power_well(dev_priv, power_well, true); 6413 6414 /* Poll for phypwrgood signal */ 6415 if (wait_for(I915_READ(DISPLAY_PHY_STATUS) & PHY_POWERGOOD(phy), 1)) 6416 DRM_ERROR("Display PHY %d is not power up\n", phy); 6417 6418 I915_WRITE(DISPLAY_PHY_CONTROL, I915_READ(DISPLAY_PHY_CONTROL) | 6419 PHY_COM_LANE_RESET_DEASSERT(phy)); 6420 } 6421 6422 static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv, 6423 struct i915_power_well *power_well) 6424 { 6425 enum dpio_phy phy; 6426 6427 WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC && 6428 power_well->data != PUNIT_POWER_WELL_DPIO_CMN_D); 6429 6430 if (power_well->data == PUNIT_POWER_WELL_DPIO_CMN_BC) { 6431 phy = DPIO_PHY0; 6432 assert_pll_disabled(dev_priv, PIPE_A); 6433 assert_pll_disabled(dev_priv, PIPE_B); 6434 } else { 6435 phy = DPIO_PHY1; 6436 assert_pll_disabled(dev_priv, PIPE_C); 6437 } 6438 6439 I915_WRITE(DISPLAY_PHY_CONTROL, I915_READ(DISPLAY_PHY_CONTROL) & 6440 ~PHY_COM_LANE_RESET_DEASSERT(phy)); 6441 6442 vlv_set_power_well(dev_priv, power_well, false); 6443 } 6444 6445 static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv, 6446 struct i915_power_well *power_well) 6447 { 6448 enum i915_pipe pipe = power_well->data; 6449 bool enabled; 6450 u32 state, ctrl; 6451 6452 mutex_lock(&dev_priv->rps.hw_lock); 6453 6454 state = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe); 6455 /* 6456 * We only ever set the power-on and power-gate states, anything 6457 * else is unexpected. 6458 */ 6459 WARN_ON(state != DP_SSS_PWR_ON(pipe) && state != DP_SSS_PWR_GATE(pipe)); 6460 enabled = state == DP_SSS_PWR_ON(pipe); 6461 6462 /* 6463 * A transient state at this point would mean some unexpected party 6464 * is poking at the power controls too. 6465 */ 6466 ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSC_MASK(pipe); 6467 WARN_ON(ctrl << 16 != state); 6468 6469 mutex_unlock(&dev_priv->rps.hw_lock); 6470 6471 return enabled; 6472 } 6473 6474 static void chv_set_pipe_power_well(struct drm_i915_private *dev_priv, 6475 struct i915_power_well *power_well, 6476 bool enable) 6477 { 6478 enum i915_pipe pipe = power_well->data; 6479 u32 state; 6480 u32 ctrl; 6481 6482 state = enable ? DP_SSS_PWR_ON(pipe) : DP_SSS_PWR_GATE(pipe); 6483 6484 mutex_lock(&dev_priv->rps.hw_lock); 6485 6486 #define COND \ 6487 ((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe)) == state) 6488 6489 if (COND) 6490 goto out; 6491 6492 ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ); 6493 ctrl &= ~DP_SSC_MASK(pipe); 6494 ctrl |= enable ? DP_SSC_PWR_ON(pipe) : DP_SSC_PWR_GATE(pipe); 6495 vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, ctrl); 6496 6497 if (wait_for(COND, 100)) 6498 DRM_ERROR("timout setting power well state %08x (%08x)\n", 6499 state, 6500 vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ)); 6501 6502 #undef COND 6503 6504 out: 6505 mutex_unlock(&dev_priv->rps.hw_lock); 6506 } 6507 6508 static void chv_pipe_power_well_sync_hw(struct drm_i915_private *dev_priv, 6509 struct i915_power_well *power_well) 6510 { 6511 chv_set_pipe_power_well(dev_priv, power_well, power_well->count > 0); 6512 } 6513 6514 static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv, 6515 struct i915_power_well *power_well) 6516 { 6517 WARN_ON_ONCE(power_well->data != PIPE_A && 6518 power_well->data != PIPE_B && 6519 power_well->data != PIPE_C); 6520 6521 chv_set_pipe_power_well(dev_priv, power_well, true); 6522 } 6523 6524 static void chv_pipe_power_well_disable(struct drm_i915_private *dev_priv, 6525 struct i915_power_well *power_well) 6526 { 6527 WARN_ON_ONCE(power_well->data != PIPE_A && 6528 power_well->data != PIPE_B && 6529 power_well->data != PIPE_C); 6530 6531 chv_set_pipe_power_well(dev_priv, power_well, false); 6532 } 6533 6534 static void check_power_well_state(struct drm_i915_private *dev_priv, 6535 struct i915_power_well *power_well) 6536 { 6537 bool enabled = power_well->ops->is_enabled(dev_priv, power_well); 6538 6539 if (power_well->always_on || !i915.disable_power_well) { 6540 if (!enabled) 6541 goto mismatch; 6542 6543 return; 6544 } 6545 6546 if (enabled != (power_well->count > 0)) 6547 goto mismatch; 6548 6549 return; 6550 6551 mismatch: 6552 WARN(1, "state mismatch for '%s' (always_on %d hw state %d use-count %d disable_power_well %d\n", 6553 power_well->name, power_well->always_on, enabled, 6554 power_well->count, i915.disable_power_well); 6555 } 6556 6557 void intel_display_power_get(struct drm_i915_private *dev_priv, 6558 enum intel_display_power_domain domain) 6559 { 6560 struct i915_power_domains *power_domains; 6561 struct i915_power_well *power_well; 6562 int i; 6563 6564 intel_runtime_pm_get(dev_priv); 6565 6566 power_domains = &dev_priv->power_domains; 6567 6568 mutex_lock(&power_domains->lock); 6569 6570 for_each_power_well(i, power_well, BIT(domain), power_domains) { 6571 if (!power_well->count++) { 6572 DRM_DEBUG_KMS("enabling %s\n", power_well->name); 6573 power_well->ops->enable(dev_priv, power_well); 6574 power_well->hw_enabled = true; 6575 } 6576 6577 check_power_well_state(dev_priv, power_well); 6578 } 6579 6580 power_domains->domain_use_count[domain]++; 6581 6582 mutex_unlock(&power_domains->lock); 6583 } 6584 6585 void intel_display_power_put(struct drm_i915_private *dev_priv, 6586 enum intel_display_power_domain domain) 6587 { 6588 struct i915_power_domains *power_domains; 6589 struct i915_power_well *power_well; 6590 int i; 6591 6592 power_domains = &dev_priv->power_domains; 6593 6594 mutex_lock(&power_domains->lock); 6595 6596 WARN_ON(!power_domains->domain_use_count[domain]); 6597 power_domains->domain_use_count[domain]--; 6598 6599 for_each_power_well_rev(i, power_well, BIT(domain), power_domains) { 6600 WARN_ON(!power_well->count); 6601 6602 if (!--power_well->count && i915.disable_power_well) { 6603 DRM_DEBUG_KMS("disabling %s\n", power_well->name); 6604 power_well->hw_enabled = false; 6605 power_well->ops->disable(dev_priv, power_well); 6606 } 6607 6608 check_power_well_state(dev_priv, power_well); 6609 } 6610 6611 mutex_unlock(&power_domains->lock); 6612 6613 intel_runtime_pm_put(dev_priv); 6614 } 6615 6616 static struct i915_power_domains *hsw_pwr; 6617 6618 /* Display audio driver power well request */ 6619 int i915_request_power_well(void) 6620 { 6621 struct drm_i915_private *dev_priv; 6622 6623 if (!hsw_pwr) 6624 return -ENODEV; 6625 6626 dev_priv = container_of(hsw_pwr, struct drm_i915_private, 6627 power_domains); 6628 intel_display_power_get(dev_priv, POWER_DOMAIN_AUDIO); 6629 return 0; 6630 } 6631 6632 /* Display audio driver power well release */ 6633 int i915_release_power_well(void) 6634 { 6635 struct drm_i915_private *dev_priv; 6636 6637 if (!hsw_pwr) 6638 return -ENODEV; 6639 6640 dev_priv = container_of(hsw_pwr, struct drm_i915_private, 6641 power_domains); 6642 intel_display_power_put(dev_priv, POWER_DOMAIN_AUDIO); 6643 return 0; 6644 } 6645 6646 /* 6647 * Private interface for the audio driver to get CDCLK in kHz. 6648 * 6649 * Caller must request power well using i915_request_power_well() prior to 6650 * making the call. 6651 */ 6652 int i915_get_cdclk_freq(void) 6653 { 6654 struct drm_i915_private *dev_priv; 6655 6656 if (!hsw_pwr) 6657 return -ENODEV; 6658 6659 dev_priv = container_of(hsw_pwr, struct drm_i915_private, 6660 power_domains); 6661 6662 return intel_ddi_get_cdclk_freq(dev_priv); 6663 } 6664 6665 #define POWER_DOMAIN_MASK (BIT(POWER_DOMAIN_NUM) - 1) 6666 6667 #define HSW_ALWAYS_ON_POWER_DOMAINS ( \ 6668 BIT(POWER_DOMAIN_PIPE_A) | \ 6669 BIT(POWER_DOMAIN_TRANSCODER_EDP) | \ 6670 BIT(POWER_DOMAIN_PORT_DDI_A_2_LANES) | \ 6671 BIT(POWER_DOMAIN_PORT_DDI_A_4_LANES) | \ 6672 BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) | \ 6673 BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \ 6674 BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) | \ 6675 BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \ 6676 BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) | \ 6677 BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) | \ 6678 BIT(POWER_DOMAIN_PORT_CRT) | \ 6679 BIT(POWER_DOMAIN_PLLS) | \ 6680 BIT(POWER_DOMAIN_INIT)) 6681 #define HSW_DISPLAY_POWER_DOMAINS ( \ 6682 (POWER_DOMAIN_MASK & ~HSW_ALWAYS_ON_POWER_DOMAINS) | \ 6683 BIT(POWER_DOMAIN_INIT)) 6684 6685 #define BDW_ALWAYS_ON_POWER_DOMAINS ( \ 6686 HSW_ALWAYS_ON_POWER_DOMAINS | \ 6687 BIT(POWER_DOMAIN_PIPE_A_PANEL_FITTER)) 6688 #define BDW_DISPLAY_POWER_DOMAINS ( \ 6689 (POWER_DOMAIN_MASK & ~BDW_ALWAYS_ON_POWER_DOMAINS) | \ 6690 BIT(POWER_DOMAIN_INIT)) 6691 6692 #define VLV_ALWAYS_ON_POWER_DOMAINS BIT(POWER_DOMAIN_INIT) 6693 #define VLV_DISPLAY_POWER_DOMAINS POWER_DOMAIN_MASK 6694 6695 #define VLV_DPIO_CMN_BC_POWER_DOMAINS ( \ 6696 BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) | \ 6697 BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \ 6698 BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) | \ 6699 BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \ 6700 BIT(POWER_DOMAIN_PORT_CRT) | \ 6701 BIT(POWER_DOMAIN_INIT)) 6702 6703 #define VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS ( \ 6704 BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) | \ 6705 BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \ 6706 BIT(POWER_DOMAIN_INIT)) 6707 6708 #define VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS ( \ 6709 BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \ 6710 BIT(POWER_DOMAIN_INIT)) 6711 6712 #define VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS ( \ 6713 BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) | \ 6714 BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \ 6715 BIT(POWER_DOMAIN_INIT)) 6716 6717 #define VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS ( \ 6718 BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \ 6719 BIT(POWER_DOMAIN_INIT)) 6720 6721 #define CHV_PIPE_A_POWER_DOMAINS ( \ 6722 BIT(POWER_DOMAIN_PIPE_A) | \ 6723 BIT(POWER_DOMAIN_INIT)) 6724 6725 #define CHV_PIPE_B_POWER_DOMAINS ( \ 6726 BIT(POWER_DOMAIN_PIPE_B) | \ 6727 BIT(POWER_DOMAIN_INIT)) 6728 6729 #define CHV_PIPE_C_POWER_DOMAINS ( \ 6730 BIT(POWER_DOMAIN_PIPE_C) | \ 6731 BIT(POWER_DOMAIN_INIT)) 6732 6733 #define CHV_DPIO_CMN_BC_POWER_DOMAINS ( \ 6734 BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) | \ 6735 BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \ 6736 BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) | \ 6737 BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \ 6738 BIT(POWER_DOMAIN_INIT)) 6739 6740 #define CHV_DPIO_CMN_D_POWER_DOMAINS ( \ 6741 BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) | \ 6742 BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) | \ 6743 BIT(POWER_DOMAIN_INIT)) 6744 6745 #define CHV_DPIO_TX_D_LANES_01_POWER_DOMAINS ( \ 6746 BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) | \ 6747 BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) | \ 6748 BIT(POWER_DOMAIN_INIT)) 6749 6750 #define CHV_DPIO_TX_D_LANES_23_POWER_DOMAINS ( \ 6751 BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) | \ 6752 BIT(POWER_DOMAIN_INIT)) 6753 6754 static const struct i915_power_well_ops i9xx_always_on_power_well_ops = { 6755 .sync_hw = i9xx_always_on_power_well_noop, 6756 .enable = i9xx_always_on_power_well_noop, 6757 .disable = i9xx_always_on_power_well_noop, 6758 .is_enabled = i9xx_always_on_power_well_enabled, 6759 }; 6760 6761 static const struct i915_power_well_ops chv_pipe_power_well_ops = { 6762 .sync_hw = chv_pipe_power_well_sync_hw, 6763 .enable = chv_pipe_power_well_enable, 6764 .disable = chv_pipe_power_well_disable, 6765 .is_enabled = chv_pipe_power_well_enabled, 6766 }; 6767 6768 static const struct i915_power_well_ops chv_dpio_cmn_power_well_ops = { 6769 .sync_hw = vlv_power_well_sync_hw, 6770 .enable = chv_dpio_cmn_power_well_enable, 6771 .disable = chv_dpio_cmn_power_well_disable, 6772 .is_enabled = vlv_power_well_enabled, 6773 }; 6774 6775 static struct i915_power_well i9xx_always_on_power_well[] = { 6776 { 6777 .name = "always-on", 6778 .always_on = 1, 6779 .domains = POWER_DOMAIN_MASK, 6780 .ops = &i9xx_always_on_power_well_ops, 6781 }, 6782 }; 6783 6784 static const struct i915_power_well_ops hsw_power_well_ops = { 6785 .sync_hw = hsw_power_well_sync_hw, 6786 .enable = hsw_power_well_enable, 6787 .disable = hsw_power_well_disable, 6788 .is_enabled = hsw_power_well_enabled, 6789 }; 6790 6791 static struct i915_power_well hsw_power_wells[] = { 6792 { 6793 .name = "always-on", 6794 .always_on = 1, 6795 .domains = HSW_ALWAYS_ON_POWER_DOMAINS, 6796 .ops = &i9xx_always_on_power_well_ops, 6797 }, 6798 #if 0 6799 { 6800 .name = "display", 6801 .domains = VLV_DISPLAY_POWER_DOMAINS, 6802 .data = PUNIT_POWER_WELL_DISP2D, 6803 .ops = &vlv_display_power_well_ops, 6804 }, 6805 #endif 6806 { 6807 .name = "display", 6808 .domains = HSW_DISPLAY_POWER_DOMAINS, 6809 .ops = &hsw_power_well_ops, 6810 }, 6811 }; 6812 6813 static struct i915_power_well bdw_power_wells[] = { 6814 { 6815 .name = "always-on", 6816 .always_on = 1, 6817 .domains = BDW_ALWAYS_ON_POWER_DOMAINS, 6818 .ops = &i9xx_always_on_power_well_ops, 6819 }, 6820 { 6821 .name = "display", 6822 .domains = BDW_DISPLAY_POWER_DOMAINS, 6823 .ops = &hsw_power_well_ops, 6824 }, 6825 }; 6826 6827 static const struct i915_power_well_ops vlv_display_power_well_ops = { 6828 .sync_hw = vlv_power_well_sync_hw, 6829 .enable = vlv_display_power_well_enable, 6830 .disable = vlv_display_power_well_disable, 6831 .is_enabled = vlv_power_well_enabled, 6832 }; 6833 6834 static const struct i915_power_well_ops vlv_dpio_cmn_power_well_ops = { 6835 .sync_hw = vlv_power_well_sync_hw, 6836 .enable = vlv_dpio_cmn_power_well_enable, 6837 .disable = vlv_dpio_cmn_power_well_disable, 6838 .is_enabled = vlv_power_well_enabled, 6839 }; 6840 6841 static const struct i915_power_well_ops vlv_dpio_power_well_ops = { 6842 .sync_hw = vlv_power_well_sync_hw, 6843 .enable = vlv_power_well_enable, 6844 .disable = vlv_power_well_disable, 6845 .is_enabled = vlv_power_well_enabled, 6846 }; 6847 6848 static struct i915_power_well vlv_power_wells[] = { 6849 { 6850 .name = "always-on", 6851 .always_on = 1, 6852 .domains = VLV_ALWAYS_ON_POWER_DOMAINS, 6853 .ops = &i9xx_always_on_power_well_ops, 6854 }, 6855 { 6856 .name = "display", 6857 .domains = VLV_DISPLAY_POWER_DOMAINS, 6858 .data = PUNIT_POWER_WELL_DISP2D, 6859 .ops = &vlv_display_power_well_ops, 6860 }, 6861 { 6862 .name = "dpio-tx-b-01", 6863 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS | 6864 VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS | 6865 VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS | 6866 VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS, 6867 .ops = &vlv_dpio_power_well_ops, 6868 .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01, 6869 }, 6870 { 6871 .name = "dpio-tx-b-23", 6872 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS | 6873 VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS | 6874 VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS | 6875 VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS, 6876 .ops = &vlv_dpio_power_well_ops, 6877 .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23, 6878 }, 6879 { 6880 .name = "dpio-tx-c-01", 6881 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS | 6882 VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS | 6883 VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS | 6884 VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS, 6885 .ops = &vlv_dpio_power_well_ops, 6886 .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01, 6887 }, 6888 { 6889 .name = "dpio-tx-c-23", 6890 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS | 6891 VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS | 6892 VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS | 6893 VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS, 6894 .ops = &vlv_dpio_power_well_ops, 6895 .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23, 6896 }, 6897 { 6898 .name = "dpio-common", 6899 .domains = VLV_DPIO_CMN_BC_POWER_DOMAINS, 6900 .data = PUNIT_POWER_WELL_DPIO_CMN_BC, 6901 .ops = &vlv_dpio_cmn_power_well_ops, 6902 }, 6903 }; 6904 6905 static struct i915_power_well chv_power_wells[] = { 6906 { 6907 .name = "always-on", 6908 .always_on = 1, 6909 .domains = VLV_ALWAYS_ON_POWER_DOMAINS, 6910 .ops = &i9xx_always_on_power_well_ops, 6911 }, 6912 #if 0 6913 { 6914 .name = "display", 6915 .domains = VLV_DISPLAY_POWER_DOMAINS, 6916 .data = PUNIT_POWER_WELL_DISP2D, 6917 .ops = &vlv_display_power_well_ops, 6918 }, 6919 { 6920 .name = "pipe-a", 6921 .domains = CHV_PIPE_A_POWER_DOMAINS, 6922 .data = PIPE_A, 6923 .ops = &chv_pipe_power_well_ops, 6924 }, 6925 { 6926 .name = "pipe-b", 6927 .domains = CHV_PIPE_B_POWER_DOMAINS, 6928 .data = PIPE_B, 6929 .ops = &chv_pipe_power_well_ops, 6930 }, 6931 { 6932 .name = "pipe-c", 6933 .domains = CHV_PIPE_C_POWER_DOMAINS, 6934 .data = PIPE_C, 6935 .ops = &chv_pipe_power_well_ops, 6936 }, 6937 #endif 6938 { 6939 .name = "dpio-common-bc", 6940 /* 6941 * XXX: cmnreset for one PHY seems to disturb the other. 6942 * As a workaround keep both powered on at the same 6943 * time for now. 6944 */ 6945 .domains = CHV_DPIO_CMN_BC_POWER_DOMAINS | CHV_DPIO_CMN_D_POWER_DOMAINS, 6946 .data = PUNIT_POWER_WELL_DPIO_CMN_BC, 6947 .ops = &chv_dpio_cmn_power_well_ops, 6948 }, 6949 { 6950 .name = "dpio-common-d", 6951 /* 6952 * XXX: cmnreset for one PHY seems to disturb the other. 6953 * As a workaround keep both powered on at the same 6954 * time for now. 6955 */ 6956 .domains = CHV_DPIO_CMN_BC_POWER_DOMAINS | CHV_DPIO_CMN_D_POWER_DOMAINS, 6957 .data = PUNIT_POWER_WELL_DPIO_CMN_D, 6958 .ops = &chv_dpio_cmn_power_well_ops, 6959 }, 6960 #if 0 6961 { 6962 .name = "dpio-tx-b-01", 6963 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS | 6964 VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS, 6965 .ops = &vlv_dpio_power_well_ops, 6966 .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01, 6967 }, 6968 { 6969 .name = "dpio-tx-b-23", 6970 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS | 6971 VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS, 6972 .ops = &vlv_dpio_power_well_ops, 6973 .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23, 6974 }, 6975 { 6976 .name = "dpio-tx-c-01", 6977 .domains = VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS | 6978 VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS, 6979 .ops = &vlv_dpio_power_well_ops, 6980 .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01, 6981 }, 6982 { 6983 .name = "dpio-tx-c-23", 6984 .domains = VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS | 6985 VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS, 6986 .ops = &vlv_dpio_power_well_ops, 6987 .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23, 6988 }, 6989 { 6990 .name = "dpio-tx-d-01", 6991 .domains = CHV_DPIO_TX_D_LANES_01_POWER_DOMAINS | 6992 CHV_DPIO_TX_D_LANES_23_POWER_DOMAINS, 6993 .ops = &vlv_dpio_power_well_ops, 6994 .data = PUNIT_POWER_WELL_DPIO_TX_D_LANES_01, 6995 }, 6996 { 6997 .name = "dpio-tx-d-23", 6998 .domains = CHV_DPIO_TX_D_LANES_01_POWER_DOMAINS | 6999 CHV_DPIO_TX_D_LANES_23_POWER_DOMAINS, 7000 .ops = &vlv_dpio_power_well_ops, 7001 .data = PUNIT_POWER_WELL_DPIO_TX_D_LANES_23, 7002 }, 7003 #endif 7004 }; 7005 7006 static struct i915_power_well *lookup_power_well(struct drm_i915_private *dev_priv, 7007 enum punit_power_well power_well_id) 7008 { 7009 struct i915_power_domains *power_domains = &dev_priv->power_domains; 7010 struct i915_power_well *power_well; 7011 int i; 7012 7013 for_each_power_well(i, power_well, POWER_DOMAIN_MASK, power_domains) { 7014 if (power_well->data == power_well_id) 7015 return power_well; 7016 } 7017 7018 return NULL; 7019 } 7020 7021 #define set_power_wells(power_domains, __power_wells) ({ \ 7022 (power_domains)->power_wells = (__power_wells); \ 7023 (power_domains)->power_well_count = ARRAY_SIZE(__power_wells); \ 7024 }) 7025 7026 int intel_power_domains_init(struct drm_i915_private *dev_priv) 7027 { 7028 struct i915_power_domains *power_domains = &dev_priv->power_domains; 7029 7030 lockinit(&power_domains->lock, "i915pl", 0, LK_CANRECURSE); 7031 7032 /* 7033 * The enabling order will be from lower to higher indexed wells, 7034 * the disabling order is reversed. 7035 */ 7036 if (IS_HASWELL(dev_priv->dev)) { 7037 set_power_wells(power_domains, hsw_power_wells); 7038 hsw_pwr = power_domains; 7039 } else if (IS_BROADWELL(dev_priv->dev)) { 7040 set_power_wells(power_domains, bdw_power_wells); 7041 hsw_pwr = power_domains; 7042 } else if (IS_CHERRYVIEW(dev_priv->dev)) { 7043 set_power_wells(power_domains, chv_power_wells); 7044 } else if (IS_VALLEYVIEW(dev_priv->dev)) { 7045 set_power_wells(power_domains, vlv_power_wells); 7046 } else { 7047 set_power_wells(power_domains, i9xx_always_on_power_well); 7048 } 7049 7050 return 0; 7051 } 7052 7053 void intel_power_domains_remove(struct drm_i915_private *dev_priv) 7054 { 7055 hsw_pwr = NULL; 7056 } 7057 7058 static void intel_power_domains_resume(struct drm_i915_private *dev_priv) 7059 { 7060 struct i915_power_domains *power_domains = &dev_priv->power_domains; 7061 struct i915_power_well *power_well; 7062 int i; 7063 7064 mutex_lock(&power_domains->lock); 7065 for_each_power_well(i, power_well, POWER_DOMAIN_MASK, power_domains) { 7066 power_well->ops->sync_hw(dev_priv, power_well); 7067 power_well->hw_enabled = power_well->ops->is_enabled(dev_priv, 7068 power_well); 7069 } 7070 mutex_unlock(&power_domains->lock); 7071 } 7072 7073 static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv) 7074 { 7075 struct i915_power_well *cmn = 7076 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC); 7077 struct i915_power_well *disp2d = 7078 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DISP2D); 7079 7080 /* nothing to do if common lane is already off */ 7081 if (!cmn->ops->is_enabled(dev_priv, cmn)) 7082 return; 7083 7084 /* If the display might be already active skip this */ 7085 if (disp2d->ops->is_enabled(dev_priv, disp2d) && 7086 I915_READ(DPIO_CTL) & DPIO_CMNRST) 7087 return; 7088 7089 DRM_DEBUG_KMS("toggling display PHY side reset\n"); 7090 7091 /* cmnlane needs DPLL registers */ 7092 disp2d->ops->enable(dev_priv, disp2d); 7093 7094 /* 7095 * From VLV2A0_DP_eDP_HDMI_DPIO_driver_vbios_notes_11.docx: 7096 * Need to assert and de-assert PHY SB reset by gating the 7097 * common lane power, then un-gating it. 7098 * Simply ungating isn't enough to reset the PHY enough to get 7099 * ports and lanes running. 7100 */ 7101 cmn->ops->disable(dev_priv, cmn); 7102 } 7103 7104 void intel_power_domains_init_hw(struct drm_i915_private *dev_priv) 7105 { 7106 struct drm_device *dev = dev_priv->dev; 7107 struct i915_power_domains *power_domains = &dev_priv->power_domains; 7108 7109 power_domains->initializing = true; 7110 7111 if (IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev)) { 7112 mutex_lock(&power_domains->lock); 7113 vlv_cmnlane_wa(dev_priv); 7114 mutex_unlock(&power_domains->lock); 7115 } 7116 7117 /* For now, we need the power well to be always enabled. */ 7118 intel_display_set_init_power(dev_priv, true); 7119 intel_power_domains_resume(dev_priv); 7120 power_domains->initializing = false; 7121 } 7122 7123 void intel_aux_display_runtime_get(struct drm_i915_private *dev_priv) 7124 { 7125 intel_runtime_pm_get(dev_priv); 7126 } 7127 7128 void intel_aux_display_runtime_put(struct drm_i915_private *dev_priv) 7129 { 7130 intel_runtime_pm_put(dev_priv); 7131 } 7132 7133 void intel_runtime_pm_get(struct drm_i915_private *dev_priv) 7134 { 7135 #if 0 7136 struct drm_device *dev = dev_priv->dev; 7137 struct device *device = &dev->pdev->dev; 7138 7139 if (!HAS_RUNTIME_PM(dev)) 7140 return; 7141 7142 pm_runtime_get_sync(device); 7143 WARN(dev_priv->pm.suspended, "Device still suspended.\n"); 7144 #endif 7145 } 7146 7147 void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv) 7148 { 7149 struct drm_device *dev = dev_priv->dev; 7150 #if 0 7151 struct device *device = &dev->pdev->dev; 7152 #endif 7153 7154 if (!HAS_RUNTIME_PM(dev)) 7155 return; 7156 7157 WARN(dev_priv->pm.suspended, "Getting nosync-ref while suspended.\n"); 7158 #if 0 7159 pm_runtime_get_noresume(device); 7160 #endif 7161 } 7162 7163 void intel_runtime_pm_put(struct drm_i915_private *dev_priv) 7164 { 7165 #if 0 7166 struct drm_device *dev = dev_priv->dev; 7167 struct device *device = &dev->pdev->dev; 7168 7169 if (!HAS_RUNTIME_PM(dev)) 7170 return; 7171 7172 pm_runtime_mark_last_busy(device); 7173 pm_runtime_put_autosuspend(device); 7174 #endif 7175 } 7176 7177 void intel_init_runtime_pm(struct drm_i915_private *dev_priv) 7178 { 7179 struct drm_device *dev = dev_priv->dev; 7180 #if 0 7181 struct device *device = &dev->pdev->dev; 7182 #endif 7183 7184 if (!HAS_RUNTIME_PM(dev)) 7185 return; 7186 7187 #if 0 7188 pm_runtime_set_active(device); 7189 7190 /* 7191 * RPM depends on RC6 to save restore the GT HW context, so make RC6 a 7192 * requirement. 7193 */ 7194 if (!intel_enable_rc6(dev)) { 7195 DRM_INFO("RC6 disabled, disabling runtime PM support\n"); 7196 return; 7197 } 7198 7199 pm_runtime_set_autosuspend_delay(device, 10000); /* 10s */ 7200 pm_runtime_mark_last_busy(device); 7201 pm_runtime_use_autosuspend(device); 7202 7203 pm_runtime_put_autosuspend(device); 7204 #endif 7205 } 7206 7207 void intel_fini_runtime_pm(struct drm_i915_private *dev_priv) 7208 { 7209 #if 0 7210 struct drm_device *dev = dev_priv->dev; 7211 struct device *device = &dev->pdev->dev; 7212 7213 if (!HAS_RUNTIME_PM(dev)) 7214 return; 7215 7216 if (!intel_enable_rc6(dev)) 7217 return; 7218 7219 /* Make sure we're not suspended first. */ 7220 pm_runtime_get_sync(device); 7221 pm_runtime_disable(device); 7222 #endif 7223 } 7224 7225 /* Set up chip specific power management-related functions */ 7226 void intel_init_pm(struct drm_device *dev) 7227 { 7228 struct drm_i915_private *dev_priv = dev->dev_private; 7229 7230 if (HAS_FBC(dev)) { 7231 if (INTEL_INFO(dev)->gen >= 7) { 7232 dev_priv->display.fbc_enabled = ironlake_fbc_enabled; 7233 dev_priv->display.enable_fbc = gen7_enable_fbc; 7234 dev_priv->display.disable_fbc = ironlake_disable_fbc; 7235 } else if (INTEL_INFO(dev)->gen >= 5) { 7236 dev_priv->display.fbc_enabled = ironlake_fbc_enabled; 7237 dev_priv->display.enable_fbc = ironlake_enable_fbc; 7238 dev_priv->display.disable_fbc = ironlake_disable_fbc; 7239 } else if (IS_GM45(dev)) { 7240 dev_priv->display.fbc_enabled = g4x_fbc_enabled; 7241 dev_priv->display.enable_fbc = g4x_enable_fbc; 7242 dev_priv->display.disable_fbc = g4x_disable_fbc; 7243 } else { 7244 dev_priv->display.fbc_enabled = i8xx_fbc_enabled; 7245 dev_priv->display.enable_fbc = i8xx_enable_fbc; 7246 dev_priv->display.disable_fbc = i8xx_disable_fbc; 7247 7248 /* This value was pulled out of someone's hat */ 7249 I915_WRITE(FBC_CONTROL, 500 << FBC_CTL_INTERVAL_SHIFT); 7250 } 7251 } 7252 7253 /* For cxsr */ 7254 if (IS_PINEVIEW(dev)) 7255 i915_pineview_get_mem_freq(dev); 7256 else if (IS_GEN5(dev)) 7257 i915_ironlake_get_mem_freq(dev); 7258 7259 /* For FIFO watermark updates */ 7260 if (HAS_PCH_SPLIT(dev)) { 7261 ilk_setup_wm_latency(dev); 7262 7263 if ((IS_GEN5(dev) && dev_priv->wm.pri_latency[1] && 7264 dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) || 7265 (!IS_GEN5(dev) && dev_priv->wm.pri_latency[0] && 7266 dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) { 7267 dev_priv->display.update_wm = ilk_update_wm; 7268 dev_priv->display.update_sprite_wm = ilk_update_sprite_wm; 7269 } else { 7270 DRM_DEBUG_KMS("Failed to read display plane latency. " 7271 "Disable CxSR\n"); 7272 } 7273 7274 if (IS_GEN5(dev)) 7275 dev_priv->display.init_clock_gating = ironlake_init_clock_gating; 7276 else if (IS_GEN6(dev)) 7277 dev_priv->display.init_clock_gating = gen6_init_clock_gating; 7278 else if (IS_IVYBRIDGE(dev)) 7279 dev_priv->display.init_clock_gating = ivybridge_init_clock_gating; 7280 else if (IS_HASWELL(dev)) 7281 dev_priv->display.init_clock_gating = haswell_init_clock_gating; 7282 else if (INTEL_INFO(dev)->gen == 8) 7283 dev_priv->display.init_clock_gating = broadwell_init_clock_gating; 7284 } else if (IS_CHERRYVIEW(dev)) { 7285 dev_priv->display.update_wm = cherryview_update_wm; 7286 dev_priv->display.update_sprite_wm = valleyview_update_sprite_wm; 7287 dev_priv->display.init_clock_gating = 7288 cherryview_init_clock_gating; 7289 } else if (IS_VALLEYVIEW(dev)) { 7290 dev_priv->display.update_wm = valleyview_update_wm; 7291 dev_priv->display.update_sprite_wm = valleyview_update_sprite_wm; 7292 dev_priv->display.init_clock_gating = 7293 valleyview_init_clock_gating; 7294 } else if (IS_PINEVIEW(dev)) { 7295 if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev), 7296 dev_priv->is_ddr3, 7297 dev_priv->fsb_freq, 7298 dev_priv->mem_freq)) { 7299 DRM_INFO("failed to find known CxSR latency " 7300 "(found ddr%s fsb freq %d, mem freq %d), " 7301 "disabling CxSR\n", 7302 (dev_priv->is_ddr3 == 1) ? "3" : "2", 7303 dev_priv->fsb_freq, dev_priv->mem_freq); 7304 /* Disable CxSR and never update its watermark again */ 7305 intel_set_memory_cxsr(dev_priv, false); 7306 dev_priv->display.update_wm = NULL; 7307 } else 7308 dev_priv->display.update_wm = pineview_update_wm; 7309 dev_priv->display.init_clock_gating = gen3_init_clock_gating; 7310 } else if (IS_G4X(dev)) { 7311 dev_priv->display.update_wm = g4x_update_wm; 7312 dev_priv->display.init_clock_gating = g4x_init_clock_gating; 7313 } else if (IS_GEN4(dev)) { 7314 dev_priv->display.update_wm = i965_update_wm; 7315 if (IS_CRESTLINE(dev)) 7316 dev_priv->display.init_clock_gating = crestline_init_clock_gating; 7317 else if (IS_BROADWATER(dev)) 7318 dev_priv->display.init_clock_gating = broadwater_init_clock_gating; 7319 } else if (IS_GEN3(dev)) { 7320 dev_priv->display.update_wm = i9xx_update_wm; 7321 dev_priv->display.get_fifo_size = i9xx_get_fifo_size; 7322 dev_priv->display.init_clock_gating = gen3_init_clock_gating; 7323 } else if (IS_GEN2(dev)) { 7324 if (INTEL_INFO(dev)->num_pipes == 1) { 7325 dev_priv->display.update_wm = i845_update_wm; 7326 dev_priv->display.get_fifo_size = i845_get_fifo_size; 7327 } else { 7328 dev_priv->display.update_wm = i9xx_update_wm; 7329 dev_priv->display.get_fifo_size = i830_get_fifo_size; 7330 } 7331 7332 if (IS_I85X(dev) || IS_I865G(dev)) 7333 dev_priv->display.init_clock_gating = i85x_init_clock_gating; 7334 else 7335 dev_priv->display.init_clock_gating = i830_init_clock_gating; 7336 } else { 7337 DRM_ERROR("unexpected fall-through in intel_init_pm\n"); 7338 } 7339 } 7340 7341 int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u8 mbox, u32 *val) 7342 { 7343 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); 7344 7345 if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) { 7346 DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n"); 7347 return -EAGAIN; 7348 } 7349 7350 I915_WRITE(GEN6_PCODE_DATA, *val); 7351 I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox); 7352 7353 if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0, 7354 500)) { 7355 DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox); 7356 return -ETIMEDOUT; 7357 } 7358 7359 *val = I915_READ(GEN6_PCODE_DATA); 7360 I915_WRITE(GEN6_PCODE_DATA, 0); 7361 7362 return 0; 7363 } 7364 7365 int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u8 mbox, u32 val) 7366 { 7367 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); 7368 7369 if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) { 7370 DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n"); 7371 return -EAGAIN; 7372 } 7373 7374 I915_WRITE(GEN6_PCODE_DATA, val); 7375 I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox); 7376 7377 if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0, 7378 500)) { 7379 DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox); 7380 return -ETIMEDOUT; 7381 } 7382 7383 I915_WRITE(GEN6_PCODE_DATA, 0); 7384 7385 return 0; 7386 } 7387 7388 static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val) 7389 { 7390 int div; 7391 7392 /* 4 x czclk */ 7393 switch (dev_priv->mem_freq) { 7394 case 800: 7395 div = 10; 7396 break; 7397 case 1066: 7398 div = 12; 7399 break; 7400 case 1333: 7401 div = 16; 7402 break; 7403 default: 7404 return -1; 7405 } 7406 7407 return DIV_ROUND_CLOSEST(dev_priv->mem_freq * (val + 6 - 0xbd), 4 * div); 7408 } 7409 7410 static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val) 7411 { 7412 int mul; 7413 7414 /* 4 x czclk */ 7415 switch (dev_priv->mem_freq) { 7416 case 800: 7417 mul = 10; 7418 break; 7419 case 1066: 7420 mul = 12; 7421 break; 7422 case 1333: 7423 mul = 16; 7424 break; 7425 default: 7426 return -1; 7427 } 7428 7429 return DIV_ROUND_CLOSEST(4 * mul * val, dev_priv->mem_freq) + 0xbd - 6; 7430 } 7431 7432 static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val) 7433 { 7434 int div, freq; 7435 7436 switch (dev_priv->rps.cz_freq) { 7437 case 200: 7438 div = 5; 7439 break; 7440 case 267: 7441 div = 6; 7442 break; 7443 case 320: 7444 case 333: 7445 case 400: 7446 div = 8; 7447 break; 7448 default: 7449 return -1; 7450 } 7451 7452 freq = (DIV_ROUND_CLOSEST((dev_priv->rps.cz_freq * val), 2 * div) / 2); 7453 7454 return freq; 7455 } 7456 7457 static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val) 7458 { 7459 int mul, opcode; 7460 7461 switch (dev_priv->rps.cz_freq) { 7462 case 200: 7463 mul = 5; 7464 break; 7465 case 267: 7466 mul = 6; 7467 break; 7468 case 320: 7469 case 333: 7470 case 400: 7471 mul = 8; 7472 break; 7473 default: 7474 return -1; 7475 } 7476 7477 /* CHV needs even values */ 7478 opcode = (DIV_ROUND_CLOSEST((val * 2 * mul), dev_priv->rps.cz_freq) * 2); 7479 7480 return opcode; 7481 } 7482 7483 int vlv_gpu_freq(struct drm_i915_private *dev_priv, int val) 7484 { 7485 int ret = -1; 7486 7487 if (IS_CHERRYVIEW(dev_priv->dev)) 7488 ret = chv_gpu_freq(dev_priv, val); 7489 else if (IS_VALLEYVIEW(dev_priv->dev)) 7490 ret = byt_gpu_freq(dev_priv, val); 7491 7492 return ret; 7493 } 7494 7495 int vlv_freq_opcode(struct drm_i915_private *dev_priv, int val) 7496 { 7497 int ret = -1; 7498 7499 if (IS_CHERRYVIEW(dev_priv->dev)) 7500 ret = chv_freq_opcode(dev_priv, val); 7501 else if (IS_VALLEYVIEW(dev_priv->dev)) 7502 ret = byt_freq_opcode(dev_priv, val); 7503 7504 return ret; 7505 } 7506 7507 void intel_pm_setup(struct drm_device *dev) 7508 { 7509 struct drm_i915_private *dev_priv = dev->dev_private; 7510 7511 lockinit(&dev_priv->rps.hw_lock, "i915 rps.hw_lock", 0, LK_CANRECURSE); 7512 7513 INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work, 7514 intel_gen6_powersave_work); 7515 7516 dev_priv->pm.suspended = false; 7517 dev_priv->pm._irqs_disabled = false; 7518 } 7519