1 /*
2 * Copyright © 2012 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eugeni Dodonov <eugeni.dodonov@intel.com>
25 *
26 */
27
28 #include <linux/module.h>
29 #include <linux/pm_runtime.h>
30
31 #include <drm/drm_atomic_helper.h>
32 #include <drm/drm_fourcc.h>
33 #include <drm/drm_plane_helper.h>
34
35 #include "display/intel_atomic.h"
36 #include "display/intel_atomic_plane.h"
37 #include "display/intel_bw.h"
38 #include "display/intel_display_types.h"
39 #include "display/intel_fbc.h"
40 #include "display/intel_sprite.h"
41 #include "display/skl_universal_plane.h"
42
43 #include "gt/intel_llc.h"
44
45 #include "i915_drv.h"
46 #include "i915_fixed.h"
47 #include "i915_irq.h"
48 #include "i915_trace.h"
49 #include "intel_pm.h"
50 #include "intel_sideband.h"
51 #include "../../../platform/x86/intel_ips.h"
52
53 /* Stores plane specific WM parameters */
54 struct skl_wm_params {
55 bool x_tiled, y_tiled;
56 bool rc_surface;
57 bool is_planar;
58 u32 width;
59 u8 cpp;
60 u32 plane_pixel_rate;
61 u32 y_min_scanlines;
62 u32 plane_bytes_per_line;
63 uint_fixed_16_16_t plane_blocks_per_line;
64 uint_fixed_16_16_t y_tile_minimum;
65 u32 linetime_us;
66 u32 dbuf_block_size;
67 };
68
69 /* used in computing the new watermarks state */
70 struct intel_wm_config {
71 unsigned int num_pipes_active;
72 bool sprites_enabled;
73 bool sprites_scaled;
74 };
75
gen9_init_clock_gating(struct drm_i915_private * dev_priv)76 static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
77 {
78 if (HAS_LLC(dev_priv)) {
79 /*
80 * WaCompressedResourceDisplayNewHashMode:skl,kbl
81 * Display WA #0390: skl,kbl
82 *
83 * Must match Sampler, Pixel Back End, and Media. See
84 * WaCompressedResourceSamplerPbeMediaNewHashMode.
85 */
86 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
87 intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) |
88 SKL_DE_COMPRESSED_HASH_MODE);
89 }
90
91 /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
92 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
93 intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
94
95 /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
96 intel_uncore_write(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
97 intel_uncore_read(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
98
99 /*
100 * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
101 * Display WA #0859: skl,bxt,kbl,glk,cfl
102 */
103 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
104 DISP_FBC_MEMORY_WAKE);
105 }
106
bxt_init_clock_gating(struct drm_i915_private * dev_priv)107 static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
108 {
109 gen9_init_clock_gating(dev_priv);
110
111 /* WaDisableSDEUnitClockGating:bxt */
112 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
113 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
114
115 /*
116 * FIXME:
117 * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
118 */
119 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
120 GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
121
122 /*
123 * Wa: Backlight PWM may stop in the asserted state, causing backlight
124 * to stay fully on.
125 */
126 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
127 PWM1_GATING_DIS | PWM2_GATING_DIS);
128
129 /*
130 * Lower the display internal timeout.
131 * This is needed to avoid any hard hangs when DSI port PLL
132 * is off and a MMIO access is attempted by any privilege
133 * application, using batch buffers or any other means.
134 */
135 intel_uncore_write(&dev_priv->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
136
137 /*
138 * WaFbcTurnOffFbcWatermark:bxt
139 * Display WA #0562: bxt
140 */
141 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
142 DISP_FBC_WM_DIS);
143
144 /*
145 * WaFbcHighMemBwCorruptionAvoidance:bxt
146 * Display WA #0883: bxt
147 */
148 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
149 ILK_DPFC_DISABLE_DUMMY0);
150 }
151
glk_init_clock_gating(struct drm_i915_private * dev_priv)152 static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
153 {
154 gen9_init_clock_gating(dev_priv);
155
156 /*
157 * WaDisablePWMClockGating:glk
158 * Backlight PWM may stop in the asserted state, causing backlight
159 * to stay fully on.
160 */
161 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
162 PWM1_GATING_DIS | PWM2_GATING_DIS);
163 }
164
pnv_get_mem_freq(struct drm_i915_private * dev_priv)165 static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
166 {
167 u32 tmp;
168
169 tmp = intel_uncore_read(&dev_priv->uncore, CLKCFG);
170
171 switch (tmp & CLKCFG_FSB_MASK) {
172 case CLKCFG_FSB_533:
173 dev_priv->fsb_freq = 533; /* 133*4 */
174 break;
175 case CLKCFG_FSB_800:
176 dev_priv->fsb_freq = 800; /* 200*4 */
177 break;
178 case CLKCFG_FSB_667:
179 dev_priv->fsb_freq = 667; /* 167*4 */
180 break;
181 case CLKCFG_FSB_400:
182 dev_priv->fsb_freq = 400; /* 100*4 */
183 break;
184 }
185
186 switch (tmp & CLKCFG_MEM_MASK) {
187 case CLKCFG_MEM_533:
188 dev_priv->mem_freq = 533;
189 break;
190 case CLKCFG_MEM_667:
191 dev_priv->mem_freq = 667;
192 break;
193 case CLKCFG_MEM_800:
194 dev_priv->mem_freq = 800;
195 break;
196 }
197
198 /* detect pineview DDR3 setting */
199 tmp = intel_uncore_read(&dev_priv->uncore, CSHRDDR3CTL);
200 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
201 }
202
ilk_get_mem_freq(struct drm_i915_private * dev_priv)203 static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
204 {
205 u16 ddrpll, csipll;
206
207 ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
208 csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
209
210 switch (ddrpll & 0xff) {
211 case 0xc:
212 dev_priv->mem_freq = 800;
213 break;
214 case 0x10:
215 dev_priv->mem_freq = 1066;
216 break;
217 case 0x14:
218 dev_priv->mem_freq = 1333;
219 break;
220 case 0x18:
221 dev_priv->mem_freq = 1600;
222 break;
223 default:
224 drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
225 ddrpll & 0xff);
226 dev_priv->mem_freq = 0;
227 break;
228 }
229
230 switch (csipll & 0x3ff) {
231 case 0x00c:
232 dev_priv->fsb_freq = 3200;
233 break;
234 case 0x00e:
235 dev_priv->fsb_freq = 3733;
236 break;
237 case 0x010:
238 dev_priv->fsb_freq = 4266;
239 break;
240 case 0x012:
241 dev_priv->fsb_freq = 4800;
242 break;
243 case 0x014:
244 dev_priv->fsb_freq = 5333;
245 break;
246 case 0x016:
247 dev_priv->fsb_freq = 5866;
248 break;
249 case 0x018:
250 dev_priv->fsb_freq = 6400;
251 break;
252 default:
253 drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
254 csipll & 0x3ff);
255 dev_priv->fsb_freq = 0;
256 break;
257 }
258 }
259
260 static const struct cxsr_latency cxsr_latency_table[] = {
261 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
262 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
263 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
264 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
265 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
266
267 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
268 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
269 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
270 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
271 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
272
273 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
274 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
275 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
276 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
277 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
278
279 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
280 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
281 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
282 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
283 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
284
285 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
286 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
287 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
288 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
289 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
290
291 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
292 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
293 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
294 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
295 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
296 };
297
intel_get_cxsr_latency(bool is_desktop,bool is_ddr3,int fsb,int mem)298 static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
299 bool is_ddr3,
300 int fsb,
301 int mem)
302 {
303 const struct cxsr_latency *latency;
304 int i;
305
306 if (fsb == 0 || mem == 0)
307 return NULL;
308
309 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
310 latency = &cxsr_latency_table[i];
311 if (is_desktop == latency->is_desktop &&
312 is_ddr3 == latency->is_ddr3 &&
313 fsb == latency->fsb_freq && mem == latency->mem_freq)
314 return latency;
315 }
316
317 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
318
319 return NULL;
320 }
321
chv_set_memory_dvfs(struct drm_i915_private * dev_priv,bool enable)322 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
323 {
324 u32 val;
325
326 vlv_punit_get(dev_priv);
327
328 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
329 if (enable)
330 val &= ~FORCE_DDR_HIGH_FREQ;
331 else
332 val |= FORCE_DDR_HIGH_FREQ;
333 val &= ~FORCE_DDR_LOW_FREQ;
334 val |= FORCE_DDR_FREQ_REQ_ACK;
335 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
336
337 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
338 FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
339 drm_err(&dev_priv->drm,
340 "timed out waiting for Punit DDR DVFS request\n");
341
342 vlv_punit_put(dev_priv);
343 }
344
chv_set_memory_pm5(struct drm_i915_private * dev_priv,bool enable)345 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
346 {
347 u32 val;
348
349 vlv_punit_get(dev_priv);
350
351 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
352 if (enable)
353 val |= DSP_MAXFIFO_PM5_ENABLE;
354 else
355 val &= ~DSP_MAXFIFO_PM5_ENABLE;
356 vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
357
358 vlv_punit_put(dev_priv);
359 }
360
361 #define FW_WM(value, plane) \
362 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
363
_intel_set_memory_cxsr(struct drm_i915_private * dev_priv,bool enable)364 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
365 {
366 bool was_enabled;
367 u32 val;
368
369 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
370 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
371 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
372 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV);
373 } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
374 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
375 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
376 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
377 } else if (IS_PINEVIEW(dev_priv)) {
378 val = intel_uncore_read(&dev_priv->uncore, DSPFW3);
379 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
380 if (enable)
381 val |= PINEVIEW_SELF_REFRESH_EN;
382 else
383 val &= ~PINEVIEW_SELF_REFRESH_EN;
384 intel_uncore_write(&dev_priv->uncore, DSPFW3, val);
385 intel_uncore_posting_read(&dev_priv->uncore, DSPFW3);
386 } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
387 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
388 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
389 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
390 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val);
391 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
392 } else if (IS_I915GM(dev_priv)) {
393 /*
394 * FIXME can't find a bit like this for 915G, and
395 * and yet it does have the related watermark in
396 * FW_BLC_SELF. What's going on?
397 */
398 was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN;
399 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
400 _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
401 intel_uncore_write(&dev_priv->uncore, INSTPM, val);
402 intel_uncore_posting_read(&dev_priv->uncore, INSTPM);
403 } else {
404 return false;
405 }
406
407 trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
408
409 drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
410 enableddisabled(enable),
411 enableddisabled(was_enabled));
412
413 return was_enabled;
414 }
415
416 /**
417 * intel_set_memory_cxsr - Configure CxSR state
418 * @dev_priv: i915 device
419 * @enable: Allow vs. disallow CxSR
420 *
421 * Allow or disallow the system to enter a special CxSR
422 * (C-state self refresh) state. What typically happens in CxSR mode
423 * is that several display FIFOs may get combined into a single larger
424 * FIFO for a particular plane (so called max FIFO mode) to allow the
425 * system to defer memory fetches longer, and the memory will enter
426 * self refresh.
427 *
428 * Note that enabling CxSR does not guarantee that the system enter
429 * this special mode, nor does it guarantee that the system stays
430 * in that mode once entered. So this just allows/disallows the system
431 * to autonomously utilize the CxSR mode. Other factors such as core
432 * C-states will affect when/if the system actually enters/exits the
433 * CxSR mode.
434 *
435 * Note that on VLV/CHV this actually only controls the max FIFO mode,
436 * and the system is free to enter/exit memory self refresh at any time
437 * even when the use of CxSR has been disallowed.
438 *
439 * While the system is actually in the CxSR/max FIFO mode, some plane
440 * control registers will not get latched on vblank. Thus in order to
441 * guarantee the system will respond to changes in the plane registers
442 * we must always disallow CxSR prior to making changes to those registers.
443 * Unfortunately the system will re-evaluate the CxSR conditions at
444 * frame start which happens after vblank start (which is when the plane
445 * registers would get latched), so we can't proceed with the plane update
446 * during the same frame where we disallowed CxSR.
447 *
448 * Certain platforms also have a deeper HPLL SR mode. Fortunately the
449 * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
450 * the hardware w.r.t. HPLL SR when writing to plane registers.
451 * Disallowing just CxSR is sufficient.
452 */
intel_set_memory_cxsr(struct drm_i915_private * dev_priv,bool enable)453 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
454 {
455 bool ret;
456
457 mutex_lock(&dev_priv->wm.wm_mutex);
458 ret = _intel_set_memory_cxsr(dev_priv, enable);
459 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
460 dev_priv->wm.vlv.cxsr = enable;
461 else if (IS_G4X(dev_priv))
462 dev_priv->wm.g4x.cxsr = enable;
463 mutex_unlock(&dev_priv->wm.wm_mutex);
464
465 return ret;
466 }
467
468 /*
469 * Latency for FIFO fetches is dependent on several factors:
470 * - memory configuration (speed, channels)
471 * - chipset
472 * - current MCH state
473 * It can be fairly high in some situations, so here we assume a fairly
474 * pessimal value. It's a tradeoff between extra memory fetches (if we
475 * set this value too high, the FIFO will fetch frequently to stay full)
476 * and power consumption (set it too low to save power and we might see
477 * FIFO underruns and display "flicker").
478 *
479 * A value of 5us seems to be a good balance; safe for very low end
480 * platforms but not overly aggressive on lower latency configs.
481 */
482 static const int pessimal_latency_ns = 5000;
483
484 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
485 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
486
vlv_get_fifo_size(struct intel_crtc_state * crtc_state)487 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
488 {
489 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
490 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
491 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
492 enum pipe pipe = crtc->pipe;
493 int sprite0_start, sprite1_start;
494 u32 dsparb, dsparb2, dsparb3;
495
496 switch (pipe) {
497 case PIPE_A:
498 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
499 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
500 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
501 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
502 break;
503 case PIPE_B:
504 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
505 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
506 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
507 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
508 break;
509 case PIPE_C:
510 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
511 dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3);
512 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
513 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
514 break;
515 default:
516 MISSING_CASE(pipe);
517 return;
518 }
519
520 fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
521 fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
522 fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
523 fifo_state->plane[PLANE_CURSOR] = 63;
524 }
525
i9xx_get_fifo_size(struct drm_i915_private * dev_priv,enum i9xx_plane_id i9xx_plane)526 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
527 enum i9xx_plane_id i9xx_plane)
528 {
529 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
530 int size;
531
532 size = dsparb & 0x7f;
533 if (i9xx_plane == PLANE_B)
534 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
535
536 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
537 dsparb, plane_name(i9xx_plane), size);
538
539 return size;
540 }
541
i830_get_fifo_size(struct drm_i915_private * dev_priv,enum i9xx_plane_id i9xx_plane)542 static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
543 enum i9xx_plane_id i9xx_plane)
544 {
545 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
546 int size;
547
548 size = dsparb & 0x1ff;
549 if (i9xx_plane == PLANE_B)
550 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
551 size >>= 1; /* Convert to cachelines */
552
553 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
554 dsparb, plane_name(i9xx_plane), size);
555
556 return size;
557 }
558
i845_get_fifo_size(struct drm_i915_private * dev_priv,enum i9xx_plane_id i9xx_plane)559 static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
560 enum i9xx_plane_id i9xx_plane)
561 {
562 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
563 int size;
564
565 size = dsparb & 0x7f;
566 size >>= 2; /* Convert to cachelines */
567
568 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
569 dsparb, plane_name(i9xx_plane), size);
570
571 return size;
572 }
573
574 /* Pineview has different values for various configs */
575 static const struct intel_watermark_params pnv_display_wm = {
576 .fifo_size = PINEVIEW_DISPLAY_FIFO,
577 .max_wm = PINEVIEW_MAX_WM,
578 .default_wm = PINEVIEW_DFT_WM,
579 .guard_size = PINEVIEW_GUARD_WM,
580 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
581 };
582
583 static const struct intel_watermark_params pnv_display_hplloff_wm = {
584 .fifo_size = PINEVIEW_DISPLAY_FIFO,
585 .max_wm = PINEVIEW_MAX_WM,
586 .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
587 .guard_size = PINEVIEW_GUARD_WM,
588 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
589 };
590
591 static const struct intel_watermark_params pnv_cursor_wm = {
592 .fifo_size = PINEVIEW_CURSOR_FIFO,
593 .max_wm = PINEVIEW_CURSOR_MAX_WM,
594 .default_wm = PINEVIEW_CURSOR_DFT_WM,
595 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
596 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
597 };
598
599 static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
600 .fifo_size = PINEVIEW_CURSOR_FIFO,
601 .max_wm = PINEVIEW_CURSOR_MAX_WM,
602 .default_wm = PINEVIEW_CURSOR_DFT_WM,
603 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
604 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
605 };
606
607 static const struct intel_watermark_params i965_cursor_wm_info = {
608 .fifo_size = I965_CURSOR_FIFO,
609 .max_wm = I965_CURSOR_MAX_WM,
610 .default_wm = I965_CURSOR_DFT_WM,
611 .guard_size = 2,
612 .cacheline_size = I915_FIFO_LINE_SIZE,
613 };
614
615 static const struct intel_watermark_params i945_wm_info = {
616 .fifo_size = I945_FIFO_SIZE,
617 .max_wm = I915_MAX_WM,
618 .default_wm = 1,
619 .guard_size = 2,
620 .cacheline_size = I915_FIFO_LINE_SIZE,
621 };
622
623 static const struct intel_watermark_params i915_wm_info = {
624 .fifo_size = I915_FIFO_SIZE,
625 .max_wm = I915_MAX_WM,
626 .default_wm = 1,
627 .guard_size = 2,
628 .cacheline_size = I915_FIFO_LINE_SIZE,
629 };
630
631 static const struct intel_watermark_params i830_a_wm_info = {
632 .fifo_size = I855GM_FIFO_SIZE,
633 .max_wm = I915_MAX_WM,
634 .default_wm = 1,
635 .guard_size = 2,
636 .cacheline_size = I830_FIFO_LINE_SIZE,
637 };
638
639 static const struct intel_watermark_params i830_bc_wm_info = {
640 .fifo_size = I855GM_FIFO_SIZE,
641 .max_wm = I915_MAX_WM/2,
642 .default_wm = 1,
643 .guard_size = 2,
644 .cacheline_size = I830_FIFO_LINE_SIZE,
645 };
646
647 static const struct intel_watermark_params i845_wm_info = {
648 .fifo_size = I830_FIFO_SIZE,
649 .max_wm = I915_MAX_WM,
650 .default_wm = 1,
651 .guard_size = 2,
652 .cacheline_size = I830_FIFO_LINE_SIZE,
653 };
654
655 /**
656 * intel_wm_method1 - Method 1 / "small buffer" watermark formula
657 * @pixel_rate: Pipe pixel rate in kHz
658 * @cpp: Plane bytes per pixel
659 * @latency: Memory wakeup latency in 0.1us units
660 *
661 * Compute the watermark using the method 1 or "small buffer"
662 * formula. The caller may additonally add extra cachelines
663 * to account for TLB misses and clock crossings.
664 *
665 * This method is concerned with the short term drain rate
666 * of the FIFO, ie. it does not account for blanking periods
667 * which would effectively reduce the average drain rate across
668 * a longer period. The name "small" refers to the fact the
669 * FIFO is relatively small compared to the amount of data
670 * fetched.
671 *
672 * The FIFO level vs. time graph might look something like:
673 *
674 * |\ |\
675 * | \ | \
676 * __---__---__ (- plane active, _ blanking)
677 * -> time
678 *
679 * or perhaps like this:
680 *
681 * |\|\ |\|\
682 * __----__----__ (- plane active, _ blanking)
683 * -> time
684 *
685 * Returns:
686 * The watermark in bytes
687 */
intel_wm_method1(unsigned int pixel_rate,unsigned int cpp,unsigned int latency)688 static unsigned int intel_wm_method1(unsigned int pixel_rate,
689 unsigned int cpp,
690 unsigned int latency)
691 {
692 u64 ret;
693
694 ret = mul_u32_u32(pixel_rate, cpp * latency);
695 ret = DIV_ROUND_UP_ULL(ret, 10000);
696
697 return ret;
698 }
699
700 /**
701 * intel_wm_method2 - Method 2 / "large buffer" watermark formula
702 * @pixel_rate: Pipe pixel rate in kHz
703 * @htotal: Pipe horizontal total
704 * @width: Plane width in pixels
705 * @cpp: Plane bytes per pixel
706 * @latency: Memory wakeup latency in 0.1us units
707 *
708 * Compute the watermark using the method 2 or "large buffer"
709 * formula. The caller may additonally add extra cachelines
710 * to account for TLB misses and clock crossings.
711 *
712 * This method is concerned with the long term drain rate
713 * of the FIFO, ie. it does account for blanking periods
714 * which effectively reduce the average drain rate across
715 * a longer period. The name "large" refers to the fact the
716 * FIFO is relatively large compared to the amount of data
717 * fetched.
718 *
719 * The FIFO level vs. time graph might look something like:
720 *
721 * |\___ |\___
722 * | \___ | \___
723 * | \ | \
724 * __ --__--__--__--__--__--__ (- plane active, _ blanking)
725 * -> time
726 *
727 * Returns:
728 * The watermark in bytes
729 */
intel_wm_method2(unsigned int pixel_rate,unsigned int htotal,unsigned int width,unsigned int cpp,unsigned int latency)730 static unsigned int intel_wm_method2(unsigned int pixel_rate,
731 unsigned int htotal,
732 unsigned int width,
733 unsigned int cpp,
734 unsigned int latency)
735 {
736 unsigned int ret;
737
738 /*
739 * FIXME remove once all users are computing
740 * watermarks in the correct place.
741 */
742 if (WARN_ON_ONCE(htotal == 0))
743 htotal = 1;
744
745 ret = (latency * pixel_rate) / (htotal * 10000);
746 ret = (ret + 1) * width * cpp;
747
748 return ret;
749 }
750
751 /**
752 * intel_calculate_wm - calculate watermark level
753 * @pixel_rate: pixel clock
754 * @wm: chip FIFO params
755 * @fifo_size: size of the FIFO buffer
756 * @cpp: bytes per pixel
757 * @latency_ns: memory latency for the platform
758 *
759 * Calculate the watermark level (the level at which the display plane will
760 * start fetching from memory again). Each chip has a different display
761 * FIFO size and allocation, so the caller needs to figure that out and pass
762 * in the correct intel_watermark_params structure.
763 *
764 * As the pixel clock runs, the FIFO will be drained at a rate that depends
765 * on the pixel size. When it reaches the watermark level, it'll start
766 * fetching FIFO line sized based chunks from memory until the FIFO fills
767 * past the watermark point. If the FIFO drains completely, a FIFO underrun
768 * will occur, and a display engine hang could result.
769 */
intel_calculate_wm(int pixel_rate,const struct intel_watermark_params * wm,int fifo_size,int cpp,unsigned int latency_ns)770 static unsigned int intel_calculate_wm(int pixel_rate,
771 const struct intel_watermark_params *wm,
772 int fifo_size, int cpp,
773 unsigned int latency_ns)
774 {
775 int entries, wm_size;
776
777 /*
778 * Note: we need to make sure we don't overflow for various clock &
779 * latency values.
780 * clocks go from a few thousand to several hundred thousand.
781 * latency is usually a few thousand
782 */
783 entries = intel_wm_method1(pixel_rate, cpp,
784 latency_ns / 100);
785 entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
786 wm->guard_size;
787 DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
788
789 wm_size = fifo_size - entries;
790 DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
791
792 /* Don't promote wm_size to unsigned... */
793 if (wm_size > wm->max_wm)
794 wm_size = wm->max_wm;
795 if (wm_size <= 0)
796 wm_size = wm->default_wm;
797
798 /*
799 * Bspec seems to indicate that the value shouldn't be lower than
800 * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
801 * Lets go for 8 which is the burst size since certain platforms
802 * already use a hardcoded 8 (which is what the spec says should be
803 * done).
804 */
805 if (wm_size <= 8)
806 wm_size = 8;
807
808 return wm_size;
809 }
810
is_disabling(int old,int new,int threshold)811 static bool is_disabling(int old, int new, int threshold)
812 {
813 return old >= threshold && new < threshold;
814 }
815
is_enabling(int old,int new,int threshold)816 static bool is_enabling(int old, int new, int threshold)
817 {
818 return old < threshold && new >= threshold;
819 }
820
intel_wm_num_levels(struct drm_i915_private * dev_priv)821 static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
822 {
823 return dev_priv->wm.max_level + 1;
824 }
825
intel_wm_plane_visible(const struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state)826 static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
827 const struct intel_plane_state *plane_state)
828 {
829 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
830
831 /* FIXME check the 'enable' instead */
832 if (!crtc_state->hw.active)
833 return false;
834
835 /*
836 * Treat cursor with fb as always visible since cursor updates
837 * can happen faster than the vrefresh rate, and the current
838 * watermark code doesn't handle that correctly. Cursor updates
839 * which set/clear the fb or change the cursor size are going
840 * to get throttled by intel_legacy_cursor_update() to work
841 * around this problem with the watermark code.
842 */
843 if (plane->id == PLANE_CURSOR)
844 return plane_state->hw.fb != NULL;
845 else
846 return plane_state->uapi.visible;
847 }
848
intel_crtc_active(struct intel_crtc * crtc)849 static bool intel_crtc_active(struct intel_crtc *crtc)
850 {
851 /* Be paranoid as we can arrive here with only partial
852 * state retrieved from the hardware during setup.
853 *
854 * We can ditch the adjusted_mode.crtc_clock check as soon
855 * as Haswell has gained clock readout/fastboot support.
856 *
857 * We can ditch the crtc->primary->state->fb check as soon as we can
858 * properly reconstruct framebuffers.
859 *
860 * FIXME: The intel_crtc->active here should be switched to
861 * crtc->state->active once we have proper CRTC states wired up
862 * for atomic.
863 */
864 return crtc->active && crtc->base.primary->state->fb &&
865 crtc->config->hw.adjusted_mode.crtc_clock;
866 }
867
single_enabled_crtc(struct drm_i915_private * dev_priv)868 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
869 {
870 struct intel_crtc *crtc, *enabled = NULL;
871
872 for_each_intel_crtc(&dev_priv->drm, crtc) {
873 if (intel_crtc_active(crtc)) {
874 if (enabled)
875 return NULL;
876 enabled = crtc;
877 }
878 }
879
880 return enabled;
881 }
882
pnv_update_wm(struct intel_crtc * unused_crtc)883 static void pnv_update_wm(struct intel_crtc *unused_crtc)
884 {
885 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
886 struct intel_crtc *crtc;
887 const struct cxsr_latency *latency;
888 u32 reg;
889 unsigned int wm;
890
891 latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
892 dev_priv->is_ddr3,
893 dev_priv->fsb_freq,
894 dev_priv->mem_freq);
895 if (!latency) {
896 drm_dbg_kms(&dev_priv->drm,
897 "Unknown FSB/MEM found, disable CxSR\n");
898 intel_set_memory_cxsr(dev_priv, false);
899 return;
900 }
901
902 crtc = single_enabled_crtc(dev_priv);
903 if (crtc) {
904 const struct drm_display_mode *pipe_mode =
905 &crtc->config->hw.pipe_mode;
906 const struct drm_framebuffer *fb =
907 crtc->base.primary->state->fb;
908 int cpp = fb->format->cpp[0];
909 int clock = pipe_mode->crtc_clock;
910
911 /* Display SR */
912 wm = intel_calculate_wm(clock, &pnv_display_wm,
913 pnv_display_wm.fifo_size,
914 cpp, latency->display_sr);
915 reg = intel_uncore_read(&dev_priv->uncore, DSPFW1);
916 reg &= ~DSPFW_SR_MASK;
917 reg |= FW_WM(wm, SR);
918 intel_uncore_write(&dev_priv->uncore, DSPFW1, reg);
919 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
920
921 /* cursor SR */
922 wm = intel_calculate_wm(clock, &pnv_cursor_wm,
923 pnv_display_wm.fifo_size,
924 4, latency->cursor_sr);
925 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
926 reg &= ~DSPFW_CURSOR_SR_MASK;
927 reg |= FW_WM(wm, CURSOR_SR);
928 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
929
930 /* Display HPLL off SR */
931 wm = intel_calculate_wm(clock, &pnv_display_hplloff_wm,
932 pnv_display_hplloff_wm.fifo_size,
933 cpp, latency->display_hpll_disable);
934 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
935 reg &= ~DSPFW_HPLL_SR_MASK;
936 reg |= FW_WM(wm, HPLL_SR);
937 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
938
939 /* cursor HPLL off SR */
940 wm = intel_calculate_wm(clock, &pnv_cursor_hplloff_wm,
941 pnv_display_hplloff_wm.fifo_size,
942 4, latency->cursor_hpll_disable);
943 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
944 reg &= ~DSPFW_HPLL_CURSOR_MASK;
945 reg |= FW_WM(wm, HPLL_CURSOR);
946 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
947 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
948
949 intel_set_memory_cxsr(dev_priv, true);
950 } else {
951 intel_set_memory_cxsr(dev_priv, false);
952 }
953 }
954
955 /*
956 * Documentation says:
957 * "If the line size is small, the TLB fetches can get in the way of the
958 * data fetches, causing some lag in the pixel data return which is not
959 * accounted for in the above formulas. The following adjustment only
960 * needs to be applied if eight whole lines fit in the buffer at once.
961 * The WM is adjusted upwards by the difference between the FIFO size
962 * and the size of 8 whole lines. This adjustment is always performed
963 * in the actual pixel depth regardless of whether FBC is enabled or not."
964 */
g4x_tlb_miss_wa(int fifo_size,int width,int cpp)965 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
966 {
967 int tlb_miss = fifo_size * 64 - width * cpp * 8;
968
969 return max(0, tlb_miss);
970 }
971
g4x_write_wm_values(struct drm_i915_private * dev_priv,const struct g4x_wm_values * wm)972 static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
973 const struct g4x_wm_values *wm)
974 {
975 enum pipe pipe;
976
977 for_each_pipe(dev_priv, pipe)
978 trace_g4x_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
979
980 intel_uncore_write(&dev_priv->uncore, DSPFW1,
981 FW_WM(wm->sr.plane, SR) |
982 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
983 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
984 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
985 intel_uncore_write(&dev_priv->uncore, DSPFW2,
986 (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
987 FW_WM(wm->sr.fbc, FBC_SR) |
988 FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
989 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
990 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
991 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
992 intel_uncore_write(&dev_priv->uncore, DSPFW3,
993 (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
994 FW_WM(wm->sr.cursor, CURSOR_SR) |
995 FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
996 FW_WM(wm->hpll.plane, HPLL_SR));
997
998 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
999 }
1000
1001 #define FW_WM_VLV(value, plane) \
1002 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
1003
vlv_write_wm_values(struct drm_i915_private * dev_priv,const struct vlv_wm_values * wm)1004 static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
1005 const struct vlv_wm_values *wm)
1006 {
1007 enum pipe pipe;
1008
1009 for_each_pipe(dev_priv, pipe) {
1010 trace_vlv_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
1011
1012 intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe),
1013 (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
1014 (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
1015 (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
1016 (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
1017 }
1018
1019 /*
1020 * Zero the (unused) WM1 watermarks, and also clear all the
1021 * high order bits so that there are no out of bounds values
1022 * present in the registers during the reprogramming.
1023 */
1024 intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0);
1025 intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0);
1026 intel_uncore_write(&dev_priv->uncore, DSPFW4, 0);
1027 intel_uncore_write(&dev_priv->uncore, DSPFW5, 0);
1028 intel_uncore_write(&dev_priv->uncore, DSPFW6, 0);
1029
1030 intel_uncore_write(&dev_priv->uncore, DSPFW1,
1031 FW_WM(wm->sr.plane, SR) |
1032 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
1033 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
1034 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
1035 intel_uncore_write(&dev_priv->uncore, DSPFW2,
1036 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
1037 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1038 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1039 intel_uncore_write(&dev_priv->uncore, DSPFW3,
1040 FW_WM(wm->sr.cursor, CURSOR_SR));
1041
1042 if (IS_CHERRYVIEW(dev_priv)) {
1043 intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV,
1044 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1045 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1046 intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV,
1047 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1048 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1049 intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV,
1050 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1051 FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1052 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1053 FW_WM(wm->sr.plane >> 9, SR_HI) |
1054 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1055 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1056 FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1057 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1058 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1059 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1060 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1061 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1062 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1063 } else {
1064 intel_uncore_write(&dev_priv->uncore, DSPFW7,
1065 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1066 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1067 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1068 FW_WM(wm->sr.plane >> 9, SR_HI) |
1069 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1070 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1071 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1072 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1073 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1074 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1075 }
1076
1077 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
1078 }
1079
1080 #undef FW_WM_VLV
1081
g4x_setup_wm_latency(struct drm_i915_private * dev_priv)1082 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1083 {
1084 /* all latencies in usec */
1085 dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1086 dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1087 dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1088
1089 dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
1090 }
1091
g4x_plane_fifo_size(enum plane_id plane_id,int level)1092 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1093 {
1094 /*
1095 * DSPCNTR[13] supposedly controls whether the
1096 * primary plane can use the FIFO space otherwise
1097 * reserved for the sprite plane. It's not 100% clear
1098 * what the actual FIFO size is, but it looks like we
1099 * can happily set both primary and sprite watermarks
1100 * up to 127 cachelines. So that would seem to mean
1101 * that either DSPCNTR[13] doesn't do anything, or that
1102 * the total FIFO is >= 256 cachelines in size. Either
1103 * way, we don't seem to have to worry about this
1104 * repartitioning as the maximum watermark value the
1105 * register can hold for each plane is lower than the
1106 * minimum FIFO size.
1107 */
1108 switch (plane_id) {
1109 case PLANE_CURSOR:
1110 return 63;
1111 case PLANE_PRIMARY:
1112 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1113 case PLANE_SPRITE0:
1114 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1115 default:
1116 MISSING_CASE(plane_id);
1117 return 0;
1118 }
1119 }
1120
g4x_fbc_fifo_size(int level)1121 static int g4x_fbc_fifo_size(int level)
1122 {
1123 switch (level) {
1124 case G4X_WM_LEVEL_SR:
1125 return 7;
1126 case G4X_WM_LEVEL_HPLL:
1127 return 15;
1128 default:
1129 MISSING_CASE(level);
1130 return 0;
1131 }
1132 }
1133
g4x_compute_wm(const struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state,int level)1134 static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1135 const struct intel_plane_state *plane_state,
1136 int level)
1137 {
1138 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1139 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1140 const struct drm_display_mode *pipe_mode =
1141 &crtc_state->hw.pipe_mode;
1142 unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
1143 unsigned int clock, htotal, cpp, width, wm;
1144
1145 if (latency == 0)
1146 return USHRT_MAX;
1147
1148 if (!intel_wm_plane_visible(crtc_state, plane_state))
1149 return 0;
1150
1151 cpp = plane_state->hw.fb->format->cpp[0];
1152
1153 /*
1154 * Not 100% sure which way ELK should go here as the
1155 * spec only says CL/CTG should assume 32bpp and BW
1156 * doesn't need to. But as these things followed the
1157 * mobile vs. desktop lines on gen3 as well, let's
1158 * assume ELK doesn't need this.
1159 *
1160 * The spec also fails to list such a restriction for
1161 * the HPLL watermark, which seems a little strange.
1162 * Let's use 32bpp for the HPLL watermark as well.
1163 */
1164 if (IS_GM45(dev_priv) && plane->id == PLANE_PRIMARY &&
1165 level != G4X_WM_LEVEL_NORMAL)
1166 cpp = max(cpp, 4u);
1167
1168 clock = pipe_mode->crtc_clock;
1169 htotal = pipe_mode->crtc_htotal;
1170
1171 width = drm_rect_width(&plane_state->uapi.dst);
1172
1173 if (plane->id == PLANE_CURSOR) {
1174 wm = intel_wm_method2(clock, htotal, width, cpp, latency);
1175 } else if (plane->id == PLANE_PRIMARY &&
1176 level == G4X_WM_LEVEL_NORMAL) {
1177 wm = intel_wm_method1(clock, cpp, latency);
1178 } else {
1179 unsigned int small, large;
1180
1181 small = intel_wm_method1(clock, cpp, latency);
1182 large = intel_wm_method2(clock, htotal, width, cpp, latency);
1183
1184 wm = min(small, large);
1185 }
1186
1187 wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1188 width, cpp);
1189
1190 wm = DIV_ROUND_UP(wm, 64) + 2;
1191
1192 return min_t(unsigned int, wm, USHRT_MAX);
1193 }
1194
g4x_raw_plane_wm_set(struct intel_crtc_state * crtc_state,int level,enum plane_id plane_id,u16 value)1195 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1196 int level, enum plane_id plane_id, u16 value)
1197 {
1198 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1199 bool dirty = false;
1200
1201 for (; level < intel_wm_num_levels(dev_priv); level++) {
1202 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1203
1204 dirty |= raw->plane[plane_id] != value;
1205 raw->plane[plane_id] = value;
1206 }
1207
1208 return dirty;
1209 }
1210
g4x_raw_fbc_wm_set(struct intel_crtc_state * crtc_state,int level,u16 value)1211 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1212 int level, u16 value)
1213 {
1214 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1215 bool dirty = false;
1216
1217 /* NORMAL level doesn't have an FBC watermark */
1218 level = max(level, G4X_WM_LEVEL_SR);
1219
1220 for (; level < intel_wm_num_levels(dev_priv); level++) {
1221 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1222
1223 dirty |= raw->fbc != value;
1224 raw->fbc = value;
1225 }
1226
1227 return dirty;
1228 }
1229
1230 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1231 const struct intel_plane_state *plane_state,
1232 u32 pri_val);
1233
g4x_raw_plane_wm_compute(struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state)1234 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1235 const struct intel_plane_state *plane_state)
1236 {
1237 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1238 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1239 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1240 enum plane_id plane_id = plane->id;
1241 bool dirty = false;
1242 int level;
1243
1244 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1245 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1246 if (plane_id == PLANE_PRIMARY)
1247 dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1248 goto out;
1249 }
1250
1251 for (level = 0; level < num_levels; level++) {
1252 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1253 int wm, max_wm;
1254
1255 wm = g4x_compute_wm(crtc_state, plane_state, level);
1256 max_wm = g4x_plane_fifo_size(plane_id, level);
1257
1258 if (wm > max_wm)
1259 break;
1260
1261 dirty |= raw->plane[plane_id] != wm;
1262 raw->plane[plane_id] = wm;
1263
1264 if (plane_id != PLANE_PRIMARY ||
1265 level == G4X_WM_LEVEL_NORMAL)
1266 continue;
1267
1268 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1269 raw->plane[plane_id]);
1270 max_wm = g4x_fbc_fifo_size(level);
1271
1272 /*
1273 * FBC wm is not mandatory as we
1274 * can always just disable its use.
1275 */
1276 if (wm > max_wm)
1277 wm = USHRT_MAX;
1278
1279 dirty |= raw->fbc != wm;
1280 raw->fbc = wm;
1281 }
1282
1283 /* mark watermarks as invalid */
1284 dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1285
1286 if (plane_id == PLANE_PRIMARY)
1287 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1288
1289 out:
1290 if (dirty) {
1291 drm_dbg_kms(&dev_priv->drm,
1292 "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1293 plane->base.name,
1294 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1295 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1296 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1297
1298 if (plane_id == PLANE_PRIMARY)
1299 drm_dbg_kms(&dev_priv->drm,
1300 "FBC watermarks: SR=%d, HPLL=%d\n",
1301 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1302 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1303 }
1304
1305 return dirty;
1306 }
1307
g4x_raw_plane_wm_is_valid(const struct intel_crtc_state * crtc_state,enum plane_id plane_id,int level)1308 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1309 enum plane_id plane_id, int level)
1310 {
1311 const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1312
1313 return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1314 }
1315
g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state * crtc_state,int level)1316 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1317 int level)
1318 {
1319 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1320
1321 if (level > dev_priv->wm.max_level)
1322 return false;
1323
1324 return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1325 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1326 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1327 }
1328
1329 /* mark all levels starting from 'level' as invalid */
g4x_invalidate_wms(struct intel_crtc * crtc,struct g4x_wm_state * wm_state,int level)1330 static void g4x_invalidate_wms(struct intel_crtc *crtc,
1331 struct g4x_wm_state *wm_state, int level)
1332 {
1333 if (level <= G4X_WM_LEVEL_NORMAL) {
1334 enum plane_id plane_id;
1335
1336 for_each_plane_id_on_crtc(crtc, plane_id)
1337 wm_state->wm.plane[plane_id] = USHRT_MAX;
1338 }
1339
1340 if (level <= G4X_WM_LEVEL_SR) {
1341 wm_state->cxsr = false;
1342 wm_state->sr.cursor = USHRT_MAX;
1343 wm_state->sr.plane = USHRT_MAX;
1344 wm_state->sr.fbc = USHRT_MAX;
1345 }
1346
1347 if (level <= G4X_WM_LEVEL_HPLL) {
1348 wm_state->hpll_en = false;
1349 wm_state->hpll.cursor = USHRT_MAX;
1350 wm_state->hpll.plane = USHRT_MAX;
1351 wm_state->hpll.fbc = USHRT_MAX;
1352 }
1353 }
1354
g4x_compute_fbc_en(const struct g4x_wm_state * wm_state,int level)1355 static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1356 int level)
1357 {
1358 if (level < G4X_WM_LEVEL_SR)
1359 return false;
1360
1361 if (level >= G4X_WM_LEVEL_SR &&
1362 wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1363 return false;
1364
1365 if (level >= G4X_WM_LEVEL_HPLL &&
1366 wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1367 return false;
1368
1369 return true;
1370 }
1371
g4x_compute_pipe_wm(struct intel_crtc_state * crtc_state)1372 static int g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1373 {
1374 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1375 struct intel_atomic_state *state =
1376 to_intel_atomic_state(crtc_state->uapi.state);
1377 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1378 int num_active_planes = hweight8(crtc_state->active_planes &
1379 ~BIT(PLANE_CURSOR));
1380 const struct g4x_pipe_wm *raw;
1381 const struct intel_plane_state *old_plane_state;
1382 const struct intel_plane_state *new_plane_state;
1383 struct intel_plane *plane;
1384 enum plane_id plane_id;
1385 int i, level;
1386 unsigned int dirty = 0;
1387
1388 for_each_oldnew_intel_plane_in_state(state, plane,
1389 old_plane_state,
1390 new_plane_state, i) {
1391 if (new_plane_state->hw.crtc != &crtc->base &&
1392 old_plane_state->hw.crtc != &crtc->base)
1393 continue;
1394
1395 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1396 dirty |= BIT(plane->id);
1397 }
1398
1399 if (!dirty)
1400 return 0;
1401
1402 level = G4X_WM_LEVEL_NORMAL;
1403 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1404 goto out;
1405
1406 raw = &crtc_state->wm.g4x.raw[level];
1407 for_each_plane_id_on_crtc(crtc, plane_id)
1408 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1409
1410 level = G4X_WM_LEVEL_SR;
1411 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1412 goto out;
1413
1414 raw = &crtc_state->wm.g4x.raw[level];
1415 wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1416 wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1417 wm_state->sr.fbc = raw->fbc;
1418
1419 wm_state->cxsr = num_active_planes == BIT(PLANE_PRIMARY);
1420
1421 level = G4X_WM_LEVEL_HPLL;
1422 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1423 goto out;
1424
1425 raw = &crtc_state->wm.g4x.raw[level];
1426 wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1427 wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1428 wm_state->hpll.fbc = raw->fbc;
1429
1430 wm_state->hpll_en = wm_state->cxsr;
1431
1432 level++;
1433
1434 out:
1435 if (level == G4X_WM_LEVEL_NORMAL)
1436 return -EINVAL;
1437
1438 /* invalidate the higher levels */
1439 g4x_invalidate_wms(crtc, wm_state, level);
1440
1441 /*
1442 * Determine if the FBC watermark(s) can be used. IF
1443 * this isn't the case we prefer to disable the FBC
1444 * watermark(s) rather than disable the SR/HPLL
1445 * level(s) entirely. 'level-1' is the highest valid
1446 * level here.
1447 */
1448 wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1449
1450 return 0;
1451 }
1452
g4x_compute_intermediate_wm(struct intel_crtc_state * new_crtc_state)1453 static int g4x_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
1454 {
1455 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
1456 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1457 struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1458 const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1459 struct intel_atomic_state *intel_state =
1460 to_intel_atomic_state(new_crtc_state->uapi.state);
1461 const struct intel_crtc_state *old_crtc_state =
1462 intel_atomic_get_old_crtc_state(intel_state, crtc);
1463 const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1464 enum plane_id plane_id;
1465
1466 if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
1467 *intermediate = *optimal;
1468
1469 intermediate->cxsr = false;
1470 intermediate->hpll_en = false;
1471 goto out;
1472 }
1473
1474 intermediate->cxsr = optimal->cxsr && active->cxsr &&
1475 !new_crtc_state->disable_cxsr;
1476 intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1477 !new_crtc_state->disable_cxsr;
1478 intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1479
1480 for_each_plane_id_on_crtc(crtc, plane_id) {
1481 intermediate->wm.plane[plane_id] =
1482 max(optimal->wm.plane[plane_id],
1483 active->wm.plane[plane_id]);
1484
1485 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1486 g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1487 }
1488
1489 intermediate->sr.plane = max(optimal->sr.plane,
1490 active->sr.plane);
1491 intermediate->sr.cursor = max(optimal->sr.cursor,
1492 active->sr.cursor);
1493 intermediate->sr.fbc = max(optimal->sr.fbc,
1494 active->sr.fbc);
1495
1496 intermediate->hpll.plane = max(optimal->hpll.plane,
1497 active->hpll.plane);
1498 intermediate->hpll.cursor = max(optimal->hpll.cursor,
1499 active->hpll.cursor);
1500 intermediate->hpll.fbc = max(optimal->hpll.fbc,
1501 active->hpll.fbc);
1502
1503 drm_WARN_ON(&dev_priv->drm,
1504 (intermediate->sr.plane >
1505 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1506 intermediate->sr.cursor >
1507 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1508 intermediate->cxsr);
1509 drm_WARN_ON(&dev_priv->drm,
1510 (intermediate->sr.plane >
1511 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1512 intermediate->sr.cursor >
1513 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1514 intermediate->hpll_en);
1515
1516 drm_WARN_ON(&dev_priv->drm,
1517 intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1518 intermediate->fbc_en && intermediate->cxsr);
1519 drm_WARN_ON(&dev_priv->drm,
1520 intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1521 intermediate->fbc_en && intermediate->hpll_en);
1522
1523 out:
1524 /*
1525 * If our intermediate WM are identical to the final WM, then we can
1526 * omit the post-vblank programming; only update if it's different.
1527 */
1528 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1529 new_crtc_state->wm.need_postvbl_update = true;
1530
1531 return 0;
1532 }
1533
g4x_merge_wm(struct drm_i915_private * dev_priv,struct g4x_wm_values * wm)1534 static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1535 struct g4x_wm_values *wm)
1536 {
1537 struct intel_crtc *crtc;
1538 int num_active_pipes = 0;
1539
1540 wm->cxsr = true;
1541 wm->hpll_en = true;
1542 wm->fbc_en = true;
1543
1544 for_each_intel_crtc(&dev_priv->drm, crtc) {
1545 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1546
1547 if (!crtc->active)
1548 continue;
1549
1550 if (!wm_state->cxsr)
1551 wm->cxsr = false;
1552 if (!wm_state->hpll_en)
1553 wm->hpll_en = false;
1554 if (!wm_state->fbc_en)
1555 wm->fbc_en = false;
1556
1557 num_active_pipes++;
1558 }
1559
1560 if (num_active_pipes != 1) {
1561 wm->cxsr = false;
1562 wm->hpll_en = false;
1563 wm->fbc_en = false;
1564 }
1565
1566 for_each_intel_crtc(&dev_priv->drm, crtc) {
1567 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1568 enum pipe pipe = crtc->pipe;
1569
1570 wm->pipe[pipe] = wm_state->wm;
1571 if (crtc->active && wm->cxsr)
1572 wm->sr = wm_state->sr;
1573 if (crtc->active && wm->hpll_en)
1574 wm->hpll = wm_state->hpll;
1575 }
1576 }
1577
g4x_program_watermarks(struct drm_i915_private * dev_priv)1578 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1579 {
1580 struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
1581 struct g4x_wm_values new_wm = {};
1582
1583 g4x_merge_wm(dev_priv, &new_wm);
1584
1585 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1586 return;
1587
1588 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1589 _intel_set_memory_cxsr(dev_priv, false);
1590
1591 g4x_write_wm_values(dev_priv, &new_wm);
1592
1593 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1594 _intel_set_memory_cxsr(dev_priv, true);
1595
1596 *old_wm = new_wm;
1597 }
1598
g4x_initial_watermarks(struct intel_atomic_state * state,struct intel_crtc * crtc)1599 static void g4x_initial_watermarks(struct intel_atomic_state *state,
1600 struct intel_crtc *crtc)
1601 {
1602 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1603 const struct intel_crtc_state *crtc_state =
1604 intel_atomic_get_new_crtc_state(state, crtc);
1605
1606 mutex_lock(&dev_priv->wm.wm_mutex);
1607 crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1608 g4x_program_watermarks(dev_priv);
1609 mutex_unlock(&dev_priv->wm.wm_mutex);
1610 }
1611
g4x_optimize_watermarks(struct intel_atomic_state * state,struct intel_crtc * crtc)1612 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1613 struct intel_crtc *crtc)
1614 {
1615 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1616 const struct intel_crtc_state *crtc_state =
1617 intel_atomic_get_new_crtc_state(state, crtc);
1618
1619 if (!crtc_state->wm.need_postvbl_update)
1620 return;
1621
1622 mutex_lock(&dev_priv->wm.wm_mutex);
1623 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1624 g4x_program_watermarks(dev_priv);
1625 mutex_unlock(&dev_priv->wm.wm_mutex);
1626 }
1627
1628 /* latency must be in 0.1us units. */
vlv_wm_method2(unsigned int pixel_rate,unsigned int htotal,unsigned int width,unsigned int cpp,unsigned int latency)1629 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1630 unsigned int htotal,
1631 unsigned int width,
1632 unsigned int cpp,
1633 unsigned int latency)
1634 {
1635 unsigned int ret;
1636
1637 ret = intel_wm_method2(pixel_rate, htotal,
1638 width, cpp, latency);
1639 ret = DIV_ROUND_UP(ret, 64);
1640
1641 return ret;
1642 }
1643
vlv_setup_wm_latency(struct drm_i915_private * dev_priv)1644 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1645 {
1646 /* all latencies in usec */
1647 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1648
1649 dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
1650
1651 if (IS_CHERRYVIEW(dev_priv)) {
1652 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1653 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1654
1655 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1656 }
1657 }
1658
vlv_compute_wm_level(const struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state,int level)1659 static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1660 const struct intel_plane_state *plane_state,
1661 int level)
1662 {
1663 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1664 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1665 const struct drm_display_mode *pipe_mode =
1666 &crtc_state->hw.pipe_mode;
1667 unsigned int clock, htotal, cpp, width, wm;
1668
1669 if (dev_priv->wm.pri_latency[level] == 0)
1670 return USHRT_MAX;
1671
1672 if (!intel_wm_plane_visible(crtc_state, plane_state))
1673 return 0;
1674
1675 cpp = plane_state->hw.fb->format->cpp[0];
1676 clock = pipe_mode->crtc_clock;
1677 htotal = pipe_mode->crtc_htotal;
1678 width = crtc_state->pipe_src_w;
1679
1680 if (plane->id == PLANE_CURSOR) {
1681 /*
1682 * FIXME the formula gives values that are
1683 * too big for the cursor FIFO, and hence we
1684 * would never be able to use cursors. For
1685 * now just hardcode the watermark.
1686 */
1687 wm = 63;
1688 } else {
1689 wm = vlv_wm_method2(clock, htotal, width, cpp,
1690 dev_priv->wm.pri_latency[level] * 10);
1691 }
1692
1693 return min_t(unsigned int, wm, USHRT_MAX);
1694 }
1695
vlv_need_sprite0_fifo_workaround(unsigned int active_planes)1696 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1697 {
1698 return (active_planes & (BIT(PLANE_SPRITE0) |
1699 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1700 }
1701
vlv_compute_fifo(struct intel_crtc_state * crtc_state)1702 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1703 {
1704 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1705 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1706 const struct g4x_pipe_wm *raw =
1707 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1708 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1709 unsigned int active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1710 int num_active_planes = hweight8(active_planes);
1711 const int fifo_size = 511;
1712 int fifo_extra, fifo_left = fifo_size;
1713 int sprite0_fifo_extra = 0;
1714 unsigned int total_rate;
1715 enum plane_id plane_id;
1716
1717 /*
1718 * When enabling sprite0 after sprite1 has already been enabled
1719 * we tend to get an underrun unless sprite0 already has some
1720 * FIFO space allcoated. Hence we always allocate at least one
1721 * cacheline for sprite0 whenever sprite1 is enabled.
1722 *
1723 * All other plane enable sequences appear immune to this problem.
1724 */
1725 if (vlv_need_sprite0_fifo_workaround(active_planes))
1726 sprite0_fifo_extra = 1;
1727
1728 total_rate = raw->plane[PLANE_PRIMARY] +
1729 raw->plane[PLANE_SPRITE0] +
1730 raw->plane[PLANE_SPRITE1] +
1731 sprite0_fifo_extra;
1732
1733 if (total_rate > fifo_size)
1734 return -EINVAL;
1735
1736 if (total_rate == 0)
1737 total_rate = 1;
1738
1739 for_each_plane_id_on_crtc(crtc, plane_id) {
1740 unsigned int rate;
1741
1742 if ((active_planes & BIT(plane_id)) == 0) {
1743 fifo_state->plane[plane_id] = 0;
1744 continue;
1745 }
1746
1747 rate = raw->plane[plane_id];
1748 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1749 fifo_left -= fifo_state->plane[plane_id];
1750 }
1751
1752 fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1753 fifo_left -= sprite0_fifo_extra;
1754
1755 fifo_state->plane[PLANE_CURSOR] = 63;
1756
1757 fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1758
1759 /* spread the remainder evenly */
1760 for_each_plane_id_on_crtc(crtc, plane_id) {
1761 int plane_extra;
1762
1763 if (fifo_left == 0)
1764 break;
1765
1766 if ((active_planes & BIT(plane_id)) == 0)
1767 continue;
1768
1769 plane_extra = min(fifo_extra, fifo_left);
1770 fifo_state->plane[plane_id] += plane_extra;
1771 fifo_left -= plane_extra;
1772 }
1773
1774 drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1775
1776 /* give it all to the first plane if none are active */
1777 if (active_planes == 0) {
1778 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1779 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1780 }
1781
1782 return 0;
1783 }
1784
1785 /* mark all levels starting from 'level' as invalid */
vlv_invalidate_wms(struct intel_crtc * crtc,struct vlv_wm_state * wm_state,int level)1786 static void vlv_invalidate_wms(struct intel_crtc *crtc,
1787 struct vlv_wm_state *wm_state, int level)
1788 {
1789 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1790
1791 for (; level < intel_wm_num_levels(dev_priv); level++) {
1792 enum plane_id plane_id;
1793
1794 for_each_plane_id_on_crtc(crtc, plane_id)
1795 wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1796
1797 wm_state->sr[level].cursor = USHRT_MAX;
1798 wm_state->sr[level].plane = USHRT_MAX;
1799 }
1800 }
1801
vlv_invert_wm_value(u16 wm,u16 fifo_size)1802 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1803 {
1804 if (wm > fifo_size)
1805 return USHRT_MAX;
1806 else
1807 return fifo_size - wm;
1808 }
1809
1810 /*
1811 * Starting from 'level' set all higher
1812 * levels to 'value' in the "raw" watermarks.
1813 */
vlv_raw_plane_wm_set(struct intel_crtc_state * crtc_state,int level,enum plane_id plane_id,u16 value)1814 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1815 int level, enum plane_id plane_id, u16 value)
1816 {
1817 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1818 int num_levels = intel_wm_num_levels(dev_priv);
1819 bool dirty = false;
1820
1821 for (; level < num_levels; level++) {
1822 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1823
1824 dirty |= raw->plane[plane_id] != value;
1825 raw->plane[plane_id] = value;
1826 }
1827
1828 return dirty;
1829 }
1830
vlv_raw_plane_wm_compute(struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state)1831 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1832 const struct intel_plane_state *plane_state)
1833 {
1834 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1835 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1836 enum plane_id plane_id = plane->id;
1837 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1838 int level;
1839 bool dirty = false;
1840
1841 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1842 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1843 goto out;
1844 }
1845
1846 for (level = 0; level < num_levels; level++) {
1847 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1848 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1849 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1850
1851 if (wm > max_wm)
1852 break;
1853
1854 dirty |= raw->plane[plane_id] != wm;
1855 raw->plane[plane_id] = wm;
1856 }
1857
1858 /* mark all higher levels as invalid */
1859 dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1860
1861 out:
1862 if (dirty)
1863 drm_dbg_kms(&dev_priv->drm,
1864 "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1865 plane->base.name,
1866 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1867 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1868 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1869
1870 return dirty;
1871 }
1872
vlv_raw_plane_wm_is_valid(const struct intel_crtc_state * crtc_state,enum plane_id plane_id,int level)1873 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1874 enum plane_id plane_id, int level)
1875 {
1876 const struct g4x_pipe_wm *raw =
1877 &crtc_state->wm.vlv.raw[level];
1878 const struct vlv_fifo_state *fifo_state =
1879 &crtc_state->wm.vlv.fifo_state;
1880
1881 return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1882 }
1883
vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state * crtc_state,int level)1884 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1885 {
1886 return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1887 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1888 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1889 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1890 }
1891
vlv_compute_pipe_wm(struct intel_crtc_state * crtc_state)1892 static int vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1893 {
1894 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1895 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1896 struct intel_atomic_state *state =
1897 to_intel_atomic_state(crtc_state->uapi.state);
1898 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1899 const struct vlv_fifo_state *fifo_state =
1900 &crtc_state->wm.vlv.fifo_state;
1901 int num_active_planes = hweight8(crtc_state->active_planes &
1902 ~BIT(PLANE_CURSOR));
1903 bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->uapi);
1904 const struct intel_plane_state *old_plane_state;
1905 const struct intel_plane_state *new_plane_state;
1906 struct intel_plane *plane;
1907 enum plane_id plane_id;
1908 int level, ret, i;
1909 unsigned int dirty = 0;
1910
1911 for_each_oldnew_intel_plane_in_state(state, plane,
1912 old_plane_state,
1913 new_plane_state, i) {
1914 if (new_plane_state->hw.crtc != &crtc->base &&
1915 old_plane_state->hw.crtc != &crtc->base)
1916 continue;
1917
1918 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1919 dirty |= BIT(plane->id);
1920 }
1921
1922 /*
1923 * DSPARB registers may have been reset due to the
1924 * power well being turned off. Make sure we restore
1925 * them to a consistent state even if no primary/sprite
1926 * planes are initially active.
1927 */
1928 if (needs_modeset)
1929 crtc_state->fifo_changed = true;
1930
1931 if (!dirty)
1932 return 0;
1933
1934 /* cursor changes don't warrant a FIFO recompute */
1935 if (dirty & ~BIT(PLANE_CURSOR)) {
1936 const struct intel_crtc_state *old_crtc_state =
1937 intel_atomic_get_old_crtc_state(state, crtc);
1938 const struct vlv_fifo_state *old_fifo_state =
1939 &old_crtc_state->wm.vlv.fifo_state;
1940
1941 ret = vlv_compute_fifo(crtc_state);
1942 if (ret)
1943 return ret;
1944
1945 if (needs_modeset ||
1946 memcmp(old_fifo_state, fifo_state,
1947 sizeof(*fifo_state)) != 0)
1948 crtc_state->fifo_changed = true;
1949 }
1950
1951 /* initially allow all levels */
1952 wm_state->num_levels = intel_wm_num_levels(dev_priv);
1953 /*
1954 * Note that enabling cxsr with no primary/sprite planes
1955 * enabled can wedge the pipe. Hence we only allow cxsr
1956 * with exactly one enabled primary/sprite plane.
1957 */
1958 wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1959
1960 for (level = 0; level < wm_state->num_levels; level++) {
1961 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1962 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1963
1964 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1965 break;
1966
1967 for_each_plane_id_on_crtc(crtc, plane_id) {
1968 wm_state->wm[level].plane[plane_id] =
1969 vlv_invert_wm_value(raw->plane[plane_id],
1970 fifo_state->plane[plane_id]);
1971 }
1972
1973 wm_state->sr[level].plane =
1974 vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1975 raw->plane[PLANE_SPRITE0],
1976 raw->plane[PLANE_SPRITE1]),
1977 sr_fifo_size);
1978
1979 wm_state->sr[level].cursor =
1980 vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1981 63);
1982 }
1983
1984 if (level == 0)
1985 return -EINVAL;
1986
1987 /* limit to only levels we can actually handle */
1988 wm_state->num_levels = level;
1989
1990 /* invalidate the higher levels */
1991 vlv_invalidate_wms(crtc, wm_state, level);
1992
1993 return 0;
1994 }
1995
1996 #define VLV_FIFO(plane, value) \
1997 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1998
vlv_atomic_update_fifo(struct intel_atomic_state * state,struct intel_crtc * crtc)1999 static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
2000 struct intel_crtc *crtc)
2001 {
2002 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2003 struct intel_uncore *uncore = &dev_priv->uncore;
2004 const struct intel_crtc_state *crtc_state =
2005 intel_atomic_get_new_crtc_state(state, crtc);
2006 const struct vlv_fifo_state *fifo_state =
2007 &crtc_state->wm.vlv.fifo_state;
2008 int sprite0_start, sprite1_start, fifo_size;
2009 u32 dsparb, dsparb2, dsparb3;
2010
2011 if (!crtc_state->fifo_changed)
2012 return;
2013
2014 sprite0_start = fifo_state->plane[PLANE_PRIMARY];
2015 sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
2016 fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
2017
2018 drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
2019 drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
2020
2021 trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
2022
2023 /*
2024 * uncore.lock serves a double purpose here. It allows us to
2025 * use the less expensive I915_{READ,WRITE}_FW() functions, and
2026 * it protects the DSPARB registers from getting clobbered by
2027 * parallel updates from multiple pipes.
2028 *
2029 * intel_pipe_update_start() has already disabled interrupts
2030 * for us, so a plain spin_lock() is sufficient here.
2031 */
2032 spin_lock(&uncore->lock);
2033
2034 switch (crtc->pipe) {
2035 case PIPE_A:
2036 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2037 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2038
2039 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
2040 VLV_FIFO(SPRITEB, 0xff));
2041 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
2042 VLV_FIFO(SPRITEB, sprite1_start));
2043
2044 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
2045 VLV_FIFO(SPRITEB_HI, 0x1));
2046 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
2047 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
2048
2049 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2050 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2051 break;
2052 case PIPE_B:
2053 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2054 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2055
2056 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2057 VLV_FIFO(SPRITED, 0xff));
2058 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2059 VLV_FIFO(SPRITED, sprite1_start));
2060
2061 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2062 VLV_FIFO(SPRITED_HI, 0xff));
2063 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2064 VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2065
2066 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2067 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2068 break;
2069 case PIPE_C:
2070 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
2071 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2072
2073 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2074 VLV_FIFO(SPRITEF, 0xff));
2075 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2076 VLV_FIFO(SPRITEF, sprite1_start));
2077
2078 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2079 VLV_FIFO(SPRITEF_HI, 0xff));
2080 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2081 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2082
2083 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2084 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2085 break;
2086 default:
2087 break;
2088 }
2089
2090 intel_uncore_posting_read_fw(uncore, DSPARB);
2091
2092 spin_unlock(&uncore->lock);
2093 }
2094
2095 #undef VLV_FIFO
2096
vlv_compute_intermediate_wm(struct intel_crtc_state * new_crtc_state)2097 static int vlv_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
2098 {
2099 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
2100 struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2101 const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2102 struct intel_atomic_state *intel_state =
2103 to_intel_atomic_state(new_crtc_state->uapi.state);
2104 const struct intel_crtc_state *old_crtc_state =
2105 intel_atomic_get_old_crtc_state(intel_state, crtc);
2106 const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2107 int level;
2108
2109 if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
2110 *intermediate = *optimal;
2111
2112 intermediate->cxsr = false;
2113 goto out;
2114 }
2115
2116 intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2117 intermediate->cxsr = optimal->cxsr && active->cxsr &&
2118 !new_crtc_state->disable_cxsr;
2119
2120 for (level = 0; level < intermediate->num_levels; level++) {
2121 enum plane_id plane_id;
2122
2123 for_each_plane_id_on_crtc(crtc, plane_id) {
2124 intermediate->wm[level].plane[plane_id] =
2125 min(optimal->wm[level].plane[plane_id],
2126 active->wm[level].plane[plane_id]);
2127 }
2128
2129 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2130 active->sr[level].plane);
2131 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2132 active->sr[level].cursor);
2133 }
2134
2135 vlv_invalidate_wms(crtc, intermediate, level);
2136
2137 out:
2138 /*
2139 * If our intermediate WM are identical to the final WM, then we can
2140 * omit the post-vblank programming; only update if it's different.
2141 */
2142 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2143 new_crtc_state->wm.need_postvbl_update = true;
2144
2145 return 0;
2146 }
2147
vlv_merge_wm(struct drm_i915_private * dev_priv,struct vlv_wm_values * wm)2148 static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2149 struct vlv_wm_values *wm)
2150 {
2151 struct intel_crtc *crtc;
2152 int num_active_pipes = 0;
2153
2154 wm->level = dev_priv->wm.max_level;
2155 wm->cxsr = true;
2156
2157 for_each_intel_crtc(&dev_priv->drm, crtc) {
2158 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2159
2160 if (!crtc->active)
2161 continue;
2162
2163 if (!wm_state->cxsr)
2164 wm->cxsr = false;
2165
2166 num_active_pipes++;
2167 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2168 }
2169
2170 if (num_active_pipes != 1)
2171 wm->cxsr = false;
2172
2173 if (num_active_pipes > 1)
2174 wm->level = VLV_WM_LEVEL_PM2;
2175
2176 for_each_intel_crtc(&dev_priv->drm, crtc) {
2177 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2178 enum pipe pipe = crtc->pipe;
2179
2180 wm->pipe[pipe] = wm_state->wm[wm->level];
2181 if (crtc->active && wm->cxsr)
2182 wm->sr = wm_state->sr[wm->level];
2183
2184 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2185 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2186 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2187 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2188 }
2189 }
2190
vlv_program_watermarks(struct drm_i915_private * dev_priv)2191 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2192 {
2193 struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
2194 struct vlv_wm_values new_wm = {};
2195
2196 vlv_merge_wm(dev_priv, &new_wm);
2197
2198 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2199 return;
2200
2201 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2202 chv_set_memory_dvfs(dev_priv, false);
2203
2204 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2205 chv_set_memory_pm5(dev_priv, false);
2206
2207 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2208 _intel_set_memory_cxsr(dev_priv, false);
2209
2210 vlv_write_wm_values(dev_priv, &new_wm);
2211
2212 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2213 _intel_set_memory_cxsr(dev_priv, true);
2214
2215 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2216 chv_set_memory_pm5(dev_priv, true);
2217
2218 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2219 chv_set_memory_dvfs(dev_priv, true);
2220
2221 *old_wm = new_wm;
2222 }
2223
vlv_initial_watermarks(struct intel_atomic_state * state,struct intel_crtc * crtc)2224 static void vlv_initial_watermarks(struct intel_atomic_state *state,
2225 struct intel_crtc *crtc)
2226 {
2227 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2228 const struct intel_crtc_state *crtc_state =
2229 intel_atomic_get_new_crtc_state(state, crtc);
2230
2231 mutex_lock(&dev_priv->wm.wm_mutex);
2232 crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2233 vlv_program_watermarks(dev_priv);
2234 mutex_unlock(&dev_priv->wm.wm_mutex);
2235 }
2236
vlv_optimize_watermarks(struct intel_atomic_state * state,struct intel_crtc * crtc)2237 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2238 struct intel_crtc *crtc)
2239 {
2240 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2241 const struct intel_crtc_state *crtc_state =
2242 intel_atomic_get_new_crtc_state(state, crtc);
2243
2244 if (!crtc_state->wm.need_postvbl_update)
2245 return;
2246
2247 mutex_lock(&dev_priv->wm.wm_mutex);
2248 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2249 vlv_program_watermarks(dev_priv);
2250 mutex_unlock(&dev_priv->wm.wm_mutex);
2251 }
2252
i965_update_wm(struct intel_crtc * unused_crtc)2253 static void i965_update_wm(struct intel_crtc *unused_crtc)
2254 {
2255 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2256 struct intel_crtc *crtc;
2257 int srwm = 1;
2258 int cursor_sr = 16;
2259 bool cxsr_enabled;
2260
2261 /* Calc sr entries for one plane configs */
2262 crtc = single_enabled_crtc(dev_priv);
2263 if (crtc) {
2264 /* self-refresh has much higher latency */
2265 static const int sr_latency_ns = 12000;
2266 const struct drm_display_mode *pipe_mode =
2267 &crtc->config->hw.pipe_mode;
2268 const struct drm_framebuffer *fb =
2269 crtc->base.primary->state->fb;
2270 int clock = pipe_mode->crtc_clock;
2271 int htotal = pipe_mode->crtc_htotal;
2272 int hdisplay = crtc->config->pipe_src_w;
2273 int cpp = fb->format->cpp[0];
2274 int entries;
2275
2276 entries = intel_wm_method2(clock, htotal,
2277 hdisplay, cpp, sr_latency_ns / 100);
2278 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2279 srwm = I965_FIFO_SIZE - entries;
2280 if (srwm < 0)
2281 srwm = 1;
2282 srwm &= 0x1ff;
2283 drm_dbg_kms(&dev_priv->drm,
2284 "self-refresh entries: %d, wm: %d\n",
2285 entries, srwm);
2286
2287 entries = intel_wm_method2(clock, htotal,
2288 crtc->base.cursor->state->crtc_w, 4,
2289 sr_latency_ns / 100);
2290 entries = DIV_ROUND_UP(entries,
2291 i965_cursor_wm_info.cacheline_size) +
2292 i965_cursor_wm_info.guard_size;
2293
2294 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2295 if (cursor_sr > i965_cursor_wm_info.max_wm)
2296 cursor_sr = i965_cursor_wm_info.max_wm;
2297
2298 drm_dbg_kms(&dev_priv->drm,
2299 "self-refresh watermark: display plane %d "
2300 "cursor %d\n", srwm, cursor_sr);
2301
2302 cxsr_enabled = true;
2303 } else {
2304 cxsr_enabled = false;
2305 /* Turn off self refresh if both pipes are enabled */
2306 intel_set_memory_cxsr(dev_priv, false);
2307 }
2308
2309 drm_dbg_kms(&dev_priv->drm,
2310 "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2311 srwm);
2312
2313 /* 965 has limitations... */
2314 intel_uncore_write(&dev_priv->uncore, DSPFW1, FW_WM(srwm, SR) |
2315 FW_WM(8, CURSORB) |
2316 FW_WM(8, PLANEB) |
2317 FW_WM(8, PLANEA));
2318 intel_uncore_write(&dev_priv->uncore, DSPFW2, FW_WM(8, CURSORA) |
2319 FW_WM(8, PLANEC_OLD));
2320 /* update cursor SR watermark */
2321 intel_uncore_write(&dev_priv->uncore, DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2322
2323 if (cxsr_enabled)
2324 intel_set_memory_cxsr(dev_priv, true);
2325 }
2326
2327 #undef FW_WM
2328
i9xx_update_wm(struct intel_crtc * unused_crtc)2329 static void i9xx_update_wm(struct intel_crtc *unused_crtc)
2330 {
2331 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2332 const struct intel_watermark_params *wm_info;
2333 u32 fwater_lo;
2334 u32 fwater_hi;
2335 int cwm, srwm = 1;
2336 int fifo_size;
2337 int planea_wm, planeb_wm;
2338 struct intel_crtc *crtc, *enabled = NULL;
2339
2340 if (IS_I945GM(dev_priv))
2341 wm_info = &i945_wm_info;
2342 else if (!IS_DISPLAY_VER(dev_priv, 2))
2343 wm_info = &i915_wm_info;
2344 else
2345 wm_info = &i830_a_wm_info;
2346
2347 fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A);
2348 crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A);
2349 if (intel_crtc_active(crtc)) {
2350 const struct drm_display_mode *pipe_mode =
2351 &crtc->config->hw.pipe_mode;
2352 const struct drm_framebuffer *fb =
2353 crtc->base.primary->state->fb;
2354 int cpp;
2355
2356 if (IS_DISPLAY_VER(dev_priv, 2))
2357 cpp = 4;
2358 else
2359 cpp = fb->format->cpp[0];
2360
2361 planea_wm = intel_calculate_wm(pipe_mode->crtc_clock,
2362 wm_info, fifo_size, cpp,
2363 pessimal_latency_ns);
2364 enabled = crtc;
2365 } else {
2366 planea_wm = fifo_size - wm_info->guard_size;
2367 if (planea_wm > (long)wm_info->max_wm)
2368 planea_wm = wm_info->max_wm;
2369 }
2370
2371 if (IS_DISPLAY_VER(dev_priv, 2))
2372 wm_info = &i830_bc_wm_info;
2373
2374 fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B);
2375 crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B);
2376 if (intel_crtc_active(crtc)) {
2377 const struct drm_display_mode *pipe_mode =
2378 &crtc->config->hw.pipe_mode;
2379 const struct drm_framebuffer *fb =
2380 crtc->base.primary->state->fb;
2381 int cpp;
2382
2383 if (IS_DISPLAY_VER(dev_priv, 2))
2384 cpp = 4;
2385 else
2386 cpp = fb->format->cpp[0];
2387
2388 planeb_wm = intel_calculate_wm(pipe_mode->crtc_clock,
2389 wm_info, fifo_size, cpp,
2390 pessimal_latency_ns);
2391 if (enabled == NULL)
2392 enabled = crtc;
2393 else
2394 enabled = NULL;
2395 } else {
2396 planeb_wm = fifo_size - wm_info->guard_size;
2397 if (planeb_wm > (long)wm_info->max_wm)
2398 planeb_wm = wm_info->max_wm;
2399 }
2400
2401 drm_dbg_kms(&dev_priv->drm,
2402 "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2403
2404 if (IS_I915GM(dev_priv) && enabled) {
2405 struct drm_i915_gem_object *obj;
2406
2407 obj = intel_fb_obj(enabled->base.primary->state->fb);
2408
2409 /* self-refresh seems busted with untiled */
2410 if (!i915_gem_object_is_tiled(obj))
2411 enabled = NULL;
2412 }
2413
2414 /*
2415 * Overlay gets an aggressive default since video jitter is bad.
2416 */
2417 cwm = 2;
2418
2419 /* Play safe and disable self-refresh before adjusting watermarks. */
2420 intel_set_memory_cxsr(dev_priv, false);
2421
2422 /* Calc sr entries for one plane configs */
2423 if (HAS_FW_BLC(dev_priv) && enabled) {
2424 /* self-refresh has much higher latency */
2425 static const int sr_latency_ns = 6000;
2426 const struct drm_display_mode *pipe_mode =
2427 &enabled->config->hw.pipe_mode;
2428 const struct drm_framebuffer *fb =
2429 enabled->base.primary->state->fb;
2430 int clock = pipe_mode->crtc_clock;
2431 int htotal = pipe_mode->crtc_htotal;
2432 int hdisplay = enabled->config->pipe_src_w;
2433 int cpp;
2434 int entries;
2435
2436 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2437 cpp = 4;
2438 else
2439 cpp = fb->format->cpp[0];
2440
2441 entries = intel_wm_method2(clock, htotal, hdisplay, cpp,
2442 sr_latency_ns / 100);
2443 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2444 drm_dbg_kms(&dev_priv->drm,
2445 "self-refresh entries: %d\n", entries);
2446 srwm = wm_info->fifo_size - entries;
2447 if (srwm < 0)
2448 srwm = 1;
2449
2450 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2451 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF,
2452 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2453 else
2454 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f);
2455 }
2456
2457 drm_dbg_kms(&dev_priv->drm,
2458 "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2459 planea_wm, planeb_wm, cwm, srwm);
2460
2461 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2462 fwater_hi = (cwm & 0x1f);
2463
2464 /* Set request length to 8 cachelines per fetch */
2465 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2466 fwater_hi = fwater_hi | (1 << 8);
2467
2468 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2469 intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi);
2470
2471 if (enabled)
2472 intel_set_memory_cxsr(dev_priv, true);
2473 }
2474
i845_update_wm(struct intel_crtc * unused_crtc)2475 static void i845_update_wm(struct intel_crtc *unused_crtc)
2476 {
2477 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2478 struct intel_crtc *crtc;
2479 const struct drm_display_mode *pipe_mode;
2480 u32 fwater_lo;
2481 int planea_wm;
2482
2483 crtc = single_enabled_crtc(dev_priv);
2484 if (crtc == NULL)
2485 return;
2486
2487 pipe_mode = &crtc->config->hw.pipe_mode;
2488 planea_wm = intel_calculate_wm(pipe_mode->crtc_clock,
2489 &i845_wm_info,
2490 dev_priv->display.get_fifo_size(dev_priv, PLANE_A),
2491 4, pessimal_latency_ns);
2492 fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff;
2493 fwater_lo |= (3<<8) | planea_wm;
2494
2495 drm_dbg_kms(&dev_priv->drm,
2496 "Setting FIFO watermarks - A: %d\n", planea_wm);
2497
2498 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2499 }
2500
2501 /* latency must be in 0.1us units. */
ilk_wm_method1(unsigned int pixel_rate,unsigned int cpp,unsigned int latency)2502 static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2503 unsigned int cpp,
2504 unsigned int latency)
2505 {
2506 unsigned int ret;
2507
2508 ret = intel_wm_method1(pixel_rate, cpp, latency);
2509 ret = DIV_ROUND_UP(ret, 64) + 2;
2510
2511 return ret;
2512 }
2513
2514 /* latency must be in 0.1us units. */
ilk_wm_method2(unsigned int pixel_rate,unsigned int htotal,unsigned int width,unsigned int cpp,unsigned int latency)2515 static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2516 unsigned int htotal,
2517 unsigned int width,
2518 unsigned int cpp,
2519 unsigned int latency)
2520 {
2521 unsigned int ret;
2522
2523 ret = intel_wm_method2(pixel_rate, htotal,
2524 width, cpp, latency);
2525 ret = DIV_ROUND_UP(ret, 64) + 2;
2526
2527 return ret;
2528 }
2529
ilk_wm_fbc(u32 pri_val,u32 horiz_pixels,u8 cpp)2530 static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2531 {
2532 /*
2533 * Neither of these should be possible since this function shouldn't be
2534 * called if the CRTC is off or the plane is invisible. But let's be
2535 * extra paranoid to avoid a potential divide-by-zero if we screw up
2536 * elsewhere in the driver.
2537 */
2538 if (WARN_ON(!cpp))
2539 return 0;
2540 if (WARN_ON(!horiz_pixels))
2541 return 0;
2542
2543 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2544 }
2545
2546 struct ilk_wm_maximums {
2547 u16 pri;
2548 u16 spr;
2549 u16 cur;
2550 u16 fbc;
2551 };
2552
2553 /*
2554 * For both WM_PIPE and WM_LP.
2555 * mem_value must be in 0.1us units.
2556 */
ilk_compute_pri_wm(const struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state,u32 mem_value,bool is_lp)2557 static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2558 const struct intel_plane_state *plane_state,
2559 u32 mem_value, bool is_lp)
2560 {
2561 u32 method1, method2;
2562 int cpp;
2563
2564 if (mem_value == 0)
2565 return U32_MAX;
2566
2567 if (!intel_wm_plane_visible(crtc_state, plane_state))
2568 return 0;
2569
2570 cpp = plane_state->hw.fb->format->cpp[0];
2571
2572 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2573
2574 if (!is_lp)
2575 return method1;
2576
2577 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2578 crtc_state->hw.pipe_mode.crtc_htotal,
2579 drm_rect_width(&plane_state->uapi.dst),
2580 cpp, mem_value);
2581
2582 return min(method1, method2);
2583 }
2584
2585 /*
2586 * For both WM_PIPE and WM_LP.
2587 * mem_value must be in 0.1us units.
2588 */
ilk_compute_spr_wm(const struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state,u32 mem_value)2589 static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2590 const struct intel_plane_state *plane_state,
2591 u32 mem_value)
2592 {
2593 u32 method1, method2;
2594 int cpp;
2595
2596 if (mem_value == 0)
2597 return U32_MAX;
2598
2599 if (!intel_wm_plane_visible(crtc_state, plane_state))
2600 return 0;
2601
2602 cpp = plane_state->hw.fb->format->cpp[0];
2603
2604 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2605 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2606 crtc_state->hw.pipe_mode.crtc_htotal,
2607 drm_rect_width(&plane_state->uapi.dst),
2608 cpp, mem_value);
2609 return min(method1, method2);
2610 }
2611
2612 /*
2613 * For both WM_PIPE and WM_LP.
2614 * mem_value must be in 0.1us units.
2615 */
ilk_compute_cur_wm(const struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state,u32 mem_value)2616 static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2617 const struct intel_plane_state *plane_state,
2618 u32 mem_value)
2619 {
2620 int cpp;
2621
2622 if (mem_value == 0)
2623 return U32_MAX;
2624
2625 if (!intel_wm_plane_visible(crtc_state, plane_state))
2626 return 0;
2627
2628 cpp = plane_state->hw.fb->format->cpp[0];
2629
2630 return ilk_wm_method2(crtc_state->pixel_rate,
2631 crtc_state->hw.pipe_mode.crtc_htotal,
2632 drm_rect_width(&plane_state->uapi.dst),
2633 cpp, mem_value);
2634 }
2635
2636 /* Only for WM_LP. */
ilk_compute_fbc_wm(const struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state,u32 pri_val)2637 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2638 const struct intel_plane_state *plane_state,
2639 u32 pri_val)
2640 {
2641 int cpp;
2642
2643 if (!intel_wm_plane_visible(crtc_state, plane_state))
2644 return 0;
2645
2646 cpp = plane_state->hw.fb->format->cpp[0];
2647
2648 return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.dst),
2649 cpp);
2650 }
2651
2652 static unsigned int
ilk_display_fifo_size(const struct drm_i915_private * dev_priv)2653 ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2654 {
2655 if (DISPLAY_VER(dev_priv) >= 8)
2656 return 3072;
2657 else if (DISPLAY_VER(dev_priv) >= 7)
2658 return 768;
2659 else
2660 return 512;
2661 }
2662
2663 static unsigned int
ilk_plane_wm_reg_max(const struct drm_i915_private * dev_priv,int level,bool is_sprite)2664 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2665 int level, bool is_sprite)
2666 {
2667 if (DISPLAY_VER(dev_priv) >= 8)
2668 /* BDW primary/sprite plane watermarks */
2669 return level == 0 ? 255 : 2047;
2670 else if (DISPLAY_VER(dev_priv) >= 7)
2671 /* IVB/HSW primary/sprite plane watermarks */
2672 return level == 0 ? 127 : 1023;
2673 else if (!is_sprite)
2674 /* ILK/SNB primary plane watermarks */
2675 return level == 0 ? 127 : 511;
2676 else
2677 /* ILK/SNB sprite plane watermarks */
2678 return level == 0 ? 63 : 255;
2679 }
2680
2681 static unsigned int
ilk_cursor_wm_reg_max(const struct drm_i915_private * dev_priv,int level)2682 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2683 {
2684 if (DISPLAY_VER(dev_priv) >= 7)
2685 return level == 0 ? 63 : 255;
2686 else
2687 return level == 0 ? 31 : 63;
2688 }
2689
ilk_fbc_wm_reg_max(const struct drm_i915_private * dev_priv)2690 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2691 {
2692 if (DISPLAY_VER(dev_priv) >= 8)
2693 return 31;
2694 else
2695 return 15;
2696 }
2697
2698 /* Calculate the maximum primary/sprite plane watermark */
ilk_plane_wm_max(const struct drm_i915_private * dev_priv,int level,const struct intel_wm_config * config,enum intel_ddb_partitioning ddb_partitioning,bool is_sprite)2699 static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2700 int level,
2701 const struct intel_wm_config *config,
2702 enum intel_ddb_partitioning ddb_partitioning,
2703 bool is_sprite)
2704 {
2705 unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2706
2707 /* if sprites aren't enabled, sprites get nothing */
2708 if (is_sprite && !config->sprites_enabled)
2709 return 0;
2710
2711 /* HSW allows LP1+ watermarks even with multiple pipes */
2712 if (level == 0 || config->num_pipes_active > 1) {
2713 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2714
2715 /*
2716 * For some reason the non self refresh
2717 * FIFO size is only half of the self
2718 * refresh FIFO size on ILK/SNB.
2719 */
2720 if (DISPLAY_VER(dev_priv) <= 6)
2721 fifo_size /= 2;
2722 }
2723
2724 if (config->sprites_enabled) {
2725 /* level 0 is always calculated with 1:1 split */
2726 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2727 if (is_sprite)
2728 fifo_size *= 5;
2729 fifo_size /= 6;
2730 } else {
2731 fifo_size /= 2;
2732 }
2733 }
2734
2735 /* clamp to max that the registers can hold */
2736 return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2737 }
2738
2739 /* Calculate the maximum cursor plane watermark */
ilk_cursor_wm_max(const struct drm_i915_private * dev_priv,int level,const struct intel_wm_config * config)2740 static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2741 int level,
2742 const struct intel_wm_config *config)
2743 {
2744 /* HSW LP1+ watermarks w/ multiple pipes */
2745 if (level > 0 && config->num_pipes_active > 1)
2746 return 64;
2747
2748 /* otherwise just report max that registers can hold */
2749 return ilk_cursor_wm_reg_max(dev_priv, level);
2750 }
2751
ilk_compute_wm_maximums(const struct drm_i915_private * dev_priv,int level,const struct intel_wm_config * config,enum intel_ddb_partitioning ddb_partitioning,struct ilk_wm_maximums * max)2752 static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2753 int level,
2754 const struct intel_wm_config *config,
2755 enum intel_ddb_partitioning ddb_partitioning,
2756 struct ilk_wm_maximums *max)
2757 {
2758 max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2759 max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2760 max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2761 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2762 }
2763
ilk_compute_wm_reg_maximums(const struct drm_i915_private * dev_priv,int level,struct ilk_wm_maximums * max)2764 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2765 int level,
2766 struct ilk_wm_maximums *max)
2767 {
2768 max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2769 max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2770 max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2771 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2772 }
2773
ilk_validate_wm_level(int level,const struct ilk_wm_maximums * max,struct intel_wm_level * result)2774 static bool ilk_validate_wm_level(int level,
2775 const struct ilk_wm_maximums *max,
2776 struct intel_wm_level *result)
2777 {
2778 bool ret;
2779
2780 /* already determined to be invalid? */
2781 if (!result->enable)
2782 return false;
2783
2784 result->enable = result->pri_val <= max->pri &&
2785 result->spr_val <= max->spr &&
2786 result->cur_val <= max->cur;
2787
2788 ret = result->enable;
2789
2790 /*
2791 * HACK until we can pre-compute everything,
2792 * and thus fail gracefully if LP0 watermarks
2793 * are exceeded...
2794 */
2795 if (level == 0 && !result->enable) {
2796 if (result->pri_val > max->pri)
2797 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2798 level, result->pri_val, max->pri);
2799 if (result->spr_val > max->spr)
2800 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2801 level, result->spr_val, max->spr);
2802 if (result->cur_val > max->cur)
2803 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2804 level, result->cur_val, max->cur);
2805
2806 result->pri_val = min_t(u32, result->pri_val, max->pri);
2807 result->spr_val = min_t(u32, result->spr_val, max->spr);
2808 result->cur_val = min_t(u32, result->cur_val, max->cur);
2809 result->enable = true;
2810 }
2811
2812 return ret;
2813 }
2814
ilk_compute_wm_level(const struct drm_i915_private * dev_priv,const struct intel_crtc * crtc,int level,struct intel_crtc_state * crtc_state,const struct intel_plane_state * pristate,const struct intel_plane_state * sprstate,const struct intel_plane_state * curstate,struct intel_wm_level * result)2815 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2816 const struct intel_crtc *crtc,
2817 int level,
2818 struct intel_crtc_state *crtc_state,
2819 const struct intel_plane_state *pristate,
2820 const struct intel_plane_state *sprstate,
2821 const struct intel_plane_state *curstate,
2822 struct intel_wm_level *result)
2823 {
2824 u16 pri_latency = dev_priv->wm.pri_latency[level];
2825 u16 spr_latency = dev_priv->wm.spr_latency[level];
2826 u16 cur_latency = dev_priv->wm.cur_latency[level];
2827
2828 /* WM1+ latency values stored in 0.5us units */
2829 if (level > 0) {
2830 pri_latency *= 5;
2831 spr_latency *= 5;
2832 cur_latency *= 5;
2833 }
2834
2835 if (pristate) {
2836 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2837 pri_latency, level);
2838 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2839 }
2840
2841 if (sprstate)
2842 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2843
2844 if (curstate)
2845 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2846
2847 result->enable = true;
2848 }
2849
intel_read_wm_latency(struct drm_i915_private * dev_priv,u16 wm[8])2850 static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2851 u16 wm[8])
2852 {
2853 struct intel_uncore *uncore = &dev_priv->uncore;
2854
2855 if (DISPLAY_VER(dev_priv) >= 9) {
2856 u32 val;
2857 int ret, i;
2858 int level, max_level = ilk_wm_max_level(dev_priv);
2859
2860 /* read the first set of memory latencies[0:3] */
2861 val = 0; /* data0 to be programmed to 0 for first set */
2862 ret = sandybridge_pcode_read(dev_priv,
2863 GEN9_PCODE_READ_MEM_LATENCY,
2864 &val, NULL);
2865
2866 if (ret) {
2867 drm_err(&dev_priv->drm,
2868 "SKL Mailbox read error = %d\n", ret);
2869 return;
2870 }
2871
2872 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2873 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2874 GEN9_MEM_LATENCY_LEVEL_MASK;
2875 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2876 GEN9_MEM_LATENCY_LEVEL_MASK;
2877 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2878 GEN9_MEM_LATENCY_LEVEL_MASK;
2879
2880 /* read the second set of memory latencies[4:7] */
2881 val = 1; /* data0 to be programmed to 1 for second set */
2882 ret = sandybridge_pcode_read(dev_priv,
2883 GEN9_PCODE_READ_MEM_LATENCY,
2884 &val, NULL);
2885 if (ret) {
2886 drm_err(&dev_priv->drm,
2887 "SKL Mailbox read error = %d\n", ret);
2888 return;
2889 }
2890
2891 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2892 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2893 GEN9_MEM_LATENCY_LEVEL_MASK;
2894 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2895 GEN9_MEM_LATENCY_LEVEL_MASK;
2896 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2897 GEN9_MEM_LATENCY_LEVEL_MASK;
2898
2899 /*
2900 * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
2901 * need to be disabled. We make sure to sanitize the values out
2902 * of the punit to satisfy this requirement.
2903 */
2904 for (level = 1; level <= max_level; level++) {
2905 if (wm[level] == 0) {
2906 for (i = level + 1; i <= max_level; i++)
2907 wm[i] = 0;
2908 break;
2909 }
2910 }
2911
2912 /*
2913 * WaWmMemoryReadLatency:skl+,glk
2914 *
2915 * punit doesn't take into account the read latency so we need
2916 * to add 2us to the various latency levels we retrieve from the
2917 * punit when level 0 response data us 0us.
2918 */
2919 if (wm[0] == 0) {
2920 wm[0] += 2;
2921 for (level = 1; level <= max_level; level++) {
2922 if (wm[level] == 0)
2923 break;
2924 wm[level] += 2;
2925 }
2926 }
2927
2928 /*
2929 * WA Level-0 adjustment for 16GB DIMMs: SKL+
2930 * If we could not get dimm info enable this WA to prevent from
2931 * any underrun. If not able to get Dimm info assume 16GB dimm
2932 * to avoid any underrun.
2933 */
2934 if (dev_priv->dram_info.wm_lv_0_adjust_needed)
2935 wm[0] += 1;
2936
2937 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2938 u64 sskpd = intel_uncore_read64(uncore, MCH_SSKPD);
2939
2940 wm[0] = (sskpd >> 56) & 0xFF;
2941 if (wm[0] == 0)
2942 wm[0] = sskpd & 0xF;
2943 wm[1] = (sskpd >> 4) & 0xFF;
2944 wm[2] = (sskpd >> 12) & 0xFF;
2945 wm[3] = (sskpd >> 20) & 0x1FF;
2946 wm[4] = (sskpd >> 32) & 0x1FF;
2947 } else if (DISPLAY_VER(dev_priv) >= 6) {
2948 u32 sskpd = intel_uncore_read(uncore, MCH_SSKPD);
2949
2950 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2951 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2952 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2953 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2954 } else if (DISPLAY_VER(dev_priv) >= 5) {
2955 u32 mltr = intel_uncore_read(uncore, MLTR_ILK);
2956
2957 /* ILK primary LP0 latency is 700 ns */
2958 wm[0] = 7;
2959 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2960 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2961 } else {
2962 MISSING_CASE(INTEL_DEVID(dev_priv));
2963 }
2964 }
2965
intel_fixup_spr_wm_latency(struct drm_i915_private * dev_priv,u16 wm[5])2966 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2967 u16 wm[5])
2968 {
2969 /* ILK sprite LP0 latency is 1300 ns */
2970 if (IS_DISPLAY_VER(dev_priv, 5))
2971 wm[0] = 13;
2972 }
2973
intel_fixup_cur_wm_latency(struct drm_i915_private * dev_priv,u16 wm[5])2974 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2975 u16 wm[5])
2976 {
2977 /* ILK cursor LP0 latency is 1300 ns */
2978 if (IS_DISPLAY_VER(dev_priv, 5))
2979 wm[0] = 13;
2980 }
2981
ilk_wm_max_level(const struct drm_i915_private * dev_priv)2982 int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2983 {
2984 /* how many WM levels are we expecting */
2985 if (DISPLAY_VER(dev_priv) >= 9)
2986 return 7;
2987 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2988 return 4;
2989 else if (DISPLAY_VER(dev_priv) >= 6)
2990 return 3;
2991 else
2992 return 2;
2993 }
2994
intel_print_wm_latency(struct drm_i915_private * dev_priv,const char * name,const u16 wm[])2995 static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
2996 const char *name,
2997 const u16 wm[])
2998 {
2999 int level, max_level = ilk_wm_max_level(dev_priv);
3000
3001 for (level = 0; level <= max_level; level++) {
3002 unsigned int latency = wm[level];
3003
3004 if (latency == 0) {
3005 drm_dbg_kms(&dev_priv->drm,
3006 "%s WM%d latency not provided\n",
3007 name, level);
3008 continue;
3009 }
3010
3011 /*
3012 * - latencies are in us on gen9.
3013 * - before then, WM1+ latency values are in 0.5us units
3014 */
3015 if (DISPLAY_VER(dev_priv) >= 9)
3016 latency *= 10;
3017 else if (level > 0)
3018 latency *= 5;
3019
3020 drm_dbg_kms(&dev_priv->drm,
3021 "%s WM%d latency %u (%u.%u usec)\n", name, level,
3022 wm[level], latency / 10, latency % 10);
3023 }
3024 }
3025
ilk_increase_wm_latency(struct drm_i915_private * dev_priv,u16 wm[5],u16 min)3026 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
3027 u16 wm[5], u16 min)
3028 {
3029 int level, max_level = ilk_wm_max_level(dev_priv);
3030
3031 if (wm[0] >= min)
3032 return false;
3033
3034 wm[0] = max(wm[0], min);
3035 for (level = 1; level <= max_level; level++)
3036 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
3037
3038 return true;
3039 }
3040
snb_wm_latency_quirk(struct drm_i915_private * dev_priv)3041 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
3042 {
3043 bool changed;
3044
3045 /*
3046 * The BIOS provided WM memory latency values are often
3047 * inadequate for high resolution displays. Adjust them.
3048 */
3049 changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
3050 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
3051 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
3052
3053 if (!changed)
3054 return;
3055
3056 drm_dbg_kms(&dev_priv->drm,
3057 "WM latency values increased to avoid potential underruns\n");
3058 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3059 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3060 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3061 }
3062
snb_wm_lp3_irq_quirk(struct drm_i915_private * dev_priv)3063 static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
3064 {
3065 /*
3066 * On some SNB machines (Thinkpad X220 Tablet at least)
3067 * LP3 usage can cause vblank interrupts to be lost.
3068 * The DEIIR bit will go high but it looks like the CPU
3069 * never gets interrupted.
3070 *
3071 * It's not clear whether other interrupt source could
3072 * be affected or if this is somehow limited to vblank
3073 * interrupts only. To play it safe we disable LP3
3074 * watermarks entirely.
3075 */
3076 if (dev_priv->wm.pri_latency[3] == 0 &&
3077 dev_priv->wm.spr_latency[3] == 0 &&
3078 dev_priv->wm.cur_latency[3] == 0)
3079 return;
3080
3081 dev_priv->wm.pri_latency[3] = 0;
3082 dev_priv->wm.spr_latency[3] = 0;
3083 dev_priv->wm.cur_latency[3] = 0;
3084
3085 drm_dbg_kms(&dev_priv->drm,
3086 "LP3 watermarks disabled due to potential for lost interrupts\n");
3087 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3088 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3089 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3090 }
3091
ilk_setup_wm_latency(struct drm_i915_private * dev_priv)3092 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3093 {
3094 intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
3095
3096 memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
3097 sizeof(dev_priv->wm.pri_latency));
3098 memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
3099 sizeof(dev_priv->wm.pri_latency));
3100
3101 intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
3102 intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
3103
3104 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3105 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3106 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3107
3108 if (IS_DISPLAY_VER(dev_priv, 6)) {
3109 snb_wm_latency_quirk(dev_priv);
3110 snb_wm_lp3_irq_quirk(dev_priv);
3111 }
3112 }
3113
skl_setup_wm_latency(struct drm_i915_private * dev_priv)3114 static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
3115 {
3116 intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
3117 intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
3118 }
3119
ilk_validate_pipe_wm(const struct drm_i915_private * dev_priv,struct intel_pipe_wm * pipe_wm)3120 static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3121 struct intel_pipe_wm *pipe_wm)
3122 {
3123 /* LP0 watermark maximums depend on this pipe alone */
3124 const struct intel_wm_config config = {
3125 .num_pipes_active = 1,
3126 .sprites_enabled = pipe_wm->sprites_enabled,
3127 .sprites_scaled = pipe_wm->sprites_scaled,
3128 };
3129 struct ilk_wm_maximums max;
3130
3131 /* LP0 watermarks always use 1/2 DDB partitioning */
3132 ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3133
3134 /* At least LP0 must be valid */
3135 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3136 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
3137 return false;
3138 }
3139
3140 return true;
3141 }
3142
3143 /* Compute new watermarks for the pipe */
ilk_compute_pipe_wm(struct intel_crtc_state * crtc_state)3144 static int ilk_compute_pipe_wm(struct intel_crtc_state *crtc_state)
3145 {
3146 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
3147 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3148 struct intel_pipe_wm *pipe_wm;
3149 struct intel_plane *plane;
3150 const struct intel_plane_state *plane_state;
3151 const struct intel_plane_state *pristate = NULL;
3152 const struct intel_plane_state *sprstate = NULL;
3153 const struct intel_plane_state *curstate = NULL;
3154 int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3155 struct ilk_wm_maximums max;
3156
3157 pipe_wm = &crtc_state->wm.ilk.optimal;
3158
3159 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3160 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
3161 pristate = plane_state;
3162 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
3163 sprstate = plane_state;
3164 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
3165 curstate = plane_state;
3166 }
3167
3168 pipe_wm->pipe_enabled = crtc_state->hw.active;
3169 if (sprstate) {
3170 pipe_wm->sprites_enabled = sprstate->uapi.visible;
3171 pipe_wm->sprites_scaled = sprstate->uapi.visible &&
3172 (drm_rect_width(&sprstate->uapi.dst) != drm_rect_width(&sprstate->uapi.src) >> 16 ||
3173 drm_rect_height(&sprstate->uapi.dst) != drm_rect_height(&sprstate->uapi.src) >> 16);
3174 }
3175
3176 usable_level = max_level;
3177
3178 /* ILK/SNB: LP2+ watermarks only w/o sprites */
3179 if (DISPLAY_VER(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3180 usable_level = 1;
3181
3182 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3183 if (pipe_wm->sprites_scaled)
3184 usable_level = 0;
3185
3186 memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3187 ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
3188 pristate, sprstate, curstate, &pipe_wm->wm[0]);
3189
3190 if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3191 return -EINVAL;
3192
3193 ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3194
3195 for (level = 1; level <= usable_level; level++) {
3196 struct intel_wm_level *wm = &pipe_wm->wm[level];
3197
3198 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
3199 pristate, sprstate, curstate, wm);
3200
3201 /*
3202 * Disable any watermark level that exceeds the
3203 * register maximums since such watermarks are
3204 * always invalid.
3205 */
3206 if (!ilk_validate_wm_level(level, &max, wm)) {
3207 memset(wm, 0, sizeof(*wm));
3208 break;
3209 }
3210 }
3211
3212 return 0;
3213 }
3214
3215 /*
3216 * Build a set of 'intermediate' watermark values that satisfy both the old
3217 * state and the new state. These can be programmed to the hardware
3218 * immediately.
3219 */
ilk_compute_intermediate_wm(struct intel_crtc_state * newstate)3220 static int ilk_compute_intermediate_wm(struct intel_crtc_state *newstate)
3221 {
3222 struct intel_crtc *intel_crtc = to_intel_crtc(newstate->uapi.crtc);
3223 struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
3224 struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate;
3225 struct intel_atomic_state *intel_state =
3226 to_intel_atomic_state(newstate->uapi.state);
3227 const struct intel_crtc_state *oldstate =
3228 intel_atomic_get_old_crtc_state(intel_state, intel_crtc);
3229 const struct intel_pipe_wm *b = &oldstate->wm.ilk.optimal;
3230 int level, max_level = ilk_wm_max_level(dev_priv);
3231
3232 /*
3233 * Start with the final, target watermarks, then combine with the
3234 * currently active watermarks to get values that are safe both before
3235 * and after the vblank.
3236 */
3237 *a = newstate->wm.ilk.optimal;
3238 if (!newstate->hw.active || drm_atomic_crtc_needs_modeset(&newstate->uapi) ||
3239 intel_state->skip_intermediate_wm)
3240 return 0;
3241
3242 a->pipe_enabled |= b->pipe_enabled;
3243 a->sprites_enabled |= b->sprites_enabled;
3244 a->sprites_scaled |= b->sprites_scaled;
3245
3246 for (level = 0; level <= max_level; level++) {
3247 struct intel_wm_level *a_wm = &a->wm[level];
3248 const struct intel_wm_level *b_wm = &b->wm[level];
3249
3250 a_wm->enable &= b_wm->enable;
3251 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3252 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3253 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3254 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3255 }
3256
3257 /*
3258 * We need to make sure that these merged watermark values are
3259 * actually a valid configuration themselves. If they're not,
3260 * there's no safe way to transition from the old state to
3261 * the new state, so we need to fail the atomic transaction.
3262 */
3263 if (!ilk_validate_pipe_wm(dev_priv, a))
3264 return -EINVAL;
3265
3266 /*
3267 * If our intermediate WM are identical to the final WM, then we can
3268 * omit the post-vblank programming; only update if it's different.
3269 */
3270 if (memcmp(a, &newstate->wm.ilk.optimal, sizeof(*a)) != 0)
3271 newstate->wm.need_postvbl_update = true;
3272
3273 return 0;
3274 }
3275
3276 /*
3277 * Merge the watermarks from all active pipes for a specific level.
3278 */
ilk_merge_wm_level(struct drm_i915_private * dev_priv,int level,struct intel_wm_level * ret_wm)3279 static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3280 int level,
3281 struct intel_wm_level *ret_wm)
3282 {
3283 const struct intel_crtc *intel_crtc;
3284
3285 ret_wm->enable = true;
3286
3287 for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3288 const struct intel_pipe_wm *active = &intel_crtc->wm.active.ilk;
3289 const struct intel_wm_level *wm = &active->wm[level];
3290
3291 if (!active->pipe_enabled)
3292 continue;
3293
3294 /*
3295 * The watermark values may have been used in the past,
3296 * so we must maintain them in the registers for some
3297 * time even if the level is now disabled.
3298 */
3299 if (!wm->enable)
3300 ret_wm->enable = false;
3301
3302 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3303 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3304 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3305 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3306 }
3307 }
3308
3309 /*
3310 * Merge all low power watermarks for all active pipes.
3311 */
ilk_wm_merge(struct drm_i915_private * dev_priv,const struct intel_wm_config * config,const struct ilk_wm_maximums * max,struct intel_pipe_wm * merged)3312 static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3313 const struct intel_wm_config *config,
3314 const struct ilk_wm_maximums *max,
3315 struct intel_pipe_wm *merged)
3316 {
3317 int level, max_level = ilk_wm_max_level(dev_priv);
3318 int last_enabled_level = max_level;
3319
3320 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3321 if ((DISPLAY_VER(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3322 config->num_pipes_active > 1)
3323 last_enabled_level = 0;
3324
3325 /* ILK: FBC WM must be disabled always */
3326 merged->fbc_wm_enabled = DISPLAY_VER(dev_priv) >= 6;
3327
3328 /* merge each WM1+ level */
3329 for (level = 1; level <= max_level; level++) {
3330 struct intel_wm_level *wm = &merged->wm[level];
3331
3332 ilk_merge_wm_level(dev_priv, level, wm);
3333
3334 if (level > last_enabled_level)
3335 wm->enable = false;
3336 else if (!ilk_validate_wm_level(level, max, wm))
3337 /* make sure all following levels get disabled */
3338 last_enabled_level = level - 1;
3339
3340 /*
3341 * The spec says it is preferred to disable
3342 * FBC WMs instead of disabling a WM level.
3343 */
3344 if (wm->fbc_val > max->fbc) {
3345 if (wm->enable)
3346 merged->fbc_wm_enabled = false;
3347 wm->fbc_val = 0;
3348 }
3349 }
3350
3351 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3352 /*
3353 * FIXME this is racy. FBC might get enabled later.
3354 * What we should check here is whether FBC can be
3355 * enabled sometime later.
3356 */
3357 if (IS_DISPLAY_VER(dev_priv, 5) && !merged->fbc_wm_enabled &&
3358 intel_fbc_is_active(dev_priv)) {
3359 for (level = 2; level <= max_level; level++) {
3360 struct intel_wm_level *wm = &merged->wm[level];
3361
3362 wm->enable = false;
3363 }
3364 }
3365 }
3366
ilk_wm_lp_to_level(int wm_lp,const struct intel_pipe_wm * pipe_wm)3367 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3368 {
3369 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3370 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3371 }
3372
3373 /* The value we need to program into the WM_LPx latency field */
ilk_wm_lp_latency(struct drm_i915_private * dev_priv,int level)3374 static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3375 int level)
3376 {
3377 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3378 return 2 * level;
3379 else
3380 return dev_priv->wm.pri_latency[level];
3381 }
3382
ilk_compute_wm_results(struct drm_i915_private * dev_priv,const struct intel_pipe_wm * merged,enum intel_ddb_partitioning partitioning,struct ilk_wm_values * results)3383 static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3384 const struct intel_pipe_wm *merged,
3385 enum intel_ddb_partitioning partitioning,
3386 struct ilk_wm_values *results)
3387 {
3388 struct intel_crtc *intel_crtc;
3389 int level, wm_lp;
3390
3391 results->enable_fbc_wm = merged->fbc_wm_enabled;
3392 results->partitioning = partitioning;
3393
3394 /* LP1+ register values */
3395 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3396 const struct intel_wm_level *r;
3397
3398 level = ilk_wm_lp_to_level(wm_lp, merged);
3399
3400 r = &merged->wm[level];
3401
3402 /*
3403 * Maintain the watermark values even if the level is
3404 * disabled. Doing otherwise could cause underruns.
3405 */
3406 results->wm_lp[wm_lp - 1] =
3407 (ilk_wm_lp_latency(dev_priv, level) << WM1_LP_LATENCY_SHIFT) |
3408 (r->pri_val << WM1_LP_SR_SHIFT) |
3409 r->cur_val;
3410
3411 if (r->enable)
3412 results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
3413
3414 if (DISPLAY_VER(dev_priv) >= 8)
3415 results->wm_lp[wm_lp - 1] |=
3416 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
3417 else
3418 results->wm_lp[wm_lp - 1] |=
3419 r->fbc_val << WM1_LP_FBC_SHIFT;
3420
3421 /*
3422 * Always set WM1S_LP_EN when spr_val != 0, even if the
3423 * level is disabled. Doing otherwise could cause underruns.
3424 */
3425 if (DISPLAY_VER(dev_priv) <= 6 && r->spr_val) {
3426 drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3427 results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
3428 } else
3429 results->wm_lp_spr[wm_lp - 1] = r->spr_val;
3430 }
3431
3432 /* LP0 register values */
3433 for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3434 enum pipe pipe = intel_crtc->pipe;
3435 const struct intel_pipe_wm *pipe_wm = &intel_crtc->wm.active.ilk;
3436 const struct intel_wm_level *r = &pipe_wm->wm[0];
3437
3438 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3439 continue;
3440
3441 results->wm_pipe[pipe] =
3442 (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
3443 (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
3444 r->cur_val;
3445 }
3446 }
3447
3448 /* Find the result with the highest level enabled. Check for enable_fbc_wm in
3449 * case both are at the same level. Prefer r1 in case they're the same. */
3450 static struct intel_pipe_wm *
ilk_find_best_result(struct drm_i915_private * dev_priv,struct intel_pipe_wm * r1,struct intel_pipe_wm * r2)3451 ilk_find_best_result(struct drm_i915_private *dev_priv,
3452 struct intel_pipe_wm *r1,
3453 struct intel_pipe_wm *r2)
3454 {
3455 int level, max_level = ilk_wm_max_level(dev_priv);
3456 int level1 = 0, level2 = 0;
3457
3458 for (level = 1; level <= max_level; level++) {
3459 if (r1->wm[level].enable)
3460 level1 = level;
3461 if (r2->wm[level].enable)
3462 level2 = level;
3463 }
3464
3465 if (level1 == level2) {
3466 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3467 return r2;
3468 else
3469 return r1;
3470 } else if (level1 > level2) {
3471 return r1;
3472 } else {
3473 return r2;
3474 }
3475 }
3476
3477 /* dirty bits used to track which watermarks need changes */
3478 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3479 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3480 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3481 #define WM_DIRTY_FBC (1 << 24)
3482 #define WM_DIRTY_DDB (1 << 25)
3483
ilk_compute_wm_dirty(struct drm_i915_private * dev_priv,const struct ilk_wm_values * old,const struct ilk_wm_values * new)3484 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3485 const struct ilk_wm_values *old,
3486 const struct ilk_wm_values *new)
3487 {
3488 unsigned int dirty = 0;
3489 enum pipe pipe;
3490 int wm_lp;
3491
3492 for_each_pipe(dev_priv, pipe) {
3493 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3494 dirty |= WM_DIRTY_PIPE(pipe);
3495 /* Must disable LP1+ watermarks too */
3496 dirty |= WM_DIRTY_LP_ALL;
3497 }
3498 }
3499
3500 if (old->enable_fbc_wm != new->enable_fbc_wm) {
3501 dirty |= WM_DIRTY_FBC;
3502 /* Must disable LP1+ watermarks too */
3503 dirty |= WM_DIRTY_LP_ALL;
3504 }
3505
3506 if (old->partitioning != new->partitioning) {
3507 dirty |= WM_DIRTY_DDB;
3508 /* Must disable LP1+ watermarks too */
3509 dirty |= WM_DIRTY_LP_ALL;
3510 }
3511
3512 /* LP1+ watermarks already deemed dirty, no need to continue */
3513 if (dirty & WM_DIRTY_LP_ALL)
3514 return dirty;
3515
3516 /* Find the lowest numbered LP1+ watermark in need of an update... */
3517 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3518 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3519 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3520 break;
3521 }
3522
3523 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3524 for (; wm_lp <= 3; wm_lp++)
3525 dirty |= WM_DIRTY_LP(wm_lp);
3526
3527 return dirty;
3528 }
3529
_ilk_disable_lp_wm(struct drm_i915_private * dev_priv,unsigned int dirty)3530 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3531 unsigned int dirty)
3532 {
3533 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3534 bool changed = false;
3535
3536 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
3537 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
3538 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]);
3539 changed = true;
3540 }
3541 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
3542 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
3543 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]);
3544 changed = true;
3545 }
3546 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
3547 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
3548 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]);
3549 changed = true;
3550 }
3551
3552 /*
3553 * Don't touch WM1S_LP_EN here.
3554 * Doing so could cause underruns.
3555 */
3556
3557 return changed;
3558 }
3559
3560 /*
3561 * The spec says we shouldn't write when we don't need, because every write
3562 * causes WMs to be re-evaluated, expending some power.
3563 */
ilk_write_wm_values(struct drm_i915_private * dev_priv,struct ilk_wm_values * results)3564 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3565 struct ilk_wm_values *results)
3566 {
3567 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3568 unsigned int dirty;
3569 u32 val;
3570
3571 dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3572 if (!dirty)
3573 return;
3574
3575 _ilk_disable_lp_wm(dev_priv, dirty);
3576
3577 if (dirty & WM_DIRTY_PIPE(PIPE_A))
3578 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]);
3579 if (dirty & WM_DIRTY_PIPE(PIPE_B))
3580 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]);
3581 if (dirty & WM_DIRTY_PIPE(PIPE_C))
3582 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]);
3583
3584 if (dirty & WM_DIRTY_DDB) {
3585 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3586 val = intel_uncore_read(&dev_priv->uncore, WM_MISC);
3587 if (results->partitioning == INTEL_DDB_PART_1_2)
3588 val &= ~WM_MISC_DATA_PARTITION_5_6;
3589 else
3590 val |= WM_MISC_DATA_PARTITION_5_6;
3591 intel_uncore_write(&dev_priv->uncore, WM_MISC, val);
3592 } else {
3593 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2);
3594 if (results->partitioning == INTEL_DDB_PART_1_2)
3595 val &= ~DISP_DATA_PARTITION_5_6;
3596 else
3597 val |= DISP_DATA_PARTITION_5_6;
3598 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val);
3599 }
3600 }
3601
3602 if (dirty & WM_DIRTY_FBC) {
3603 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL);
3604 if (results->enable_fbc_wm)
3605 val &= ~DISP_FBC_WM_DIS;
3606 else
3607 val |= DISP_FBC_WM_DIS;
3608 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, val);
3609 }
3610
3611 if (dirty & WM_DIRTY_LP(1) &&
3612 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3613 intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]);
3614
3615 if (DISPLAY_VER(dev_priv) >= 7) {
3616 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3617 intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]);
3618 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3619 intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]);
3620 }
3621
3622 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3623 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]);
3624 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3625 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]);
3626 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3627 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]);
3628
3629 dev_priv->wm.hw = *results;
3630 }
3631
ilk_disable_lp_wm(struct drm_i915_private * dev_priv)3632 bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3633 {
3634 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3635 }
3636
intel_enabled_dbuf_slices_mask(struct drm_i915_private * dev_priv)3637 u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *dev_priv)
3638 {
3639 int i;
3640 int max_slices = INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
3641 u8 enabled_slices_mask = 0;
3642
3643 for (i = 0; i < max_slices; i++) {
3644 if (intel_uncore_read(&dev_priv->uncore, DBUF_CTL_S(i)) & DBUF_POWER_STATE)
3645 enabled_slices_mask |= BIT(i);
3646 }
3647
3648 return enabled_slices_mask;
3649 }
3650
3651 /*
3652 * FIXME: We still don't have the proper code detect if we need to apply the WA,
3653 * so assume we'll always need it in order to avoid underruns.
3654 */
skl_needs_memory_bw_wa(struct drm_i915_private * dev_priv)3655 static bool skl_needs_memory_bw_wa(struct drm_i915_private *dev_priv)
3656 {
3657 return IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv);
3658 }
3659
3660 static bool
intel_has_sagv(struct drm_i915_private * dev_priv)3661 intel_has_sagv(struct drm_i915_private *dev_priv)
3662 {
3663 return (IS_GEN9_BC(dev_priv) || DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv)) &&
3664 dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED;
3665 }
3666
3667 static void
skl_setup_sagv_block_time(struct drm_i915_private * dev_priv)3668 skl_setup_sagv_block_time(struct drm_i915_private *dev_priv)
3669 {
3670 if (DISPLAY_VER(dev_priv) >= 12) {
3671 u32 val = 0;
3672 int ret;
3673
3674 ret = sandybridge_pcode_read(dev_priv,
3675 GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
3676 &val, NULL);
3677 if (!ret) {
3678 dev_priv->sagv_block_time_us = val;
3679 return;
3680 }
3681
3682 drm_dbg(&dev_priv->drm, "Couldn't read SAGV block time!\n");
3683 } else if (IS_DISPLAY_VER(dev_priv, 11)) {
3684 dev_priv->sagv_block_time_us = 10;
3685 return;
3686 } else if (IS_DISPLAY_VER(dev_priv, 10)) {
3687 dev_priv->sagv_block_time_us = 20;
3688 return;
3689 } else if (IS_DISPLAY_VER(dev_priv, 9)) {
3690 dev_priv->sagv_block_time_us = 30;
3691 return;
3692 } else {
3693 MISSING_CASE(DISPLAY_VER(dev_priv));
3694 }
3695
3696 /* Default to an unusable block time */
3697 dev_priv->sagv_block_time_us = -1;
3698 }
3699
3700 /*
3701 * SAGV dynamically adjusts the system agent voltage and clock frequencies
3702 * depending on power and performance requirements. The display engine access
3703 * to system memory is blocked during the adjustment time. Because of the
3704 * blocking time, having this enabled can cause full system hangs and/or pipe
3705 * underruns if we don't meet all of the following requirements:
3706 *
3707 * - <= 1 pipe enabled
3708 * - All planes can enable watermarks for latencies >= SAGV engine block time
3709 * - We're not using an interlaced display configuration
3710 */
3711 static int
intel_enable_sagv(struct drm_i915_private * dev_priv)3712 intel_enable_sagv(struct drm_i915_private *dev_priv)
3713 {
3714 int ret;
3715
3716 if (!intel_has_sagv(dev_priv))
3717 return 0;
3718
3719 if (dev_priv->sagv_status == I915_SAGV_ENABLED)
3720 return 0;
3721
3722 drm_dbg_kms(&dev_priv->drm, "Enabling SAGV\n");
3723 ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3724 GEN9_SAGV_ENABLE);
3725
3726 /* We don't need to wait for SAGV when enabling */
3727
3728 /*
3729 * Some skl systems, pre-release machines in particular,
3730 * don't actually have SAGV.
3731 */
3732 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3733 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3734 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3735 return 0;
3736 } else if (ret < 0) {
3737 drm_err(&dev_priv->drm, "Failed to enable SAGV\n");
3738 return ret;
3739 }
3740
3741 dev_priv->sagv_status = I915_SAGV_ENABLED;
3742 return 0;
3743 }
3744
3745 static int
intel_disable_sagv(struct drm_i915_private * dev_priv)3746 intel_disable_sagv(struct drm_i915_private *dev_priv)
3747 {
3748 int ret;
3749
3750 if (!intel_has_sagv(dev_priv))
3751 return 0;
3752
3753 if (dev_priv->sagv_status == I915_SAGV_DISABLED)
3754 return 0;
3755
3756 drm_dbg_kms(&dev_priv->drm, "Disabling SAGV\n");
3757 /* bspec says to keep retrying for at least 1 ms */
3758 ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3759 GEN9_SAGV_DISABLE,
3760 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
3761 1);
3762 /*
3763 * Some skl systems, pre-release machines in particular,
3764 * don't actually have SAGV.
3765 */
3766 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3767 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3768 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3769 return 0;
3770 } else if (ret < 0) {
3771 drm_err(&dev_priv->drm, "Failed to disable SAGV (%d)\n", ret);
3772 return ret;
3773 }
3774
3775 dev_priv->sagv_status = I915_SAGV_DISABLED;
3776 return 0;
3777 }
3778
intel_sagv_pre_plane_update(struct intel_atomic_state * state)3779 void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
3780 {
3781 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3782 const struct intel_bw_state *new_bw_state;
3783 const struct intel_bw_state *old_bw_state;
3784 u32 new_mask = 0;
3785
3786 /*
3787 * Just return if we can't control SAGV or don't have it.
3788 * This is different from situation when we have SAGV but just can't
3789 * afford it due to DBuf limitation - in case if SAGV is completely
3790 * disabled in a BIOS, we are not even allowed to send a PCode request,
3791 * as it will throw an error. So have to check it here.
3792 */
3793 if (!intel_has_sagv(dev_priv))
3794 return;
3795
3796 new_bw_state = intel_atomic_get_new_bw_state(state);
3797 if (!new_bw_state)
3798 return;
3799
3800 if (DISPLAY_VER(dev_priv) < 11 && !intel_can_enable_sagv(dev_priv, new_bw_state)) {
3801 intel_disable_sagv(dev_priv);
3802 return;
3803 }
3804
3805 old_bw_state = intel_atomic_get_old_bw_state(state);
3806 /*
3807 * Nothing to mask
3808 */
3809 if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
3810 return;
3811
3812 new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
3813
3814 /*
3815 * If new mask is zero - means there is nothing to mask,
3816 * we can only unmask, which should be done in unmask.
3817 */
3818 if (!new_mask)
3819 return;
3820
3821 /*
3822 * Restrict required qgv points before updating the configuration.
3823 * According to BSpec we can't mask and unmask qgv points at the same
3824 * time. Also masking should be done before updating the configuration
3825 * and unmasking afterwards.
3826 */
3827 icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3828 }
3829
intel_sagv_post_plane_update(struct intel_atomic_state * state)3830 void intel_sagv_post_plane_update(struct intel_atomic_state *state)
3831 {
3832 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3833 const struct intel_bw_state *new_bw_state;
3834 const struct intel_bw_state *old_bw_state;
3835 u32 new_mask = 0;
3836
3837 /*
3838 * Just return if we can't control SAGV or don't have it.
3839 * This is different from situation when we have SAGV but just can't
3840 * afford it due to DBuf limitation - in case if SAGV is completely
3841 * disabled in a BIOS, we are not even allowed to send a PCode request,
3842 * as it will throw an error. So have to check it here.
3843 */
3844 if (!intel_has_sagv(dev_priv))
3845 return;
3846
3847 new_bw_state = intel_atomic_get_new_bw_state(state);
3848 if (!new_bw_state)
3849 return;
3850
3851 if (DISPLAY_VER(dev_priv) < 11 && intel_can_enable_sagv(dev_priv, new_bw_state)) {
3852 intel_enable_sagv(dev_priv);
3853 return;
3854 }
3855
3856 old_bw_state = intel_atomic_get_old_bw_state(state);
3857 /*
3858 * Nothing to unmask
3859 */
3860 if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
3861 return;
3862
3863 new_mask = new_bw_state->qgv_points_mask;
3864
3865 /*
3866 * Allow required qgv points after updating the configuration.
3867 * According to BSpec we can't mask and unmask qgv points at the same
3868 * time. Also masking should be done before updating the configuration
3869 * and unmasking afterwards.
3870 */
3871 icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3872 }
3873
skl_crtc_can_enable_sagv(const struct intel_crtc_state * crtc_state)3874 static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3875 {
3876 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3877 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3878 enum plane_id plane_id;
3879 int max_level = INT_MAX;
3880
3881 if (!intel_has_sagv(dev_priv))
3882 return false;
3883
3884 if (!crtc_state->hw.active)
3885 return true;
3886
3887 if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE)
3888 return false;
3889
3890 for_each_plane_id_on_crtc(crtc, plane_id) {
3891 const struct skl_plane_wm *wm =
3892 &crtc_state->wm.skl.optimal.planes[plane_id];
3893 int level;
3894
3895 /* Skip this plane if it's not enabled */
3896 if (!wm->wm[0].enable)
3897 continue;
3898
3899 /* Find the highest enabled wm level for this plane */
3900 for (level = ilk_wm_max_level(dev_priv);
3901 !wm->wm[level].enable; --level)
3902 { }
3903
3904 /* Highest common enabled wm level for all planes */
3905 max_level = min(level, max_level);
3906 }
3907
3908 /* No enabled planes? */
3909 if (max_level == INT_MAX)
3910 return true;
3911
3912 for_each_plane_id_on_crtc(crtc, plane_id) {
3913 const struct skl_plane_wm *wm =
3914 &crtc_state->wm.skl.optimal.planes[plane_id];
3915
3916 /*
3917 * All enabled planes must have enabled a common wm level that
3918 * can tolerate memory latencies higher than sagv_block_time_us
3919 */
3920 if (wm->wm[0].enable && !wm->wm[max_level].can_sagv)
3921 return false;
3922 }
3923
3924 return true;
3925 }
3926
tgl_crtc_can_enable_sagv(const struct intel_crtc_state * crtc_state)3927 static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3928 {
3929 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3930 enum plane_id plane_id;
3931
3932 if (!crtc_state->hw.active)
3933 return true;
3934
3935 for_each_plane_id_on_crtc(crtc, plane_id) {
3936 const struct skl_plane_wm *wm =
3937 &crtc_state->wm.skl.optimal.planes[plane_id];
3938
3939 if (wm->wm[0].enable && !wm->sagv.wm0.enable)
3940 return false;
3941 }
3942
3943 return true;
3944 }
3945
intel_crtc_can_enable_sagv(const struct intel_crtc_state * crtc_state)3946 static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3947 {
3948 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3949 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3950
3951 if (DISPLAY_VER(dev_priv) >= 12)
3952 return tgl_crtc_can_enable_sagv(crtc_state);
3953 else
3954 return skl_crtc_can_enable_sagv(crtc_state);
3955 }
3956
intel_can_enable_sagv(struct drm_i915_private * dev_priv,const struct intel_bw_state * bw_state)3957 bool intel_can_enable_sagv(struct drm_i915_private *dev_priv,
3958 const struct intel_bw_state *bw_state)
3959 {
3960 if (DISPLAY_VER(dev_priv) < 11 &&
3961 bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
3962 return false;
3963
3964 return bw_state->pipe_sagv_reject == 0;
3965 }
3966
intel_compute_sagv_mask(struct intel_atomic_state * state)3967 static int intel_compute_sagv_mask(struct intel_atomic_state *state)
3968 {
3969 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3970 int ret;
3971 struct intel_crtc *crtc;
3972 struct intel_crtc_state *new_crtc_state;
3973 struct intel_bw_state *new_bw_state = NULL;
3974 const struct intel_bw_state *old_bw_state = NULL;
3975 int i;
3976
3977 for_each_new_intel_crtc_in_state(state, crtc,
3978 new_crtc_state, i) {
3979 new_bw_state = intel_atomic_get_bw_state(state);
3980 if (IS_ERR(new_bw_state))
3981 return PTR_ERR(new_bw_state);
3982
3983 old_bw_state = intel_atomic_get_old_bw_state(state);
3984
3985 if (intel_crtc_can_enable_sagv(new_crtc_state))
3986 new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
3987 else
3988 new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
3989 }
3990
3991 if (!new_bw_state)
3992 return 0;
3993
3994 new_bw_state->active_pipes =
3995 intel_calc_active_pipes(state, old_bw_state->active_pipes);
3996
3997 if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
3998 ret = intel_atomic_lock_global_state(&new_bw_state->base);
3999 if (ret)
4000 return ret;
4001 }
4002
4003 for_each_new_intel_crtc_in_state(state, crtc,
4004 new_crtc_state, i) {
4005 struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
4006
4007 /*
4008 * We store use_sagv_wm in the crtc state rather than relying on
4009 * that bw state since we have no convenient way to get at the
4010 * latter from the plane commit hooks (especially in the legacy
4011 * cursor case)
4012 */
4013 pipe_wm->use_sagv_wm = DISPLAY_VER(dev_priv) >= 12 &&
4014 intel_can_enable_sagv(dev_priv, new_bw_state);
4015 }
4016
4017 if (intel_can_enable_sagv(dev_priv, new_bw_state) !=
4018 intel_can_enable_sagv(dev_priv, old_bw_state)) {
4019 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
4020 if (ret)
4021 return ret;
4022 } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
4023 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4024 if (ret)
4025 return ret;
4026 }
4027
4028 return 0;
4029 }
4030
intel_dbuf_size(struct drm_i915_private * dev_priv)4031 static int intel_dbuf_size(struct drm_i915_private *dev_priv)
4032 {
4033 int ddb_size = INTEL_INFO(dev_priv)->ddb_size;
4034
4035 drm_WARN_ON(&dev_priv->drm, ddb_size == 0);
4036
4037 if (DISPLAY_VER(dev_priv) < 11)
4038 return ddb_size - 4; /* 4 blocks for bypass path allocation */
4039
4040 return ddb_size;
4041 }
4042
intel_dbuf_slice_size(struct drm_i915_private * dev_priv)4043 static int intel_dbuf_slice_size(struct drm_i915_private *dev_priv)
4044 {
4045 return intel_dbuf_size(dev_priv) /
4046 INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
4047 }
4048
4049 static void
skl_ddb_entry_for_slices(struct drm_i915_private * dev_priv,u8 slice_mask,struct skl_ddb_entry * ddb)4050 skl_ddb_entry_for_slices(struct drm_i915_private *dev_priv, u8 slice_mask,
4051 struct skl_ddb_entry *ddb)
4052 {
4053 int slice_size = intel_dbuf_slice_size(dev_priv);
4054
4055 if (!slice_mask) {
4056 ddb->start = 0;
4057 ddb->end = 0;
4058 return;
4059 }
4060
4061 ddb->start = (ffs(slice_mask) - 1) * slice_size;
4062 ddb->end = fls(slice_mask) * slice_size;
4063
4064 WARN_ON(ddb->start >= ddb->end);
4065 WARN_ON(ddb->end > intel_dbuf_size(dev_priv));
4066 }
4067
skl_ddb_dbuf_slice_mask(struct drm_i915_private * dev_priv,const struct skl_ddb_entry * entry)4068 u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *dev_priv,
4069 const struct skl_ddb_entry *entry)
4070 {
4071 u32 slice_mask = 0;
4072 u16 ddb_size = intel_dbuf_size(dev_priv);
4073 u16 num_supported_slices = INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
4074 u16 slice_size = ddb_size / num_supported_slices;
4075 u16 start_slice;
4076 u16 end_slice;
4077
4078 if (!skl_ddb_entry_size(entry))
4079 return 0;
4080
4081 start_slice = entry->start / slice_size;
4082 end_slice = (entry->end - 1) / slice_size;
4083
4084 /*
4085 * Per plane DDB entry can in a really worst case be on multiple slices
4086 * but single entry is anyway contigious.
4087 */
4088 while (start_slice <= end_slice) {
4089 slice_mask |= BIT(start_slice);
4090 start_slice++;
4091 }
4092
4093 return slice_mask;
4094 }
4095
intel_crtc_ddb_weight(const struct intel_crtc_state * crtc_state)4096 static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state)
4097 {
4098 const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
4099 int hdisplay, vdisplay;
4100
4101 if (!crtc_state->hw.active)
4102 return 0;
4103
4104 /*
4105 * Watermark/ddb requirement highly depends upon width of the
4106 * framebuffer, So instead of allocating DDB equally among pipes
4107 * distribute DDB based on resolution/width of the display.
4108 */
4109 drm_mode_get_hv_timing(pipe_mode, &hdisplay, &vdisplay);
4110
4111 return hdisplay;
4112 }
4113
intel_crtc_dbuf_weights(const struct intel_dbuf_state * dbuf_state,enum pipe for_pipe,unsigned int * weight_start,unsigned int * weight_end,unsigned int * weight_total)4114 static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state,
4115 enum pipe for_pipe,
4116 unsigned int *weight_start,
4117 unsigned int *weight_end,
4118 unsigned int *weight_total)
4119 {
4120 struct drm_i915_private *dev_priv =
4121 to_i915(dbuf_state->base.state->base.dev);
4122 enum pipe pipe;
4123
4124 *weight_start = 0;
4125 *weight_end = 0;
4126 *weight_total = 0;
4127
4128 for_each_pipe(dev_priv, pipe) {
4129 int weight = dbuf_state->weight[pipe];
4130
4131 /*
4132 * Do not account pipes using other slice sets
4133 * luckily as of current BSpec slice sets do not partially
4134 * intersect(pipes share either same one slice or same slice set
4135 * i.e no partial intersection), so it is enough to check for
4136 * equality for now.
4137 */
4138 if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe])
4139 continue;
4140
4141 *weight_total += weight;
4142 if (pipe < for_pipe) {
4143 *weight_start += weight;
4144 *weight_end += weight;
4145 } else if (pipe == for_pipe) {
4146 *weight_end += weight;
4147 }
4148 }
4149 }
4150
4151 static int
skl_crtc_allocate_ddb(struct intel_atomic_state * state,struct intel_crtc * crtc)4152 skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc)
4153 {
4154 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4155 unsigned int weight_total, weight_start, weight_end;
4156 const struct intel_dbuf_state *old_dbuf_state =
4157 intel_atomic_get_old_dbuf_state(state);
4158 struct intel_dbuf_state *new_dbuf_state =
4159 intel_atomic_get_new_dbuf_state(state);
4160 struct intel_crtc_state *crtc_state;
4161 struct skl_ddb_entry ddb_slices;
4162 enum pipe pipe = crtc->pipe;
4163 u32 ddb_range_size;
4164 u32 dbuf_slice_mask;
4165 u32 start, end;
4166 int ret;
4167
4168 if (new_dbuf_state->weight[pipe] == 0) {
4169 new_dbuf_state->ddb[pipe].start = 0;
4170 new_dbuf_state->ddb[pipe].end = 0;
4171 goto out;
4172 }
4173
4174 dbuf_slice_mask = new_dbuf_state->slices[pipe];
4175
4176 skl_ddb_entry_for_slices(dev_priv, dbuf_slice_mask, &ddb_slices);
4177 ddb_range_size = skl_ddb_entry_size(&ddb_slices);
4178
4179 intel_crtc_dbuf_weights(new_dbuf_state, pipe,
4180 &weight_start, &weight_end, &weight_total);
4181
4182 start = ddb_range_size * weight_start / weight_total;
4183 end = ddb_range_size * weight_end / weight_total;
4184
4185 new_dbuf_state->ddb[pipe].start = ddb_slices.start + start;
4186 new_dbuf_state->ddb[pipe].end = ddb_slices.start + end;
4187
4188 out:
4189 if (skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe],
4190 &new_dbuf_state->ddb[pipe]))
4191 return 0;
4192
4193 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
4194 if (ret)
4195 return ret;
4196
4197 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
4198 if (IS_ERR(crtc_state))
4199 return PTR_ERR(crtc_state);
4200
4201 crtc_state->wm.skl.ddb = new_dbuf_state->ddb[pipe];
4202
4203 drm_dbg_kms(&dev_priv->drm,
4204 "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n",
4205 crtc->base.base.id, crtc->base.name,
4206 old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe],
4207 old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end,
4208 new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end,
4209 old_dbuf_state->active_pipes, new_dbuf_state->active_pipes);
4210
4211 return 0;
4212 }
4213
4214 static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
4215 int width, const struct drm_format_info *format,
4216 u64 modifier, unsigned int rotation,
4217 u32 plane_pixel_rate, struct skl_wm_params *wp,
4218 int color_plane);
4219 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
4220 int level,
4221 unsigned int latency,
4222 const struct skl_wm_params *wp,
4223 const struct skl_wm_level *result_prev,
4224 struct skl_wm_level *result /* out */);
4225
4226 static unsigned int
skl_cursor_allocation(const struct intel_crtc_state * crtc_state,int num_active)4227 skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
4228 int num_active)
4229 {
4230 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4231 int level, max_level = ilk_wm_max_level(dev_priv);
4232 struct skl_wm_level wm = {};
4233 int ret, min_ddb_alloc = 0;
4234 struct skl_wm_params wp;
4235
4236 ret = skl_compute_wm_params(crtc_state, 256,
4237 drm_format_info(DRM_FORMAT_ARGB8888),
4238 DRM_FORMAT_MOD_LINEAR,
4239 DRM_MODE_ROTATE_0,
4240 crtc_state->pixel_rate, &wp, 0);
4241 drm_WARN_ON(&dev_priv->drm, ret);
4242
4243 for (level = 0; level <= max_level; level++) {
4244 unsigned int latency = dev_priv->wm.skl_latency[level];
4245
4246 skl_compute_plane_wm(crtc_state, level, latency, &wp, &wm, &wm);
4247 if (wm.min_ddb_alloc == U16_MAX)
4248 break;
4249
4250 min_ddb_alloc = wm.min_ddb_alloc;
4251 }
4252
4253 return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
4254 }
4255
skl_ddb_entry_init_from_hw(struct drm_i915_private * dev_priv,struct skl_ddb_entry * entry,u32 reg)4256 static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv,
4257 struct skl_ddb_entry *entry, u32 reg)
4258 {
4259
4260 entry->start = reg & DDB_ENTRY_MASK;
4261 entry->end = (reg >> DDB_ENTRY_END_SHIFT) & DDB_ENTRY_MASK;
4262
4263 if (entry->end)
4264 entry->end += 1;
4265 }
4266
4267 static void
skl_ddb_get_hw_plane_state(struct drm_i915_private * dev_priv,const enum pipe pipe,const enum plane_id plane_id,struct skl_ddb_entry * ddb_y,struct skl_ddb_entry * ddb_uv)4268 skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
4269 const enum pipe pipe,
4270 const enum plane_id plane_id,
4271 struct skl_ddb_entry *ddb_y,
4272 struct skl_ddb_entry *ddb_uv)
4273 {
4274 u32 val, val2;
4275 u32 fourcc = 0;
4276
4277 /* Cursor doesn't support NV12/planar, so no extra calculation needed */
4278 if (plane_id == PLANE_CURSOR) {
4279 val = intel_uncore_read(&dev_priv->uncore, CUR_BUF_CFG(pipe));
4280 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4281 return;
4282 }
4283
4284 val = intel_uncore_read(&dev_priv->uncore, PLANE_CTL(pipe, plane_id));
4285
4286 /* No DDB allocated for disabled planes */
4287 if (val & PLANE_CTL_ENABLE)
4288 fourcc = skl_format_to_fourcc(val & PLANE_CTL_FORMAT_MASK,
4289 val & PLANE_CTL_ORDER_RGBX,
4290 val & PLANE_CTL_ALPHA_MASK);
4291
4292 if (DISPLAY_VER(dev_priv) >= 11) {
4293 val = intel_uncore_read(&dev_priv->uncore, PLANE_BUF_CFG(pipe, plane_id));
4294 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4295 } else {
4296 val = intel_uncore_read(&dev_priv->uncore, PLANE_BUF_CFG(pipe, plane_id));
4297 val2 = intel_uncore_read(&dev_priv->uncore, PLANE_NV12_BUF_CFG(pipe, plane_id));
4298
4299 if (fourcc &&
4300 drm_format_info_is_yuv_semiplanar(drm_format_info(fourcc)))
4301 swap(val, val2);
4302
4303 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4304 skl_ddb_entry_init_from_hw(dev_priv, ddb_uv, val2);
4305 }
4306 }
4307
skl_pipe_ddb_get_hw_state(struct intel_crtc * crtc,struct skl_ddb_entry * ddb_y,struct skl_ddb_entry * ddb_uv)4308 void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
4309 struct skl_ddb_entry *ddb_y,
4310 struct skl_ddb_entry *ddb_uv)
4311 {
4312 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4313 enum intel_display_power_domain power_domain;
4314 enum pipe pipe = crtc->pipe;
4315 intel_wakeref_t wakeref;
4316 enum plane_id plane_id;
4317
4318 power_domain = POWER_DOMAIN_PIPE(pipe);
4319 wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4320 if (!wakeref)
4321 return;
4322
4323 for_each_plane_id_on_crtc(crtc, plane_id)
4324 skl_ddb_get_hw_plane_state(dev_priv, pipe,
4325 plane_id,
4326 &ddb_y[plane_id],
4327 &ddb_uv[plane_id]);
4328
4329 intel_display_power_put(dev_priv, power_domain, wakeref);
4330 }
4331
4332 /*
4333 * Determines the downscale amount of a plane for the purposes of watermark calculations.
4334 * The bspec defines downscale amount as:
4335 *
4336 * """
4337 * Horizontal down scale amount = maximum[1, Horizontal source size /
4338 * Horizontal destination size]
4339 * Vertical down scale amount = maximum[1, Vertical source size /
4340 * Vertical destination size]
4341 * Total down scale amount = Horizontal down scale amount *
4342 * Vertical down scale amount
4343 * """
4344 *
4345 * Return value is provided in 16.16 fixed point form to retain fractional part.
4346 * Caller should take care of dividing & rounding off the value.
4347 */
4348 static uint_fixed_16_16_t
skl_plane_downscale_amount(const struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state)4349 skl_plane_downscale_amount(const struct intel_crtc_state *crtc_state,
4350 const struct intel_plane_state *plane_state)
4351 {
4352 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4353 u32 src_w, src_h, dst_w, dst_h;
4354 uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
4355 uint_fixed_16_16_t downscale_h, downscale_w;
4356
4357 if (drm_WARN_ON(&dev_priv->drm,
4358 !intel_wm_plane_visible(crtc_state, plane_state)))
4359 return u32_to_fixed16(0);
4360
4361 /*
4362 * Src coordinates are already rotated by 270 degrees for
4363 * the 90/270 degree plane rotation cases (to match the
4364 * GTT mapping), hence no need to account for rotation here.
4365 *
4366 * n.b., src is 16.16 fixed point, dst is whole integer.
4367 */
4368 src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
4369 src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
4370 dst_w = drm_rect_width(&plane_state->uapi.dst);
4371 dst_h = drm_rect_height(&plane_state->uapi.dst);
4372
4373 fp_w_ratio = div_fixed16(src_w, dst_w);
4374 fp_h_ratio = div_fixed16(src_h, dst_h);
4375 downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
4376 downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4377
4378 return mul_fixed16(downscale_w, downscale_h);
4379 }
4380
4381 struct dbuf_slice_conf_entry {
4382 u8 active_pipes;
4383 u8 dbuf_mask[I915_MAX_PIPES];
4384 };
4385
4386 /*
4387 * Table taken from Bspec 12716
4388 * Pipes do have some preferred DBuf slice affinity,
4389 * plus there are some hardcoded requirements on how
4390 * those should be distributed for multipipe scenarios.
4391 * For more DBuf slices algorithm can get even more messy
4392 * and less readable, so decided to use a table almost
4393 * as is from BSpec itself - that way it is at least easier
4394 * to compare, change and check.
4395 */
4396 static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
4397 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
4398 {
4399 {
4400 .active_pipes = BIT(PIPE_A),
4401 .dbuf_mask = {
4402 [PIPE_A] = BIT(DBUF_S1),
4403 },
4404 },
4405 {
4406 .active_pipes = BIT(PIPE_B),
4407 .dbuf_mask = {
4408 [PIPE_B] = BIT(DBUF_S1),
4409 },
4410 },
4411 {
4412 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4413 .dbuf_mask = {
4414 [PIPE_A] = BIT(DBUF_S1),
4415 [PIPE_B] = BIT(DBUF_S2),
4416 },
4417 },
4418 {
4419 .active_pipes = BIT(PIPE_C),
4420 .dbuf_mask = {
4421 [PIPE_C] = BIT(DBUF_S2),
4422 },
4423 },
4424 {
4425 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4426 .dbuf_mask = {
4427 [PIPE_A] = BIT(DBUF_S1),
4428 [PIPE_C] = BIT(DBUF_S2),
4429 },
4430 },
4431 {
4432 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4433 .dbuf_mask = {
4434 [PIPE_B] = BIT(DBUF_S1),
4435 [PIPE_C] = BIT(DBUF_S2),
4436 },
4437 },
4438 {
4439 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4440 .dbuf_mask = {
4441 [PIPE_A] = BIT(DBUF_S1),
4442 [PIPE_B] = BIT(DBUF_S1),
4443 [PIPE_C] = BIT(DBUF_S2),
4444 },
4445 },
4446 {}
4447 };
4448
4449 /*
4450 * Table taken from Bspec 49255
4451 * Pipes do have some preferred DBuf slice affinity,
4452 * plus there are some hardcoded requirements on how
4453 * those should be distributed for multipipe scenarios.
4454 * For more DBuf slices algorithm can get even more messy
4455 * and less readable, so decided to use a table almost
4456 * as is from BSpec itself - that way it is at least easier
4457 * to compare, change and check.
4458 */
4459 static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
4460 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
4461 {
4462 {
4463 .active_pipes = BIT(PIPE_A),
4464 .dbuf_mask = {
4465 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4466 },
4467 },
4468 {
4469 .active_pipes = BIT(PIPE_B),
4470 .dbuf_mask = {
4471 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4472 },
4473 },
4474 {
4475 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4476 .dbuf_mask = {
4477 [PIPE_A] = BIT(DBUF_S2),
4478 [PIPE_B] = BIT(DBUF_S1),
4479 },
4480 },
4481 {
4482 .active_pipes = BIT(PIPE_C),
4483 .dbuf_mask = {
4484 [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
4485 },
4486 },
4487 {
4488 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4489 .dbuf_mask = {
4490 [PIPE_A] = BIT(DBUF_S1),
4491 [PIPE_C] = BIT(DBUF_S2),
4492 },
4493 },
4494 {
4495 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4496 .dbuf_mask = {
4497 [PIPE_B] = BIT(DBUF_S1),
4498 [PIPE_C] = BIT(DBUF_S2),
4499 },
4500 },
4501 {
4502 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4503 .dbuf_mask = {
4504 [PIPE_A] = BIT(DBUF_S1),
4505 [PIPE_B] = BIT(DBUF_S1),
4506 [PIPE_C] = BIT(DBUF_S2),
4507 },
4508 },
4509 {
4510 .active_pipes = BIT(PIPE_D),
4511 .dbuf_mask = {
4512 [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
4513 },
4514 },
4515 {
4516 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4517 .dbuf_mask = {
4518 [PIPE_A] = BIT(DBUF_S1),
4519 [PIPE_D] = BIT(DBUF_S2),
4520 },
4521 },
4522 {
4523 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4524 .dbuf_mask = {
4525 [PIPE_B] = BIT(DBUF_S1),
4526 [PIPE_D] = BIT(DBUF_S2),
4527 },
4528 },
4529 {
4530 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4531 .dbuf_mask = {
4532 [PIPE_A] = BIT(DBUF_S1),
4533 [PIPE_B] = BIT(DBUF_S1),
4534 [PIPE_D] = BIT(DBUF_S2),
4535 },
4536 },
4537 {
4538 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4539 .dbuf_mask = {
4540 [PIPE_C] = BIT(DBUF_S1),
4541 [PIPE_D] = BIT(DBUF_S2),
4542 },
4543 },
4544 {
4545 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4546 .dbuf_mask = {
4547 [PIPE_A] = BIT(DBUF_S1),
4548 [PIPE_C] = BIT(DBUF_S2),
4549 [PIPE_D] = BIT(DBUF_S2),
4550 },
4551 },
4552 {
4553 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4554 .dbuf_mask = {
4555 [PIPE_B] = BIT(DBUF_S1),
4556 [PIPE_C] = BIT(DBUF_S2),
4557 [PIPE_D] = BIT(DBUF_S2),
4558 },
4559 },
4560 {
4561 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4562 .dbuf_mask = {
4563 [PIPE_A] = BIT(DBUF_S1),
4564 [PIPE_B] = BIT(DBUF_S1),
4565 [PIPE_C] = BIT(DBUF_S2),
4566 [PIPE_D] = BIT(DBUF_S2),
4567 },
4568 },
4569 {}
4570 };
4571
compute_dbuf_slices(enum pipe pipe,u8 active_pipes,const struct dbuf_slice_conf_entry * dbuf_slices)4572 static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes,
4573 const struct dbuf_slice_conf_entry *dbuf_slices)
4574 {
4575 int i;
4576
4577 for (i = 0; i < dbuf_slices[i].active_pipes; i++) {
4578 if (dbuf_slices[i].active_pipes == active_pipes)
4579 return dbuf_slices[i].dbuf_mask[pipe];
4580 }
4581 return 0;
4582 }
4583
4584 /*
4585 * This function finds an entry with same enabled pipe configuration and
4586 * returns correspondent DBuf slice mask as stated in BSpec for particular
4587 * platform.
4588 */
icl_compute_dbuf_slices(enum pipe pipe,u8 active_pipes)4589 static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4590 {
4591 /*
4592 * FIXME: For ICL this is still a bit unclear as prev BSpec revision
4593 * required calculating "pipe ratio" in order to determine
4594 * if one or two slices can be used for single pipe configurations
4595 * as additional constraint to the existing table.
4596 * However based on recent info, it should be not "pipe ratio"
4597 * but rather ratio between pixel_rate and cdclk with additional
4598 * constants, so for now we are using only table until this is
4599 * clarified. Also this is the reason why crtc_state param is
4600 * still here - we will need it once those additional constraints
4601 * pop up.
4602 */
4603 return compute_dbuf_slices(pipe, active_pipes, icl_allowed_dbufs);
4604 }
4605
tgl_compute_dbuf_slices(enum pipe pipe,u8 active_pipes)4606 static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4607 {
4608 return compute_dbuf_slices(pipe, active_pipes, tgl_allowed_dbufs);
4609 }
4610
skl_compute_dbuf_slices(struct intel_crtc * crtc,u8 active_pipes)4611 static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes)
4612 {
4613 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4614 enum pipe pipe = crtc->pipe;
4615
4616 if (IS_DISPLAY_VER(dev_priv, 12))
4617 return tgl_compute_dbuf_slices(pipe, active_pipes);
4618 else if (IS_DISPLAY_VER(dev_priv, 11))
4619 return icl_compute_dbuf_slices(pipe, active_pipes);
4620 /*
4621 * For anything else just return one slice yet.
4622 * Should be extended for other platforms.
4623 */
4624 return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
4625 }
4626
4627 static u64
skl_plane_relative_data_rate(const struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state,int color_plane)4628 skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state,
4629 const struct intel_plane_state *plane_state,
4630 int color_plane)
4631 {
4632 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
4633 const struct drm_framebuffer *fb = plane_state->hw.fb;
4634 u32 data_rate;
4635 u32 width = 0, height = 0;
4636 uint_fixed_16_16_t down_scale_amount;
4637 u64 rate;
4638
4639 if (!plane_state->uapi.visible)
4640 return 0;
4641
4642 if (plane->id == PLANE_CURSOR)
4643 return 0;
4644
4645 if (color_plane == 1 &&
4646 !intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier))
4647 return 0;
4648
4649 /*
4650 * Src coordinates are already rotated by 270 degrees for
4651 * the 90/270 degree plane rotation cases (to match the
4652 * GTT mapping), hence no need to account for rotation here.
4653 */
4654 width = drm_rect_width(&plane_state->uapi.src) >> 16;
4655 height = drm_rect_height(&plane_state->uapi.src) >> 16;
4656
4657 /* UV plane does 1/2 pixel sub-sampling */
4658 if (color_plane == 1) {
4659 width /= 2;
4660 height /= 2;
4661 }
4662
4663 data_rate = width * height;
4664
4665 down_scale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
4666
4667 rate = mul_round_up_u32_fixed16(data_rate, down_scale_amount);
4668
4669 rate *= fb->format->cpp[color_plane];
4670 return rate;
4671 }
4672
4673 static u64
skl_get_total_relative_data_rate(struct intel_atomic_state * state,struct intel_crtc * crtc)4674 skl_get_total_relative_data_rate(struct intel_atomic_state *state,
4675 struct intel_crtc *crtc)
4676 {
4677 struct intel_crtc_state *crtc_state =
4678 intel_atomic_get_new_crtc_state(state, crtc);
4679 const struct intel_plane_state *plane_state;
4680 struct intel_plane *plane;
4681 u64 total_data_rate = 0;
4682 enum plane_id plane_id;
4683 int i;
4684
4685 /* Calculate and cache data rate for each plane */
4686 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
4687 if (plane->pipe != crtc->pipe)
4688 continue;
4689
4690 plane_id = plane->id;
4691
4692 /* packed/y */
4693 crtc_state->plane_data_rate[plane_id] =
4694 skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4695
4696 /* uv-plane */
4697 crtc_state->uv_plane_data_rate[plane_id] =
4698 skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4699 }
4700
4701 for_each_plane_id_on_crtc(crtc, plane_id) {
4702 total_data_rate += crtc_state->plane_data_rate[plane_id];
4703 total_data_rate += crtc_state->uv_plane_data_rate[plane_id];
4704 }
4705
4706 return total_data_rate;
4707 }
4708
4709 static u64
icl_get_total_relative_data_rate(struct intel_atomic_state * state,struct intel_crtc * crtc)4710 icl_get_total_relative_data_rate(struct intel_atomic_state *state,
4711 struct intel_crtc *crtc)
4712 {
4713 struct intel_crtc_state *crtc_state =
4714 intel_atomic_get_new_crtc_state(state, crtc);
4715 const struct intel_plane_state *plane_state;
4716 struct intel_plane *plane;
4717 u64 total_data_rate = 0;
4718 enum plane_id plane_id;
4719 int i;
4720
4721 /* Calculate and cache data rate for each plane */
4722 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
4723 if (plane->pipe != crtc->pipe)
4724 continue;
4725
4726 plane_id = plane->id;
4727
4728 if (!plane_state->planar_linked_plane) {
4729 crtc_state->plane_data_rate[plane_id] =
4730 skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4731 } else {
4732 enum plane_id y_plane_id;
4733
4734 /*
4735 * The slave plane might not iterate in
4736 * intel_atomic_crtc_state_for_each_plane_state(),
4737 * and needs the master plane state which may be
4738 * NULL if we try get_new_plane_state(), so we
4739 * always calculate from the master.
4740 */
4741 if (plane_state->planar_slave)
4742 continue;
4743
4744 /* Y plane rate is calculated on the slave */
4745 y_plane_id = plane_state->planar_linked_plane->id;
4746 crtc_state->plane_data_rate[y_plane_id] =
4747 skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4748
4749 crtc_state->plane_data_rate[plane_id] =
4750 skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4751 }
4752 }
4753
4754 for_each_plane_id_on_crtc(crtc, plane_id)
4755 total_data_rate += crtc_state->plane_data_rate[plane_id];
4756
4757 return total_data_rate;
4758 }
4759
4760 const struct skl_wm_level *
skl_plane_wm_level(const struct skl_pipe_wm * pipe_wm,enum plane_id plane_id,int level)4761 skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm,
4762 enum plane_id plane_id,
4763 int level)
4764 {
4765 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
4766
4767 if (level == 0 && pipe_wm->use_sagv_wm)
4768 return &wm->sagv.wm0;
4769
4770 return &wm->wm[level];
4771 }
4772
4773 const struct skl_wm_level *
skl_plane_trans_wm(const struct skl_pipe_wm * pipe_wm,enum plane_id plane_id)4774 skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm,
4775 enum plane_id plane_id)
4776 {
4777 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
4778
4779 if (pipe_wm->use_sagv_wm)
4780 return &wm->sagv.trans_wm;
4781
4782 return &wm->trans_wm;
4783 }
4784
4785 /*
4786 * We only disable the watermarks for each plane if
4787 * they exceed the ddb allocation of said plane. This
4788 * is done so that we don't end up touching cursor
4789 * watermarks needlessly when some other plane reduces
4790 * our max possible watermark level.
4791 *
4792 * Bspec has this to say about the PLANE_WM enable bit:
4793 * "All the watermarks at this level for all enabled
4794 * planes must be enabled before the level will be used."
4795 * So this is actually safe to do.
4796 */
4797 static void
skl_check_wm_level(struct skl_wm_level * wm,u64 total)4798 skl_check_wm_level(struct skl_wm_level *wm, u64 total)
4799 {
4800 if (wm->min_ddb_alloc > total)
4801 memset(wm, 0, sizeof(*wm));
4802 }
4803
4804 static void
skl_check_nv12_wm_level(struct skl_wm_level * wm,struct skl_wm_level * uv_wm,u64 total,u64 uv_total)4805 skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm,
4806 u64 total, u64 uv_total)
4807 {
4808 if (wm->min_ddb_alloc > total ||
4809 uv_wm->min_ddb_alloc > uv_total) {
4810 memset(wm, 0, sizeof(*wm));
4811 memset(uv_wm, 0, sizeof(*uv_wm));
4812 }
4813 }
4814
4815 static int
skl_allocate_plane_ddb(struct intel_atomic_state * state,struct intel_crtc * crtc)4816 skl_allocate_plane_ddb(struct intel_atomic_state *state,
4817 struct intel_crtc *crtc)
4818 {
4819 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4820 struct intel_crtc_state *crtc_state =
4821 intel_atomic_get_new_crtc_state(state, crtc);
4822 const struct intel_dbuf_state *dbuf_state =
4823 intel_atomic_get_new_dbuf_state(state);
4824 const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe];
4825 int num_active = hweight8(dbuf_state->active_pipes);
4826 u16 alloc_size, start = 0;
4827 u16 total[I915_MAX_PLANES] = {};
4828 u16 uv_total[I915_MAX_PLANES] = {};
4829 u64 total_data_rate;
4830 enum plane_id plane_id;
4831 u32 blocks;
4832 int level;
4833
4834 /* Clear the partitioning for disabled planes. */
4835 memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
4836 memset(crtc_state->wm.skl.plane_ddb_uv, 0, sizeof(crtc_state->wm.skl.plane_ddb_uv));
4837
4838 if (!crtc_state->hw.active)
4839 return 0;
4840
4841 if (DISPLAY_VER(dev_priv) >= 11)
4842 total_data_rate =
4843 icl_get_total_relative_data_rate(state, crtc);
4844 else
4845 total_data_rate =
4846 skl_get_total_relative_data_rate(state, crtc);
4847
4848 alloc_size = skl_ddb_entry_size(alloc);
4849 if (alloc_size == 0)
4850 return 0;
4851
4852 /* Allocate fixed number of blocks for cursor. */
4853 total[PLANE_CURSOR] = skl_cursor_allocation(crtc_state, num_active);
4854 alloc_size -= total[PLANE_CURSOR];
4855 crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].start =
4856 alloc->end - total[PLANE_CURSOR];
4857 crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].end = alloc->end;
4858
4859 if (total_data_rate == 0)
4860 return 0;
4861
4862 /*
4863 * Find the highest watermark level for which we can satisfy the block
4864 * requirement of active planes.
4865 */
4866 for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) {
4867 blocks = 0;
4868 for_each_plane_id_on_crtc(crtc, plane_id) {
4869 const struct skl_plane_wm *wm =
4870 &crtc_state->wm.skl.optimal.planes[plane_id];
4871
4872 if (plane_id == PLANE_CURSOR) {
4873 if (wm->wm[level].min_ddb_alloc > total[PLANE_CURSOR]) {
4874 drm_WARN_ON(&dev_priv->drm,
4875 wm->wm[level].min_ddb_alloc != U16_MAX);
4876 blocks = U32_MAX;
4877 break;
4878 }
4879 continue;
4880 }
4881
4882 blocks += wm->wm[level].min_ddb_alloc;
4883 blocks += wm->uv_wm[level].min_ddb_alloc;
4884 }
4885
4886 if (blocks <= alloc_size) {
4887 alloc_size -= blocks;
4888 break;
4889 }
4890 }
4891
4892 if (level < 0) {
4893 drm_dbg_kms(&dev_priv->drm,
4894 "Requested display configuration exceeds system DDB limitations");
4895 drm_dbg_kms(&dev_priv->drm, "minimum required %d/%d\n",
4896 blocks, alloc_size);
4897 return -EINVAL;
4898 }
4899
4900 /*
4901 * Grant each plane the blocks it requires at the highest achievable
4902 * watermark level, plus an extra share of the leftover blocks
4903 * proportional to its relative data rate.
4904 */
4905 for_each_plane_id_on_crtc(crtc, plane_id) {
4906 const struct skl_plane_wm *wm =
4907 &crtc_state->wm.skl.optimal.planes[plane_id];
4908 u64 rate;
4909 u16 extra;
4910
4911 if (plane_id == PLANE_CURSOR)
4912 continue;
4913
4914 /*
4915 * We've accounted for all active planes; remaining planes are
4916 * all disabled.
4917 */
4918 if (total_data_rate == 0)
4919 break;
4920
4921 rate = crtc_state->plane_data_rate[plane_id];
4922 extra = min_t(u16, alloc_size,
4923 DIV64_U64_ROUND_UP(alloc_size * rate,
4924 total_data_rate));
4925 total[plane_id] = wm->wm[level].min_ddb_alloc + extra;
4926 alloc_size -= extra;
4927 total_data_rate -= rate;
4928
4929 if (total_data_rate == 0)
4930 break;
4931
4932 rate = crtc_state->uv_plane_data_rate[plane_id];
4933 extra = min_t(u16, alloc_size,
4934 DIV64_U64_ROUND_UP(alloc_size * rate,
4935 total_data_rate));
4936 uv_total[plane_id] = wm->uv_wm[level].min_ddb_alloc + extra;
4937 alloc_size -= extra;
4938 total_data_rate -= rate;
4939 }
4940 drm_WARN_ON(&dev_priv->drm, alloc_size != 0 || total_data_rate != 0);
4941
4942 /* Set the actual DDB start/end points for each plane */
4943 start = alloc->start;
4944 for_each_plane_id_on_crtc(crtc, plane_id) {
4945 struct skl_ddb_entry *plane_alloc =
4946 &crtc_state->wm.skl.plane_ddb_y[plane_id];
4947 struct skl_ddb_entry *uv_plane_alloc =
4948 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
4949
4950 if (plane_id == PLANE_CURSOR)
4951 continue;
4952
4953 /* Gen11+ uses a separate plane for UV watermarks */
4954 drm_WARN_ON(&dev_priv->drm,
4955 DISPLAY_VER(dev_priv) >= 11 && uv_total[plane_id]);
4956
4957 /* Leave disabled planes at (0,0) */
4958 if (total[plane_id]) {
4959 plane_alloc->start = start;
4960 start += total[plane_id];
4961 plane_alloc->end = start;
4962 }
4963
4964 if (uv_total[plane_id]) {
4965 uv_plane_alloc->start = start;
4966 start += uv_total[plane_id];
4967 uv_plane_alloc->end = start;
4968 }
4969 }
4970
4971 /*
4972 * When we calculated watermark values we didn't know how high
4973 * of a level we'd actually be able to hit, so we just marked
4974 * all levels as "enabled." Go back now and disable the ones
4975 * that aren't actually possible.
4976 */
4977 for (level++; level <= ilk_wm_max_level(dev_priv); level++) {
4978 for_each_plane_id_on_crtc(crtc, plane_id) {
4979 struct skl_plane_wm *wm =
4980 &crtc_state->wm.skl.optimal.planes[plane_id];
4981
4982 skl_check_nv12_wm_level(&wm->wm[level], &wm->uv_wm[level],
4983 total[plane_id], uv_total[plane_id]);
4984
4985 /*
4986 * Wa_1408961008:icl, ehl
4987 * Underruns with WM1+ disabled
4988 */
4989 if (IS_DISPLAY_VER(dev_priv, 11) &&
4990 level == 1 && wm->wm[0].enable) {
4991 wm->wm[level].blocks = wm->wm[0].blocks;
4992 wm->wm[level].lines = wm->wm[0].lines;
4993 wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
4994 }
4995 }
4996 }
4997
4998 /*
4999 * Go back and disable the transition and SAGV watermarks
5000 * if it turns out we don't have enough DDB blocks for them.
5001 */
5002 for_each_plane_id_on_crtc(crtc, plane_id) {
5003 struct skl_plane_wm *wm =
5004 &crtc_state->wm.skl.optimal.planes[plane_id];
5005
5006 skl_check_wm_level(&wm->trans_wm, total[plane_id]);
5007 skl_check_wm_level(&wm->sagv.wm0, total[plane_id]);
5008 skl_check_wm_level(&wm->sagv.trans_wm, total[plane_id]);
5009 }
5010
5011 return 0;
5012 }
5013
5014 /*
5015 * The max latency should be 257 (max the punit can code is 255 and we add 2us
5016 * for the read latency) and cpp should always be <= 8, so that
5017 * should allow pixel_rate up to ~2 GHz which seems sufficient since max
5018 * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
5019 */
5020 static uint_fixed_16_16_t
skl_wm_method1(const struct drm_i915_private * dev_priv,u32 pixel_rate,u8 cpp,u32 latency,u32 dbuf_block_size)5021 skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate,
5022 u8 cpp, u32 latency, u32 dbuf_block_size)
5023 {
5024 u32 wm_intermediate_val;
5025 uint_fixed_16_16_t ret;
5026
5027 if (latency == 0)
5028 return FP_16_16_MAX;
5029
5030 wm_intermediate_val = latency * pixel_rate * cpp;
5031 ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
5032
5033 if (DISPLAY_VER(dev_priv) >= 10)
5034 ret = add_fixed16_u32(ret, 1);
5035
5036 return ret;
5037 }
5038
5039 static uint_fixed_16_16_t
skl_wm_method2(u32 pixel_rate,u32 pipe_htotal,u32 latency,uint_fixed_16_16_t plane_blocks_per_line)5040 skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
5041 uint_fixed_16_16_t plane_blocks_per_line)
5042 {
5043 u32 wm_intermediate_val;
5044 uint_fixed_16_16_t ret;
5045
5046 if (latency == 0)
5047 return FP_16_16_MAX;
5048
5049 wm_intermediate_val = latency * pixel_rate;
5050 wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
5051 pipe_htotal * 1000);
5052 ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
5053 return ret;
5054 }
5055
5056 static uint_fixed_16_16_t
intel_get_linetime_us(const struct intel_crtc_state * crtc_state)5057 intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
5058 {
5059 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5060 u32 pixel_rate;
5061 u32 crtc_htotal;
5062 uint_fixed_16_16_t linetime_us;
5063
5064 if (!crtc_state->hw.active)
5065 return u32_to_fixed16(0);
5066
5067 pixel_rate = crtc_state->pixel_rate;
5068
5069 if (drm_WARN_ON(&dev_priv->drm, pixel_rate == 0))
5070 return u32_to_fixed16(0);
5071
5072 crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal;
5073 linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
5074
5075 return linetime_us;
5076 }
5077
5078 static int
skl_compute_wm_params(const struct intel_crtc_state * crtc_state,int width,const struct drm_format_info * format,u64 modifier,unsigned int rotation,u32 plane_pixel_rate,struct skl_wm_params * wp,int color_plane)5079 skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
5080 int width, const struct drm_format_info *format,
5081 u64 modifier, unsigned int rotation,
5082 u32 plane_pixel_rate, struct skl_wm_params *wp,
5083 int color_plane)
5084 {
5085 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5086 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5087 u32 interm_pbpl;
5088
5089 /* only planar format has two planes */
5090 if (color_plane == 1 &&
5091 !intel_format_info_is_yuv_semiplanar(format, modifier)) {
5092 drm_dbg_kms(&dev_priv->drm,
5093 "Non planar format have single plane\n");
5094 return -EINVAL;
5095 }
5096
5097 wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED ||
5098 modifier == I915_FORMAT_MOD_Yf_TILED ||
5099 modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5100 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5101 wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
5102 wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5103 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5104 wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
5105
5106 wp->width = width;
5107 if (color_plane == 1 && wp->is_planar)
5108 wp->width /= 2;
5109
5110 wp->cpp = format->cpp[color_plane];
5111 wp->plane_pixel_rate = plane_pixel_rate;
5112
5113 if (DISPLAY_VER(dev_priv) >= 11 &&
5114 modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 1)
5115 wp->dbuf_block_size = 256;
5116 else
5117 wp->dbuf_block_size = 512;
5118
5119 if (drm_rotation_90_or_270(rotation)) {
5120 switch (wp->cpp) {
5121 case 1:
5122 wp->y_min_scanlines = 16;
5123 break;
5124 case 2:
5125 wp->y_min_scanlines = 8;
5126 break;
5127 case 4:
5128 wp->y_min_scanlines = 4;
5129 break;
5130 default:
5131 MISSING_CASE(wp->cpp);
5132 return -EINVAL;
5133 }
5134 } else {
5135 wp->y_min_scanlines = 4;
5136 }
5137
5138 if (skl_needs_memory_bw_wa(dev_priv))
5139 wp->y_min_scanlines *= 2;
5140
5141 wp->plane_bytes_per_line = wp->width * wp->cpp;
5142 if (wp->y_tiled) {
5143 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
5144 wp->y_min_scanlines,
5145 wp->dbuf_block_size);
5146
5147 if (DISPLAY_VER(dev_priv) >= 10)
5148 interm_pbpl++;
5149
5150 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
5151 wp->y_min_scanlines);
5152 } else {
5153 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
5154 wp->dbuf_block_size);
5155
5156 if (!wp->x_tiled || DISPLAY_VER(dev_priv) >= 10)
5157 interm_pbpl++;
5158
5159 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
5160 }
5161
5162 wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
5163 wp->plane_blocks_per_line);
5164
5165 wp->linetime_us = fixed16_to_u32_round_up(
5166 intel_get_linetime_us(crtc_state));
5167
5168 return 0;
5169 }
5170
5171 static int
skl_compute_plane_wm_params(const struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state,struct skl_wm_params * wp,int color_plane)5172 skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
5173 const struct intel_plane_state *plane_state,
5174 struct skl_wm_params *wp, int color_plane)
5175 {
5176 const struct drm_framebuffer *fb = plane_state->hw.fb;
5177 int width;
5178
5179 /*
5180 * Src coordinates are already rotated by 270 degrees for
5181 * the 90/270 degree plane rotation cases (to match the
5182 * GTT mapping), hence no need to account for rotation here.
5183 */
5184 width = drm_rect_width(&plane_state->uapi.src) >> 16;
5185
5186 return skl_compute_wm_params(crtc_state, width,
5187 fb->format, fb->modifier,
5188 plane_state->hw.rotation,
5189 intel_plane_pixel_rate(crtc_state, plane_state),
5190 wp, color_plane);
5191 }
5192
skl_wm_has_lines(struct drm_i915_private * dev_priv,int level)5193 static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level)
5194 {
5195 if (DISPLAY_VER(dev_priv) >= 10)
5196 return true;
5197
5198 /* The number of lines are ignored for the level 0 watermark. */
5199 return level > 0;
5200 }
5201
skl_compute_plane_wm(const struct intel_crtc_state * crtc_state,int level,unsigned int latency,const struct skl_wm_params * wp,const struct skl_wm_level * result_prev,struct skl_wm_level * result)5202 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
5203 int level,
5204 unsigned int latency,
5205 const struct skl_wm_params *wp,
5206 const struct skl_wm_level *result_prev,
5207 struct skl_wm_level *result /* out */)
5208 {
5209 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5210 uint_fixed_16_16_t method1, method2;
5211 uint_fixed_16_16_t selected_result;
5212 u32 blocks, lines, min_ddb_alloc = 0;
5213
5214 if (latency == 0) {
5215 /* reject it */
5216 result->min_ddb_alloc = U16_MAX;
5217 return;
5218 }
5219
5220 /*
5221 * WaIncreaseLatencyIPCEnabled: kbl,cfl
5222 * Display WA #1141: kbl,cfl
5223 */
5224 if ((IS_KABYLAKE(dev_priv) ||
5225 IS_COFFEELAKE(dev_priv) ||
5226 IS_COMETLAKE(dev_priv)) &&
5227 dev_priv->ipc_enabled)
5228 latency += 4;
5229
5230 if (skl_needs_memory_bw_wa(dev_priv) && wp->x_tiled)
5231 latency += 15;
5232
5233 method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
5234 wp->cpp, latency, wp->dbuf_block_size);
5235 method2 = skl_wm_method2(wp->plane_pixel_rate,
5236 crtc_state->hw.pipe_mode.crtc_htotal,
5237 latency,
5238 wp->plane_blocks_per_line);
5239
5240 if (wp->y_tiled) {
5241 selected_result = max_fixed16(method2, wp->y_tile_minimum);
5242 } else {
5243 if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal /
5244 wp->dbuf_block_size < 1) &&
5245 (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
5246 selected_result = method2;
5247 } else if (latency >= wp->linetime_us) {
5248 if (IS_DISPLAY_VER(dev_priv, 9))
5249 selected_result = min_fixed16(method1, method2);
5250 else
5251 selected_result = method2;
5252 } else {
5253 selected_result = method1;
5254 }
5255 }
5256
5257 blocks = fixed16_to_u32_round_up(selected_result) + 1;
5258 lines = div_round_up_fixed16(selected_result,
5259 wp->plane_blocks_per_line);
5260
5261 if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv)) {
5262 /* Display WA #1125: skl,bxt,kbl */
5263 if (level == 0 && wp->rc_surface)
5264 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
5265
5266 /* Display WA #1126: skl,bxt,kbl */
5267 if (level >= 1 && level <= 7) {
5268 if (wp->y_tiled) {
5269 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
5270 lines += wp->y_min_scanlines;
5271 } else {
5272 blocks++;
5273 }
5274
5275 /*
5276 * Make sure result blocks for higher latency levels are
5277 * atleast as high as level below the current level.
5278 * Assumption in DDB algorithm optimization for special
5279 * cases. Also covers Display WA #1125 for RC.
5280 */
5281 if (result_prev->blocks > blocks)
5282 blocks = result_prev->blocks;
5283 }
5284 }
5285
5286 if (DISPLAY_VER(dev_priv) >= 11) {
5287 if (wp->y_tiled) {
5288 int extra_lines;
5289
5290 if (lines % wp->y_min_scanlines == 0)
5291 extra_lines = wp->y_min_scanlines;
5292 else
5293 extra_lines = wp->y_min_scanlines * 2 -
5294 lines % wp->y_min_scanlines;
5295
5296 min_ddb_alloc = mul_round_up_u32_fixed16(lines + extra_lines,
5297 wp->plane_blocks_per_line);
5298 } else {
5299 min_ddb_alloc = blocks + DIV_ROUND_UP(blocks, 10);
5300 }
5301 }
5302
5303 if (!skl_wm_has_lines(dev_priv, level))
5304 lines = 0;
5305
5306 if (lines > 31) {
5307 /* reject it */
5308 result->min_ddb_alloc = U16_MAX;
5309 return;
5310 }
5311
5312 /*
5313 * If lines is valid, assume we can use this watermark level
5314 * for now. We'll come back and disable it after we calculate the
5315 * DDB allocation if it turns out we don't actually have enough
5316 * blocks to satisfy it.
5317 */
5318 result->blocks = blocks;
5319 result->lines = lines;
5320 /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
5321 result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1;
5322 result->enable = true;
5323
5324 if (DISPLAY_VER(dev_priv) < 12)
5325 result->can_sagv = latency >= dev_priv->sagv_block_time_us;
5326 }
5327
5328 static void
skl_compute_wm_levels(const struct intel_crtc_state * crtc_state,const struct skl_wm_params * wm_params,struct skl_wm_level * levels)5329 skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
5330 const struct skl_wm_params *wm_params,
5331 struct skl_wm_level *levels)
5332 {
5333 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5334 int level, max_level = ilk_wm_max_level(dev_priv);
5335 struct skl_wm_level *result_prev = &levels[0];
5336
5337 for (level = 0; level <= max_level; level++) {
5338 struct skl_wm_level *result = &levels[level];
5339 unsigned int latency = dev_priv->wm.skl_latency[level];
5340
5341 skl_compute_plane_wm(crtc_state, level, latency,
5342 wm_params, result_prev, result);
5343
5344 result_prev = result;
5345 }
5346 }
5347
tgl_compute_sagv_wm(const struct intel_crtc_state * crtc_state,const struct skl_wm_params * wm_params,struct skl_plane_wm * plane_wm)5348 static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
5349 const struct skl_wm_params *wm_params,
5350 struct skl_plane_wm *plane_wm)
5351 {
5352 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5353 struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0;
5354 struct skl_wm_level *levels = plane_wm->wm;
5355 unsigned int latency = dev_priv->wm.skl_latency[0] + dev_priv->sagv_block_time_us;
5356
5357 skl_compute_plane_wm(crtc_state, 0, latency,
5358 wm_params, &levels[0],
5359 sagv_wm);
5360 }
5361
skl_compute_transition_wm(struct drm_i915_private * dev_priv,struct skl_wm_level * trans_wm,const struct skl_wm_level * wm0,const struct skl_wm_params * wp)5362 static void skl_compute_transition_wm(struct drm_i915_private *dev_priv,
5363 struct skl_wm_level *trans_wm,
5364 const struct skl_wm_level *wm0,
5365 const struct skl_wm_params *wp)
5366 {
5367 u16 trans_min, trans_amount, trans_y_tile_min;
5368 u16 wm0_blocks, trans_offset, blocks;
5369
5370 /* Transition WM don't make any sense if ipc is disabled */
5371 if (!dev_priv->ipc_enabled)
5372 return;
5373
5374 /*
5375 * WaDisableTWM:skl,kbl,cfl,bxt
5376 * Transition WM are not recommended by HW team for GEN9
5377 */
5378 if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv))
5379 return;
5380
5381 if (DISPLAY_VER(dev_priv) >= 11)
5382 trans_min = 4;
5383 else
5384 trans_min = 14;
5385
5386 /* Display WA #1140: glk,cnl */
5387 if (IS_DISPLAY_VER(dev_priv, 10))
5388 trans_amount = 0;
5389 else
5390 trans_amount = 10; /* This is configurable amount */
5391
5392 trans_offset = trans_min + trans_amount;
5393
5394 /*
5395 * The spec asks for Selected Result Blocks for wm0 (the real value),
5396 * not Result Blocks (the integer value). Pay attention to the capital
5397 * letters. The value wm_l0->blocks is actually Result Blocks, but
5398 * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
5399 * and since we later will have to get the ceiling of the sum in the
5400 * transition watermarks calculation, we can just pretend Selected
5401 * Result Blocks is Result Blocks minus 1 and it should work for the
5402 * current platforms.
5403 */
5404 wm0_blocks = wm0->blocks - 1;
5405
5406 if (wp->y_tiled) {
5407 trans_y_tile_min =
5408 (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
5409 blocks = max(wm0_blocks, trans_y_tile_min) + trans_offset;
5410 } else {
5411 blocks = wm0_blocks + trans_offset;
5412 }
5413 blocks++;
5414
5415 /*
5416 * Just assume we can enable the transition watermark. After
5417 * computing the DDB we'll come back and disable it if that
5418 * assumption turns out to be false.
5419 */
5420 trans_wm->blocks = blocks;
5421 trans_wm->min_ddb_alloc = max_t(u16, wm0->min_ddb_alloc, blocks + 1);
5422 trans_wm->enable = true;
5423 }
5424
skl_build_plane_wm_single(struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state,enum plane_id plane_id,int color_plane)5425 static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
5426 const struct intel_plane_state *plane_state,
5427 enum plane_id plane_id, int color_plane)
5428 {
5429 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5430 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5431 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5432 struct skl_wm_params wm_params;
5433 int ret;
5434
5435 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5436 &wm_params, color_plane);
5437 if (ret)
5438 return ret;
5439
5440 skl_compute_wm_levels(crtc_state, &wm_params, wm->wm);
5441
5442 skl_compute_transition_wm(dev_priv, &wm->trans_wm,
5443 &wm->wm[0], &wm_params);
5444
5445 if (DISPLAY_VER(dev_priv) >= 12) {
5446 tgl_compute_sagv_wm(crtc_state, &wm_params, wm);
5447
5448 skl_compute_transition_wm(dev_priv, &wm->sagv.trans_wm,
5449 &wm->sagv.wm0, &wm_params);
5450 }
5451
5452 return 0;
5453 }
5454
skl_build_plane_wm_uv(struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state,enum plane_id plane_id)5455 static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
5456 const struct intel_plane_state *plane_state,
5457 enum plane_id plane_id)
5458 {
5459 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5460 struct skl_wm_params wm_params;
5461 int ret;
5462
5463 wm->is_planar = true;
5464
5465 /* uv plane watermarks must also be validated for NV12/Planar */
5466 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5467 &wm_params, 1);
5468 if (ret)
5469 return ret;
5470
5471 skl_compute_wm_levels(crtc_state, &wm_params, wm->uv_wm);
5472
5473 return 0;
5474 }
5475
skl_build_plane_wm(struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state)5476 static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
5477 const struct intel_plane_state *plane_state)
5478 {
5479 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5480 enum plane_id plane_id = plane->id;
5481 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5482 const struct drm_framebuffer *fb = plane_state->hw.fb;
5483 int ret;
5484
5485 memset(wm, 0, sizeof(*wm));
5486
5487 if (!intel_wm_plane_visible(crtc_state, plane_state))
5488 return 0;
5489
5490 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5491 plane_id, 0);
5492 if (ret)
5493 return ret;
5494
5495 if (fb->format->is_yuv && fb->format->num_planes > 1) {
5496 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
5497 plane_id);
5498 if (ret)
5499 return ret;
5500 }
5501
5502 return 0;
5503 }
5504
icl_build_plane_wm(struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state)5505 static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
5506 const struct intel_plane_state *plane_state)
5507 {
5508 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5509 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5510 enum plane_id plane_id = plane->id;
5511 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5512 int ret;
5513
5514 /* Watermarks calculated in master */
5515 if (plane_state->planar_slave)
5516 return 0;
5517
5518 memset(wm, 0, sizeof(*wm));
5519
5520 if (plane_state->planar_linked_plane) {
5521 const struct drm_framebuffer *fb = plane_state->hw.fb;
5522 enum plane_id y_plane_id = plane_state->planar_linked_plane->id;
5523
5524 drm_WARN_ON(&dev_priv->drm,
5525 !intel_wm_plane_visible(crtc_state, plane_state));
5526 drm_WARN_ON(&dev_priv->drm, !fb->format->is_yuv ||
5527 fb->format->num_planes == 1);
5528
5529 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5530 y_plane_id, 0);
5531 if (ret)
5532 return ret;
5533
5534 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5535 plane_id, 1);
5536 if (ret)
5537 return ret;
5538 } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
5539 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5540 plane_id, 0);
5541 if (ret)
5542 return ret;
5543 }
5544
5545 return 0;
5546 }
5547
skl_build_pipe_wm(struct intel_atomic_state * state,struct intel_crtc * crtc)5548 static int skl_build_pipe_wm(struct intel_atomic_state *state,
5549 struct intel_crtc *crtc)
5550 {
5551 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5552 struct intel_crtc_state *crtc_state =
5553 intel_atomic_get_new_crtc_state(state, crtc);
5554 const struct intel_plane_state *plane_state;
5555 struct intel_plane *plane;
5556 int ret, i;
5557
5558 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
5559 /*
5560 * FIXME should perhaps check {old,new}_plane_crtc->hw.crtc
5561 * instead but we don't populate that correctly for NV12 Y
5562 * planes so for now hack this.
5563 */
5564 if (plane->pipe != crtc->pipe)
5565 continue;
5566
5567 if (DISPLAY_VER(dev_priv) >= 11)
5568 ret = icl_build_plane_wm(crtc_state, plane_state);
5569 else
5570 ret = skl_build_plane_wm(crtc_state, plane_state);
5571 if (ret)
5572 return ret;
5573 }
5574
5575 crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw;
5576
5577 return 0;
5578 }
5579
skl_ddb_entry_write(struct drm_i915_private * dev_priv,i915_reg_t reg,const struct skl_ddb_entry * entry)5580 static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
5581 i915_reg_t reg,
5582 const struct skl_ddb_entry *entry)
5583 {
5584 if (entry->end)
5585 intel_de_write_fw(dev_priv, reg,
5586 (entry->end - 1) << 16 | entry->start);
5587 else
5588 intel_de_write_fw(dev_priv, reg, 0);
5589 }
5590
skl_write_wm_level(struct drm_i915_private * dev_priv,i915_reg_t reg,const struct skl_wm_level * level)5591 static void skl_write_wm_level(struct drm_i915_private *dev_priv,
5592 i915_reg_t reg,
5593 const struct skl_wm_level *level)
5594 {
5595 u32 val = 0;
5596
5597 if (level->enable)
5598 val |= PLANE_WM_EN;
5599 if (level->ignore_lines)
5600 val |= PLANE_WM_IGNORE_LINES;
5601 val |= level->blocks;
5602 val |= level->lines << PLANE_WM_LINES_SHIFT;
5603
5604 intel_de_write_fw(dev_priv, reg, val);
5605 }
5606
skl_write_plane_wm(struct intel_plane * plane,const struct intel_crtc_state * crtc_state)5607 void skl_write_plane_wm(struct intel_plane *plane,
5608 const struct intel_crtc_state *crtc_state)
5609 {
5610 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5611 int level, max_level = ilk_wm_max_level(dev_priv);
5612 enum plane_id plane_id = plane->id;
5613 enum pipe pipe = plane->pipe;
5614 const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5615 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
5616 const struct skl_ddb_entry *ddb_y =
5617 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5618 const struct skl_ddb_entry *ddb_uv =
5619 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
5620
5621 for (level = 0; level <= max_level; level++)
5622 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
5623 skl_plane_wm_level(pipe_wm, plane_id, level));
5624
5625 skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
5626 skl_plane_trans_wm(pipe_wm, plane_id));
5627
5628 if (DISPLAY_VER(dev_priv) >= 11) {
5629 skl_ddb_entry_write(dev_priv,
5630 PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5631 return;
5632 }
5633
5634 if (wm->is_planar)
5635 swap(ddb_y, ddb_uv);
5636
5637 skl_ddb_entry_write(dev_priv,
5638 PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5639 skl_ddb_entry_write(dev_priv,
5640 PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_uv);
5641 }
5642
skl_write_cursor_wm(struct intel_plane * plane,const struct intel_crtc_state * crtc_state)5643 void skl_write_cursor_wm(struct intel_plane *plane,
5644 const struct intel_crtc_state *crtc_state)
5645 {
5646 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5647 int level, max_level = ilk_wm_max_level(dev_priv);
5648 enum plane_id plane_id = plane->id;
5649 enum pipe pipe = plane->pipe;
5650 const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5651 const struct skl_ddb_entry *ddb =
5652 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5653
5654 for (level = 0; level <= max_level; level++)
5655 skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
5656 skl_plane_wm_level(pipe_wm, plane_id, level));
5657
5658 skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe),
5659 skl_plane_trans_wm(pipe_wm, plane_id));
5660
5661 skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb);
5662 }
5663
skl_wm_level_equals(const struct skl_wm_level * l1,const struct skl_wm_level * l2)5664 bool skl_wm_level_equals(const struct skl_wm_level *l1,
5665 const struct skl_wm_level *l2)
5666 {
5667 return l1->enable == l2->enable &&
5668 l1->ignore_lines == l2->ignore_lines &&
5669 l1->lines == l2->lines &&
5670 l1->blocks == l2->blocks;
5671 }
5672
skl_plane_wm_equals(struct drm_i915_private * dev_priv,const struct skl_plane_wm * wm1,const struct skl_plane_wm * wm2)5673 static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv,
5674 const struct skl_plane_wm *wm1,
5675 const struct skl_plane_wm *wm2)
5676 {
5677 int level, max_level = ilk_wm_max_level(dev_priv);
5678
5679 for (level = 0; level <= max_level; level++) {
5680 /*
5681 * We don't check uv_wm as the hardware doesn't actually
5682 * use it. It only gets used for calculating the required
5683 * ddb allocation.
5684 */
5685 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
5686 return false;
5687 }
5688
5689 return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm) &&
5690 skl_wm_level_equals(&wm1->sagv.wm0, &wm2->sagv.wm0) &&
5691 skl_wm_level_equals(&wm1->sagv.trans_wm, &wm2->sagv.trans_wm);
5692 }
5693
skl_ddb_entries_overlap(const struct skl_ddb_entry * a,const struct skl_ddb_entry * b)5694 static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
5695 const struct skl_ddb_entry *b)
5696 {
5697 return a->start < b->end && b->start < a->end;
5698 }
5699
skl_ddb_entry_union(struct skl_ddb_entry * a,const struct skl_ddb_entry * b)5700 static void skl_ddb_entry_union(struct skl_ddb_entry *a,
5701 const struct skl_ddb_entry *b)
5702 {
5703 if (a->end && b->end) {
5704 a->start = min(a->start, b->start);
5705 a->end = max(a->end, b->end);
5706 } else if (b->end) {
5707 a->start = b->start;
5708 a->end = b->end;
5709 }
5710 }
5711
skl_ddb_allocation_overlaps(const struct skl_ddb_entry * ddb,const struct skl_ddb_entry * entries,int num_entries,int ignore_idx)5712 bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
5713 const struct skl_ddb_entry *entries,
5714 int num_entries, int ignore_idx)
5715 {
5716 int i;
5717
5718 for (i = 0; i < num_entries; i++) {
5719 if (i != ignore_idx &&
5720 skl_ddb_entries_overlap(ddb, &entries[i]))
5721 return true;
5722 }
5723
5724 return false;
5725 }
5726
5727 static int
skl_ddb_add_affected_planes(const struct intel_crtc_state * old_crtc_state,struct intel_crtc_state * new_crtc_state)5728 skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
5729 struct intel_crtc_state *new_crtc_state)
5730 {
5731 struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
5732 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
5733 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5734 struct intel_plane *plane;
5735
5736 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5737 struct intel_plane_state *plane_state;
5738 enum plane_id plane_id = plane->id;
5739
5740 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
5741 &new_crtc_state->wm.skl.plane_ddb_y[plane_id]) &&
5742 skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_uv[plane_id],
5743 &new_crtc_state->wm.skl.plane_ddb_uv[plane_id]))
5744 continue;
5745
5746 plane_state = intel_atomic_get_plane_state(state, plane);
5747 if (IS_ERR(plane_state))
5748 return PTR_ERR(plane_state);
5749
5750 new_crtc_state->update_planes |= BIT(plane_id);
5751 }
5752
5753 return 0;
5754 }
5755
intel_dbuf_enabled_slices(const struct intel_dbuf_state * dbuf_state)5756 static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state)
5757 {
5758 struct drm_i915_private *dev_priv = to_i915(dbuf_state->base.state->base.dev);
5759 u8 enabled_slices;
5760 enum pipe pipe;
5761
5762 /*
5763 * FIXME: For now we always enable slice S1 as per
5764 * the Bspec display initialization sequence.
5765 */
5766 enabled_slices = BIT(DBUF_S1);
5767
5768 for_each_pipe(dev_priv, pipe)
5769 enabled_slices |= dbuf_state->slices[pipe];
5770
5771 return enabled_slices;
5772 }
5773
5774 static int
skl_compute_ddb(struct intel_atomic_state * state)5775 skl_compute_ddb(struct intel_atomic_state *state)
5776 {
5777 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5778 const struct intel_dbuf_state *old_dbuf_state;
5779 struct intel_dbuf_state *new_dbuf_state = NULL;
5780 const struct intel_crtc_state *old_crtc_state;
5781 struct intel_crtc_state *new_crtc_state;
5782 struct intel_crtc *crtc;
5783 int ret, i;
5784
5785 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
5786 new_dbuf_state = intel_atomic_get_dbuf_state(state);
5787 if (IS_ERR(new_dbuf_state))
5788 return PTR_ERR(new_dbuf_state);
5789
5790 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
5791 break;
5792 }
5793
5794 if (!new_dbuf_state)
5795 return 0;
5796
5797 new_dbuf_state->active_pipes =
5798 intel_calc_active_pipes(state, old_dbuf_state->active_pipes);
5799
5800 if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
5801 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
5802 if (ret)
5803 return ret;
5804 }
5805
5806 for_each_intel_crtc(&dev_priv->drm, crtc) {
5807 enum pipe pipe = crtc->pipe;
5808
5809 new_dbuf_state->slices[pipe] =
5810 skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes);
5811
5812 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe])
5813 continue;
5814
5815 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
5816 if (ret)
5817 return ret;
5818 }
5819
5820 new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state);
5821
5822 if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices) {
5823 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
5824 if (ret)
5825 return ret;
5826
5827 drm_dbg_kms(&dev_priv->drm,
5828 "Enabled dbuf slices 0x%x -> 0x%x (out of %d dbuf slices)\n",
5829 old_dbuf_state->enabled_slices,
5830 new_dbuf_state->enabled_slices,
5831 INTEL_INFO(dev_priv)->num_supported_dbuf_slices);
5832 }
5833
5834 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
5835 enum pipe pipe = crtc->pipe;
5836
5837 new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(new_crtc_state);
5838
5839 if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe])
5840 continue;
5841
5842 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
5843 if (ret)
5844 return ret;
5845 }
5846
5847 for_each_intel_crtc(&dev_priv->drm, crtc) {
5848 ret = skl_crtc_allocate_ddb(state, crtc);
5849 if (ret)
5850 return ret;
5851 }
5852
5853 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5854 new_crtc_state, i) {
5855 ret = skl_allocate_plane_ddb(state, crtc);
5856 if (ret)
5857 return ret;
5858
5859 ret = skl_ddb_add_affected_planes(old_crtc_state,
5860 new_crtc_state);
5861 if (ret)
5862 return ret;
5863 }
5864
5865 return 0;
5866 }
5867
enast(bool enable)5868 static char enast(bool enable)
5869 {
5870 return enable ? '*' : ' ';
5871 }
5872
5873 static void
skl_print_wm_changes(struct intel_atomic_state * state)5874 skl_print_wm_changes(struct intel_atomic_state *state)
5875 {
5876 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5877 const struct intel_crtc_state *old_crtc_state;
5878 const struct intel_crtc_state *new_crtc_state;
5879 struct intel_plane *plane;
5880 struct intel_crtc *crtc;
5881 int i;
5882
5883 if (!drm_debug_enabled(DRM_UT_KMS))
5884 return;
5885
5886 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5887 new_crtc_state, i) {
5888 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
5889
5890 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
5891 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
5892
5893 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5894 enum plane_id plane_id = plane->id;
5895 const struct skl_ddb_entry *old, *new;
5896
5897 old = &old_crtc_state->wm.skl.plane_ddb_y[plane_id];
5898 new = &new_crtc_state->wm.skl.plane_ddb_y[plane_id];
5899
5900 if (skl_ddb_entry_equal(old, new))
5901 continue;
5902
5903 drm_dbg_kms(&dev_priv->drm,
5904 "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
5905 plane->base.base.id, plane->base.name,
5906 old->start, old->end, new->start, new->end,
5907 skl_ddb_entry_size(old), skl_ddb_entry_size(new));
5908 }
5909
5910 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5911 enum plane_id plane_id = plane->id;
5912 const struct skl_plane_wm *old_wm, *new_wm;
5913
5914 old_wm = &old_pipe_wm->planes[plane_id];
5915 new_wm = &new_pipe_wm->planes[plane_id];
5916
5917 if (skl_plane_wm_equals(dev_priv, old_wm, new_wm))
5918 continue;
5919
5920 drm_dbg_kms(&dev_priv->drm,
5921 "[PLANE:%d:%s] level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm"
5922 " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n",
5923 plane->base.base.id, plane->base.name,
5924 enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable),
5925 enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable),
5926 enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable),
5927 enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable),
5928 enast(old_wm->trans_wm.enable),
5929 enast(old_wm->sagv.wm0.enable),
5930 enast(old_wm->sagv.trans_wm.enable),
5931 enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable),
5932 enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable),
5933 enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable),
5934 enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable),
5935 enast(new_wm->trans_wm.enable),
5936 enast(new_wm->sagv.wm0.enable),
5937 enast(new_wm->sagv.trans_wm.enable));
5938
5939 drm_dbg_kms(&dev_priv->drm,
5940 "[PLANE:%d:%s] lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d"
5941 " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n",
5942 plane->base.base.id, plane->base.name,
5943 enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines,
5944 enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines,
5945 enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines,
5946 enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines,
5947 enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines,
5948 enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines,
5949 enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines,
5950 enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines,
5951 enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines,
5952 enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines,
5953 enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines,
5954 enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines,
5955 enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines,
5956 enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines,
5957 enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines,
5958 enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines,
5959 enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines,
5960 enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines,
5961 enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines,
5962 enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines,
5963 enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines,
5964 enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines);
5965
5966 drm_dbg_kms(&dev_priv->drm,
5967 "[PLANE:%d:%s] blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
5968 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
5969 plane->base.base.id, plane->base.name,
5970 old_wm->wm[0].blocks, old_wm->wm[1].blocks,
5971 old_wm->wm[2].blocks, old_wm->wm[3].blocks,
5972 old_wm->wm[4].blocks, old_wm->wm[5].blocks,
5973 old_wm->wm[6].blocks, old_wm->wm[7].blocks,
5974 old_wm->trans_wm.blocks,
5975 old_wm->sagv.wm0.blocks,
5976 old_wm->sagv.trans_wm.blocks,
5977 new_wm->wm[0].blocks, new_wm->wm[1].blocks,
5978 new_wm->wm[2].blocks, new_wm->wm[3].blocks,
5979 new_wm->wm[4].blocks, new_wm->wm[5].blocks,
5980 new_wm->wm[6].blocks, new_wm->wm[7].blocks,
5981 new_wm->trans_wm.blocks,
5982 new_wm->sagv.wm0.blocks,
5983 new_wm->sagv.trans_wm.blocks);
5984
5985 drm_dbg_kms(&dev_priv->drm,
5986 "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
5987 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
5988 plane->base.base.id, plane->base.name,
5989 old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
5990 old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
5991 old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
5992 old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
5993 old_wm->trans_wm.min_ddb_alloc,
5994 old_wm->sagv.wm0.min_ddb_alloc,
5995 old_wm->sagv.trans_wm.min_ddb_alloc,
5996 new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
5997 new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
5998 new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
5999 new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
6000 new_wm->trans_wm.min_ddb_alloc,
6001 new_wm->sagv.wm0.min_ddb_alloc,
6002 new_wm->sagv.trans_wm.min_ddb_alloc);
6003 }
6004 }
6005 }
6006
skl_plane_selected_wm_equals(struct intel_plane * plane,const struct skl_pipe_wm * old_pipe_wm,const struct skl_pipe_wm * new_pipe_wm)6007 static bool skl_plane_selected_wm_equals(struct intel_plane *plane,
6008 const struct skl_pipe_wm *old_pipe_wm,
6009 const struct skl_pipe_wm *new_pipe_wm)
6010 {
6011 struct drm_i915_private *i915 = to_i915(plane->base.dev);
6012 int level, max_level = ilk_wm_max_level(i915);
6013
6014 for (level = 0; level <= max_level; level++) {
6015 /*
6016 * We don't check uv_wm as the hardware doesn't actually
6017 * use it. It only gets used for calculating the required
6018 * ddb allocation.
6019 */
6020 if (!skl_wm_level_equals(skl_plane_wm_level(old_pipe_wm, plane->id, level),
6021 skl_plane_wm_level(new_pipe_wm, plane->id, level)))
6022 return false;
6023 }
6024
6025 return skl_wm_level_equals(skl_plane_trans_wm(old_pipe_wm, plane->id),
6026 skl_plane_trans_wm(new_pipe_wm, plane->id));
6027 }
6028
6029 /*
6030 * To make sure the cursor watermark registers are always consistent
6031 * with our computed state the following scenario needs special
6032 * treatment:
6033 *
6034 * 1. enable cursor
6035 * 2. move cursor entirely offscreen
6036 * 3. disable cursor
6037 *
6038 * Step 2. does call .disable_plane() but does not zero the watermarks
6039 * (since we consider an offscreen cursor still active for the purposes
6040 * of watermarks). Step 3. would not normally call .disable_plane()
6041 * because the actual plane visibility isn't changing, and we don't
6042 * deallocate the cursor ddb until the pipe gets disabled. So we must
6043 * force step 3. to call .disable_plane() to update the watermark
6044 * registers properly.
6045 *
6046 * Other planes do not suffer from this issues as their watermarks are
6047 * calculated based on the actual plane visibility. The only time this
6048 * can trigger for the other planes is during the initial readout as the
6049 * default value of the watermarks registers is not zero.
6050 */
skl_wm_add_affected_planes(struct intel_atomic_state * state,struct intel_crtc * crtc)6051 static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
6052 struct intel_crtc *crtc)
6053 {
6054 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6055 const struct intel_crtc_state *old_crtc_state =
6056 intel_atomic_get_old_crtc_state(state, crtc);
6057 struct intel_crtc_state *new_crtc_state =
6058 intel_atomic_get_new_crtc_state(state, crtc);
6059 struct intel_plane *plane;
6060
6061 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6062 struct intel_plane_state *plane_state;
6063 enum plane_id plane_id = plane->id;
6064
6065 /*
6066 * Force a full wm update for every plane on modeset.
6067 * Required because the reset value of the wm registers
6068 * is non-zero, whereas we want all disabled planes to
6069 * have zero watermarks. So if we turn off the relevant
6070 * power well the hardware state will go out of sync
6071 * with the software state.
6072 */
6073 if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) &&
6074 skl_plane_selected_wm_equals(plane,
6075 &old_crtc_state->wm.skl.optimal,
6076 &new_crtc_state->wm.skl.optimal))
6077 continue;
6078
6079 plane_state = intel_atomic_get_plane_state(state, plane);
6080 if (IS_ERR(plane_state))
6081 return PTR_ERR(plane_state);
6082
6083 new_crtc_state->update_planes |= BIT(plane_id);
6084 }
6085
6086 return 0;
6087 }
6088
6089 static int
skl_compute_wm(struct intel_atomic_state * state)6090 skl_compute_wm(struct intel_atomic_state *state)
6091 {
6092 struct intel_crtc *crtc;
6093 struct intel_crtc_state *new_crtc_state;
6094 int ret, i;
6095
6096 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6097 ret = skl_build_pipe_wm(state, crtc);
6098 if (ret)
6099 return ret;
6100 }
6101
6102 ret = skl_compute_ddb(state);
6103 if (ret)
6104 return ret;
6105
6106 ret = intel_compute_sagv_mask(state);
6107 if (ret)
6108 return ret;
6109
6110 /*
6111 * skl_compute_ddb() will have adjusted the final watermarks
6112 * based on how much ddb is available. Now we can actually
6113 * check if the final watermarks changed.
6114 */
6115 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6116 ret = skl_wm_add_affected_planes(state, crtc);
6117 if (ret)
6118 return ret;
6119 }
6120
6121 skl_print_wm_changes(state);
6122
6123 return 0;
6124 }
6125
ilk_compute_wm_config(struct drm_i915_private * dev_priv,struct intel_wm_config * config)6126 static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
6127 struct intel_wm_config *config)
6128 {
6129 struct intel_crtc *crtc;
6130
6131 /* Compute the currently _active_ config */
6132 for_each_intel_crtc(&dev_priv->drm, crtc) {
6133 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
6134
6135 if (!wm->pipe_enabled)
6136 continue;
6137
6138 config->sprites_enabled |= wm->sprites_enabled;
6139 config->sprites_scaled |= wm->sprites_scaled;
6140 config->num_pipes_active++;
6141 }
6142 }
6143
ilk_program_watermarks(struct drm_i915_private * dev_priv)6144 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
6145 {
6146 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
6147 struct ilk_wm_maximums max;
6148 struct intel_wm_config config = {};
6149 struct ilk_wm_values results = {};
6150 enum intel_ddb_partitioning partitioning;
6151
6152 ilk_compute_wm_config(dev_priv, &config);
6153
6154 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
6155 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
6156
6157 /* 5/6 split only in single pipe config on IVB+ */
6158 if (DISPLAY_VER(dev_priv) >= 7 &&
6159 config.num_pipes_active == 1 && config.sprites_enabled) {
6160 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
6161 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
6162
6163 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
6164 } else {
6165 best_lp_wm = &lp_wm_1_2;
6166 }
6167
6168 partitioning = (best_lp_wm == &lp_wm_1_2) ?
6169 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
6170
6171 ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
6172
6173 ilk_write_wm_values(dev_priv, &results);
6174 }
6175
ilk_initial_watermarks(struct intel_atomic_state * state,struct intel_crtc * crtc)6176 static void ilk_initial_watermarks(struct intel_atomic_state *state,
6177 struct intel_crtc *crtc)
6178 {
6179 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6180 const struct intel_crtc_state *crtc_state =
6181 intel_atomic_get_new_crtc_state(state, crtc);
6182
6183 mutex_lock(&dev_priv->wm.wm_mutex);
6184 crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
6185 ilk_program_watermarks(dev_priv);
6186 mutex_unlock(&dev_priv->wm.wm_mutex);
6187 }
6188
ilk_optimize_watermarks(struct intel_atomic_state * state,struct intel_crtc * crtc)6189 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
6190 struct intel_crtc *crtc)
6191 {
6192 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6193 const struct intel_crtc_state *crtc_state =
6194 intel_atomic_get_new_crtc_state(state, crtc);
6195
6196 if (!crtc_state->wm.need_postvbl_update)
6197 return;
6198
6199 mutex_lock(&dev_priv->wm.wm_mutex);
6200 crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
6201 ilk_program_watermarks(dev_priv);
6202 mutex_unlock(&dev_priv->wm.wm_mutex);
6203 }
6204
skl_wm_level_from_reg_val(u32 val,struct skl_wm_level * level)6205 static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
6206 {
6207 level->enable = val & PLANE_WM_EN;
6208 level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
6209 level->blocks = val & PLANE_WM_BLOCKS_MASK;
6210 level->lines = (val >> PLANE_WM_LINES_SHIFT) &
6211 PLANE_WM_LINES_MASK;
6212 }
6213
skl_pipe_wm_get_hw_state(struct intel_crtc * crtc,struct skl_pipe_wm * out)6214 void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
6215 struct skl_pipe_wm *out)
6216 {
6217 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6218 enum pipe pipe = crtc->pipe;
6219 int level, max_level;
6220 enum plane_id plane_id;
6221 u32 val;
6222
6223 max_level = ilk_wm_max_level(dev_priv);
6224
6225 for_each_plane_id_on_crtc(crtc, plane_id) {
6226 struct skl_plane_wm *wm = &out->planes[plane_id];
6227
6228 for (level = 0; level <= max_level; level++) {
6229 if (plane_id != PLANE_CURSOR)
6230 val = intel_uncore_read(&dev_priv->uncore, PLANE_WM(pipe, plane_id, level));
6231 else
6232 val = intel_uncore_read(&dev_priv->uncore, CUR_WM(pipe, level));
6233
6234 skl_wm_level_from_reg_val(val, &wm->wm[level]);
6235 }
6236
6237 if (plane_id != PLANE_CURSOR)
6238 val = intel_uncore_read(&dev_priv->uncore, PLANE_WM_TRANS(pipe, plane_id));
6239 else
6240 val = intel_uncore_read(&dev_priv->uncore, CUR_WM_TRANS(pipe));
6241
6242 skl_wm_level_from_reg_val(val, &wm->trans_wm);
6243
6244 if (DISPLAY_VER(dev_priv) >= 12) {
6245 wm->sagv.wm0 = wm->wm[0];
6246 wm->sagv.trans_wm = wm->trans_wm;
6247 }
6248 }
6249 }
6250
skl_wm_get_hw_state(struct drm_i915_private * dev_priv)6251 void skl_wm_get_hw_state(struct drm_i915_private *dev_priv)
6252 {
6253 struct intel_dbuf_state *dbuf_state =
6254 to_intel_dbuf_state(dev_priv->dbuf.obj.state);
6255 struct intel_crtc *crtc;
6256
6257 for_each_intel_crtc(&dev_priv->drm, crtc) {
6258 struct intel_crtc_state *crtc_state =
6259 to_intel_crtc_state(crtc->base.state);
6260 enum pipe pipe = crtc->pipe;
6261 enum plane_id plane_id;
6262
6263 skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
6264 crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal;
6265
6266 memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe]));
6267
6268 for_each_plane_id_on_crtc(crtc, plane_id) {
6269 struct skl_ddb_entry *ddb_y =
6270 &crtc_state->wm.skl.plane_ddb_y[plane_id];
6271 struct skl_ddb_entry *ddb_uv =
6272 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
6273
6274 skl_ddb_get_hw_plane_state(dev_priv, crtc->pipe,
6275 plane_id, ddb_y, ddb_uv);
6276
6277 skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_y);
6278 skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_uv);
6279 }
6280
6281 dbuf_state->slices[pipe] =
6282 skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes);
6283
6284 dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state);
6285
6286 crtc_state->wm.skl.ddb = dbuf_state->ddb[pipe];
6287
6288 drm_dbg_kms(&dev_priv->drm,
6289 "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x\n",
6290 crtc->base.base.id, crtc->base.name,
6291 dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start,
6292 dbuf_state->ddb[pipe].end, dbuf_state->active_pipes);
6293 }
6294
6295 dbuf_state->enabled_slices = dev_priv->dbuf.enabled_slices;
6296 }
6297
ilk_pipe_wm_get_hw_state(struct intel_crtc * crtc)6298 static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
6299 {
6300 struct drm_device *dev = crtc->base.dev;
6301 struct drm_i915_private *dev_priv = to_i915(dev);
6302 struct ilk_wm_values *hw = &dev_priv->wm.hw;
6303 struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
6304 struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
6305 enum pipe pipe = crtc->pipe;
6306
6307 hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe));
6308
6309 memset(active, 0, sizeof(*active));
6310
6311 active->pipe_enabled = crtc->active;
6312
6313 if (active->pipe_enabled) {
6314 u32 tmp = hw->wm_pipe[pipe];
6315
6316 /*
6317 * For active pipes LP0 watermark is marked as
6318 * enabled, and LP1+ watermaks as disabled since
6319 * we can't really reverse compute them in case
6320 * multiple pipes are active.
6321 */
6322 active->wm[0].enable = true;
6323 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
6324 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
6325 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
6326 } else {
6327 int level, max_level = ilk_wm_max_level(dev_priv);
6328
6329 /*
6330 * For inactive pipes, all watermark levels
6331 * should be marked as enabled but zeroed,
6332 * which is what we'd compute them to.
6333 */
6334 for (level = 0; level <= max_level; level++)
6335 active->wm[level].enable = true;
6336 }
6337
6338 crtc->wm.active.ilk = *active;
6339 }
6340
6341 #define _FW_WM(value, plane) \
6342 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
6343 #define _FW_WM_VLV(value, plane) \
6344 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
6345
g4x_read_wm_values(struct drm_i915_private * dev_priv,struct g4x_wm_values * wm)6346 static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
6347 struct g4x_wm_values *wm)
6348 {
6349 u32 tmp;
6350
6351 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
6352 wm->sr.plane = _FW_WM(tmp, SR);
6353 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6354 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
6355 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
6356
6357 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
6358 wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
6359 wm->sr.fbc = _FW_WM(tmp, FBC_SR);
6360 wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
6361 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
6362 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6363 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
6364
6365 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
6366 wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
6367 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6368 wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
6369 wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
6370 }
6371
vlv_read_wm_values(struct drm_i915_private * dev_priv,struct vlv_wm_values * wm)6372 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
6373 struct vlv_wm_values *wm)
6374 {
6375 enum pipe pipe;
6376 u32 tmp;
6377
6378 for_each_pipe(dev_priv, pipe) {
6379 tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe));
6380
6381 wm->ddl[pipe].plane[PLANE_PRIMARY] =
6382 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6383 wm->ddl[pipe].plane[PLANE_CURSOR] =
6384 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6385 wm->ddl[pipe].plane[PLANE_SPRITE0] =
6386 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6387 wm->ddl[pipe].plane[PLANE_SPRITE1] =
6388 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6389 }
6390
6391 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
6392 wm->sr.plane = _FW_WM(tmp, SR);
6393 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6394 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
6395 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
6396
6397 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
6398 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
6399 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6400 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
6401
6402 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
6403 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6404
6405 if (IS_CHERRYVIEW(dev_priv)) {
6406 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV);
6407 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6408 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6409
6410 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV);
6411 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
6412 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
6413
6414 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV);
6415 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
6416 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
6417
6418 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
6419 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6420 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
6421 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
6422 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
6423 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6424 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6425 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6426 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6427 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6428 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6429 } else {
6430 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7);
6431 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6432 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6433
6434 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
6435 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6436 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6437 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6438 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6439 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6440 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6441 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6442 }
6443 }
6444
6445 #undef _FW_WM
6446 #undef _FW_WM_VLV
6447
g4x_wm_get_hw_state(struct drm_i915_private * dev_priv)6448 void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
6449 {
6450 struct g4x_wm_values *wm = &dev_priv->wm.g4x;
6451 struct intel_crtc *crtc;
6452
6453 g4x_read_wm_values(dev_priv, wm);
6454
6455 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
6456
6457 for_each_intel_crtc(&dev_priv->drm, crtc) {
6458 struct intel_crtc_state *crtc_state =
6459 to_intel_crtc_state(crtc->base.state);
6460 struct g4x_wm_state *active = &crtc->wm.active.g4x;
6461 struct g4x_pipe_wm *raw;
6462 enum pipe pipe = crtc->pipe;
6463 enum plane_id plane_id;
6464 int level, max_level;
6465
6466 active->cxsr = wm->cxsr;
6467 active->hpll_en = wm->hpll_en;
6468 active->fbc_en = wm->fbc_en;
6469
6470 active->sr = wm->sr;
6471 active->hpll = wm->hpll;
6472
6473 for_each_plane_id_on_crtc(crtc, plane_id) {
6474 active->wm.plane[plane_id] =
6475 wm->pipe[pipe].plane[plane_id];
6476 }
6477
6478 if (wm->cxsr && wm->hpll_en)
6479 max_level = G4X_WM_LEVEL_HPLL;
6480 else if (wm->cxsr)
6481 max_level = G4X_WM_LEVEL_SR;
6482 else
6483 max_level = G4X_WM_LEVEL_NORMAL;
6484
6485 level = G4X_WM_LEVEL_NORMAL;
6486 raw = &crtc_state->wm.g4x.raw[level];
6487 for_each_plane_id_on_crtc(crtc, plane_id)
6488 raw->plane[plane_id] = active->wm.plane[plane_id];
6489
6490 if (++level > max_level)
6491 goto out;
6492
6493 raw = &crtc_state->wm.g4x.raw[level];
6494 raw->plane[PLANE_PRIMARY] = active->sr.plane;
6495 raw->plane[PLANE_CURSOR] = active->sr.cursor;
6496 raw->plane[PLANE_SPRITE0] = 0;
6497 raw->fbc = active->sr.fbc;
6498
6499 if (++level > max_level)
6500 goto out;
6501
6502 raw = &crtc_state->wm.g4x.raw[level];
6503 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
6504 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
6505 raw->plane[PLANE_SPRITE0] = 0;
6506 raw->fbc = active->hpll.fbc;
6507
6508 out:
6509 for_each_plane_id_on_crtc(crtc, plane_id)
6510 g4x_raw_plane_wm_set(crtc_state, level,
6511 plane_id, USHRT_MAX);
6512 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
6513
6514 crtc_state->wm.g4x.optimal = *active;
6515 crtc_state->wm.g4x.intermediate = *active;
6516
6517 drm_dbg_kms(&dev_priv->drm,
6518 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
6519 pipe_name(pipe),
6520 wm->pipe[pipe].plane[PLANE_PRIMARY],
6521 wm->pipe[pipe].plane[PLANE_CURSOR],
6522 wm->pipe[pipe].plane[PLANE_SPRITE0]);
6523 }
6524
6525 drm_dbg_kms(&dev_priv->drm,
6526 "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
6527 wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
6528 drm_dbg_kms(&dev_priv->drm,
6529 "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
6530 wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
6531 drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
6532 yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en));
6533 }
6534
g4x_wm_sanitize(struct drm_i915_private * dev_priv)6535 void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
6536 {
6537 struct intel_plane *plane;
6538 struct intel_crtc *crtc;
6539
6540 mutex_lock(&dev_priv->wm.wm_mutex);
6541
6542 for_each_intel_plane(&dev_priv->drm, plane) {
6543 struct intel_crtc *crtc =
6544 intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6545 struct intel_crtc_state *crtc_state =
6546 to_intel_crtc_state(crtc->base.state);
6547 struct intel_plane_state *plane_state =
6548 to_intel_plane_state(plane->base.state);
6549 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
6550 enum plane_id plane_id = plane->id;
6551 int level;
6552
6553 if (plane_state->uapi.visible)
6554 continue;
6555
6556 for (level = 0; level < 3; level++) {
6557 struct g4x_pipe_wm *raw =
6558 &crtc_state->wm.g4x.raw[level];
6559
6560 raw->plane[plane_id] = 0;
6561 wm_state->wm.plane[plane_id] = 0;
6562 }
6563
6564 if (plane_id == PLANE_PRIMARY) {
6565 for (level = 0; level < 3; level++) {
6566 struct g4x_pipe_wm *raw =
6567 &crtc_state->wm.g4x.raw[level];
6568 raw->fbc = 0;
6569 }
6570
6571 wm_state->sr.fbc = 0;
6572 wm_state->hpll.fbc = 0;
6573 wm_state->fbc_en = false;
6574 }
6575 }
6576
6577 for_each_intel_crtc(&dev_priv->drm, crtc) {
6578 struct intel_crtc_state *crtc_state =
6579 to_intel_crtc_state(crtc->base.state);
6580
6581 crtc_state->wm.g4x.intermediate =
6582 crtc_state->wm.g4x.optimal;
6583 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
6584 }
6585
6586 g4x_program_watermarks(dev_priv);
6587
6588 mutex_unlock(&dev_priv->wm.wm_mutex);
6589 }
6590
vlv_wm_get_hw_state(struct drm_i915_private * dev_priv)6591 void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
6592 {
6593 struct vlv_wm_values *wm = &dev_priv->wm.vlv;
6594 struct intel_crtc *crtc;
6595 u32 val;
6596
6597 vlv_read_wm_values(dev_priv, wm);
6598
6599 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
6600 wm->level = VLV_WM_LEVEL_PM2;
6601
6602 if (IS_CHERRYVIEW(dev_priv)) {
6603 vlv_punit_get(dev_priv);
6604
6605 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
6606 if (val & DSP_MAXFIFO_PM5_ENABLE)
6607 wm->level = VLV_WM_LEVEL_PM5;
6608
6609 /*
6610 * If DDR DVFS is disabled in the BIOS, Punit
6611 * will never ack the request. So if that happens
6612 * assume we don't have to enable/disable DDR DVFS
6613 * dynamically. To test that just set the REQ_ACK
6614 * bit to poke the Punit, but don't change the
6615 * HIGH/LOW bits so that we don't actually change
6616 * the current state.
6617 */
6618 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6619 val |= FORCE_DDR_FREQ_REQ_ACK;
6620 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
6621
6622 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
6623 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
6624 drm_dbg_kms(&dev_priv->drm,
6625 "Punit not acking DDR DVFS request, "
6626 "assuming DDR DVFS is disabled\n");
6627 dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
6628 } else {
6629 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6630 if ((val & FORCE_DDR_HIGH_FREQ) == 0)
6631 wm->level = VLV_WM_LEVEL_DDR_DVFS;
6632 }
6633
6634 vlv_punit_put(dev_priv);
6635 }
6636
6637 for_each_intel_crtc(&dev_priv->drm, crtc) {
6638 struct intel_crtc_state *crtc_state =
6639 to_intel_crtc_state(crtc->base.state);
6640 struct vlv_wm_state *active = &crtc->wm.active.vlv;
6641 const struct vlv_fifo_state *fifo_state =
6642 &crtc_state->wm.vlv.fifo_state;
6643 enum pipe pipe = crtc->pipe;
6644 enum plane_id plane_id;
6645 int level;
6646
6647 vlv_get_fifo_size(crtc_state);
6648
6649 active->num_levels = wm->level + 1;
6650 active->cxsr = wm->cxsr;
6651
6652 for (level = 0; level < active->num_levels; level++) {
6653 struct g4x_pipe_wm *raw =
6654 &crtc_state->wm.vlv.raw[level];
6655
6656 active->sr[level].plane = wm->sr.plane;
6657 active->sr[level].cursor = wm->sr.cursor;
6658
6659 for_each_plane_id_on_crtc(crtc, plane_id) {
6660 active->wm[level].plane[plane_id] =
6661 wm->pipe[pipe].plane[plane_id];
6662
6663 raw->plane[plane_id] =
6664 vlv_invert_wm_value(active->wm[level].plane[plane_id],
6665 fifo_state->plane[plane_id]);
6666 }
6667 }
6668
6669 for_each_plane_id_on_crtc(crtc, plane_id)
6670 vlv_raw_plane_wm_set(crtc_state, level,
6671 plane_id, USHRT_MAX);
6672 vlv_invalidate_wms(crtc, active, level);
6673
6674 crtc_state->wm.vlv.optimal = *active;
6675 crtc_state->wm.vlv.intermediate = *active;
6676
6677 drm_dbg_kms(&dev_priv->drm,
6678 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
6679 pipe_name(pipe),
6680 wm->pipe[pipe].plane[PLANE_PRIMARY],
6681 wm->pipe[pipe].plane[PLANE_CURSOR],
6682 wm->pipe[pipe].plane[PLANE_SPRITE0],
6683 wm->pipe[pipe].plane[PLANE_SPRITE1]);
6684 }
6685
6686 drm_dbg_kms(&dev_priv->drm,
6687 "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
6688 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
6689 }
6690
vlv_wm_sanitize(struct drm_i915_private * dev_priv)6691 void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
6692 {
6693 struct intel_plane *plane;
6694 struct intel_crtc *crtc;
6695
6696 mutex_lock(&dev_priv->wm.wm_mutex);
6697
6698 for_each_intel_plane(&dev_priv->drm, plane) {
6699 struct intel_crtc *crtc =
6700 intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6701 struct intel_crtc_state *crtc_state =
6702 to_intel_crtc_state(crtc->base.state);
6703 struct intel_plane_state *plane_state =
6704 to_intel_plane_state(plane->base.state);
6705 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
6706 const struct vlv_fifo_state *fifo_state =
6707 &crtc_state->wm.vlv.fifo_state;
6708 enum plane_id plane_id = plane->id;
6709 int level;
6710
6711 if (plane_state->uapi.visible)
6712 continue;
6713
6714 for (level = 0; level < wm_state->num_levels; level++) {
6715 struct g4x_pipe_wm *raw =
6716 &crtc_state->wm.vlv.raw[level];
6717
6718 raw->plane[plane_id] = 0;
6719
6720 wm_state->wm[level].plane[plane_id] =
6721 vlv_invert_wm_value(raw->plane[plane_id],
6722 fifo_state->plane[plane_id]);
6723 }
6724 }
6725
6726 for_each_intel_crtc(&dev_priv->drm, crtc) {
6727 struct intel_crtc_state *crtc_state =
6728 to_intel_crtc_state(crtc->base.state);
6729
6730 crtc_state->wm.vlv.intermediate =
6731 crtc_state->wm.vlv.optimal;
6732 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
6733 }
6734
6735 vlv_program_watermarks(dev_priv);
6736
6737 mutex_unlock(&dev_priv->wm.wm_mutex);
6738 }
6739
6740 /*
6741 * FIXME should probably kill this and improve
6742 * the real watermark readout/sanitation instead
6743 */
ilk_init_lp_watermarks(struct drm_i915_private * dev_priv)6744 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
6745 {
6746 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK) & ~WM1_LP_SR_EN);
6747 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK) & ~WM1_LP_SR_EN);
6748 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK) & ~WM1_LP_SR_EN);
6749
6750 /*
6751 * Don't touch WM1S_LP_EN here.
6752 * Doing so could cause underruns.
6753 */
6754 }
6755
ilk_wm_get_hw_state(struct drm_i915_private * dev_priv)6756 void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
6757 {
6758 struct ilk_wm_values *hw = &dev_priv->wm.hw;
6759 struct intel_crtc *crtc;
6760
6761 ilk_init_lp_watermarks(dev_priv);
6762
6763 for_each_intel_crtc(&dev_priv->drm, crtc)
6764 ilk_pipe_wm_get_hw_state(crtc);
6765
6766 hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK);
6767 hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK);
6768 hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK);
6769
6770 hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK);
6771 if (DISPLAY_VER(dev_priv) >= 7) {
6772 hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB);
6773 hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB);
6774 }
6775
6776 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
6777 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
6778 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6779 else if (IS_IVYBRIDGE(dev_priv))
6780 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
6781 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6782
6783 hw->enable_fbc_wm =
6784 !(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
6785 }
6786
6787 /**
6788 * intel_update_watermarks - update FIFO watermark values based on current modes
6789 * @crtc: the #intel_crtc on which to compute the WM
6790 *
6791 * Calculate watermark values for the various WM regs based on current mode
6792 * and plane configuration.
6793 *
6794 * There are several cases to deal with here:
6795 * - normal (i.e. non-self-refresh)
6796 * - self-refresh (SR) mode
6797 * - lines are large relative to FIFO size (buffer can hold up to 2)
6798 * - lines are small relative to FIFO size (buffer can hold more than 2
6799 * lines), so need to account for TLB latency
6800 *
6801 * The normal calculation is:
6802 * watermark = dotclock * bytes per pixel * latency
6803 * where latency is platform & configuration dependent (we assume pessimal
6804 * values here).
6805 *
6806 * The SR calculation is:
6807 * watermark = (trunc(latency/line time)+1) * surface width *
6808 * bytes per pixel
6809 * where
6810 * line time = htotal / dotclock
6811 * surface width = hdisplay for normal plane and 64 for cursor
6812 * and latency is assumed to be high, as above.
6813 *
6814 * The final value programmed to the register should always be rounded up,
6815 * and include an extra 2 entries to account for clock crossings.
6816 *
6817 * We don't use the sprite, so we can ignore that. And on Crestline we have
6818 * to set the non-SR watermarks to 8.
6819 */
intel_update_watermarks(struct intel_crtc * crtc)6820 void intel_update_watermarks(struct intel_crtc *crtc)
6821 {
6822 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6823
6824 if (dev_priv->display.update_wm)
6825 dev_priv->display.update_wm(crtc);
6826 }
6827
intel_enable_ipc(struct drm_i915_private * dev_priv)6828 void intel_enable_ipc(struct drm_i915_private *dev_priv)
6829 {
6830 u32 val;
6831
6832 if (!HAS_IPC(dev_priv))
6833 return;
6834
6835 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2);
6836
6837 if (dev_priv->ipc_enabled)
6838 val |= DISP_IPC_ENABLE;
6839 else
6840 val &= ~DISP_IPC_ENABLE;
6841
6842 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val);
6843 }
6844
intel_can_enable_ipc(struct drm_i915_private * dev_priv)6845 static bool intel_can_enable_ipc(struct drm_i915_private *dev_priv)
6846 {
6847 /* Display WA #0477 WaDisableIPC: skl */
6848 if (IS_SKYLAKE(dev_priv))
6849 return false;
6850
6851 /* Display WA #1141: SKL:all KBL:all CFL */
6852 if (IS_KABYLAKE(dev_priv) ||
6853 IS_COFFEELAKE(dev_priv) ||
6854 IS_COMETLAKE(dev_priv))
6855 return dev_priv->dram_info.symmetric_memory;
6856
6857 return true;
6858 }
6859
intel_init_ipc(struct drm_i915_private * dev_priv)6860 void intel_init_ipc(struct drm_i915_private *dev_priv)
6861 {
6862 if (!HAS_IPC(dev_priv))
6863 return;
6864
6865 dev_priv->ipc_enabled = intel_can_enable_ipc(dev_priv);
6866
6867 intel_enable_ipc(dev_priv);
6868 }
6869
ibx_init_clock_gating(struct drm_i915_private * dev_priv)6870 static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
6871 {
6872 /*
6873 * On Ibex Peak and Cougar Point, we need to disable clock
6874 * gating for the panel power sequencer or it will fail to
6875 * start up when no ports are active.
6876 */
6877 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
6878 }
6879
g4x_disable_trickle_feed(struct drm_i915_private * dev_priv)6880 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
6881 {
6882 enum pipe pipe;
6883
6884 for_each_pipe(dev_priv, pipe) {
6885 intel_uncore_write(&dev_priv->uncore, DSPCNTR(pipe),
6886 intel_uncore_read(&dev_priv->uncore, DSPCNTR(pipe)) |
6887 DISPPLANE_TRICKLE_FEED_DISABLE);
6888
6889 intel_uncore_write(&dev_priv->uncore, DSPSURF(pipe), intel_uncore_read(&dev_priv->uncore, DSPSURF(pipe)));
6890 intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe));
6891 }
6892 }
6893
ilk_init_clock_gating(struct drm_i915_private * dev_priv)6894 static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
6895 {
6896 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6897
6898 /*
6899 * Required for FBC
6900 * WaFbcDisableDpfcClockGating:ilk
6901 */
6902 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
6903 ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
6904 ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
6905
6906 intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS0,
6907 MARIUNIT_CLOCK_GATE_DISABLE |
6908 SVSMUNIT_CLOCK_GATE_DISABLE);
6909 intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS1,
6910 VFMUNIT_CLOCK_GATE_DISABLE);
6911
6912 /*
6913 * According to the spec the following bits should be set in
6914 * order to enable memory self-refresh
6915 * The bit 22/21 of 0x42004
6916 * The bit 5 of 0x42020
6917 * The bit 15 of 0x45000
6918 */
6919 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
6920 (intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
6921 ILK_DPARB_GATE | ILK_VSDPFD_FULL));
6922 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
6923 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL,
6924 (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
6925 DISP_FBC_WM_DIS));
6926
6927 /*
6928 * Based on the document from hardware guys the following bits
6929 * should be set unconditionally in order to enable FBC.
6930 * The bit 22 of 0x42000
6931 * The bit 22 of 0x42004
6932 * The bit 7,8,9 of 0x42020.
6933 */
6934 if (IS_IRONLAKE_M(dev_priv)) {
6935 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
6936 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
6937 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
6938 ILK_FBCQ_DIS);
6939 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
6940 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
6941 ILK_DPARB_GATE);
6942 }
6943
6944 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
6945
6946 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
6947 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
6948 ILK_ELPIN_409_SELECT);
6949
6950 g4x_disable_trickle_feed(dev_priv);
6951
6952 ibx_init_clock_gating(dev_priv);
6953 }
6954
cpt_init_clock_gating(struct drm_i915_private * dev_priv)6955 static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
6956 {
6957 enum pipe pipe;
6958 u32 val;
6959
6960 /*
6961 * On Ibex Peak and Cougar Point, we need to disable clock
6962 * gating for the panel power sequencer or it will fail to
6963 * start up when no ports are active.
6964 */
6965 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
6966 PCH_DPLUNIT_CLOCK_GATE_DISABLE |
6967 PCH_CPUNIT_CLOCK_GATE_DISABLE);
6968 intel_uncore_write(&dev_priv->uncore, SOUTH_CHICKEN2, intel_uncore_read(&dev_priv->uncore, SOUTH_CHICKEN2) |
6969 DPLS_EDP_PPS_FIX_DIS);
6970 /* The below fixes the weird display corruption, a few pixels shifted
6971 * downward, on (only) LVDS of some HP laptops with IVY.
6972 */
6973 for_each_pipe(dev_priv, pipe) {
6974 val = intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN2(pipe));
6975 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
6976 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
6977 if (dev_priv->vbt.fdi_rx_polarity_inverted)
6978 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
6979 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
6980 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
6981 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN2(pipe), val);
6982 }
6983 /* WADP0ClockGatingDisable */
6984 for_each_pipe(dev_priv, pipe) {
6985 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(pipe),
6986 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
6987 }
6988 }
6989
gen6_check_mch_setup(struct drm_i915_private * dev_priv)6990 static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
6991 {
6992 u32 tmp;
6993
6994 tmp = intel_uncore_read(&dev_priv->uncore, MCH_SSKPD);
6995 if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
6996 drm_dbg_kms(&dev_priv->drm,
6997 "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
6998 tmp);
6999 }
7000
gen6_init_clock_gating(struct drm_i915_private * dev_priv)7001 static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
7002 {
7003 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
7004
7005 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
7006
7007 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7008 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7009 ILK_ELPIN_409_SELECT);
7010
7011 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
7012 intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7013 GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
7014 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7015
7016 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
7017 * gating disable must be set. Failure to set it results in
7018 * flickering pixels due to Z write ordering failures after
7019 * some amount of runtime in the Mesa "fire" demo, and Unigine
7020 * Sanctuary and Tropics, and apparently anything else with
7021 * alpha test or pixel discard.
7022 *
7023 * According to the spec, bit 11 (RCCUNIT) must also be set,
7024 * but we didn't debug actual testcases to find it out.
7025 *
7026 * WaDisableRCCUnitClockGating:snb
7027 * WaDisableRCPBUnitClockGating:snb
7028 */
7029 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7030 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
7031 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
7032
7033 /*
7034 * According to the spec the following bits should be
7035 * set in order to enable memory self-refresh and fbc:
7036 * The bit21 and bit22 of 0x42000
7037 * The bit21 and bit22 of 0x42004
7038 * The bit5 and bit7 of 0x42020
7039 * The bit14 of 0x70180
7040 * The bit14 of 0x71180
7041 *
7042 * WaFbcAsynchFlipDisableFbcQueue:snb
7043 */
7044 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7045 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7046 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
7047 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7048 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7049 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
7050 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D,
7051 intel_uncore_read(&dev_priv->uncore, ILK_DSPCLK_GATE_D) |
7052 ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
7053 ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
7054
7055 g4x_disable_trickle_feed(dev_priv);
7056
7057 cpt_init_clock_gating(dev_priv);
7058
7059 gen6_check_mch_setup(dev_priv);
7060 }
7061
lpt_init_clock_gating(struct drm_i915_private * dev_priv)7062 static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
7063 {
7064 /*
7065 * TODO: this bit should only be enabled when really needed, then
7066 * disabled when not needed anymore in order to save power.
7067 */
7068 if (HAS_PCH_LPT_LP(dev_priv))
7069 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D,
7070 intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) |
7071 PCH_LP_PARTITION_LEVEL_DISABLE);
7072
7073 /* WADPOClockGatingDisable:hsw */
7074 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A),
7075 intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A)) |
7076 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
7077 }
7078
lpt_suspend_hw(struct drm_i915_private * dev_priv)7079 static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
7080 {
7081 if (HAS_PCH_LPT_LP(dev_priv)) {
7082 u32 val = intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D);
7083
7084 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
7085 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, val);
7086 }
7087 }
7088
gen8_set_l3sqc_credits(struct drm_i915_private * dev_priv,int general_prio_credits,int high_prio_credits)7089 static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
7090 int general_prio_credits,
7091 int high_prio_credits)
7092 {
7093 u32 misccpctl;
7094 u32 val;
7095
7096 /* WaTempDisableDOPClkGating:bdw */
7097 misccpctl = intel_uncore_read(&dev_priv->uncore, GEN7_MISCCPCTL);
7098 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
7099
7100 val = intel_uncore_read(&dev_priv->uncore, GEN8_L3SQCREG1);
7101 val &= ~L3_PRIO_CREDITS_MASK;
7102 val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
7103 val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
7104 intel_uncore_write(&dev_priv->uncore, GEN8_L3SQCREG1, val);
7105
7106 /*
7107 * Wait at least 100 clocks before re-enabling clock gating.
7108 * See the definition of L3SQCREG1 in BSpec.
7109 */
7110 intel_uncore_posting_read(&dev_priv->uncore, GEN8_L3SQCREG1);
7111 udelay(1);
7112 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl);
7113 }
7114
icl_init_clock_gating(struct drm_i915_private * dev_priv)7115 static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
7116 {
7117 /* Wa_1409120013:icl,ehl */
7118 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN,
7119 ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7120
7121 /* This is not an Wa. Enable to reduce Sampler power */
7122 intel_uncore_write(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN,
7123 intel_uncore_read(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE);
7124
7125 /*Wa_14010594013:icl, ehl */
7126 intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
7127 0, CNL_DELAY_PMRSP);
7128 }
7129
gen12lp_init_clock_gating(struct drm_i915_private * dev_priv)7130 static void gen12lp_init_clock_gating(struct drm_i915_private *dev_priv)
7131 {
7132 /* Wa_1409120013:tgl,rkl,adl_s,dg1 */
7133 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN,
7134 ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7135
7136 /* Wa_1409825376:tgl (pre-prod)*/
7137 if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B1))
7138 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) |
7139 TGL_VRH_GATING_DIS);
7140
7141 /* Wa_14011059788:tgl,rkl,adl_s,dg1 */
7142 intel_uncore_rmw(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN,
7143 0, DFR_DISABLE);
7144 }
7145
dg1_init_clock_gating(struct drm_i915_private * dev_priv)7146 static void dg1_init_clock_gating(struct drm_i915_private *dev_priv)
7147 {
7148 gen12lp_init_clock_gating(dev_priv);
7149
7150 /* Wa_1409836686:dg1[a0] */
7151 if (IS_DG1_REVID(dev_priv, DG1_REVID_A0, DG1_REVID_A0))
7152 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) |
7153 DPT_GATING_DIS);
7154 }
7155
cnp_init_clock_gating(struct drm_i915_private * dev_priv)7156 static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
7157 {
7158 if (!HAS_PCH_CNP(dev_priv))
7159 return;
7160
7161 /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
7162 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) |
7163 CNP_PWM_CGE_GATING_DISABLE);
7164 }
7165
cnl_init_clock_gating(struct drm_i915_private * dev_priv)7166 static void cnl_init_clock_gating(struct drm_i915_private *dev_priv)
7167 {
7168 u32 val;
7169 cnp_init_clock_gating(dev_priv);
7170
7171 /* This is not an Wa. Enable for better image quality */
7172 intel_uncore_write(&dev_priv->uncore, _3D_CHICKEN3,
7173 _MASKED_BIT_ENABLE(_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE));
7174
7175 /* WaEnableChickenDCPR:cnl */
7176 intel_uncore_write(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
7177 intel_uncore_read(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
7178
7179 /*
7180 * WaFbcWakeMemOn:cnl
7181 * Display WA #0859: cnl
7182 */
7183 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7184 DISP_FBC_MEMORY_WAKE);
7185
7186 val = intel_uncore_read(&dev_priv->uncore, SLICE_UNIT_LEVEL_CLKGATE);
7187 /* ReadHitWriteOnlyDisable:cnl */
7188 val |= RCCUNIT_CLKGATE_DIS;
7189 intel_uncore_write(&dev_priv->uncore, SLICE_UNIT_LEVEL_CLKGATE, val);
7190
7191 /* Wa_2201832410:cnl */
7192 val = intel_uncore_read(&dev_priv->uncore, SUBSLICE_UNIT_LEVEL_CLKGATE);
7193 val |= GWUNIT_CLKGATE_DIS;
7194 intel_uncore_write(&dev_priv->uncore, SUBSLICE_UNIT_LEVEL_CLKGATE, val);
7195
7196 /* WaDisableVFclkgate:cnl */
7197 /* WaVFUnitClockGatingDisable:cnl */
7198 val = intel_uncore_read(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE);
7199 val |= VFUNIT_CLKGATE_DIS;
7200 intel_uncore_write(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE, val);
7201 }
7202
cfl_init_clock_gating(struct drm_i915_private * dev_priv)7203 static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
7204 {
7205 cnp_init_clock_gating(dev_priv);
7206 gen9_init_clock_gating(dev_priv);
7207
7208 /* WAC6entrylatency:cfl */
7209 intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7210 FBC_LLC_FULLY_OPEN);
7211
7212 /*
7213 * WaFbcTurnOffFbcWatermark:cfl
7214 * Display WA #0562: cfl
7215 */
7216 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7217 DISP_FBC_WM_DIS);
7218
7219 /*
7220 * WaFbcNukeOnHostModify:cfl
7221 * Display WA #0873: cfl
7222 */
7223 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
7224 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7225 }
7226
kbl_init_clock_gating(struct drm_i915_private * dev_priv)7227 static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
7228 {
7229 gen9_init_clock_gating(dev_priv);
7230
7231 /* WAC6entrylatency:kbl */
7232 intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7233 FBC_LLC_FULLY_OPEN);
7234
7235 /* WaDisableSDEUnitClockGating:kbl */
7236 if (IS_KBL_GT_STEP(dev_priv, 0, STEP_B0))
7237 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7238 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7239
7240 /* WaDisableGamClockGating:kbl */
7241 if (IS_KBL_GT_STEP(dev_priv, 0, STEP_B0))
7242 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7243 GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
7244
7245 /*
7246 * WaFbcTurnOffFbcWatermark:kbl
7247 * Display WA #0562: kbl
7248 */
7249 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7250 DISP_FBC_WM_DIS);
7251
7252 /*
7253 * WaFbcNukeOnHostModify:kbl
7254 * Display WA #0873: kbl
7255 */
7256 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
7257 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7258 }
7259
skl_init_clock_gating(struct drm_i915_private * dev_priv)7260 static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
7261 {
7262 gen9_init_clock_gating(dev_priv);
7263
7264 /* WaDisableDopClockGating:skl */
7265 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, intel_uncore_read(&dev_priv->uncore, GEN7_MISCCPCTL) &
7266 ~GEN7_DOP_CLOCK_GATE_ENABLE);
7267
7268 /* WAC6entrylatency:skl */
7269 intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7270 FBC_LLC_FULLY_OPEN);
7271
7272 /*
7273 * WaFbcTurnOffFbcWatermark:skl
7274 * Display WA #0562: skl
7275 */
7276 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7277 DISP_FBC_WM_DIS);
7278
7279 /*
7280 * WaFbcNukeOnHostModify:skl
7281 * Display WA #0873: skl
7282 */
7283 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
7284 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7285
7286 /*
7287 * WaFbcHighMemBwCorruptionAvoidance:skl
7288 * Display WA #0883: skl
7289 */
7290 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
7291 ILK_DPFC_DISABLE_DUMMY0);
7292 }
7293
bdw_init_clock_gating(struct drm_i915_private * dev_priv)7294 static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
7295 {
7296 enum pipe pipe;
7297
7298 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7299 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A),
7300 intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) |
7301 HSW_FBCQ_DIS);
7302
7303 /* WaSwitchSolVfFArbitrationPriority:bdw */
7304 intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7305
7306 /* WaPsrDPAMaskVBlankInSRD:bdw */
7307 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
7308 intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
7309
7310 for_each_pipe(dev_priv, pipe) {
7311 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
7312 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
7313 intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe)) |
7314 BDW_DPRS_MASK_VBLANK_SRD);
7315
7316 /* Undocumented but fixes async flip + VT-d corruption */
7317 if (intel_vtd_active())
7318 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
7319 HSW_PRI_STRETCH_MAX_MASK, HSW_PRI_STRETCH_MAX_X1);
7320 }
7321
7322 /* WaVSRefCountFullforceMissDisable:bdw */
7323 /* WaDSRefCountFullforceMissDisable:bdw */
7324 intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
7325 intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) &
7326 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7327
7328 intel_uncore_write(&dev_priv->uncore, GEN6_RC_SLEEP_PSMI_CONTROL,
7329 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7330
7331 /* WaDisableSDEUnitClockGating:bdw */
7332 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7333 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7334
7335 /* WaProgramL3SqcReg1Default:bdw */
7336 gen8_set_l3sqc_credits(dev_priv, 30, 2);
7337
7338 /* WaKVMNotificationOnConfigChange:bdw */
7339 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR2_1, intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR2_1)
7340 | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
7341
7342 lpt_init_clock_gating(dev_priv);
7343
7344 /* WaDisableDopClockGating:bdw
7345 *
7346 * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
7347 * clock gating.
7348 */
7349 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
7350 intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
7351 }
7352
hsw_init_clock_gating(struct drm_i915_private * dev_priv)7353 static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
7354 {
7355 enum pipe pipe;
7356
7357 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7358 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A),
7359 intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) |
7360 HSW_FBCQ_DIS);
7361
7362 for_each_pipe(dev_priv, pipe) {
7363 /* Undocumented but fixes async flip + VT-d corruption */
7364 if (intel_vtd_active())
7365 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
7366 HSW_PRI_STRETCH_MAX_MASK, HSW_PRI_STRETCH_MAX_X1);
7367 }
7368
7369 /* This is required by WaCatErrorRejectionIssue:hsw */
7370 intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7371 intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7372 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7373
7374 /* WaSwitchSolVfFArbitrationPriority:hsw */
7375 intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7376
7377 lpt_init_clock_gating(dev_priv);
7378 }
7379
ivb_init_clock_gating(struct drm_i915_private * dev_priv)7380 static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
7381 {
7382 u32 snpcr;
7383
7384 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
7385
7386 /* WaFbcAsynchFlipDisableFbcQueue:ivb */
7387 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7388 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7389 ILK_FBCQ_DIS);
7390
7391 /* WaDisableBackToBackFlipFix:ivb */
7392 intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
7393 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7394 CHICKEN3_DGMG_DONE_FIX_DISABLE);
7395
7396 if (IS_IVB_GT1(dev_priv))
7397 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7398 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7399 else {
7400 /* must write both registers */
7401 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7402 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7403 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2_GT2,
7404 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7405 }
7406
7407 /*
7408 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7409 * This implements the WaDisableRCZUnitClockGating:ivb workaround.
7410 */
7411 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7412 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7413
7414 /* This is required by WaCatErrorRejectionIssue:ivb */
7415 intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7416 intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7417 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7418
7419 g4x_disable_trickle_feed(dev_priv);
7420
7421 snpcr = intel_uncore_read(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR);
7422 snpcr &= ~GEN6_MBC_SNPCR_MASK;
7423 snpcr |= GEN6_MBC_SNPCR_MED;
7424 intel_uncore_write(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR, snpcr);
7425
7426 if (!HAS_PCH_NOP(dev_priv))
7427 cpt_init_clock_gating(dev_priv);
7428
7429 gen6_check_mch_setup(dev_priv);
7430 }
7431
vlv_init_clock_gating(struct drm_i915_private * dev_priv)7432 static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
7433 {
7434 /* WaDisableBackToBackFlipFix:vlv */
7435 intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
7436 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7437 CHICKEN3_DGMG_DONE_FIX_DISABLE);
7438
7439 /* WaDisableDopClockGating:vlv */
7440 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7441 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7442
7443 /* This is required by WaCatErrorRejectionIssue:vlv */
7444 intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7445 intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7446 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7447
7448 /*
7449 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7450 * This implements the WaDisableRCZUnitClockGating:vlv workaround.
7451 */
7452 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7453 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7454
7455 /* WaDisableL3Bank2xClockGate:vlv
7456 * Disabling L3 clock gating- MMIO 940c[25] = 1
7457 * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
7458 intel_uncore_write(&dev_priv->uncore, GEN7_UCGCTL4,
7459 intel_uncore_read(&dev_priv->uncore, GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
7460
7461 /*
7462 * WaDisableVLVClockGating_VBIIssue:vlv
7463 * Disable clock gating on th GCFG unit to prevent a delay
7464 * in the reporting of vblank events.
7465 */
7466 intel_uncore_write(&dev_priv->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
7467 }
7468
chv_init_clock_gating(struct drm_i915_private * dev_priv)7469 static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
7470 {
7471 /* WaVSRefCountFullforceMissDisable:chv */
7472 /* WaDSRefCountFullforceMissDisable:chv */
7473 intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
7474 intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) &
7475 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7476
7477 /* WaDisableSemaphoreAndSyncFlipWait:chv */
7478 intel_uncore_write(&dev_priv->uncore, GEN6_RC_SLEEP_PSMI_CONTROL,
7479 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7480
7481 /* WaDisableCSUnitClockGating:chv */
7482 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7483 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7484
7485 /* WaDisableSDEUnitClockGating:chv */
7486 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7487 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7488
7489 /*
7490 * WaProgramL3SqcReg1Default:chv
7491 * See gfxspecs/Related Documents/Performance Guide/
7492 * LSQC Setting Recommendations.
7493 */
7494 gen8_set_l3sqc_credits(dev_priv, 38, 2);
7495 }
7496
g4x_init_clock_gating(struct drm_i915_private * dev_priv)7497 static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
7498 {
7499 u32 dspclk_gate;
7500
7501 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, 0);
7502 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
7503 GS_UNIT_CLOCK_GATE_DISABLE |
7504 CL_UNIT_CLOCK_GATE_DISABLE);
7505 intel_uncore_write(&dev_priv->uncore, RAMCLK_GATE_D, 0);
7506 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
7507 OVRUNIT_CLOCK_GATE_DISABLE |
7508 OVCUNIT_CLOCK_GATE_DISABLE;
7509 if (IS_GM45(dev_priv))
7510 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
7511 intel_uncore_write(&dev_priv->uncore, DSPCLK_GATE_D, dspclk_gate);
7512
7513 g4x_disable_trickle_feed(dev_priv);
7514 }
7515
i965gm_init_clock_gating(struct drm_i915_private * dev_priv)7516 static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
7517 {
7518 struct intel_uncore *uncore = &dev_priv->uncore;
7519
7520 intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
7521 intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
7522 intel_uncore_write(uncore, DSPCLK_GATE_D, 0);
7523 intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
7524 intel_uncore_write16(uncore, DEUC, 0);
7525 intel_uncore_write(uncore,
7526 MI_ARB_STATE,
7527 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7528 }
7529
i965g_init_clock_gating(struct drm_i915_private * dev_priv)7530 static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
7531 {
7532 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
7533 I965_RCC_CLOCK_GATE_DISABLE |
7534 I965_RCPB_CLOCK_GATE_DISABLE |
7535 I965_ISC_CLOCK_GATE_DISABLE |
7536 I965_FBC_CLOCK_GATE_DISABLE);
7537 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, 0);
7538 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
7539 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7540 }
7541
gen3_init_clock_gating(struct drm_i915_private * dev_priv)7542 static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
7543 {
7544 u32 dstate = intel_uncore_read(&dev_priv->uncore, D_STATE);
7545
7546 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
7547 DSTATE_DOT_CLOCK_GATING;
7548 intel_uncore_write(&dev_priv->uncore, D_STATE, dstate);
7549
7550 if (IS_PINEVIEW(dev_priv))
7551 intel_uncore_write(&dev_priv->uncore, ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
7552
7553 /* IIR "flip pending" means done if this bit is set */
7554 intel_uncore_write(&dev_priv->uncore, ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
7555
7556 /* interrupts should cause a wake up from C3 */
7557 intel_uncore_write(&dev_priv->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
7558
7559 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
7560 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
7561
7562 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
7563 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7564 }
7565
i85x_init_clock_gating(struct drm_i915_private * dev_priv)7566 static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
7567 {
7568 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
7569
7570 /* interrupts should cause a wake up from C3 */
7571 intel_uncore_write(&dev_priv->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
7572 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
7573
7574 intel_uncore_write(&dev_priv->uncore, MEM_MODE,
7575 _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
7576
7577 /*
7578 * Have FBC ignore 3D activity since we use software
7579 * render tracking, and otherwise a pure 3D workload
7580 * (even if it just renders a single frame and then does
7581 * abosultely nothing) would not allow FBC to recompress
7582 * until a 2D blit occurs.
7583 */
7584 intel_uncore_write(&dev_priv->uncore, SCPD0,
7585 _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
7586 }
7587
i830_init_clock_gating(struct drm_i915_private * dev_priv)7588 static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
7589 {
7590 intel_uncore_write(&dev_priv->uncore, MEM_MODE,
7591 _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
7592 _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
7593 }
7594
intel_init_clock_gating(struct drm_i915_private * dev_priv)7595 void intel_init_clock_gating(struct drm_i915_private *dev_priv)
7596 {
7597 dev_priv->display.init_clock_gating(dev_priv);
7598 }
7599
intel_suspend_hw(struct drm_i915_private * dev_priv)7600 void intel_suspend_hw(struct drm_i915_private *dev_priv)
7601 {
7602 if (HAS_PCH_LPT(dev_priv))
7603 lpt_suspend_hw(dev_priv);
7604 }
7605
nop_init_clock_gating(struct drm_i915_private * dev_priv)7606 static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
7607 {
7608 drm_dbg_kms(&dev_priv->drm,
7609 "No clock gating settings or workarounds applied.\n");
7610 }
7611
7612 /**
7613 * intel_init_clock_gating_hooks - setup the clock gating hooks
7614 * @dev_priv: device private
7615 *
7616 * Setup the hooks that configure which clocks of a given platform can be
7617 * gated and also apply various GT and display specific workarounds for these
7618 * platforms. Note that some GT specific workarounds are applied separately
7619 * when GPU contexts or batchbuffers start their execution.
7620 */
intel_init_clock_gating_hooks(struct drm_i915_private * dev_priv)7621 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
7622 {
7623 if (IS_DG1(dev_priv))
7624 dev_priv->display.init_clock_gating = dg1_init_clock_gating;
7625 else if (IS_GEN(dev_priv, 12))
7626 dev_priv->display.init_clock_gating = gen12lp_init_clock_gating;
7627 else if (IS_GEN(dev_priv, 11))
7628 dev_priv->display.init_clock_gating = icl_init_clock_gating;
7629 else if (IS_CANNONLAKE(dev_priv))
7630 dev_priv->display.init_clock_gating = cnl_init_clock_gating;
7631 else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv))
7632 dev_priv->display.init_clock_gating = cfl_init_clock_gating;
7633 else if (IS_SKYLAKE(dev_priv))
7634 dev_priv->display.init_clock_gating = skl_init_clock_gating;
7635 else if (IS_KABYLAKE(dev_priv))
7636 dev_priv->display.init_clock_gating = kbl_init_clock_gating;
7637 else if (IS_BROXTON(dev_priv))
7638 dev_priv->display.init_clock_gating = bxt_init_clock_gating;
7639 else if (IS_GEMINILAKE(dev_priv))
7640 dev_priv->display.init_clock_gating = glk_init_clock_gating;
7641 else if (IS_BROADWELL(dev_priv))
7642 dev_priv->display.init_clock_gating = bdw_init_clock_gating;
7643 else if (IS_CHERRYVIEW(dev_priv))
7644 dev_priv->display.init_clock_gating = chv_init_clock_gating;
7645 else if (IS_HASWELL(dev_priv))
7646 dev_priv->display.init_clock_gating = hsw_init_clock_gating;
7647 else if (IS_IVYBRIDGE(dev_priv))
7648 dev_priv->display.init_clock_gating = ivb_init_clock_gating;
7649 else if (IS_VALLEYVIEW(dev_priv))
7650 dev_priv->display.init_clock_gating = vlv_init_clock_gating;
7651 else if (IS_GEN(dev_priv, 6))
7652 dev_priv->display.init_clock_gating = gen6_init_clock_gating;
7653 else if (IS_GEN(dev_priv, 5))
7654 dev_priv->display.init_clock_gating = ilk_init_clock_gating;
7655 else if (IS_G4X(dev_priv))
7656 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
7657 else if (IS_I965GM(dev_priv))
7658 dev_priv->display.init_clock_gating = i965gm_init_clock_gating;
7659 else if (IS_I965G(dev_priv))
7660 dev_priv->display.init_clock_gating = i965g_init_clock_gating;
7661 else if (IS_GEN(dev_priv, 3))
7662 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
7663 else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
7664 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
7665 else if (IS_GEN(dev_priv, 2))
7666 dev_priv->display.init_clock_gating = i830_init_clock_gating;
7667 else {
7668 MISSING_CASE(INTEL_DEVID(dev_priv));
7669 dev_priv->display.init_clock_gating = nop_init_clock_gating;
7670 }
7671 }
7672
7673 /* Set up chip specific power management-related functions */
intel_init_pm(struct drm_i915_private * dev_priv)7674 void intel_init_pm(struct drm_i915_private *dev_priv)
7675 {
7676 /* For cxsr */
7677 if (IS_PINEVIEW(dev_priv))
7678 pnv_get_mem_freq(dev_priv);
7679 else if (IS_GEN(dev_priv, 5))
7680 ilk_get_mem_freq(dev_priv);
7681
7682 if (intel_has_sagv(dev_priv))
7683 skl_setup_sagv_block_time(dev_priv);
7684
7685 /* For FIFO watermark updates */
7686 if (DISPLAY_VER(dev_priv) >= 9) {
7687 skl_setup_wm_latency(dev_priv);
7688 dev_priv->display.compute_global_watermarks = skl_compute_wm;
7689 } else if (HAS_PCH_SPLIT(dev_priv)) {
7690 ilk_setup_wm_latency(dev_priv);
7691
7692 if ((IS_DISPLAY_VER(dev_priv, 5) && dev_priv->wm.pri_latency[1] &&
7693 dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
7694 (!IS_DISPLAY_VER(dev_priv, 5) && dev_priv->wm.pri_latency[0] &&
7695 dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
7696 dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
7697 dev_priv->display.compute_intermediate_wm =
7698 ilk_compute_intermediate_wm;
7699 dev_priv->display.initial_watermarks =
7700 ilk_initial_watermarks;
7701 dev_priv->display.optimize_watermarks =
7702 ilk_optimize_watermarks;
7703 } else {
7704 drm_dbg_kms(&dev_priv->drm,
7705 "Failed to read display plane latency. "
7706 "Disable CxSR\n");
7707 }
7708 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
7709 vlv_setup_wm_latency(dev_priv);
7710 dev_priv->display.compute_pipe_wm = vlv_compute_pipe_wm;
7711 dev_priv->display.compute_intermediate_wm = vlv_compute_intermediate_wm;
7712 dev_priv->display.initial_watermarks = vlv_initial_watermarks;
7713 dev_priv->display.optimize_watermarks = vlv_optimize_watermarks;
7714 dev_priv->display.atomic_update_watermarks = vlv_atomic_update_fifo;
7715 } else if (IS_G4X(dev_priv)) {
7716 g4x_setup_wm_latency(dev_priv);
7717 dev_priv->display.compute_pipe_wm = g4x_compute_pipe_wm;
7718 dev_priv->display.compute_intermediate_wm = g4x_compute_intermediate_wm;
7719 dev_priv->display.initial_watermarks = g4x_initial_watermarks;
7720 dev_priv->display.optimize_watermarks = g4x_optimize_watermarks;
7721 } else if (IS_PINEVIEW(dev_priv)) {
7722 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
7723 dev_priv->is_ddr3,
7724 dev_priv->fsb_freq,
7725 dev_priv->mem_freq)) {
7726 drm_info(&dev_priv->drm,
7727 "failed to find known CxSR latency "
7728 "(found ddr%s fsb freq %d, mem freq %d), "
7729 "disabling CxSR\n",
7730 (dev_priv->is_ddr3 == 1) ? "3" : "2",
7731 dev_priv->fsb_freq, dev_priv->mem_freq);
7732 /* Disable CxSR and never update its watermark again */
7733 intel_set_memory_cxsr(dev_priv, false);
7734 dev_priv->display.update_wm = NULL;
7735 } else
7736 dev_priv->display.update_wm = pnv_update_wm;
7737 } else if (IS_DISPLAY_VER(dev_priv, 4)) {
7738 dev_priv->display.update_wm = i965_update_wm;
7739 } else if (IS_DISPLAY_VER(dev_priv, 3)) {
7740 dev_priv->display.update_wm = i9xx_update_wm;
7741 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
7742 } else if (IS_DISPLAY_VER(dev_priv, 2)) {
7743 if (INTEL_NUM_PIPES(dev_priv) == 1) {
7744 dev_priv->display.update_wm = i845_update_wm;
7745 dev_priv->display.get_fifo_size = i845_get_fifo_size;
7746 } else {
7747 dev_priv->display.update_wm = i9xx_update_wm;
7748 dev_priv->display.get_fifo_size = i830_get_fifo_size;
7749 }
7750 } else {
7751 drm_err(&dev_priv->drm,
7752 "unexpected fall-through in %s\n", __func__);
7753 }
7754 }
7755
intel_pm_setup(struct drm_i915_private * dev_priv)7756 void intel_pm_setup(struct drm_i915_private *dev_priv)
7757 {
7758 dev_priv->runtime_pm.suspended = false;
7759 atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
7760 }
7761
intel_dbuf_duplicate_state(struct intel_global_obj * obj)7762 static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
7763 {
7764 struct intel_dbuf_state *dbuf_state;
7765
7766 dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
7767 if (!dbuf_state)
7768 return NULL;
7769
7770 return &dbuf_state->base;
7771 }
7772
intel_dbuf_destroy_state(struct intel_global_obj * obj,struct intel_global_state * state)7773 static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
7774 struct intel_global_state *state)
7775 {
7776 kfree(state);
7777 }
7778
7779 static const struct intel_global_state_funcs intel_dbuf_funcs = {
7780 .atomic_duplicate_state = intel_dbuf_duplicate_state,
7781 .atomic_destroy_state = intel_dbuf_destroy_state,
7782 };
7783
7784 struct intel_dbuf_state *
intel_atomic_get_dbuf_state(struct intel_atomic_state * state)7785 intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
7786 {
7787 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7788 struct intel_global_state *dbuf_state;
7789
7790 dbuf_state = intel_atomic_get_global_obj_state(state, &dev_priv->dbuf.obj);
7791 if (IS_ERR(dbuf_state))
7792 return ERR_CAST(dbuf_state);
7793
7794 return to_intel_dbuf_state(dbuf_state);
7795 }
7796
intel_dbuf_init(struct drm_i915_private * dev_priv)7797 int intel_dbuf_init(struct drm_i915_private *dev_priv)
7798 {
7799 struct intel_dbuf_state *dbuf_state;
7800
7801 dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
7802 if (!dbuf_state)
7803 return -ENOMEM;
7804
7805 intel_atomic_global_obj_init(dev_priv, &dev_priv->dbuf.obj,
7806 &dbuf_state->base, &intel_dbuf_funcs);
7807
7808 return 0;
7809 }
7810
intel_dbuf_pre_plane_update(struct intel_atomic_state * state)7811 void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
7812 {
7813 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7814 const struct intel_dbuf_state *new_dbuf_state =
7815 intel_atomic_get_new_dbuf_state(state);
7816 const struct intel_dbuf_state *old_dbuf_state =
7817 intel_atomic_get_old_dbuf_state(state);
7818
7819 if (!new_dbuf_state ||
7820 new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
7821 return;
7822
7823 WARN_ON(!new_dbuf_state->base.changed);
7824
7825 gen9_dbuf_slices_update(dev_priv,
7826 old_dbuf_state->enabled_slices |
7827 new_dbuf_state->enabled_slices);
7828 }
7829
intel_dbuf_post_plane_update(struct intel_atomic_state * state)7830 void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
7831 {
7832 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7833 const struct intel_dbuf_state *new_dbuf_state =
7834 intel_atomic_get_new_dbuf_state(state);
7835 const struct intel_dbuf_state *old_dbuf_state =
7836 intel_atomic_get_old_dbuf_state(state);
7837
7838 if (!new_dbuf_state ||
7839 new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
7840 return;
7841
7842 WARN_ON(!new_dbuf_state->base.changed);
7843
7844 gen9_dbuf_slices_update(dev_priv,
7845 new_dbuf_state->enabled_slices);
7846 }
7847