1 /* $NetBSD: radeon_r600.c,v 1.6 2021/12/18 23:45:43 riastradh Exp $ */
2
3 /*
4 * Copyright 2008 Advanced Micro Devices, Inc.
5 * Copyright 2008 Red Hat Inc.
6 * Copyright 2009 Jerome Glisse.
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the "Software"),
10 * to deal in the Software without restriction, including without limitation
11 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
12 * and/or sell copies of the Software, and to permit persons to whom the
13 * Software is furnished to do so, subject to the following conditions:
14 *
15 * The above copyright notice and this permission notice shall be included in
16 * all copies or substantial portions of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
22 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
23 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
24 * OTHER DEALINGS IN THE SOFTWARE.
25 *
26 * Authors: Dave Airlie
27 * Alex Deucher
28 * Jerome Glisse
29 */
30
31 #include <sys/cdefs.h>
32 __KERNEL_RCSID(0, "$NetBSD: radeon_r600.c,v 1.6 2021/12/18 23:45:43 riastradh Exp $");
33
34 #include <linux/firmware.h>
35 #include <linux/module.h>
36 #include <linux/pci.h>
37 #include <linux/slab.h>
38 #include <linux/seq_file.h>
39
40 #include <drm/drm_debugfs.h>
41 #include <drm/drm_device.h>
42 #include <drm/drm_vblank.h>
43 #include <drm/radeon_drm.h>
44
45 #include "atom.h"
46 #include "avivod.h"
47 #include "r600d.h"
48 #include "radeon.h"
49 #include "radeon_asic.h"
50 #include "radeon_audio.h"
51 #include "radeon_mode.h"
52 #include "radeon_ucode.h"
53
54 #include <linux/nbsd-namespace.h>
55
56 /* Firmware Names */
57 MODULE_FIRMWARE("radeon/R600_pfp.bin");
58 MODULE_FIRMWARE("radeon/R600_me.bin");
59 MODULE_FIRMWARE("radeon/RV610_pfp.bin");
60 MODULE_FIRMWARE("radeon/RV610_me.bin");
61 MODULE_FIRMWARE("radeon/RV630_pfp.bin");
62 MODULE_FIRMWARE("radeon/RV630_me.bin");
63 MODULE_FIRMWARE("radeon/RV620_pfp.bin");
64 MODULE_FIRMWARE("radeon/RV620_me.bin");
65 MODULE_FIRMWARE("radeon/RV635_pfp.bin");
66 MODULE_FIRMWARE("radeon/RV635_me.bin");
67 MODULE_FIRMWARE("radeon/RV670_pfp.bin");
68 MODULE_FIRMWARE("radeon/RV670_me.bin");
69 MODULE_FIRMWARE("radeon/RS780_pfp.bin");
70 MODULE_FIRMWARE("radeon/RS780_me.bin");
71 MODULE_FIRMWARE("radeon/RV770_pfp.bin");
72 MODULE_FIRMWARE("radeon/RV770_me.bin");
73 MODULE_FIRMWARE("radeon/RV770_smc.bin");
74 MODULE_FIRMWARE("radeon/RV730_pfp.bin");
75 MODULE_FIRMWARE("radeon/RV730_me.bin");
76 MODULE_FIRMWARE("radeon/RV730_smc.bin");
77 MODULE_FIRMWARE("radeon/RV740_smc.bin");
78 MODULE_FIRMWARE("radeon/RV710_pfp.bin");
79 MODULE_FIRMWARE("radeon/RV710_me.bin");
80 MODULE_FIRMWARE("radeon/RV710_smc.bin");
81 MODULE_FIRMWARE("radeon/R600_rlc.bin");
82 MODULE_FIRMWARE("radeon/R700_rlc.bin");
83 MODULE_FIRMWARE("radeon/CEDAR_pfp.bin");
84 MODULE_FIRMWARE("radeon/CEDAR_me.bin");
85 MODULE_FIRMWARE("radeon/CEDAR_rlc.bin");
86 MODULE_FIRMWARE("radeon/CEDAR_smc.bin");
87 MODULE_FIRMWARE("radeon/REDWOOD_pfp.bin");
88 MODULE_FIRMWARE("radeon/REDWOOD_me.bin");
89 MODULE_FIRMWARE("radeon/REDWOOD_rlc.bin");
90 MODULE_FIRMWARE("radeon/REDWOOD_smc.bin");
91 MODULE_FIRMWARE("radeon/JUNIPER_pfp.bin");
92 MODULE_FIRMWARE("radeon/JUNIPER_me.bin");
93 MODULE_FIRMWARE("radeon/JUNIPER_rlc.bin");
94 MODULE_FIRMWARE("radeon/JUNIPER_smc.bin");
95 MODULE_FIRMWARE("radeon/CYPRESS_pfp.bin");
96 MODULE_FIRMWARE("radeon/CYPRESS_me.bin");
97 MODULE_FIRMWARE("radeon/CYPRESS_rlc.bin");
98 MODULE_FIRMWARE("radeon/CYPRESS_smc.bin");
99 MODULE_FIRMWARE("radeon/PALM_pfp.bin");
100 MODULE_FIRMWARE("radeon/PALM_me.bin");
101 MODULE_FIRMWARE("radeon/SUMO_rlc.bin");
102 MODULE_FIRMWARE("radeon/SUMO_pfp.bin");
103 MODULE_FIRMWARE("radeon/SUMO_me.bin");
104 MODULE_FIRMWARE("radeon/SUMO2_pfp.bin");
105 MODULE_FIRMWARE("radeon/SUMO2_me.bin");
106
107 static const u32 crtc_offsets[2] =
108 {
109 0,
110 AVIVO_D2CRTC_H_TOTAL - AVIVO_D1CRTC_H_TOTAL
111 };
112
113 int r600_debugfs_mc_info_init(struct radeon_device *rdev);
114
115 /* r600,rv610,rv630,rv620,rv635,rv670 */
116 int r600_mc_wait_for_idle(struct radeon_device *rdev);
117 static void r600_gpu_init(struct radeon_device *rdev);
118 void r600_fini(struct radeon_device *rdev);
119 void r600_irq_disable(struct radeon_device *rdev);
120 static void r600_pcie_gen2_enable(struct radeon_device *rdev);
121 extern int evergreen_rlc_resume(struct radeon_device *rdev);
122 extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
123
124 /*
125 * Indirect registers accessor
126 */
r600_rcu_rreg(struct radeon_device * rdev,u32 reg)127 u32 r600_rcu_rreg(struct radeon_device *rdev, u32 reg)
128 {
129 unsigned long flags;
130 u32 r;
131
132 spin_lock_irqsave(&rdev->rcu_idx_lock, flags);
133 WREG32(R600_RCU_INDEX, ((reg) & 0x1fff));
134 r = RREG32(R600_RCU_DATA);
135 spin_unlock_irqrestore(&rdev->rcu_idx_lock, flags);
136 return r;
137 }
138
r600_rcu_wreg(struct radeon_device * rdev,u32 reg,u32 v)139 void r600_rcu_wreg(struct radeon_device *rdev, u32 reg, u32 v)
140 {
141 unsigned long flags;
142
143 spin_lock_irqsave(&rdev->rcu_idx_lock, flags);
144 WREG32(R600_RCU_INDEX, ((reg) & 0x1fff));
145 WREG32(R600_RCU_DATA, (v));
146 spin_unlock_irqrestore(&rdev->rcu_idx_lock, flags);
147 }
148
r600_uvd_ctx_rreg(struct radeon_device * rdev,u32 reg)149 u32 r600_uvd_ctx_rreg(struct radeon_device *rdev, u32 reg)
150 {
151 unsigned long flags;
152 u32 r;
153
154 spin_lock_irqsave(&rdev->uvd_idx_lock, flags);
155 WREG32(R600_UVD_CTX_INDEX, ((reg) & 0x1ff));
156 r = RREG32(R600_UVD_CTX_DATA);
157 spin_unlock_irqrestore(&rdev->uvd_idx_lock, flags);
158 return r;
159 }
160
r600_uvd_ctx_wreg(struct radeon_device * rdev,u32 reg,u32 v)161 void r600_uvd_ctx_wreg(struct radeon_device *rdev, u32 reg, u32 v)
162 {
163 unsigned long flags;
164
165 spin_lock_irqsave(&rdev->uvd_idx_lock, flags);
166 WREG32(R600_UVD_CTX_INDEX, ((reg) & 0x1ff));
167 WREG32(R600_UVD_CTX_DATA, (v));
168 spin_unlock_irqrestore(&rdev->uvd_idx_lock, flags);
169 }
170
171 /**
172 * r600_get_allowed_info_register - fetch the register for the info ioctl
173 *
174 * @rdev: radeon_device pointer
175 * @reg: register offset in bytes
176 * @val: register value
177 *
178 * Returns 0 for success or -EINVAL for an invalid register
179 *
180 */
r600_get_allowed_info_register(struct radeon_device * rdev,u32 reg,u32 * val)181 int r600_get_allowed_info_register(struct radeon_device *rdev,
182 u32 reg, u32 *val)
183 {
184 switch (reg) {
185 case GRBM_STATUS:
186 case GRBM_STATUS2:
187 case R_000E50_SRBM_STATUS:
188 case DMA_STATUS_REG:
189 case UVD_STATUS:
190 *val = RREG32(reg);
191 return 0;
192 default:
193 return -EINVAL;
194 }
195 }
196
197 /**
198 * r600_get_xclk - get the xclk
199 *
200 * @rdev: radeon_device pointer
201 *
202 * Returns the reference clock used by the gfx engine
203 * (r6xx, IGPs, APUs).
204 */
r600_get_xclk(struct radeon_device * rdev)205 u32 r600_get_xclk(struct radeon_device *rdev)
206 {
207 return rdev->clock.spll.reference_freq;
208 }
209
r600_set_uvd_clocks(struct radeon_device * rdev,u32 vclk,u32 dclk)210 int r600_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
211 {
212 unsigned fb_div = 0, ref_div, vclk_div = 0, dclk_div = 0;
213 int r;
214
215 /* bypass vclk and dclk with bclk */
216 WREG32_P(CG_UPLL_FUNC_CNTL_2,
217 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
218 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
219
220 /* assert BYPASS_EN, deassert UPLL_RESET, UPLL_SLEEP and UPLL_CTLREQ */
221 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~(
222 UPLL_RESET_MASK | UPLL_SLEEP_MASK | UPLL_CTLREQ_MASK));
223
224 if (rdev->family >= CHIP_RS780)
225 WREG32_P(GFX_MACRO_BYPASS_CNTL, UPLL_BYPASS_CNTL,
226 ~UPLL_BYPASS_CNTL);
227
228 if (!vclk || !dclk) {
229 /* keep the Bypass mode, put PLL to sleep */
230 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
231 return 0;
232 }
233
234 if (rdev->clock.spll.reference_freq == 10000)
235 ref_div = 34;
236 else
237 ref_div = 4;
238
239 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 50000, 160000,
240 ref_div + 1, 0xFFF, 2, 30, ~0,
241 &fb_div, &vclk_div, &dclk_div);
242 if (r)
243 return r;
244
245 if (rdev->family >= CHIP_RV670 && rdev->family < CHIP_RS780)
246 fb_div >>= 1;
247 else
248 fb_div |= 1;
249
250 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
251 if (r)
252 return r;
253
254 /* assert PLL_RESET */
255 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
256
257 /* For RS780 we have to choose ref clk */
258 if (rdev->family >= CHIP_RS780)
259 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_REFCLK_SRC_SEL_MASK,
260 ~UPLL_REFCLK_SRC_SEL_MASK);
261
262 /* set the required fb, ref and post divder values */
263 WREG32_P(CG_UPLL_FUNC_CNTL,
264 UPLL_FB_DIV(fb_div) |
265 UPLL_REF_DIV(ref_div),
266 ~(UPLL_FB_DIV_MASK | UPLL_REF_DIV_MASK));
267 WREG32_P(CG_UPLL_FUNC_CNTL_2,
268 UPLL_SW_HILEN(vclk_div >> 1) |
269 UPLL_SW_LOLEN((vclk_div >> 1) + (vclk_div & 1)) |
270 UPLL_SW_HILEN2(dclk_div >> 1) |
271 UPLL_SW_LOLEN2((dclk_div >> 1) + (dclk_div & 1)) |
272 UPLL_DIVEN_MASK | UPLL_DIVEN2_MASK,
273 ~UPLL_SW_MASK);
274
275 /* give the PLL some time to settle */
276 mdelay(15);
277
278 /* deassert PLL_RESET */
279 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
280
281 mdelay(15);
282
283 /* deassert BYPASS EN */
284 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
285
286 if (rdev->family >= CHIP_RS780)
287 WREG32_P(GFX_MACRO_BYPASS_CNTL, 0, ~UPLL_BYPASS_CNTL);
288
289 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
290 if (r)
291 return r;
292
293 /* switch VCLK and DCLK selection */
294 WREG32_P(CG_UPLL_FUNC_CNTL_2,
295 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
296 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
297
298 mdelay(100);
299
300 return 0;
301 }
302
dce3_program_fmt(struct drm_encoder * encoder)303 void dce3_program_fmt(struct drm_encoder *encoder)
304 {
305 struct drm_device *dev = encoder->dev;
306 struct radeon_device *rdev = dev->dev_private;
307 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
308 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
309 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
310 int bpc = 0;
311 u32 tmp = 0;
312 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
313
314 if (connector) {
315 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
316 bpc = radeon_get_monitor_bpc(connector);
317 dither = radeon_connector->dither;
318 }
319
320 /* LVDS FMT is set up by atom */
321 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
322 return;
323
324 /* not needed for analog */
325 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
326 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
327 return;
328
329 if (bpc == 0)
330 return;
331
332 switch (bpc) {
333 case 6:
334 if (dither == RADEON_FMT_DITHER_ENABLE)
335 /* XXX sort out optimal dither settings */
336 tmp |= FMT_SPATIAL_DITHER_EN;
337 else
338 tmp |= FMT_TRUNCATE_EN;
339 break;
340 case 8:
341 if (dither == RADEON_FMT_DITHER_ENABLE)
342 /* XXX sort out optimal dither settings */
343 tmp |= (FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
344 else
345 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
346 break;
347 case 10:
348 default:
349 /* not needed */
350 break;
351 }
352
353 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
354 }
355
356 /* get temperature in millidegrees */
rv6xx_get_temp(struct radeon_device * rdev)357 int rv6xx_get_temp(struct radeon_device *rdev)
358 {
359 u32 temp = (RREG32(CG_THERMAL_STATUS) & ASIC_T_MASK) >>
360 ASIC_T_SHIFT;
361 int actual_temp = temp & 0xff;
362
363 if (temp & 0x100)
364 actual_temp -= 256;
365
366 return actual_temp * 1000;
367 }
368
r600_pm_get_dynpm_state(struct radeon_device * rdev)369 void r600_pm_get_dynpm_state(struct radeon_device *rdev)
370 {
371 int i;
372
373 rdev->pm.dynpm_can_upclock = true;
374 rdev->pm.dynpm_can_downclock = true;
375
376 /* power state array is low to high, default is first */
377 if ((rdev->flags & RADEON_IS_IGP) || (rdev->family == CHIP_R600)) {
378 int min_power_state_index = 0;
379
380 if (rdev->pm.num_power_states > 2)
381 min_power_state_index = 1;
382
383 switch (rdev->pm.dynpm_planned_action) {
384 case DYNPM_ACTION_MINIMUM:
385 rdev->pm.requested_power_state_index = min_power_state_index;
386 rdev->pm.requested_clock_mode_index = 0;
387 rdev->pm.dynpm_can_downclock = false;
388 break;
389 case DYNPM_ACTION_DOWNCLOCK:
390 if (rdev->pm.current_power_state_index == min_power_state_index) {
391 rdev->pm.requested_power_state_index = rdev->pm.current_power_state_index;
392 rdev->pm.dynpm_can_downclock = false;
393 } else {
394 if (rdev->pm.active_crtc_count > 1) {
395 for (i = 0; i < rdev->pm.num_power_states; i++) {
396 if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
397 continue;
398 else if (i >= rdev->pm.current_power_state_index) {
399 rdev->pm.requested_power_state_index =
400 rdev->pm.current_power_state_index;
401 break;
402 } else {
403 rdev->pm.requested_power_state_index = i;
404 break;
405 }
406 }
407 } else {
408 if (rdev->pm.current_power_state_index == 0)
409 rdev->pm.requested_power_state_index =
410 rdev->pm.num_power_states - 1;
411 else
412 rdev->pm.requested_power_state_index =
413 rdev->pm.current_power_state_index - 1;
414 }
415 }
416 rdev->pm.requested_clock_mode_index = 0;
417 /* don't use the power state if crtcs are active and no display flag is set */
418 if ((rdev->pm.active_crtc_count > 0) &&
419 (rdev->pm.power_state[rdev->pm.requested_power_state_index].
420 clock_info[rdev->pm.requested_clock_mode_index].flags &
421 RADEON_PM_MODE_NO_DISPLAY)) {
422 rdev->pm.requested_power_state_index++;
423 }
424 break;
425 case DYNPM_ACTION_UPCLOCK:
426 if (rdev->pm.current_power_state_index == (rdev->pm.num_power_states - 1)) {
427 rdev->pm.requested_power_state_index = rdev->pm.current_power_state_index;
428 rdev->pm.dynpm_can_upclock = false;
429 } else {
430 if (rdev->pm.active_crtc_count > 1) {
431 for (i = (rdev->pm.num_power_states - 1); i >= 0; i--) {
432 if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
433 continue;
434 else if (i <= rdev->pm.current_power_state_index) {
435 rdev->pm.requested_power_state_index =
436 rdev->pm.current_power_state_index;
437 break;
438 } else {
439 rdev->pm.requested_power_state_index = i;
440 break;
441 }
442 }
443 } else
444 rdev->pm.requested_power_state_index =
445 rdev->pm.current_power_state_index + 1;
446 }
447 rdev->pm.requested_clock_mode_index = 0;
448 break;
449 case DYNPM_ACTION_DEFAULT:
450 rdev->pm.requested_power_state_index = rdev->pm.default_power_state_index;
451 rdev->pm.requested_clock_mode_index = 0;
452 rdev->pm.dynpm_can_upclock = false;
453 break;
454 case DYNPM_ACTION_NONE:
455 default:
456 DRM_ERROR("Requested mode for not defined action\n");
457 return;
458 }
459 } else {
460 /* XXX select a power state based on AC/DC, single/dualhead, etc. */
461 /* for now just select the first power state and switch between clock modes */
462 /* power state array is low to high, default is first (0) */
463 if (rdev->pm.active_crtc_count > 1) {
464 rdev->pm.requested_power_state_index = -1;
465 /* start at 1 as we don't want the default mode */
466 for (i = 1; i < rdev->pm.num_power_states; i++) {
467 if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
468 continue;
469 else if ((rdev->pm.power_state[i].type == POWER_STATE_TYPE_PERFORMANCE) ||
470 (rdev->pm.power_state[i].type == POWER_STATE_TYPE_BATTERY)) {
471 rdev->pm.requested_power_state_index = i;
472 break;
473 }
474 }
475 /* if nothing selected, grab the default state. */
476 if (rdev->pm.requested_power_state_index == -1)
477 rdev->pm.requested_power_state_index = 0;
478 } else
479 rdev->pm.requested_power_state_index = 1;
480
481 switch (rdev->pm.dynpm_planned_action) {
482 case DYNPM_ACTION_MINIMUM:
483 rdev->pm.requested_clock_mode_index = 0;
484 rdev->pm.dynpm_can_downclock = false;
485 break;
486 case DYNPM_ACTION_DOWNCLOCK:
487 if (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index) {
488 if (rdev->pm.current_clock_mode_index == 0) {
489 rdev->pm.requested_clock_mode_index = 0;
490 rdev->pm.dynpm_can_downclock = false;
491 } else
492 rdev->pm.requested_clock_mode_index =
493 rdev->pm.current_clock_mode_index - 1;
494 } else {
495 rdev->pm.requested_clock_mode_index = 0;
496 rdev->pm.dynpm_can_downclock = false;
497 }
498 /* don't use the power state if crtcs are active and no display flag is set */
499 if ((rdev->pm.active_crtc_count > 0) &&
500 (rdev->pm.power_state[rdev->pm.requested_power_state_index].
501 clock_info[rdev->pm.requested_clock_mode_index].flags &
502 RADEON_PM_MODE_NO_DISPLAY)) {
503 rdev->pm.requested_clock_mode_index++;
504 }
505 break;
506 case DYNPM_ACTION_UPCLOCK:
507 if (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index) {
508 if (rdev->pm.current_clock_mode_index ==
509 (rdev->pm.power_state[rdev->pm.requested_power_state_index].num_clock_modes - 1)) {
510 rdev->pm.requested_clock_mode_index = rdev->pm.current_clock_mode_index;
511 rdev->pm.dynpm_can_upclock = false;
512 } else
513 rdev->pm.requested_clock_mode_index =
514 rdev->pm.current_clock_mode_index + 1;
515 } else {
516 rdev->pm.requested_clock_mode_index =
517 rdev->pm.power_state[rdev->pm.requested_power_state_index].num_clock_modes - 1;
518 rdev->pm.dynpm_can_upclock = false;
519 }
520 break;
521 case DYNPM_ACTION_DEFAULT:
522 rdev->pm.requested_power_state_index = rdev->pm.default_power_state_index;
523 rdev->pm.requested_clock_mode_index = 0;
524 rdev->pm.dynpm_can_upclock = false;
525 break;
526 case DYNPM_ACTION_NONE:
527 default:
528 DRM_ERROR("Requested mode for not defined action\n");
529 return;
530 }
531 }
532
533 DRM_DEBUG_DRIVER("Requested: e: %d m: %d p: %d\n",
534 rdev->pm.power_state[rdev->pm.requested_power_state_index].
535 clock_info[rdev->pm.requested_clock_mode_index].sclk,
536 rdev->pm.power_state[rdev->pm.requested_power_state_index].
537 clock_info[rdev->pm.requested_clock_mode_index].mclk,
538 rdev->pm.power_state[rdev->pm.requested_power_state_index].
539 pcie_lanes);
540 }
541
rs780_pm_init_profile(struct radeon_device * rdev)542 void rs780_pm_init_profile(struct radeon_device *rdev)
543 {
544 if (rdev->pm.num_power_states == 2) {
545 /* default */
546 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
547 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
548 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
549 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
550 /* low sh */
551 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 0;
552 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 0;
553 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
554 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
555 /* mid sh */
556 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 0;
557 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 0;
558 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
559 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
560 /* high sh */
561 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 0;
562 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 1;
563 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
564 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
565 /* low mh */
566 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 0;
567 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 0;
568 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
569 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
570 /* mid mh */
571 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 0;
572 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 0;
573 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
574 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
575 /* high mh */
576 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 0;
577 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 1;
578 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
579 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
580 } else if (rdev->pm.num_power_states == 3) {
581 /* default */
582 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
583 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
584 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
585 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
586 /* low sh */
587 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 1;
588 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 1;
589 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
590 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
591 /* mid sh */
592 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 1;
593 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 1;
594 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
595 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
596 /* high sh */
597 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 1;
598 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 2;
599 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
600 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
601 /* low mh */
602 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 1;
603 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 1;
604 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
605 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
606 /* mid mh */
607 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 1;
608 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 1;
609 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
610 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
611 /* high mh */
612 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 1;
613 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 2;
614 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
615 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
616 } else {
617 /* default */
618 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
619 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
620 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
621 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
622 /* low sh */
623 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 2;
624 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 2;
625 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
626 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
627 /* mid sh */
628 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 2;
629 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 2;
630 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
631 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
632 /* high sh */
633 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 2;
634 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 3;
635 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
636 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
637 /* low mh */
638 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 2;
639 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 0;
640 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
641 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
642 /* mid mh */
643 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 2;
644 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 0;
645 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
646 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
647 /* high mh */
648 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 2;
649 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 3;
650 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
651 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
652 }
653 }
654
r600_pm_init_profile(struct radeon_device * rdev)655 void r600_pm_init_profile(struct radeon_device *rdev)
656 {
657 int idx;
658
659 if (rdev->family == CHIP_R600) {
660 /* XXX */
661 /* default */
662 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
663 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
664 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
665 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
666 /* low sh */
667 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
668 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
669 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
670 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
671 /* mid sh */
672 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
673 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
674 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
675 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
676 /* high sh */
677 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
678 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
679 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
680 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
681 /* low mh */
682 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
683 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
684 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
685 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
686 /* mid mh */
687 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
688 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
689 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
690 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
691 /* high mh */
692 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
693 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
694 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
695 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
696 } else {
697 if (rdev->pm.num_power_states < 4) {
698 /* default */
699 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
700 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
701 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
702 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
703 /* low sh */
704 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 1;
705 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 1;
706 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
707 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
708 /* mid sh */
709 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 1;
710 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 1;
711 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
712 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
713 /* high sh */
714 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 1;
715 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 1;
716 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
717 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
718 /* low mh */
719 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 2;
720 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 2;
721 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
722 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
723 /* low mh */
724 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 2;
725 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 2;
726 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
727 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
728 /* high mh */
729 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 2;
730 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 2;
731 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
732 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
733 } else {
734 /* default */
735 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
736 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
737 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
738 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
739 /* low sh */
740 if (rdev->flags & RADEON_IS_MOBILITY)
741 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
742 else
743 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
744 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
745 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
746 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
747 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
748 /* mid sh */
749 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
750 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
751 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
752 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
753 /* high sh */
754 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
755 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
756 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
757 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
758 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
759 /* low mh */
760 if (rdev->flags & RADEON_IS_MOBILITY)
761 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 1);
762 else
763 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 1);
764 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
765 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
766 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
767 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
768 /* mid mh */
769 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
770 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
771 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
772 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
773 /* high mh */
774 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 1);
775 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
776 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
777 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
778 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
779 }
780 }
781 }
782
r600_pm_misc(struct radeon_device * rdev)783 void r600_pm_misc(struct radeon_device *rdev)
784 {
785 int req_ps_idx = rdev->pm.requested_power_state_index;
786 int req_cm_idx = rdev->pm.requested_clock_mode_index;
787 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
788 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
789
790 if ((voltage->type == VOLTAGE_SW) && voltage->voltage) {
791 /* 0xff01 is a flag rather then an actual voltage */
792 if (voltage->voltage == 0xff01)
793 return;
794 if (voltage->voltage != rdev->pm.current_vddc) {
795 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
796 rdev->pm.current_vddc = voltage->voltage;
797 DRM_DEBUG_DRIVER("Setting: v: %d\n", voltage->voltage);
798 }
799 }
800 }
801
r600_gui_idle(struct radeon_device * rdev)802 bool r600_gui_idle(struct radeon_device *rdev)
803 {
804 if (RREG32(GRBM_STATUS) & GUI_ACTIVE)
805 return false;
806 else
807 return true;
808 }
809
810 /* hpd for digital panel detect/disconnect */
r600_hpd_sense(struct radeon_device * rdev,enum radeon_hpd_id hpd)811 bool r600_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
812 {
813 bool connected = false;
814
815 if (ASIC_IS_DCE3(rdev)) {
816 switch (hpd) {
817 case RADEON_HPD_1:
818 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
819 connected = true;
820 break;
821 case RADEON_HPD_2:
822 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
823 connected = true;
824 break;
825 case RADEON_HPD_3:
826 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
827 connected = true;
828 break;
829 case RADEON_HPD_4:
830 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
831 connected = true;
832 break;
833 /* DCE 3.2 */
834 case RADEON_HPD_5:
835 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
836 connected = true;
837 break;
838 case RADEON_HPD_6:
839 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
840 connected = true;
841 break;
842 default:
843 break;
844 }
845 } else {
846 switch (hpd) {
847 case RADEON_HPD_1:
848 if (RREG32(DC_HOT_PLUG_DETECT1_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
849 connected = true;
850 break;
851 case RADEON_HPD_2:
852 if (RREG32(DC_HOT_PLUG_DETECT2_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
853 connected = true;
854 break;
855 case RADEON_HPD_3:
856 if (RREG32(DC_HOT_PLUG_DETECT3_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
857 connected = true;
858 break;
859 default:
860 break;
861 }
862 }
863 return connected;
864 }
865
r600_hpd_set_polarity(struct radeon_device * rdev,enum radeon_hpd_id hpd)866 void r600_hpd_set_polarity(struct radeon_device *rdev,
867 enum radeon_hpd_id hpd)
868 {
869 u32 tmp;
870 bool connected = r600_hpd_sense(rdev, hpd);
871
872 if (ASIC_IS_DCE3(rdev)) {
873 switch (hpd) {
874 case RADEON_HPD_1:
875 tmp = RREG32(DC_HPD1_INT_CONTROL);
876 if (connected)
877 tmp &= ~DC_HPDx_INT_POLARITY;
878 else
879 tmp |= DC_HPDx_INT_POLARITY;
880 WREG32(DC_HPD1_INT_CONTROL, tmp);
881 break;
882 case RADEON_HPD_2:
883 tmp = RREG32(DC_HPD2_INT_CONTROL);
884 if (connected)
885 tmp &= ~DC_HPDx_INT_POLARITY;
886 else
887 tmp |= DC_HPDx_INT_POLARITY;
888 WREG32(DC_HPD2_INT_CONTROL, tmp);
889 break;
890 case RADEON_HPD_3:
891 tmp = RREG32(DC_HPD3_INT_CONTROL);
892 if (connected)
893 tmp &= ~DC_HPDx_INT_POLARITY;
894 else
895 tmp |= DC_HPDx_INT_POLARITY;
896 WREG32(DC_HPD3_INT_CONTROL, tmp);
897 break;
898 case RADEON_HPD_4:
899 tmp = RREG32(DC_HPD4_INT_CONTROL);
900 if (connected)
901 tmp &= ~DC_HPDx_INT_POLARITY;
902 else
903 tmp |= DC_HPDx_INT_POLARITY;
904 WREG32(DC_HPD4_INT_CONTROL, tmp);
905 break;
906 case RADEON_HPD_5:
907 tmp = RREG32(DC_HPD5_INT_CONTROL);
908 if (connected)
909 tmp &= ~DC_HPDx_INT_POLARITY;
910 else
911 tmp |= DC_HPDx_INT_POLARITY;
912 WREG32(DC_HPD5_INT_CONTROL, tmp);
913 break;
914 /* DCE 3.2 */
915 case RADEON_HPD_6:
916 tmp = RREG32(DC_HPD6_INT_CONTROL);
917 if (connected)
918 tmp &= ~DC_HPDx_INT_POLARITY;
919 else
920 tmp |= DC_HPDx_INT_POLARITY;
921 WREG32(DC_HPD6_INT_CONTROL, tmp);
922 break;
923 default:
924 break;
925 }
926 } else {
927 switch (hpd) {
928 case RADEON_HPD_1:
929 tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
930 if (connected)
931 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
932 else
933 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
934 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
935 break;
936 case RADEON_HPD_2:
937 tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
938 if (connected)
939 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
940 else
941 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
942 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
943 break;
944 case RADEON_HPD_3:
945 tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
946 if (connected)
947 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
948 else
949 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
950 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
951 break;
952 default:
953 break;
954 }
955 }
956 }
957
r600_hpd_init(struct radeon_device * rdev)958 void r600_hpd_init(struct radeon_device *rdev)
959 {
960 struct drm_device *dev = rdev->ddev;
961 struct drm_connector *connector;
962 unsigned enable = 0;
963
964 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
965 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
966
967 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
968 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
969 /* don't try to enable hpd on eDP or LVDS avoid breaking the
970 * aux dp channel on imac and help (but not completely fix)
971 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
972 */
973 continue;
974 }
975 if (ASIC_IS_DCE3(rdev)) {
976 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) | DC_HPDx_RX_INT_TIMER(0xfa);
977 if (ASIC_IS_DCE32(rdev))
978 tmp |= DC_HPDx_EN;
979
980 switch (radeon_connector->hpd.hpd) {
981 case RADEON_HPD_1:
982 WREG32(DC_HPD1_CONTROL, tmp);
983 break;
984 case RADEON_HPD_2:
985 WREG32(DC_HPD2_CONTROL, tmp);
986 break;
987 case RADEON_HPD_3:
988 WREG32(DC_HPD3_CONTROL, tmp);
989 break;
990 case RADEON_HPD_4:
991 WREG32(DC_HPD4_CONTROL, tmp);
992 break;
993 /* DCE 3.2 */
994 case RADEON_HPD_5:
995 WREG32(DC_HPD5_CONTROL, tmp);
996 break;
997 case RADEON_HPD_6:
998 WREG32(DC_HPD6_CONTROL, tmp);
999 break;
1000 default:
1001 break;
1002 }
1003 } else {
1004 switch (radeon_connector->hpd.hpd) {
1005 case RADEON_HPD_1:
1006 WREG32(DC_HOT_PLUG_DETECT1_CONTROL, DC_HOT_PLUG_DETECTx_EN);
1007 break;
1008 case RADEON_HPD_2:
1009 WREG32(DC_HOT_PLUG_DETECT2_CONTROL, DC_HOT_PLUG_DETECTx_EN);
1010 break;
1011 case RADEON_HPD_3:
1012 WREG32(DC_HOT_PLUG_DETECT3_CONTROL, DC_HOT_PLUG_DETECTx_EN);
1013 break;
1014 default:
1015 break;
1016 }
1017 }
1018 if (radeon_connector->hpd.hpd != RADEON_HPD_NONE)
1019 enable |= 1 << radeon_connector->hpd.hpd;
1020 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
1021 }
1022 radeon_irq_kms_enable_hpd(rdev, enable);
1023 }
1024
r600_hpd_fini(struct radeon_device * rdev)1025 void r600_hpd_fini(struct radeon_device *rdev)
1026 {
1027 struct drm_device *dev = rdev->ddev;
1028 struct drm_connector *connector;
1029 unsigned disable = 0;
1030
1031 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1032 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1033 if (ASIC_IS_DCE3(rdev)) {
1034 switch (radeon_connector->hpd.hpd) {
1035 case RADEON_HPD_1:
1036 WREG32(DC_HPD1_CONTROL, 0);
1037 break;
1038 case RADEON_HPD_2:
1039 WREG32(DC_HPD2_CONTROL, 0);
1040 break;
1041 case RADEON_HPD_3:
1042 WREG32(DC_HPD3_CONTROL, 0);
1043 break;
1044 case RADEON_HPD_4:
1045 WREG32(DC_HPD4_CONTROL, 0);
1046 break;
1047 /* DCE 3.2 */
1048 case RADEON_HPD_5:
1049 WREG32(DC_HPD5_CONTROL, 0);
1050 break;
1051 case RADEON_HPD_6:
1052 WREG32(DC_HPD6_CONTROL, 0);
1053 break;
1054 default:
1055 break;
1056 }
1057 } else {
1058 switch (radeon_connector->hpd.hpd) {
1059 case RADEON_HPD_1:
1060 WREG32(DC_HOT_PLUG_DETECT1_CONTROL, 0);
1061 break;
1062 case RADEON_HPD_2:
1063 WREG32(DC_HOT_PLUG_DETECT2_CONTROL, 0);
1064 break;
1065 case RADEON_HPD_3:
1066 WREG32(DC_HOT_PLUG_DETECT3_CONTROL, 0);
1067 break;
1068 default:
1069 break;
1070 }
1071 }
1072 if (radeon_connector->hpd.hpd != RADEON_HPD_NONE)
1073 disable |= 1 << radeon_connector->hpd.hpd;
1074 }
1075 radeon_irq_kms_disable_hpd(rdev, disable);
1076 }
1077
1078 #ifdef __NetBSD__
1079 /*
1080 * XXX Can't use bus_space here because this is all mapped through the
1081 * radeon_bo abstraction. Can't assume we're x86 because this is
1082 * AMD/ATI Radeon, not Intel.
1083 */
1084
1085 # define __iomem volatile
1086 # define readl fake_readl
1087
1088 static inline uint32_t
fake_readl(const void __iomem * ptr)1089 fake_readl(const void __iomem *ptr)
1090 {
1091 uint32_t v;
1092
1093 v = *(const uint32_t __iomem *)ptr;
1094 membar_consumer();
1095
1096 return v;
1097 }
1098 #endif
1099
1100 /*
1101 * R600 PCIE GART
1102 */
r600_pcie_gart_tlb_flush(struct radeon_device * rdev)1103 void r600_pcie_gart_tlb_flush(struct radeon_device *rdev)
1104 {
1105 unsigned i;
1106 u32 tmp;
1107
1108 /* flush hdp cache so updates hit vram */
1109 if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
1110 !(rdev->flags & RADEON_IS_AGP)) {
1111 void __iomem *ptr = rdev->gart.ptr;
1112
1113 /* r7xx hw bug. write to HDP_DEBUG1 followed by fb read
1114 * rather than write to HDP_REG_COHERENCY_FLUSH_CNTL
1115 * This seems to cause problems on some AGP cards. Just use the old
1116 * method for them.
1117 */
1118 WREG32(HDP_DEBUG1, 0);
1119 (void)readl(ptr);
1120 } else
1121 WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
1122
1123 WREG32(VM_CONTEXT0_INVALIDATION_LOW_ADDR, rdev->mc.gtt_start >> 12);
1124 WREG32(VM_CONTEXT0_INVALIDATION_HIGH_ADDR, (rdev->mc.gtt_end - 1) >> 12);
1125 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
1126 for (i = 0; i < rdev->usec_timeout; i++) {
1127 /* read MC_STATUS */
1128 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
1129 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
1130 if (tmp == 2) {
1131 pr_warn("[drm] r600 flush TLB failed\n");
1132 return;
1133 }
1134 if (tmp) {
1135 return;
1136 }
1137 udelay(1);
1138 }
1139 }
1140
1141 #ifdef __NetBSD__
1142 # undef __iomem
1143 # undef readl
1144 #endif
1145
r600_pcie_gart_init(struct radeon_device * rdev)1146 int r600_pcie_gart_init(struct radeon_device *rdev)
1147 {
1148 int r;
1149
1150 if (rdev->gart.robj) {
1151 WARN(1, "R600 PCIE GART already initialized\n");
1152 return 0;
1153 }
1154 /* Initialize common gart structure */
1155 r = radeon_gart_init(rdev);
1156 if (r)
1157 return r;
1158 rdev->gart.table_size = rdev->gart.num_gpu_pages * 8;
1159 return radeon_gart_table_vram_alloc(rdev);
1160 }
1161
r600_pcie_gart_enable(struct radeon_device * rdev)1162 static int r600_pcie_gart_enable(struct radeon_device *rdev)
1163 {
1164 u32 tmp;
1165 int r, i;
1166
1167 if (rdev->gart.robj == NULL) {
1168 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
1169 return -EINVAL;
1170 }
1171 r = radeon_gart_table_vram_pin(rdev);
1172 if (r)
1173 return r;
1174
1175 /* Setup L2 cache */
1176 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
1177 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1178 EFFECTIVE_L2_QUEUE_SIZE(7));
1179 WREG32(VM_L2_CNTL2, 0);
1180 WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
1181 /* Setup TLB control */
1182 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
1183 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1184 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
1185 ENABLE_WAIT_L2_QUERY;
1186 WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
1187 WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
1188 WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp | ENABLE_L1_STRICT_ORDERING);
1189 WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
1190 WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
1191 WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
1192 WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
1193 WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
1194 WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
1195 WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
1196 WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
1197 WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
1198 WREG32(MC_VM_L1_TLB_MCB_RD_UVD_CNTL, tmp);
1199 WREG32(MC_VM_L1_TLB_MCB_WR_UVD_CNTL, tmp);
1200 WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
1201 WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
1202 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1203 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1204 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1205 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1206 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
1207 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1208 (u32)(rdev->dummy_page.addr >> 12));
1209 for (i = 1; i < 7; i++)
1210 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
1211
1212 r600_pcie_gart_tlb_flush(rdev);
1213 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1214 (unsigned)(rdev->mc.gtt_size >> 20),
1215 (unsigned long long)rdev->gart.table_addr);
1216 rdev->gart.ready = true;
1217 return 0;
1218 }
1219
r600_pcie_gart_disable(struct radeon_device * rdev)1220 static void r600_pcie_gart_disable(struct radeon_device *rdev)
1221 {
1222 u32 tmp;
1223 int i;
1224
1225 /* Disable all tables */
1226 for (i = 0; i < 7; i++)
1227 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
1228
1229 /* Disable L2 cache */
1230 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
1231 EFFECTIVE_L2_QUEUE_SIZE(7));
1232 WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
1233 /* Setup L1 TLB control */
1234 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
1235 ENABLE_WAIT_L2_QUERY;
1236 WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
1237 WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
1238 WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
1239 WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
1240 WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
1241 WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
1242 WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
1243 WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
1244 WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp);
1245 WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp);
1246 WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
1247 WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
1248 WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp);
1249 WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
1250 WREG32(MC_VM_L1_TLB_MCB_RD_UVD_CNTL, tmp);
1251 WREG32(MC_VM_L1_TLB_MCB_WR_UVD_CNTL, tmp);
1252 radeon_gart_table_vram_unpin(rdev);
1253 }
1254
r600_pcie_gart_fini(struct radeon_device * rdev)1255 static void r600_pcie_gart_fini(struct radeon_device *rdev)
1256 {
1257 radeon_gart_fini(rdev);
1258 r600_pcie_gart_disable(rdev);
1259 radeon_gart_table_vram_free(rdev);
1260 }
1261
r600_agp_enable(struct radeon_device * rdev)1262 static void r600_agp_enable(struct radeon_device *rdev)
1263 {
1264 u32 tmp;
1265 int i;
1266
1267 /* Setup L2 cache */
1268 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
1269 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1270 EFFECTIVE_L2_QUEUE_SIZE(7));
1271 WREG32(VM_L2_CNTL2, 0);
1272 WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
1273 /* Setup TLB control */
1274 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
1275 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1276 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
1277 ENABLE_WAIT_L2_QUERY;
1278 WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
1279 WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
1280 WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp | ENABLE_L1_STRICT_ORDERING);
1281 WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
1282 WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
1283 WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
1284 WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
1285 WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
1286 WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
1287 WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
1288 WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
1289 WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
1290 WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
1291 WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
1292 for (i = 0; i < 7; i++)
1293 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
1294 }
1295
r600_mc_wait_for_idle(struct radeon_device * rdev)1296 int r600_mc_wait_for_idle(struct radeon_device *rdev)
1297 {
1298 unsigned i;
1299 u32 tmp;
1300
1301 for (i = 0; i < rdev->usec_timeout; i++) {
1302 /* read MC_STATUS */
1303 tmp = RREG32(R_000E50_SRBM_STATUS) & 0x3F00;
1304 if (!tmp)
1305 return 0;
1306 udelay(1);
1307 }
1308 return -1;
1309 }
1310
rs780_mc_rreg(struct radeon_device * rdev,uint32_t reg)1311 uint32_t rs780_mc_rreg(struct radeon_device *rdev, uint32_t reg)
1312 {
1313 unsigned long flags;
1314 uint32_t r;
1315
1316 spin_lock_irqsave(&rdev->mc_idx_lock, flags);
1317 WREG32(R_0028F8_MC_INDEX, S_0028F8_MC_IND_ADDR(reg));
1318 r = RREG32(R_0028FC_MC_DATA);
1319 WREG32(R_0028F8_MC_INDEX, ~C_0028F8_MC_IND_ADDR);
1320 spin_unlock_irqrestore(&rdev->mc_idx_lock, flags);
1321 return r;
1322 }
1323
rs780_mc_wreg(struct radeon_device * rdev,uint32_t reg,uint32_t v)1324 void rs780_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v)
1325 {
1326 unsigned long flags;
1327
1328 spin_lock_irqsave(&rdev->mc_idx_lock, flags);
1329 WREG32(R_0028F8_MC_INDEX, S_0028F8_MC_IND_ADDR(reg) |
1330 S_0028F8_MC_IND_WR_EN(1));
1331 WREG32(R_0028FC_MC_DATA, v);
1332 WREG32(R_0028F8_MC_INDEX, 0x7F);
1333 spin_unlock_irqrestore(&rdev->mc_idx_lock, flags);
1334 }
1335
r600_mc_program(struct radeon_device * rdev)1336 static void r600_mc_program(struct radeon_device *rdev)
1337 {
1338 struct rv515_mc_save save;
1339 u32 tmp;
1340 int i, j;
1341
1342 /* Initialize HDP */
1343 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1344 WREG32((0x2c14 + j), 0x00000000);
1345 WREG32((0x2c18 + j), 0x00000000);
1346 WREG32((0x2c1c + j), 0x00000000);
1347 WREG32((0x2c20 + j), 0x00000000);
1348 WREG32((0x2c24 + j), 0x00000000);
1349 }
1350 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
1351
1352 rv515_mc_stop(rdev, &save);
1353 if (r600_mc_wait_for_idle(rdev)) {
1354 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1355 }
1356 /* Lockout access through VGA aperture (doesn't exist before R600) */
1357 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
1358 /* Update configuration */
1359 if (rdev->flags & RADEON_IS_AGP) {
1360 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
1361 /* VRAM before AGP */
1362 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1363 rdev->mc.vram_start >> 12);
1364 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1365 rdev->mc.gtt_end >> 12);
1366 } else {
1367 /* VRAM after AGP */
1368 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1369 rdev->mc.gtt_start >> 12);
1370 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1371 rdev->mc.vram_end >> 12);
1372 }
1373 } else {
1374 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, rdev->mc.vram_start >> 12);
1375 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, rdev->mc.vram_end >> 12);
1376 }
1377 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
1378 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
1379 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
1380 WREG32(MC_VM_FB_LOCATION, tmp);
1381 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1382 WREG32(HDP_NONSURFACE_INFO, (2 << 7));
1383 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1384 if (rdev->flags & RADEON_IS_AGP) {
1385 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 22);
1386 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 22);
1387 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
1388 } else {
1389 WREG32(MC_VM_AGP_BASE, 0);
1390 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
1391 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
1392 }
1393 if (r600_mc_wait_for_idle(rdev)) {
1394 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1395 }
1396 rv515_mc_resume(rdev, &save);
1397 /* we need to own VRAM, so turn off the VGA renderer here
1398 * to stop it overwriting our objects */
1399 rv515_vga_render_disable(rdev);
1400 }
1401
1402 /**
1403 * r600_vram_gtt_location - try to find VRAM & GTT location
1404 * @rdev: radeon device structure holding all necessary informations
1405 * @mc: memory controller structure holding memory informations
1406 *
1407 * Function will place try to place VRAM at same place as in CPU (PCI)
1408 * address space as some GPU seems to have issue when we reprogram at
1409 * different address space.
1410 *
1411 * If there is not enough space to fit the unvisible VRAM after the
1412 * aperture then we limit the VRAM size to the aperture.
1413 *
1414 * If we are using AGP then place VRAM adjacent to AGP aperture are we need
1415 * them to be in one from GPU point of view so that we can program GPU to
1416 * catch access outside them (weird GPU policy see ??).
1417 *
1418 * This function will never fails, worst case are limiting VRAM or GTT.
1419 *
1420 * Note: GTT start, end, size should be initialized before calling this
1421 * function on AGP platform.
1422 */
r600_vram_gtt_location(struct radeon_device * rdev,struct radeon_mc * mc)1423 static void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
1424 {
1425 u64 size_bf, size_af;
1426
1427 if (mc->mc_vram_size > 0xE0000000) {
1428 /* leave room for at least 512M GTT */
1429 dev_warn(rdev->dev, "limiting VRAM\n");
1430 mc->real_vram_size = 0xE0000000;
1431 mc->mc_vram_size = 0xE0000000;
1432 }
1433 if (rdev->flags & RADEON_IS_AGP) {
1434 size_bf = mc->gtt_start;
1435 size_af = mc->mc_mask - mc->gtt_end;
1436 if (size_bf > size_af) {
1437 if (mc->mc_vram_size > size_bf) {
1438 dev_warn(rdev->dev, "limiting VRAM\n");
1439 mc->real_vram_size = size_bf;
1440 mc->mc_vram_size = size_bf;
1441 }
1442 mc->vram_start = mc->gtt_start - mc->mc_vram_size;
1443 } else {
1444 if (mc->mc_vram_size > size_af) {
1445 dev_warn(rdev->dev, "limiting VRAM\n");
1446 mc->real_vram_size = size_af;
1447 mc->mc_vram_size = size_af;
1448 }
1449 mc->vram_start = mc->gtt_end + 1;
1450 }
1451 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
1452 dev_info(rdev->dev, "VRAM: %"PRIu64"M 0x%08"PRIX64" - 0x%08"PRIX64" (%"PRIu64"M used)\n",
1453 mc->mc_vram_size >> 20, mc->vram_start,
1454 mc->vram_end, mc->real_vram_size >> 20);
1455 } else {
1456 u64 base = 0;
1457 if (rdev->flags & RADEON_IS_IGP) {
1458 base = RREG32(MC_VM_FB_LOCATION) & 0xFFFF;
1459 base <<= 24;
1460 }
1461 radeon_vram_location(rdev, &rdev->mc, base);
1462 rdev->mc.gtt_base_align = 0;
1463 radeon_gtt_location(rdev, mc);
1464 }
1465 }
1466
r600_mc_init(struct radeon_device * rdev)1467 static int r600_mc_init(struct radeon_device *rdev)
1468 {
1469 u32 tmp;
1470 int chansize, numchan;
1471 uint32_t h_addr, l_addr;
1472 unsigned long long k8_addr;
1473
1474 /* Get VRAM informations */
1475 rdev->mc.vram_is_ddr = true;
1476 tmp = RREG32(RAMCFG);
1477 if (tmp & CHANSIZE_OVERRIDE) {
1478 chansize = 16;
1479 } else if (tmp & CHANSIZE_MASK) {
1480 chansize = 64;
1481 } else {
1482 chansize = 32;
1483 }
1484 tmp = RREG32(CHMAP);
1485 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1486 case 0:
1487 default:
1488 numchan = 1;
1489 break;
1490 case 1:
1491 numchan = 2;
1492 break;
1493 case 2:
1494 numchan = 4;
1495 break;
1496 case 3:
1497 numchan = 8;
1498 break;
1499 }
1500 rdev->mc.vram_width = numchan * chansize;
1501 /* Could aper size report 0 ? */
1502 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
1503 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1504 /* Setup GPU memory space */
1505 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
1506 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
1507 rdev->mc.visible_vram_size = rdev->mc.aper_size;
1508 r600_vram_gtt_location(rdev, &rdev->mc);
1509
1510 if (rdev->flags & RADEON_IS_IGP) {
1511 rs690_pm_info(rdev);
1512 rdev->mc.igp_sideport_enabled = radeon_atombios_sideport_present(rdev);
1513
1514 if (rdev->family == CHIP_RS780 || rdev->family == CHIP_RS880) {
1515 /* Use K8 direct mapping for fast fb access. */
1516 rdev->fastfb_working = false;
1517 h_addr = G_000012_K8_ADDR_EXT(RREG32_MC(R_000012_MC_MISC_UMA_CNTL));
1518 l_addr = RREG32_MC(R_000011_K8_FB_LOCATION);
1519 k8_addr = ((unsigned long long)h_addr) << 32 | l_addr;
1520 #if defined(CONFIG_X86_32) && !defined(CONFIG_X86_PAE)
1521 if (k8_addr + rdev->mc.visible_vram_size < 0x100000000ULL)
1522 #endif
1523 {
1524 /* FastFB shall be used with UMA memory. Here it is simply disabled when sideport
1525 * memory is present.
1526 */
1527 if (rdev->mc.igp_sideport_enabled == false && radeon_fastfb == 1) {
1528 DRM_INFO("Direct mapping: aper base at 0x%llx, replaced by direct mapping base 0x%llx.\n",
1529 (unsigned long long)rdev->mc.aper_base, k8_addr);
1530 rdev->mc.aper_base = (resource_size_t)k8_addr;
1531 rdev->fastfb_working = true;
1532 }
1533 }
1534 }
1535 }
1536
1537 radeon_update_bandwidth_info(rdev);
1538 return 0;
1539 }
1540
r600_vram_scratch_init(struct radeon_device * rdev)1541 int r600_vram_scratch_init(struct radeon_device *rdev)
1542 {
1543 int r;
1544
1545 if (rdev->vram_scratch.robj == NULL) {
1546 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE,
1547 PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
1548 0, NULL, NULL, &rdev->vram_scratch.robj);
1549 if (r) {
1550 return r;
1551 }
1552 }
1553
1554 r = radeon_bo_reserve(rdev->vram_scratch.robj, false);
1555 if (unlikely(r != 0))
1556 return r;
1557 r = radeon_bo_pin(rdev->vram_scratch.robj,
1558 RADEON_GEM_DOMAIN_VRAM, &rdev->vram_scratch.gpu_addr);
1559 if (r) {
1560 radeon_bo_unreserve(rdev->vram_scratch.robj);
1561 return r;
1562 }
1563 r = radeon_bo_kmap(rdev->vram_scratch.robj,
1564 (void **)__UNVOLATILE(&rdev->vram_scratch.ptr));
1565 if (r)
1566 radeon_bo_unpin(rdev->vram_scratch.robj);
1567 radeon_bo_unreserve(rdev->vram_scratch.robj);
1568
1569 return r;
1570 }
1571
r600_vram_scratch_fini(struct radeon_device * rdev)1572 void r600_vram_scratch_fini(struct radeon_device *rdev)
1573 {
1574 int r;
1575
1576 if (rdev->vram_scratch.robj == NULL) {
1577 return;
1578 }
1579 r = radeon_bo_reserve(rdev->vram_scratch.robj, false);
1580 if (likely(r == 0)) {
1581 radeon_bo_kunmap(rdev->vram_scratch.robj);
1582 radeon_bo_unpin(rdev->vram_scratch.robj);
1583 radeon_bo_unreserve(rdev->vram_scratch.robj);
1584 }
1585 radeon_bo_unref(&rdev->vram_scratch.robj);
1586 }
1587
r600_set_bios_scratch_engine_hung(struct radeon_device * rdev,bool hung)1588 void r600_set_bios_scratch_engine_hung(struct radeon_device *rdev, bool hung)
1589 {
1590 u32 tmp = RREG32(R600_BIOS_3_SCRATCH);
1591
1592 if (hung)
1593 tmp |= ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1594 else
1595 tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1596
1597 WREG32(R600_BIOS_3_SCRATCH, tmp);
1598 }
1599
r600_print_gpu_status_regs(struct radeon_device * rdev)1600 static void r600_print_gpu_status_regs(struct radeon_device *rdev)
1601 {
1602 dev_info(rdev->dev, " R_008010_GRBM_STATUS = 0x%08X\n",
1603 RREG32(R_008010_GRBM_STATUS));
1604 dev_info(rdev->dev, " R_008014_GRBM_STATUS2 = 0x%08X\n",
1605 RREG32(R_008014_GRBM_STATUS2));
1606 dev_info(rdev->dev, " R_000E50_SRBM_STATUS = 0x%08X\n",
1607 RREG32(R_000E50_SRBM_STATUS));
1608 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
1609 RREG32(CP_STALLED_STAT1));
1610 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
1611 RREG32(CP_STALLED_STAT2));
1612 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
1613 RREG32(CP_BUSY_STAT));
1614 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
1615 RREG32(CP_STAT));
1616 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
1617 RREG32(DMA_STATUS_REG));
1618 }
1619
r600_is_display_hung(struct radeon_device * rdev)1620 static bool r600_is_display_hung(struct radeon_device *rdev)
1621 {
1622 u32 crtc_hung = 0;
1623 u32 crtc_status[2];
1624 u32 i, j, tmp;
1625
1626 for (i = 0; i < rdev->num_crtc; i++) {
1627 if (RREG32(AVIVO_D1CRTC_CONTROL + crtc_offsets[i]) & AVIVO_CRTC_EN) {
1628 crtc_status[i] = RREG32(AVIVO_D1CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
1629 crtc_hung |= (1 << i);
1630 }
1631 }
1632
1633 for (j = 0; j < 10; j++) {
1634 for (i = 0; i < rdev->num_crtc; i++) {
1635 if (crtc_hung & (1 << i)) {
1636 tmp = RREG32(AVIVO_D1CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
1637 if (tmp != crtc_status[i])
1638 crtc_hung &= ~(1 << i);
1639 }
1640 }
1641 if (crtc_hung == 0)
1642 return false;
1643 udelay(100);
1644 }
1645
1646 return true;
1647 }
1648
r600_gpu_check_soft_reset(struct radeon_device * rdev)1649 u32 r600_gpu_check_soft_reset(struct radeon_device *rdev)
1650 {
1651 u32 reset_mask = 0;
1652 u32 tmp;
1653
1654 /* GRBM_STATUS */
1655 tmp = RREG32(R_008010_GRBM_STATUS);
1656 if (rdev->family >= CHIP_RV770) {
1657 if (G_008010_PA_BUSY(tmp) | G_008010_SC_BUSY(tmp) |
1658 G_008010_SH_BUSY(tmp) | G_008010_SX_BUSY(tmp) |
1659 G_008010_TA_BUSY(tmp) | G_008010_VGT_BUSY(tmp) |
1660 G_008010_DB03_BUSY(tmp) | G_008010_CB03_BUSY(tmp) |
1661 G_008010_SPI03_BUSY(tmp) | G_008010_VGT_BUSY_NO_DMA(tmp))
1662 reset_mask |= RADEON_RESET_GFX;
1663 } else {
1664 if (G_008010_PA_BUSY(tmp) | G_008010_SC_BUSY(tmp) |
1665 G_008010_SH_BUSY(tmp) | G_008010_SX_BUSY(tmp) |
1666 G_008010_TA03_BUSY(tmp) | G_008010_VGT_BUSY(tmp) |
1667 G_008010_DB03_BUSY(tmp) | G_008010_CB03_BUSY(tmp) |
1668 G_008010_SPI03_BUSY(tmp) | G_008010_VGT_BUSY_NO_DMA(tmp))
1669 reset_mask |= RADEON_RESET_GFX;
1670 }
1671
1672 if (G_008010_CF_RQ_PENDING(tmp) | G_008010_PF_RQ_PENDING(tmp) |
1673 G_008010_CP_BUSY(tmp) | G_008010_CP_COHERENCY_BUSY(tmp))
1674 reset_mask |= RADEON_RESET_CP;
1675
1676 if (G_008010_GRBM_EE_BUSY(tmp))
1677 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1678
1679 /* DMA_STATUS_REG */
1680 tmp = RREG32(DMA_STATUS_REG);
1681 if (!(tmp & DMA_IDLE))
1682 reset_mask |= RADEON_RESET_DMA;
1683
1684 /* SRBM_STATUS */
1685 tmp = RREG32(R_000E50_SRBM_STATUS);
1686 if (G_000E50_RLC_RQ_PENDING(tmp) | G_000E50_RLC_BUSY(tmp))
1687 reset_mask |= RADEON_RESET_RLC;
1688
1689 if (G_000E50_IH_BUSY(tmp))
1690 reset_mask |= RADEON_RESET_IH;
1691
1692 if (G_000E50_SEM_BUSY(tmp))
1693 reset_mask |= RADEON_RESET_SEM;
1694
1695 if (G_000E50_GRBM_RQ_PENDING(tmp))
1696 reset_mask |= RADEON_RESET_GRBM;
1697
1698 if (G_000E50_VMC_BUSY(tmp))
1699 reset_mask |= RADEON_RESET_VMC;
1700
1701 if (G_000E50_MCB_BUSY(tmp) | G_000E50_MCDZ_BUSY(tmp) |
1702 G_000E50_MCDY_BUSY(tmp) | G_000E50_MCDX_BUSY(tmp) |
1703 G_000E50_MCDW_BUSY(tmp))
1704 reset_mask |= RADEON_RESET_MC;
1705
1706 if (r600_is_display_hung(rdev))
1707 reset_mask |= RADEON_RESET_DISPLAY;
1708
1709 /* Skip MC reset as it's mostly likely not hung, just busy */
1710 if (reset_mask & RADEON_RESET_MC) {
1711 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1712 reset_mask &= ~RADEON_RESET_MC;
1713 }
1714
1715 return reset_mask;
1716 }
1717
r600_gpu_soft_reset(struct radeon_device * rdev,u32 reset_mask)1718 static void r600_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1719 {
1720 struct rv515_mc_save save;
1721 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1722 u32 tmp;
1723
1724 if (reset_mask == 0)
1725 return;
1726
1727 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1728
1729 r600_print_gpu_status_regs(rdev);
1730
1731 /* Disable CP parsing/prefetching */
1732 if (rdev->family >= CHIP_RV770)
1733 WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1) | S_0086D8_CP_PFP_HALT(1));
1734 else
1735 WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1736
1737 /* disable the RLC */
1738 WREG32(RLC_CNTL, 0);
1739
1740 if (reset_mask & RADEON_RESET_DMA) {
1741 /* Disable DMA */
1742 tmp = RREG32(DMA_RB_CNTL);
1743 tmp &= ~DMA_RB_ENABLE;
1744 WREG32(DMA_RB_CNTL, tmp);
1745 }
1746
1747 mdelay(50);
1748
1749 rv515_mc_stop(rdev, &save);
1750 if (r600_mc_wait_for_idle(rdev)) {
1751 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1752 }
1753
1754 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1755 if (rdev->family >= CHIP_RV770)
1756 grbm_soft_reset |= S_008020_SOFT_RESET_DB(1) |
1757 S_008020_SOFT_RESET_CB(1) |
1758 S_008020_SOFT_RESET_PA(1) |
1759 S_008020_SOFT_RESET_SC(1) |
1760 S_008020_SOFT_RESET_SPI(1) |
1761 S_008020_SOFT_RESET_SX(1) |
1762 S_008020_SOFT_RESET_SH(1) |
1763 S_008020_SOFT_RESET_TC(1) |
1764 S_008020_SOFT_RESET_TA(1) |
1765 S_008020_SOFT_RESET_VC(1) |
1766 S_008020_SOFT_RESET_VGT(1);
1767 else
1768 grbm_soft_reset |= S_008020_SOFT_RESET_CR(1) |
1769 S_008020_SOFT_RESET_DB(1) |
1770 S_008020_SOFT_RESET_CB(1) |
1771 S_008020_SOFT_RESET_PA(1) |
1772 S_008020_SOFT_RESET_SC(1) |
1773 S_008020_SOFT_RESET_SMX(1) |
1774 S_008020_SOFT_RESET_SPI(1) |
1775 S_008020_SOFT_RESET_SX(1) |
1776 S_008020_SOFT_RESET_SH(1) |
1777 S_008020_SOFT_RESET_TC(1) |
1778 S_008020_SOFT_RESET_TA(1) |
1779 S_008020_SOFT_RESET_VC(1) |
1780 S_008020_SOFT_RESET_VGT(1);
1781 }
1782
1783 if (reset_mask & RADEON_RESET_CP) {
1784 grbm_soft_reset |= S_008020_SOFT_RESET_CP(1) |
1785 S_008020_SOFT_RESET_VGT(1);
1786
1787 srbm_soft_reset |= S_000E60_SOFT_RESET_GRBM(1);
1788 }
1789
1790 if (reset_mask & RADEON_RESET_DMA) {
1791 if (rdev->family >= CHIP_RV770)
1792 srbm_soft_reset |= RV770_SOFT_RESET_DMA;
1793 else
1794 srbm_soft_reset |= SOFT_RESET_DMA;
1795 }
1796
1797 if (reset_mask & RADEON_RESET_RLC)
1798 srbm_soft_reset |= S_000E60_SOFT_RESET_RLC(1);
1799
1800 if (reset_mask & RADEON_RESET_SEM)
1801 srbm_soft_reset |= S_000E60_SOFT_RESET_SEM(1);
1802
1803 if (reset_mask & RADEON_RESET_IH)
1804 srbm_soft_reset |= S_000E60_SOFT_RESET_IH(1);
1805
1806 if (reset_mask & RADEON_RESET_GRBM)
1807 srbm_soft_reset |= S_000E60_SOFT_RESET_GRBM(1);
1808
1809 if (!(rdev->flags & RADEON_IS_IGP)) {
1810 if (reset_mask & RADEON_RESET_MC)
1811 srbm_soft_reset |= S_000E60_SOFT_RESET_MC(1);
1812 }
1813
1814 if (reset_mask & RADEON_RESET_VMC)
1815 srbm_soft_reset |= S_000E60_SOFT_RESET_VMC(1);
1816
1817 if (grbm_soft_reset) {
1818 tmp = RREG32(R_008020_GRBM_SOFT_RESET);
1819 tmp |= grbm_soft_reset;
1820 dev_info(rdev->dev, "R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
1821 WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1822 tmp = RREG32(R_008020_GRBM_SOFT_RESET);
1823
1824 udelay(50);
1825
1826 tmp &= ~grbm_soft_reset;
1827 WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1828 tmp = RREG32(R_008020_GRBM_SOFT_RESET);
1829 }
1830
1831 if (srbm_soft_reset) {
1832 tmp = RREG32(SRBM_SOFT_RESET);
1833 tmp |= srbm_soft_reset;
1834 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1835 WREG32(SRBM_SOFT_RESET, tmp);
1836 tmp = RREG32(SRBM_SOFT_RESET);
1837
1838 udelay(50);
1839
1840 tmp &= ~srbm_soft_reset;
1841 WREG32(SRBM_SOFT_RESET, tmp);
1842 tmp = RREG32(SRBM_SOFT_RESET);
1843 }
1844
1845 /* Wait a little for things to settle down */
1846 mdelay(1);
1847
1848 rv515_mc_resume(rdev, &save);
1849 udelay(50);
1850
1851 r600_print_gpu_status_regs(rdev);
1852 }
1853
r600_gpu_pci_config_reset(struct radeon_device * rdev)1854 static void r600_gpu_pci_config_reset(struct radeon_device *rdev)
1855 {
1856 struct rv515_mc_save save;
1857 u32 tmp, i;
1858
1859 dev_info(rdev->dev, "GPU pci config reset\n");
1860
1861 /* disable dpm? */
1862
1863 /* Disable CP parsing/prefetching */
1864 if (rdev->family >= CHIP_RV770)
1865 WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1) | S_0086D8_CP_PFP_HALT(1));
1866 else
1867 WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1868
1869 /* disable the RLC */
1870 WREG32(RLC_CNTL, 0);
1871
1872 /* Disable DMA */
1873 tmp = RREG32(DMA_RB_CNTL);
1874 tmp &= ~DMA_RB_ENABLE;
1875 WREG32(DMA_RB_CNTL, tmp);
1876
1877 mdelay(50);
1878
1879 /* set mclk/sclk to bypass */
1880 if (rdev->family >= CHIP_RV770)
1881 rv770_set_clk_bypass_mode(rdev);
1882 /* disable BM */
1883 pci_clear_master(rdev->pdev);
1884 /* disable mem access */
1885 rv515_mc_stop(rdev, &save);
1886 if (r600_mc_wait_for_idle(rdev)) {
1887 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1888 }
1889
1890 /* BIF reset workaround. Not sure if this is needed on 6xx */
1891 tmp = RREG32(BUS_CNTL);
1892 tmp |= VGA_COHE_SPEC_TIMER_DIS;
1893 WREG32(BUS_CNTL, tmp);
1894
1895 tmp = RREG32(BIF_SCRATCH0);
1896
1897 /* reset */
1898 radeon_pci_config_reset(rdev);
1899 mdelay(1);
1900
1901 /* BIF reset workaround. Not sure if this is needed on 6xx */
1902 tmp = SOFT_RESET_BIF;
1903 WREG32(SRBM_SOFT_RESET, tmp);
1904 mdelay(1);
1905 WREG32(SRBM_SOFT_RESET, 0);
1906
1907 /* wait for asic to come out of reset */
1908 for (i = 0; i < rdev->usec_timeout; i++) {
1909 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
1910 break;
1911 udelay(1);
1912 }
1913 }
1914
r600_asic_reset(struct radeon_device * rdev,bool hard)1915 int r600_asic_reset(struct radeon_device *rdev, bool hard)
1916 {
1917 u32 reset_mask;
1918
1919 if (hard) {
1920 r600_gpu_pci_config_reset(rdev);
1921 return 0;
1922 }
1923
1924 reset_mask = r600_gpu_check_soft_reset(rdev);
1925
1926 if (reset_mask)
1927 r600_set_bios_scratch_engine_hung(rdev, true);
1928
1929 /* try soft reset */
1930 r600_gpu_soft_reset(rdev, reset_mask);
1931
1932 reset_mask = r600_gpu_check_soft_reset(rdev);
1933
1934 /* try pci config reset */
1935 if (reset_mask && radeon_hard_reset)
1936 r600_gpu_pci_config_reset(rdev);
1937
1938 reset_mask = r600_gpu_check_soft_reset(rdev);
1939
1940 if (!reset_mask)
1941 r600_set_bios_scratch_engine_hung(rdev, false);
1942
1943 return 0;
1944 }
1945
1946 /**
1947 * r600_gfx_is_lockup - Check if the GFX engine is locked up
1948 *
1949 * @rdev: radeon_device pointer
1950 * @ring: radeon_ring structure holding ring information
1951 *
1952 * Check if the GFX engine is locked up.
1953 * Returns true if the engine appears to be locked up, false if not.
1954 */
r600_gfx_is_lockup(struct radeon_device * rdev,struct radeon_ring * ring)1955 bool r600_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1956 {
1957 u32 reset_mask = r600_gpu_check_soft_reset(rdev);
1958
1959 if (!(reset_mask & (RADEON_RESET_GFX |
1960 RADEON_RESET_COMPUTE |
1961 RADEON_RESET_CP))) {
1962 radeon_ring_lockup_update(rdev, ring);
1963 return false;
1964 }
1965 return radeon_ring_test_lockup(rdev, ring);
1966 }
1967
r6xx_remap_render_backend(struct radeon_device * rdev,u32 tiling_pipe_num,u32 max_rb_num,u32 total_max_rb_num,u32 disabled_rb_mask)1968 u32 r6xx_remap_render_backend(struct radeon_device *rdev,
1969 u32 tiling_pipe_num,
1970 u32 max_rb_num,
1971 u32 total_max_rb_num,
1972 u32 disabled_rb_mask)
1973 {
1974 u32 rendering_pipe_num, rb_num_width, req_rb_num;
1975 u32 pipe_rb_ratio, pipe_rb_remain, tmp;
1976 u32 data = 0, mask = 1 << (max_rb_num - 1);
1977 unsigned i, j;
1978
1979 /* mask out the RBs that don't exist on that asic */
1980 tmp = disabled_rb_mask | ((0xff << max_rb_num) & 0xff);
1981 /* make sure at least one RB is available */
1982 if ((tmp & 0xff) != 0xff)
1983 disabled_rb_mask = tmp;
1984
1985 rendering_pipe_num = 1 << tiling_pipe_num;
1986 req_rb_num = total_max_rb_num - r600_count_pipe_bits(disabled_rb_mask);
1987 BUG_ON(rendering_pipe_num < req_rb_num);
1988
1989 pipe_rb_ratio = rendering_pipe_num / req_rb_num;
1990 pipe_rb_remain = rendering_pipe_num - pipe_rb_ratio * req_rb_num;
1991
1992 if (rdev->family <= CHIP_RV740) {
1993 /* r6xx/r7xx */
1994 rb_num_width = 2;
1995 } else {
1996 /* eg+ */
1997 rb_num_width = 4;
1998 }
1999
2000 for (i = 0; i < max_rb_num; i++) {
2001 if (!(mask & disabled_rb_mask)) {
2002 for (j = 0; j < pipe_rb_ratio; j++) {
2003 data <<= rb_num_width;
2004 data |= max_rb_num - i - 1;
2005 }
2006 if (pipe_rb_remain) {
2007 data <<= rb_num_width;
2008 data |= max_rb_num - i - 1;
2009 pipe_rb_remain--;
2010 }
2011 }
2012 mask >>= 1;
2013 }
2014
2015 return data;
2016 }
2017
r600_count_pipe_bits(uint32_t val)2018 int r600_count_pipe_bits(uint32_t val)
2019 {
2020 return hweight32(val);
2021 }
2022
r600_gpu_init(struct radeon_device * rdev)2023 static void r600_gpu_init(struct radeon_device *rdev)
2024 {
2025 u32 tiling_config;
2026 u32 ramcfg;
2027 u32 cc_gc_shader_pipe_config;
2028 u32 tmp;
2029 int i, j;
2030 u32 sq_config;
2031 u32 sq_gpr_resource_mgmt_1 = 0;
2032 u32 sq_gpr_resource_mgmt_2 = 0;
2033 u32 sq_thread_resource_mgmt = 0;
2034 u32 sq_stack_resource_mgmt_1 = 0;
2035 u32 sq_stack_resource_mgmt_2 = 0;
2036 u32 disabled_rb_mask;
2037
2038 rdev->config.r600.tiling_group_size = 256;
2039 switch (rdev->family) {
2040 case CHIP_R600:
2041 rdev->config.r600.max_pipes = 4;
2042 rdev->config.r600.max_tile_pipes = 8;
2043 rdev->config.r600.max_simds = 4;
2044 rdev->config.r600.max_backends = 4;
2045 rdev->config.r600.max_gprs = 256;
2046 rdev->config.r600.max_threads = 192;
2047 rdev->config.r600.max_stack_entries = 256;
2048 rdev->config.r600.max_hw_contexts = 8;
2049 rdev->config.r600.max_gs_threads = 16;
2050 rdev->config.r600.sx_max_export_size = 128;
2051 rdev->config.r600.sx_max_export_pos_size = 16;
2052 rdev->config.r600.sx_max_export_smx_size = 128;
2053 rdev->config.r600.sq_num_cf_insts = 2;
2054 break;
2055 case CHIP_RV630:
2056 case CHIP_RV635:
2057 rdev->config.r600.max_pipes = 2;
2058 rdev->config.r600.max_tile_pipes = 2;
2059 rdev->config.r600.max_simds = 3;
2060 rdev->config.r600.max_backends = 1;
2061 rdev->config.r600.max_gprs = 128;
2062 rdev->config.r600.max_threads = 192;
2063 rdev->config.r600.max_stack_entries = 128;
2064 rdev->config.r600.max_hw_contexts = 8;
2065 rdev->config.r600.max_gs_threads = 4;
2066 rdev->config.r600.sx_max_export_size = 128;
2067 rdev->config.r600.sx_max_export_pos_size = 16;
2068 rdev->config.r600.sx_max_export_smx_size = 128;
2069 rdev->config.r600.sq_num_cf_insts = 2;
2070 break;
2071 case CHIP_RV610:
2072 case CHIP_RV620:
2073 case CHIP_RS780:
2074 case CHIP_RS880:
2075 rdev->config.r600.max_pipes = 1;
2076 rdev->config.r600.max_tile_pipes = 1;
2077 rdev->config.r600.max_simds = 2;
2078 rdev->config.r600.max_backends = 1;
2079 rdev->config.r600.max_gprs = 128;
2080 rdev->config.r600.max_threads = 192;
2081 rdev->config.r600.max_stack_entries = 128;
2082 rdev->config.r600.max_hw_contexts = 4;
2083 rdev->config.r600.max_gs_threads = 4;
2084 rdev->config.r600.sx_max_export_size = 128;
2085 rdev->config.r600.sx_max_export_pos_size = 16;
2086 rdev->config.r600.sx_max_export_smx_size = 128;
2087 rdev->config.r600.sq_num_cf_insts = 1;
2088 break;
2089 case CHIP_RV670:
2090 rdev->config.r600.max_pipes = 4;
2091 rdev->config.r600.max_tile_pipes = 4;
2092 rdev->config.r600.max_simds = 4;
2093 rdev->config.r600.max_backends = 4;
2094 rdev->config.r600.max_gprs = 192;
2095 rdev->config.r600.max_threads = 192;
2096 rdev->config.r600.max_stack_entries = 256;
2097 rdev->config.r600.max_hw_contexts = 8;
2098 rdev->config.r600.max_gs_threads = 16;
2099 rdev->config.r600.sx_max_export_size = 128;
2100 rdev->config.r600.sx_max_export_pos_size = 16;
2101 rdev->config.r600.sx_max_export_smx_size = 128;
2102 rdev->config.r600.sq_num_cf_insts = 2;
2103 break;
2104 default:
2105 break;
2106 }
2107
2108 /* Initialize HDP */
2109 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2110 WREG32((0x2c14 + j), 0x00000000);
2111 WREG32((0x2c18 + j), 0x00000000);
2112 WREG32((0x2c1c + j), 0x00000000);
2113 WREG32((0x2c20 + j), 0x00000000);
2114 WREG32((0x2c24 + j), 0x00000000);
2115 }
2116
2117 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2118
2119 /* Setup tiling */
2120 tiling_config = 0;
2121 ramcfg = RREG32(RAMCFG);
2122 switch (rdev->config.r600.max_tile_pipes) {
2123 case 1:
2124 tiling_config |= PIPE_TILING(0);
2125 break;
2126 case 2:
2127 tiling_config |= PIPE_TILING(1);
2128 break;
2129 case 4:
2130 tiling_config |= PIPE_TILING(2);
2131 break;
2132 case 8:
2133 tiling_config |= PIPE_TILING(3);
2134 break;
2135 default:
2136 break;
2137 }
2138 rdev->config.r600.tiling_npipes = rdev->config.r600.max_tile_pipes;
2139 rdev->config.r600.tiling_nbanks = 4 << ((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
2140 tiling_config |= BANK_TILING((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
2141 tiling_config |= GROUP_SIZE((ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
2142
2143 tmp = (ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
2144 if (tmp > 3) {
2145 tiling_config |= ROW_TILING(3);
2146 tiling_config |= SAMPLE_SPLIT(3);
2147 } else {
2148 tiling_config |= ROW_TILING(tmp);
2149 tiling_config |= SAMPLE_SPLIT(tmp);
2150 }
2151 tiling_config |= BANK_SWAPS(1);
2152
2153 cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0x00ffff00;
2154 tmp = rdev->config.r600.max_simds -
2155 r600_count_pipe_bits((cc_gc_shader_pipe_config >> 16) & R6XX_MAX_SIMDS_MASK);
2156 rdev->config.r600.active_simds = tmp;
2157
2158 disabled_rb_mask = (RREG32(CC_RB_BACKEND_DISABLE) >> 16) & R6XX_MAX_BACKENDS_MASK;
2159 tmp = 0;
2160 for (i = 0; i < rdev->config.r600.max_backends; i++)
2161 tmp |= (1 << i);
2162 /* if all the backends are disabled, fix it up here */
2163 if ((disabled_rb_mask & tmp) == tmp) {
2164 for (i = 0; i < rdev->config.r600.max_backends; i++)
2165 disabled_rb_mask &= ~(1 << i);
2166 }
2167 tmp = (tiling_config & PIPE_TILING__MASK) >> PIPE_TILING__SHIFT;
2168 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.r600.max_backends,
2169 R6XX_MAX_BACKENDS, disabled_rb_mask);
2170 tiling_config |= tmp << 16;
2171 rdev->config.r600.backend_map = tmp;
2172
2173 rdev->config.r600.tile_config = tiling_config;
2174 WREG32(GB_TILING_CONFIG, tiling_config);
2175 WREG32(DCP_TILING_CONFIG, tiling_config & 0xffff);
2176 WREG32(HDP_TILING_CONFIG, tiling_config & 0xffff);
2177 WREG32(DMA_TILING_CONFIG, tiling_config & 0xffff);
2178
2179 tmp = R6XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
2180 WREG32(VGT_OUT_DEALLOC_CNTL, (tmp * 4) & DEALLOC_DIST_MASK);
2181 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((tmp * 4) - 2) & VTX_REUSE_DEPTH_MASK);
2182
2183 /* Setup some CP states */
2184 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) | ROQ_IB2_START(0x2b)));
2185 WREG32(CP_MEQ_THRESHOLDS, (MEQ_END(0x40) | ROQ_END(0x40)));
2186
2187 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO | SYNC_GRADIENT |
2188 SYNC_WALKER | SYNC_ALIGNER));
2189 /* Setup various GPU states */
2190 if (rdev->family == CHIP_RV670)
2191 WREG32(ARB_GDEC_RD_CNTL, 0x00000021);
2192
2193 tmp = RREG32(SX_DEBUG_1);
2194 tmp |= SMX_EVENT_RELEASE;
2195 if ((rdev->family > CHIP_R600))
2196 tmp |= ENABLE_NEW_SMX_ADDRESS;
2197 WREG32(SX_DEBUG_1, tmp);
2198
2199 if (((rdev->family) == CHIP_R600) ||
2200 ((rdev->family) == CHIP_RV630) ||
2201 ((rdev->family) == CHIP_RV610) ||
2202 ((rdev->family) == CHIP_RV620) ||
2203 ((rdev->family) == CHIP_RS780) ||
2204 ((rdev->family) == CHIP_RS880)) {
2205 WREG32(DB_DEBUG, PREZ_MUST_WAIT_FOR_POSTZ_DONE);
2206 } else {
2207 WREG32(DB_DEBUG, 0);
2208 }
2209 WREG32(DB_WATERMARKS, (DEPTH_FREE(4) | DEPTH_CACHELINE_FREE(16) |
2210 DEPTH_FLUSH(16) | DEPTH_PENDING_FREE(4)));
2211
2212 WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
2213 WREG32(VGT_NUM_INSTANCES, 0);
2214
2215 WREG32(SPI_CONFIG_CNTL, GPR_WRITE_PRIORITY(0));
2216 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(0));
2217
2218 tmp = RREG32(SQ_MS_FIFO_SIZES);
2219 if (((rdev->family) == CHIP_RV610) ||
2220 ((rdev->family) == CHIP_RV620) ||
2221 ((rdev->family) == CHIP_RS780) ||
2222 ((rdev->family) == CHIP_RS880)) {
2223 tmp = (CACHE_FIFO_SIZE(0xa) |
2224 FETCH_FIFO_HIWATER(0xa) |
2225 DONE_FIFO_HIWATER(0xe0) |
2226 ALU_UPDATE_FIFO_HIWATER(0x8));
2227 } else if (((rdev->family) == CHIP_R600) ||
2228 ((rdev->family) == CHIP_RV630)) {
2229 tmp &= ~DONE_FIFO_HIWATER(0xff);
2230 tmp |= DONE_FIFO_HIWATER(0x4);
2231 }
2232 WREG32(SQ_MS_FIFO_SIZES, tmp);
2233
2234 /* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
2235 * should be adjusted as needed by the 2D/3D drivers. This just sets default values
2236 */
2237 sq_config = RREG32(SQ_CONFIG);
2238 sq_config &= ~(PS_PRIO(3) |
2239 VS_PRIO(3) |
2240 GS_PRIO(3) |
2241 ES_PRIO(3U));
2242 sq_config |= (DX9_CONSTS |
2243 VC_ENABLE |
2244 PS_PRIO(0) |
2245 VS_PRIO(1) |
2246 GS_PRIO(2) |
2247 ES_PRIO(3U));
2248
2249 if ((rdev->family) == CHIP_R600) {
2250 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(124) |
2251 NUM_VS_GPRS(124) |
2252 NUM_CLAUSE_TEMP_GPRS(4));
2253 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(0) |
2254 NUM_ES_GPRS(0));
2255 sq_thread_resource_mgmt = (NUM_PS_THREADS(136) |
2256 NUM_VS_THREADS(48) |
2257 NUM_GS_THREADS(4) |
2258 NUM_ES_THREADS(4));
2259 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(128) |
2260 NUM_VS_STACK_ENTRIES(128));
2261 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(0) |
2262 NUM_ES_STACK_ENTRIES(0));
2263 } else if (((rdev->family) == CHIP_RV610) ||
2264 ((rdev->family) == CHIP_RV620) ||
2265 ((rdev->family) == CHIP_RS780) ||
2266 ((rdev->family) == CHIP_RS880)) {
2267 /* no vertex cache */
2268 sq_config &= ~VC_ENABLE;
2269
2270 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
2271 NUM_VS_GPRS(44) |
2272 NUM_CLAUSE_TEMP_GPRS(2));
2273 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) |
2274 NUM_ES_GPRS(17));
2275 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
2276 NUM_VS_THREADS(78) |
2277 NUM_GS_THREADS(4) |
2278 NUM_ES_THREADS(31));
2279 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) |
2280 NUM_VS_STACK_ENTRIES(40));
2281 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) |
2282 NUM_ES_STACK_ENTRIES(16));
2283 } else if (((rdev->family) == CHIP_RV630) ||
2284 ((rdev->family) == CHIP_RV635)) {
2285 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
2286 NUM_VS_GPRS(44) |
2287 NUM_CLAUSE_TEMP_GPRS(2));
2288 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(18) |
2289 NUM_ES_GPRS(18));
2290 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
2291 NUM_VS_THREADS(78) |
2292 NUM_GS_THREADS(4) |
2293 NUM_ES_THREADS(31));
2294 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) |
2295 NUM_VS_STACK_ENTRIES(40));
2296 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) |
2297 NUM_ES_STACK_ENTRIES(16));
2298 } else if ((rdev->family) == CHIP_RV670) {
2299 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
2300 NUM_VS_GPRS(44) |
2301 NUM_CLAUSE_TEMP_GPRS(2));
2302 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) |
2303 NUM_ES_GPRS(17));
2304 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
2305 NUM_VS_THREADS(78) |
2306 NUM_GS_THREADS(4) |
2307 NUM_ES_THREADS(31));
2308 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(64) |
2309 NUM_VS_STACK_ENTRIES(64));
2310 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(64) |
2311 NUM_ES_STACK_ENTRIES(64));
2312 }
2313
2314 WREG32(SQ_CONFIG, sq_config);
2315 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
2316 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
2317 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
2318 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
2319 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
2320
2321 if (((rdev->family) == CHIP_RV610) ||
2322 ((rdev->family) == CHIP_RV620) ||
2323 ((rdev->family) == CHIP_RS780) ||
2324 ((rdev->family) == CHIP_RS880)) {
2325 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(TC_ONLY));
2326 } else {
2327 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC));
2328 }
2329
2330 /* More default values. 2D/3D driver should adjust as needed */
2331 WREG32(PA_SC_AA_SAMPLE_LOCS_2S, (S0_X(0xc) | S0_Y(0x4) |
2332 S1_X(0x4) | S1_Y(0xc)));
2333 WREG32(PA_SC_AA_SAMPLE_LOCS_4S, (S0_X(0xe) | S0_Y(0xe) |
2334 S1_X(0x2) | S1_Y(0x2) |
2335 S2_X(0xa) | S2_Y(0x6) |
2336 S3_X(0x6) | S3_Y(0xaU)));
2337 WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD0, (S0_X(0xe) | S0_Y(0xb) |
2338 S1_X(0x4) | S1_Y(0xc) |
2339 S2_X(0x1) | S2_Y(0x6) |
2340 S3_X(0xa) | S3_Y(0xeU)));
2341 WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD1, (S4_X(0x6) | S4_Y(0x1) |
2342 S5_X(0x0) | S5_Y(0x0) |
2343 S6_X(0xb) | S6_Y(0x4) |
2344 S7_X(0x7) | S7_Y(0x8U)));
2345
2346 WREG32(VGT_STRMOUT_EN, 0);
2347 tmp = rdev->config.r600.max_pipes * 16;
2348 switch (rdev->family) {
2349 case CHIP_RV610:
2350 case CHIP_RV620:
2351 case CHIP_RS780:
2352 case CHIP_RS880:
2353 tmp += 32;
2354 break;
2355 case CHIP_RV670:
2356 tmp += 128;
2357 break;
2358 default:
2359 break;
2360 }
2361 if (tmp > 256) {
2362 tmp = 256;
2363 }
2364 WREG32(VGT_ES_PER_GS, 128);
2365 WREG32(VGT_GS_PER_ES, tmp);
2366 WREG32(VGT_GS_PER_VS, 2);
2367 WREG32(VGT_GS_VERTEX_REUSE, 16);
2368
2369 /* more default values. 2D/3D driver should adjust as needed */
2370 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2371 WREG32(VGT_STRMOUT_EN, 0);
2372 WREG32(SX_MISC, 0);
2373 WREG32(PA_SC_MODE_CNTL, 0);
2374 WREG32(PA_SC_AA_CONFIG, 0);
2375 WREG32(PA_SC_LINE_STIPPLE, 0);
2376 WREG32(SPI_INPUT_Z, 0);
2377 WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
2378 WREG32(CB_COLOR7_FRAG, 0);
2379
2380 /* Clear render buffer base addresses */
2381 WREG32(CB_COLOR0_BASE, 0);
2382 WREG32(CB_COLOR1_BASE, 0);
2383 WREG32(CB_COLOR2_BASE, 0);
2384 WREG32(CB_COLOR3_BASE, 0);
2385 WREG32(CB_COLOR4_BASE, 0);
2386 WREG32(CB_COLOR5_BASE, 0);
2387 WREG32(CB_COLOR6_BASE, 0);
2388 WREG32(CB_COLOR7_BASE, 0);
2389 WREG32(CB_COLOR7_FRAG, 0);
2390
2391 switch (rdev->family) {
2392 case CHIP_RV610:
2393 case CHIP_RV620:
2394 case CHIP_RS780:
2395 case CHIP_RS880:
2396 tmp = TC_L2_SIZE(8);
2397 break;
2398 case CHIP_RV630:
2399 case CHIP_RV635:
2400 tmp = TC_L2_SIZE(4);
2401 break;
2402 case CHIP_R600:
2403 tmp = TC_L2_SIZE(0) | L2_DISABLE_LATE_HIT;
2404 break;
2405 default:
2406 tmp = TC_L2_SIZE(0);
2407 break;
2408 }
2409 WREG32(TC_CNTL, tmp);
2410
2411 tmp = RREG32(HDP_HOST_PATH_CNTL);
2412 WREG32(HDP_HOST_PATH_CNTL, tmp);
2413
2414 tmp = RREG32(ARB_POP);
2415 tmp |= ENABLE_TC128;
2416 WREG32(ARB_POP, tmp);
2417
2418 WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
2419 WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA |
2420 NUM_CLIP_SEQ(3)));
2421 WREG32(PA_SC_ENHANCE, FORCE_EOV_MAX_CLK_CNT(4095));
2422 WREG32(VC_ENHANCE, 0);
2423 }
2424
2425
2426 /*
2427 * Indirect registers accessor
2428 */
r600_pciep_rreg(struct radeon_device * rdev,u32 reg)2429 u32 r600_pciep_rreg(struct radeon_device *rdev, u32 reg)
2430 {
2431 unsigned long flags;
2432 u32 r;
2433
2434 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
2435 WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
2436 (void)RREG32(PCIE_PORT_INDEX);
2437 r = RREG32(PCIE_PORT_DATA);
2438 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
2439 return r;
2440 }
2441
r600_pciep_wreg(struct radeon_device * rdev,u32 reg,u32 v)2442 void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
2443 {
2444 unsigned long flags;
2445
2446 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
2447 WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
2448 (void)RREG32(PCIE_PORT_INDEX);
2449 WREG32(PCIE_PORT_DATA, (v));
2450 (void)RREG32(PCIE_PORT_DATA);
2451 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
2452 }
2453
2454 /*
2455 * CP & Ring
2456 */
r600_cp_stop(struct radeon_device * rdev)2457 void r600_cp_stop(struct radeon_device *rdev)
2458 {
2459 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
2460 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
2461 WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
2462 WREG32(SCRATCH_UMSK, 0);
2463 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
2464 }
2465
r600_init_microcode(struct radeon_device * rdev)2466 int r600_init_microcode(struct radeon_device *rdev)
2467 {
2468 const char *chip_name;
2469 const char *rlc_chip_name;
2470 const char *smc_chip_name = "RV770";
2471 size_t pfp_req_size, me_req_size, rlc_req_size, smc_req_size = 0;
2472 char fw_name[30];
2473 int err;
2474
2475 DRM_DEBUG("\n");
2476
2477 switch (rdev->family) {
2478 case CHIP_R600:
2479 chip_name = "R600";
2480 rlc_chip_name = "R600";
2481 break;
2482 case CHIP_RV610:
2483 chip_name = "RV610";
2484 rlc_chip_name = "R600";
2485 break;
2486 case CHIP_RV630:
2487 chip_name = "RV630";
2488 rlc_chip_name = "R600";
2489 break;
2490 case CHIP_RV620:
2491 chip_name = "RV620";
2492 rlc_chip_name = "R600";
2493 break;
2494 case CHIP_RV635:
2495 chip_name = "RV635";
2496 rlc_chip_name = "R600";
2497 break;
2498 case CHIP_RV670:
2499 chip_name = "RV670";
2500 rlc_chip_name = "R600";
2501 break;
2502 case CHIP_RS780:
2503 case CHIP_RS880:
2504 chip_name = "RS780";
2505 rlc_chip_name = "R600";
2506 break;
2507 case CHIP_RV770:
2508 chip_name = "RV770";
2509 rlc_chip_name = "R700";
2510 smc_chip_name = "RV770";
2511 smc_req_size = ALIGN(RV770_SMC_UCODE_SIZE, 4);
2512 break;
2513 case CHIP_RV730:
2514 chip_name = "RV730";
2515 rlc_chip_name = "R700";
2516 smc_chip_name = "RV730";
2517 smc_req_size = ALIGN(RV730_SMC_UCODE_SIZE, 4);
2518 break;
2519 case CHIP_RV710:
2520 chip_name = "RV710";
2521 rlc_chip_name = "R700";
2522 smc_chip_name = "RV710";
2523 smc_req_size = ALIGN(RV710_SMC_UCODE_SIZE, 4);
2524 break;
2525 case CHIP_RV740:
2526 chip_name = "RV730";
2527 rlc_chip_name = "R700";
2528 smc_chip_name = "RV740";
2529 smc_req_size = ALIGN(RV740_SMC_UCODE_SIZE, 4);
2530 break;
2531 case CHIP_CEDAR:
2532 chip_name = "CEDAR";
2533 rlc_chip_name = "CEDAR";
2534 smc_chip_name = "CEDAR";
2535 smc_req_size = ALIGN(CEDAR_SMC_UCODE_SIZE, 4);
2536 break;
2537 case CHIP_REDWOOD:
2538 chip_name = "REDWOOD";
2539 rlc_chip_name = "REDWOOD";
2540 smc_chip_name = "REDWOOD";
2541 smc_req_size = ALIGN(REDWOOD_SMC_UCODE_SIZE, 4);
2542 break;
2543 case CHIP_JUNIPER:
2544 chip_name = "JUNIPER";
2545 rlc_chip_name = "JUNIPER";
2546 smc_chip_name = "JUNIPER";
2547 smc_req_size = ALIGN(JUNIPER_SMC_UCODE_SIZE, 4);
2548 break;
2549 case CHIP_CYPRESS:
2550 case CHIP_HEMLOCK:
2551 chip_name = "CYPRESS";
2552 rlc_chip_name = "CYPRESS";
2553 smc_chip_name = "CYPRESS";
2554 smc_req_size = ALIGN(CYPRESS_SMC_UCODE_SIZE, 4);
2555 break;
2556 case CHIP_PALM:
2557 chip_name = "PALM";
2558 rlc_chip_name = "SUMO";
2559 break;
2560 case CHIP_SUMO:
2561 chip_name = "SUMO";
2562 rlc_chip_name = "SUMO";
2563 break;
2564 case CHIP_SUMO2:
2565 chip_name = "SUMO2";
2566 rlc_chip_name = "SUMO";
2567 break;
2568 default: BUG();
2569 }
2570
2571 if (rdev->family >= CHIP_CEDAR) {
2572 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
2573 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
2574 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
2575 } else if (rdev->family >= CHIP_RV770) {
2576 pfp_req_size = R700_PFP_UCODE_SIZE * 4;
2577 me_req_size = R700_PM4_UCODE_SIZE * 4;
2578 rlc_req_size = R700_RLC_UCODE_SIZE * 4;
2579 } else {
2580 pfp_req_size = R600_PFP_UCODE_SIZE * 4;
2581 me_req_size = R600_PM4_UCODE_SIZE * 12;
2582 rlc_req_size = R600_RLC_UCODE_SIZE * 4;
2583 }
2584
2585 DRM_INFO("Loading %s Microcode\n", chip_name);
2586
2587 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
2588 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2589 if (err)
2590 goto out;
2591 if (rdev->pfp_fw->size != pfp_req_size) {
2592 pr_err("r600_cp: Bogus length %zu in firmware \"%s\"\n",
2593 rdev->pfp_fw->size, fw_name);
2594 err = -EINVAL;
2595 goto out;
2596 }
2597
2598 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2599 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2600 if (err)
2601 goto out;
2602 if (rdev->me_fw->size != me_req_size) {
2603 pr_err("r600_cp: Bogus length %zu in firmware \"%s\"\n",
2604 rdev->me_fw->size, fw_name);
2605 err = -EINVAL;
2606 }
2607
2608 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
2609 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2610 if (err)
2611 goto out;
2612 if (rdev->rlc_fw->size != rlc_req_size) {
2613 pr_err("r600_rlc: Bogus length %zu in firmware \"%s\"\n",
2614 rdev->rlc_fw->size, fw_name);
2615 err = -EINVAL;
2616 }
2617
2618 if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_HEMLOCK)) {
2619 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", smc_chip_name);
2620 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2621 if (err) {
2622 pr_err("smc: error loading firmware \"%s\"\n", fw_name);
2623 release_firmware(rdev->smc_fw);
2624 rdev->smc_fw = NULL;
2625 err = 0;
2626 } else if (rdev->smc_fw->size != smc_req_size) {
2627 pr_err("smc: Bogus length %zu in firmware \"%s\"\n",
2628 rdev->smc_fw->size, fw_name);
2629 err = -EINVAL;
2630 }
2631 }
2632
2633 out:
2634 if (err) {
2635 if (err != -EINVAL)
2636 pr_err("r600_cp: Failed to load firmware \"%s\"\n",
2637 fw_name);
2638 release_firmware(rdev->pfp_fw);
2639 rdev->pfp_fw = NULL;
2640 release_firmware(rdev->me_fw);
2641 rdev->me_fw = NULL;
2642 release_firmware(rdev->rlc_fw);
2643 rdev->rlc_fw = NULL;
2644 release_firmware(rdev->smc_fw);
2645 rdev->smc_fw = NULL;
2646 }
2647 return err;
2648 }
2649
r600_gfx_get_rptr(struct radeon_device * rdev,struct radeon_ring * ring)2650 u32 r600_gfx_get_rptr(struct radeon_device *rdev,
2651 struct radeon_ring *ring)
2652 {
2653 u32 rptr;
2654
2655 if (rdev->wb.enabled)
2656 rptr = rdev->wb.wb[ring->rptr_offs/4];
2657 else
2658 rptr = RREG32(R600_CP_RB_RPTR);
2659
2660 return rptr;
2661 }
2662
r600_gfx_get_wptr(struct radeon_device * rdev,struct radeon_ring * ring)2663 u32 r600_gfx_get_wptr(struct radeon_device *rdev,
2664 struct radeon_ring *ring)
2665 {
2666 return RREG32(R600_CP_RB_WPTR);
2667 }
2668
r600_gfx_set_wptr(struct radeon_device * rdev,struct radeon_ring * ring)2669 void r600_gfx_set_wptr(struct radeon_device *rdev,
2670 struct radeon_ring *ring)
2671 {
2672 WREG32(R600_CP_RB_WPTR, ring->wptr);
2673 (void)RREG32(R600_CP_RB_WPTR);
2674 }
2675
r600_cp_load_microcode(struct radeon_device * rdev)2676 static int r600_cp_load_microcode(struct radeon_device *rdev)
2677 {
2678 const __be32 *fw_data;
2679 int i;
2680
2681 if (!rdev->me_fw || !rdev->pfp_fw)
2682 return -EINVAL;
2683
2684 r600_cp_stop(rdev);
2685
2686 WREG32(CP_RB_CNTL,
2687 #ifdef __BIG_ENDIAN
2688 BUF_SWAP_32BIT |
2689 #endif
2690 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2691
2692 /* Reset cp */
2693 WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
2694 RREG32(GRBM_SOFT_RESET);
2695 mdelay(15);
2696 WREG32(GRBM_SOFT_RESET, 0);
2697
2698 WREG32(CP_ME_RAM_WADDR, 0);
2699
2700 fw_data = (const __be32 *)rdev->me_fw->data;
2701 WREG32(CP_ME_RAM_WADDR, 0);
2702 for (i = 0; i < R600_PM4_UCODE_SIZE * 3; i++)
2703 WREG32(CP_ME_RAM_DATA,
2704 be32_to_cpup(fw_data++));
2705
2706 fw_data = (const __be32 *)rdev->pfp_fw->data;
2707 WREG32(CP_PFP_UCODE_ADDR, 0);
2708 for (i = 0; i < R600_PFP_UCODE_SIZE; i++)
2709 WREG32(CP_PFP_UCODE_DATA,
2710 be32_to_cpup(fw_data++));
2711
2712 WREG32(CP_PFP_UCODE_ADDR, 0);
2713 WREG32(CP_ME_RAM_WADDR, 0);
2714 WREG32(CP_ME_RAM_RADDR, 0);
2715 return 0;
2716 }
2717
r600_cp_start(struct radeon_device * rdev)2718 int r600_cp_start(struct radeon_device *rdev)
2719 {
2720 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2721 int r;
2722 uint32_t cp_me;
2723
2724 r = radeon_ring_lock(rdev, ring, 7);
2725 if (r) {
2726 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2727 return r;
2728 }
2729 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2730 radeon_ring_write(ring, 0x1);
2731 if (rdev->family >= CHIP_RV770) {
2732 radeon_ring_write(ring, 0x0);
2733 radeon_ring_write(ring, rdev->config.rv770.max_hw_contexts - 1);
2734 } else {
2735 radeon_ring_write(ring, 0x3);
2736 radeon_ring_write(ring, rdev->config.r600.max_hw_contexts - 1);
2737 }
2738 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2739 radeon_ring_write(ring, 0);
2740 radeon_ring_write(ring, 0);
2741 radeon_ring_unlock_commit(rdev, ring, false);
2742
2743 cp_me = 0xff;
2744 WREG32(R_0086D8_CP_ME_CNTL, cp_me);
2745 return 0;
2746 }
2747
r600_cp_resume(struct radeon_device * rdev)2748 int r600_cp_resume(struct radeon_device *rdev)
2749 {
2750 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2751 u32 tmp;
2752 u32 rb_bufsz;
2753 int r;
2754
2755 /* Reset cp */
2756 WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
2757 RREG32(GRBM_SOFT_RESET);
2758 mdelay(15);
2759 WREG32(GRBM_SOFT_RESET, 0);
2760
2761 /* Set ring buffer size */
2762 rb_bufsz = order_base_2(ring->ring_size / 8);
2763 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2764 #ifdef __BIG_ENDIAN
2765 tmp |= BUF_SWAP_32BIT;
2766 #endif
2767 WREG32(CP_RB_CNTL, tmp);
2768 WREG32(CP_SEM_WAIT_TIMER, 0x0);
2769
2770 /* Set the write pointer delay */
2771 WREG32(CP_RB_WPTR_DELAY, 0);
2772
2773 /* Initialize the ring buffer's read and write pointers */
2774 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2775 WREG32(CP_RB_RPTR_WR, 0);
2776 ring->wptr = 0;
2777 WREG32(CP_RB_WPTR, ring->wptr);
2778
2779 /* set the wb address whether it's enabled or not */
2780 WREG32(CP_RB_RPTR_ADDR,
2781 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
2782 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2783 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2784
2785 if (rdev->wb.enabled)
2786 WREG32(SCRATCH_UMSK, 0xff);
2787 else {
2788 tmp |= RB_NO_UPDATE;
2789 WREG32(SCRATCH_UMSK, 0);
2790 }
2791
2792 mdelay(1);
2793 WREG32(CP_RB_CNTL, tmp);
2794
2795 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
2796 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2797
2798 r600_cp_start(rdev);
2799 ring->ready = true;
2800 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
2801 if (r) {
2802 ring->ready = false;
2803 return r;
2804 }
2805
2806 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
2807 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
2808
2809 return 0;
2810 }
2811
r600_ring_init(struct radeon_device * rdev,struct radeon_ring * ring,unsigned ring_size)2812 void r600_ring_init(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ring_size)
2813 {
2814 u32 rb_bufsz;
2815 int r;
2816
2817 /* Align ring size */
2818 rb_bufsz = order_base_2(ring_size / 8);
2819 ring_size = (1 << (rb_bufsz + 1)) * 4;
2820 ring->ring_size = ring_size;
2821 ring->align_mask = 16 - 1;
2822
2823 if (radeon_ring_supports_scratch_reg(rdev, ring)) {
2824 r = radeon_scratch_get(rdev, &ring->rptr_save_reg);
2825 if (r) {
2826 DRM_ERROR("failed to get scratch reg for rptr save (%d).\n", r);
2827 ring->rptr_save_reg = 0;
2828 }
2829 }
2830 }
2831
r600_cp_fini(struct radeon_device * rdev)2832 void r600_cp_fini(struct radeon_device *rdev)
2833 {
2834 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2835 r600_cp_stop(rdev);
2836 radeon_ring_fini(rdev, ring);
2837 radeon_scratch_free(rdev, ring->rptr_save_reg);
2838 }
2839
2840 /*
2841 * GPU scratch registers helpers function.
2842 */
r600_scratch_init(struct radeon_device * rdev)2843 void r600_scratch_init(struct radeon_device *rdev)
2844 {
2845 int i;
2846
2847 rdev->scratch.num_reg = 7;
2848 rdev->scratch.reg_base = SCRATCH_REG0;
2849 for (i = 0; i < rdev->scratch.num_reg; i++) {
2850 rdev->scratch.free[i] = true;
2851 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
2852 }
2853 }
2854
r600_ring_test(struct radeon_device * rdev,struct radeon_ring * ring)2855 int r600_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
2856 {
2857 uint32_t scratch;
2858 uint32_t tmp = 0;
2859 unsigned i;
2860 int r;
2861
2862 r = radeon_scratch_get(rdev, &scratch);
2863 if (r) {
2864 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
2865 return r;
2866 }
2867 WREG32(scratch, 0xCAFEDEAD);
2868 r = radeon_ring_lock(rdev, ring, 3);
2869 if (r) {
2870 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
2871 radeon_scratch_free(rdev, scratch);
2872 return r;
2873 }
2874 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2875 radeon_ring_write(ring, ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
2876 radeon_ring_write(ring, 0xDEADBEEF);
2877 radeon_ring_unlock_commit(rdev, ring, false);
2878 for (i = 0; i < rdev->usec_timeout; i++) {
2879 tmp = RREG32(scratch);
2880 if (tmp == 0xDEADBEEF)
2881 break;
2882 udelay(1);
2883 }
2884 if (i < rdev->usec_timeout) {
2885 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
2886 } else {
2887 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
2888 ring->idx, scratch, tmp);
2889 r = -EINVAL;
2890 }
2891 radeon_scratch_free(rdev, scratch);
2892 return r;
2893 }
2894
2895 /*
2896 * CP fences/semaphores
2897 */
2898
r600_fence_ring_emit(struct radeon_device * rdev,struct radeon_fence * fence)2899 void r600_fence_ring_emit(struct radeon_device *rdev,
2900 struct radeon_fence *fence)
2901 {
2902 struct radeon_ring *ring = &rdev->ring[fence->ring];
2903 u32 cp_coher_cntl = PACKET3_TC_ACTION_ENA | PACKET3_VC_ACTION_ENA |
2904 PACKET3_SH_ACTION_ENA;
2905
2906 if (rdev->family >= CHIP_RV770)
2907 cp_coher_cntl |= PACKET3_FULL_CACHE_ENA;
2908
2909 if (rdev->wb.use_event) {
2910 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
2911 /* flush read cache over gart */
2912 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2913 radeon_ring_write(ring, cp_coher_cntl);
2914 radeon_ring_write(ring, 0xFFFFFFFF);
2915 radeon_ring_write(ring, 0);
2916 radeon_ring_write(ring, 10); /* poll interval */
2917 /* EVENT_WRITE_EOP - flush caches, send int */
2918 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
2919 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
2920 radeon_ring_write(ring, lower_32_bits(addr));
2921 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
2922 radeon_ring_write(ring, fence->seq);
2923 radeon_ring_write(ring, 0);
2924 } else {
2925 /* flush read cache over gart */
2926 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2927 radeon_ring_write(ring, cp_coher_cntl);
2928 radeon_ring_write(ring, 0xFFFFFFFF);
2929 radeon_ring_write(ring, 0);
2930 radeon_ring_write(ring, 10); /* poll interval */
2931 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
2932 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT) | EVENT_INDEX(0));
2933 /* wait for 3D idle clean */
2934 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2935 radeon_ring_write(ring, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
2936 radeon_ring_write(ring, WAIT_3D_IDLE_bit | WAIT_3D_IDLECLEAN_bit);
2937 /* Emit fence sequence & fire IRQ */
2938 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2939 radeon_ring_write(ring, ((rdev->fence_drv[fence->ring].scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
2940 radeon_ring_write(ring, fence->seq);
2941 /* CP_INTERRUPT packet 3 no longer exists, use packet 0 */
2942 radeon_ring_write(ring, PACKET0(CP_INT_STATUS, 0));
2943 radeon_ring_write(ring, RB_INT_STAT);
2944 }
2945 }
2946
2947 /**
2948 * r600_semaphore_ring_emit - emit a semaphore on the CP ring
2949 *
2950 * @rdev: radeon_device pointer
2951 * @ring: radeon ring buffer object
2952 * @semaphore: radeon semaphore object
2953 * @emit_wait: Is this a sempahore wait?
2954 *
2955 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
2956 * from running ahead of semaphore waits.
2957 */
r600_semaphore_ring_emit(struct radeon_device * rdev,struct radeon_ring * ring,struct radeon_semaphore * semaphore,bool emit_wait)2958 bool r600_semaphore_ring_emit(struct radeon_device *rdev,
2959 struct radeon_ring *ring,
2960 struct radeon_semaphore *semaphore,
2961 bool emit_wait)
2962 {
2963 uint64_t addr = semaphore->gpu_addr;
2964 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
2965
2966 if (rdev->family < CHIP_CAYMAN)
2967 sel |= PACKET3_SEM_WAIT_ON_SIGNAL;
2968
2969 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
2970 radeon_ring_write(ring, lower_32_bits(addr));
2971 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | sel);
2972
2973 /* PFP_SYNC_ME packet only exists on 7xx+, only enable it on eg+ */
2974 if (emit_wait && (rdev->family >= CHIP_CEDAR)) {
2975 /* Prevent the PFP from running ahead of the semaphore wait */
2976 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
2977 radeon_ring_write(ring, 0x0);
2978 }
2979
2980 return true;
2981 }
2982
2983 /**
2984 * r600_copy_cpdma - copy pages using the CP DMA engine
2985 *
2986 * @rdev: radeon_device pointer
2987 * @src_offset: src GPU address
2988 * @dst_offset: dst GPU address
2989 * @num_gpu_pages: number of GPU pages to xfer
2990 * @fence: radeon fence object
2991 *
2992 * Copy GPU paging using the CP DMA engine (r6xx+).
2993 * Used by the radeon ttm implementation to move pages if
2994 * registered as the asic copy callback.
2995 */
r600_copy_cpdma(struct radeon_device * rdev,uint64_t src_offset,uint64_t dst_offset,unsigned num_gpu_pages,struct dma_resv * resv)2996 struct radeon_fence *r600_copy_cpdma(struct radeon_device *rdev,
2997 uint64_t src_offset, uint64_t dst_offset,
2998 unsigned num_gpu_pages,
2999 struct dma_resv *resv)
3000 {
3001 struct radeon_fence *fence;
3002 struct radeon_sync sync;
3003 int ring_index = rdev->asic->copy.blit_ring_index;
3004 struct radeon_ring *ring = &rdev->ring[ring_index];
3005 u32 size_in_bytes, cur_size_in_bytes, tmp;
3006 int i, num_loops;
3007 int r = 0;
3008
3009 radeon_sync_create(&sync);
3010
3011 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3012 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3013 r = radeon_ring_lock(rdev, ring, num_loops * 6 + 24);
3014 if (r) {
3015 DRM_ERROR("radeon: moving bo (%d).\n", r);
3016 radeon_sync_free(rdev, &sync, NULL);
3017 return ERR_PTR(r);
3018 }
3019
3020 radeon_sync_resv(rdev, &sync, resv, false);
3021 radeon_sync_rings(rdev, &sync, ring->idx);
3022
3023 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3024 radeon_ring_write(ring, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
3025 radeon_ring_write(ring, WAIT_3D_IDLE_bit);
3026 for (i = 0; i < num_loops; i++) {
3027 cur_size_in_bytes = size_in_bytes;
3028 if (cur_size_in_bytes > 0x1fffff)
3029 cur_size_in_bytes = 0x1fffff;
3030 size_in_bytes -= cur_size_in_bytes;
3031 tmp = upper_32_bits(src_offset) & 0xff;
3032 if (size_in_bytes == 0)
3033 tmp |= PACKET3_CP_DMA_CP_SYNC;
3034 radeon_ring_write(ring, PACKET3(PACKET3_CP_DMA, 4));
3035 radeon_ring_write(ring, lower_32_bits(src_offset));
3036 radeon_ring_write(ring, tmp);
3037 radeon_ring_write(ring, lower_32_bits(dst_offset));
3038 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
3039 radeon_ring_write(ring, cur_size_in_bytes);
3040 src_offset += cur_size_in_bytes;
3041 dst_offset += cur_size_in_bytes;
3042 }
3043 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3044 radeon_ring_write(ring, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
3045 radeon_ring_write(ring, WAIT_CP_DMA_IDLE_bit);
3046
3047 r = radeon_fence_emit(rdev, &fence, ring->idx);
3048 if (r) {
3049 radeon_ring_unlock_undo(rdev, ring);
3050 radeon_sync_free(rdev, &sync, NULL);
3051 return ERR_PTR(r);
3052 }
3053
3054 radeon_ring_unlock_commit(rdev, ring, false);
3055 radeon_sync_free(rdev, &sync, fence);
3056
3057 return fence;
3058 }
3059
r600_set_surface_reg(struct radeon_device * rdev,int reg,uint32_t tiling_flags,uint32_t pitch,uint32_t offset,uint32_t obj_size)3060 int r600_set_surface_reg(struct radeon_device *rdev, int reg,
3061 uint32_t tiling_flags, uint32_t pitch,
3062 uint32_t offset, uint32_t obj_size)
3063 {
3064 /* FIXME: implement */
3065 return 0;
3066 }
3067
r600_clear_surface_reg(struct radeon_device * rdev,int reg)3068 void r600_clear_surface_reg(struct radeon_device *rdev, int reg)
3069 {
3070 /* FIXME: implement */
3071 }
3072
r600_uvd_init(struct radeon_device * rdev)3073 static void r600_uvd_init(struct radeon_device *rdev)
3074 {
3075 int r;
3076
3077 if (!rdev->has_uvd)
3078 return;
3079
3080 r = radeon_uvd_init(rdev);
3081 if (r) {
3082 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
3083 /*
3084 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
3085 * to early fails uvd_v1_0_resume() and thus nothing happens
3086 * there. So it is pointless to try to go through that code
3087 * hence why we disable uvd here.
3088 */
3089 rdev->has_uvd = false;
3090 return;
3091 }
3092 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
3093 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
3094 }
3095
r600_uvd_start(struct radeon_device * rdev)3096 static void r600_uvd_start(struct radeon_device *rdev)
3097 {
3098 int r;
3099
3100 if (!rdev->has_uvd)
3101 return;
3102
3103 r = uvd_v1_0_resume(rdev);
3104 if (r) {
3105 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
3106 goto error;
3107 }
3108 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
3109 if (r) {
3110 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
3111 goto error;
3112 }
3113 return;
3114
3115 error:
3116 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
3117 }
3118
r600_uvd_resume(struct radeon_device * rdev)3119 static void r600_uvd_resume(struct radeon_device *rdev)
3120 {
3121 struct radeon_ring *ring;
3122 int r;
3123
3124 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
3125 return;
3126
3127 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
3128 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
3129 if (r) {
3130 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
3131 return;
3132 }
3133 r = uvd_v1_0_init(rdev);
3134 if (r) {
3135 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
3136 return;
3137 }
3138 }
3139
r600_startup(struct radeon_device * rdev)3140 static int r600_startup(struct radeon_device *rdev)
3141 {
3142 struct radeon_ring *ring;
3143 int r;
3144
3145 /* enable pcie gen2 link */
3146 r600_pcie_gen2_enable(rdev);
3147
3148 /* scratch needs to be initialized before MC */
3149 r = r600_vram_scratch_init(rdev);
3150 if (r)
3151 return r;
3152
3153 r600_mc_program(rdev);
3154
3155 if (rdev->flags & RADEON_IS_AGP) {
3156 r600_agp_enable(rdev);
3157 } else {
3158 r = r600_pcie_gart_enable(rdev);
3159 if (r)
3160 return r;
3161 }
3162 r600_gpu_init(rdev);
3163
3164 /* allocate wb buffer */
3165 r = radeon_wb_init(rdev);
3166 if (r)
3167 return r;
3168
3169 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
3170 if (r) {
3171 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
3172 return r;
3173 }
3174
3175 r600_uvd_start(rdev);
3176
3177 /* Enable IRQ */
3178 if (!rdev->irq.installed) {
3179 r = radeon_irq_kms_init(rdev);
3180 if (r)
3181 return r;
3182 }
3183
3184 r = r600_irq_init(rdev);
3185 if (r) {
3186 DRM_ERROR("radeon: IH init failed (%d).\n", r);
3187 radeon_irq_kms_fini(rdev);
3188 return r;
3189 }
3190 r600_irq_set(rdev);
3191
3192 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3193 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
3194 RADEON_CP_PACKET2);
3195 if (r)
3196 return r;
3197
3198 r = r600_cp_load_microcode(rdev);
3199 if (r)
3200 return r;
3201 r = r600_cp_resume(rdev);
3202 if (r)
3203 return r;
3204
3205 r600_uvd_resume(rdev);
3206
3207 r = radeon_ib_pool_init(rdev);
3208 if (r) {
3209 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
3210 return r;
3211 }
3212
3213 r = radeon_audio_init(rdev);
3214 if (r) {
3215 DRM_ERROR("radeon: audio init failed\n");
3216 return r;
3217 }
3218
3219 return 0;
3220 }
3221
r600_vga_set_state(struct radeon_device * rdev,bool state)3222 void r600_vga_set_state(struct radeon_device *rdev, bool state)
3223 {
3224 uint32_t temp;
3225
3226 temp = RREG32(CONFIG_CNTL);
3227 if (!state) {
3228 temp &= ~(1<<0);
3229 temp |= (1<<1);
3230 } else {
3231 temp &= ~(1<<1);
3232 }
3233 WREG32(CONFIG_CNTL, temp);
3234 }
3235
r600_resume(struct radeon_device * rdev)3236 int r600_resume(struct radeon_device *rdev)
3237 {
3238 int r;
3239
3240 /* Do not reset GPU before posting, on r600 hw unlike on r500 hw,
3241 * posting will perform necessary task to bring back GPU into good
3242 * shape.
3243 */
3244 /* post card */
3245 atom_asic_init(rdev->mode_info.atom_context);
3246
3247 if (rdev->pm.pm_method == PM_METHOD_DPM)
3248 radeon_pm_resume(rdev);
3249
3250 rdev->accel_working = true;
3251 r = r600_startup(rdev);
3252 if (r) {
3253 DRM_ERROR("r600 startup failed on resume\n");
3254 rdev->accel_working = false;
3255 return r;
3256 }
3257
3258 return r;
3259 }
3260
r600_suspend(struct radeon_device * rdev)3261 int r600_suspend(struct radeon_device *rdev)
3262 {
3263 radeon_pm_suspend(rdev);
3264 radeon_audio_fini(rdev);
3265 r600_cp_stop(rdev);
3266 if (rdev->has_uvd) {
3267 uvd_v1_0_fini(rdev);
3268 radeon_uvd_suspend(rdev);
3269 }
3270 r600_irq_suspend(rdev);
3271 radeon_wb_disable(rdev);
3272 r600_pcie_gart_disable(rdev);
3273
3274 return 0;
3275 }
3276
3277 /* Plan is to move initialization in that function and use
3278 * helper function so that radeon_device_init pretty much
3279 * do nothing more than calling asic specific function. This
3280 * should also allow to remove a bunch of callback function
3281 * like vram_info.
3282 */
r600_init(struct radeon_device * rdev)3283 int r600_init(struct radeon_device *rdev)
3284 {
3285 int r;
3286
3287 if (r600_debugfs_mc_info_init(rdev)) {
3288 DRM_ERROR("Failed to register debugfs file for mc !\n");
3289 }
3290 /* Read BIOS */
3291 if (!radeon_get_bios(rdev)) {
3292 if (ASIC_IS_AVIVO(rdev))
3293 return -EINVAL;
3294 }
3295 /* Must be an ATOMBIOS */
3296 if (!rdev->is_atom_bios) {
3297 dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
3298 return -EINVAL;
3299 }
3300 r = radeon_atombios_init(rdev);
3301 if (r)
3302 return r;
3303 /* Post card if necessary */
3304 if (!radeon_card_posted(rdev)) {
3305 if (!rdev->bios) {
3306 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
3307 return -EINVAL;
3308 }
3309 DRM_INFO("GPU not posted. posting now...\n");
3310 atom_asic_init(rdev->mode_info.atom_context);
3311 }
3312 /* Initialize scratch registers */
3313 r600_scratch_init(rdev);
3314 /* Initialize surface registers */
3315 radeon_surface_init(rdev);
3316 /* Initialize clocks */
3317 radeon_get_clock_info(rdev->ddev);
3318 /* Fence driver */
3319 r = radeon_fence_driver_init(rdev);
3320 if (r)
3321 return r;
3322 if (rdev->flags & RADEON_IS_AGP) {
3323 r = radeon_agp_init(rdev);
3324 if (r)
3325 radeon_agp_disable(rdev);
3326 }
3327 r = r600_mc_init(rdev);
3328 if (r)
3329 return r;
3330 /* Memory manager */
3331 r = radeon_bo_init(rdev);
3332 if (r)
3333 return r;
3334
3335 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
3336 r = r600_init_microcode(rdev);
3337 if (r) {
3338 DRM_ERROR("Failed to load firmware!\n");
3339 return r;
3340 }
3341 }
3342
3343 /* Initialize power management */
3344 radeon_pm_init(rdev);
3345
3346 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
3347 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
3348
3349 r600_uvd_init(rdev);
3350
3351 rdev->ih.ring_obj = NULL;
3352 r600_ih_ring_init(rdev, 64 * 1024);
3353
3354 r = r600_pcie_gart_init(rdev);
3355 if (r)
3356 return r;
3357
3358 rdev->accel_working = true;
3359 r = r600_startup(rdev);
3360 if (r) {
3361 dev_err(rdev->dev, "disabling GPU acceleration\n");
3362 r600_cp_fini(rdev);
3363 r600_irq_fini(rdev);
3364 radeon_wb_fini(rdev);
3365 radeon_ib_pool_fini(rdev);
3366 radeon_irq_kms_fini(rdev);
3367 r600_pcie_gart_fini(rdev);
3368 rdev->accel_working = false;
3369 }
3370
3371 return 0;
3372 }
3373
r600_fini(struct radeon_device * rdev)3374 void r600_fini(struct radeon_device *rdev)
3375 {
3376 radeon_pm_fini(rdev);
3377 radeon_audio_fini(rdev);
3378 r600_cp_fini(rdev);
3379 r600_irq_fini(rdev);
3380 if (rdev->has_uvd) {
3381 uvd_v1_0_fini(rdev);
3382 radeon_uvd_fini(rdev);
3383 }
3384 radeon_wb_fini(rdev);
3385 radeon_ib_pool_fini(rdev);
3386 radeon_irq_kms_fini(rdev);
3387 r600_pcie_gart_fini(rdev);
3388 r600_vram_scratch_fini(rdev);
3389 radeon_agp_fini(rdev);
3390 radeon_gem_fini(rdev);
3391 radeon_fence_driver_fini(rdev);
3392 radeon_bo_fini(rdev);
3393 radeon_atombios_fini(rdev);
3394 kfree(rdev->bios);
3395 rdev->bios = NULL;
3396 }
3397
3398
3399 /*
3400 * CS stuff
3401 */
r600_ring_ib_execute(struct radeon_device * rdev,struct radeon_ib * ib)3402 void r600_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3403 {
3404 struct radeon_ring *ring = &rdev->ring[ib->ring];
3405 u32 next_rptr;
3406
3407 if (ring->rptr_save_reg) {
3408 next_rptr = ring->wptr + 3 + 4;
3409 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3410 radeon_ring_write(ring, ((ring->rptr_save_reg -
3411 PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
3412 radeon_ring_write(ring, next_rptr);
3413 } else if (rdev->wb.enabled) {
3414 next_rptr = ring->wptr + 5 + 4;
3415 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
3416 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3417 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
3418 radeon_ring_write(ring, next_rptr);
3419 radeon_ring_write(ring, 0);
3420 }
3421
3422 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
3423 radeon_ring_write(ring,
3424 #ifdef __BIG_ENDIAN
3425 (2 << 0) |
3426 #endif
3427 (ib->gpu_addr & 0xFFFFFFFC));
3428 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
3429 radeon_ring_write(ring, ib->length_dw);
3430 }
3431
r600_ib_test(struct radeon_device * rdev,struct radeon_ring * ring)3432 int r600_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3433 {
3434 struct radeon_ib ib;
3435 uint32_t scratch;
3436 uint32_t tmp = 0;
3437 unsigned i;
3438 int r;
3439
3440 r = radeon_scratch_get(rdev, &scratch);
3441 if (r) {
3442 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3443 return r;
3444 }
3445 WREG32(scratch, 0xCAFEDEAD);
3446 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3447 if (r) {
3448 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3449 goto free_scratch;
3450 }
3451 ib.ptr[0] = PACKET3(PACKET3_SET_CONFIG_REG, 1);
3452 ib.ptr[1] = ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
3453 ib.ptr[2] = 0xDEADBEEF;
3454 ib.length_dw = 3;
3455 r = radeon_ib_schedule(rdev, &ib, NULL, false);
3456 if (r) {
3457 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3458 goto free_ib;
3459 }
3460 r = radeon_fence_wait_timeout(ib.fence, false, usecs_to_jiffies(
3461 RADEON_USEC_IB_TEST_TIMEOUT));
3462 if (r < 0) {
3463 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3464 goto free_ib;
3465 } else if (r == 0) {
3466 DRM_ERROR("radeon: fence wait timed out.\n");
3467 r = -ETIMEDOUT;
3468 goto free_ib;
3469 }
3470 r = 0;
3471 for (i = 0; i < rdev->usec_timeout; i++) {
3472 tmp = RREG32(scratch);
3473 if (tmp == 0xDEADBEEF)
3474 break;
3475 udelay(1);
3476 }
3477 if (i < rdev->usec_timeout) {
3478 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3479 } else {
3480 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3481 scratch, tmp);
3482 r = -EINVAL;
3483 }
3484 free_ib:
3485 radeon_ib_free(rdev, &ib);
3486 free_scratch:
3487 radeon_scratch_free(rdev, scratch);
3488 return r;
3489 }
3490
3491 /*
3492 * Interrupts
3493 *
3494 * Interrupts use a ring buffer on r6xx/r7xx hardware. It works pretty
3495 * the same as the CP ring buffer, but in reverse. Rather than the CPU
3496 * writing to the ring and the GPU consuming, the GPU writes to the ring
3497 * and host consumes. As the host irq handler processes interrupts, it
3498 * increments the rptr. When the rptr catches up with the wptr, all the
3499 * current interrupts have been processed.
3500 */
3501
r600_ih_ring_init(struct radeon_device * rdev,unsigned ring_size)3502 void r600_ih_ring_init(struct radeon_device *rdev, unsigned ring_size)
3503 {
3504 u32 rb_bufsz;
3505
3506 /* Align ring size */
3507 rb_bufsz = order_base_2(ring_size / 4);
3508 ring_size = (1 << rb_bufsz) * 4;
3509 rdev->ih.ring_size = ring_size;
3510 rdev->ih.ptr_mask = rdev->ih.ring_size - 1;
3511 rdev->ih.rptr = 0;
3512 }
3513
r600_ih_ring_alloc(struct radeon_device * rdev)3514 int r600_ih_ring_alloc(struct radeon_device *rdev)
3515 {
3516 int r;
3517
3518 /* Allocate ring buffer */
3519 if (rdev->ih.ring_obj == NULL) {
3520 r = radeon_bo_create(rdev, rdev->ih.ring_size,
3521 PAGE_SIZE, true,
3522 RADEON_GEM_DOMAIN_GTT, 0,
3523 NULL, NULL, &rdev->ih.ring_obj);
3524 if (r) {
3525 DRM_ERROR("radeon: failed to create ih ring buffer (%d).\n", r);
3526 return r;
3527 }
3528 r = radeon_bo_reserve(rdev->ih.ring_obj, false);
3529 if (unlikely(r != 0))
3530 return r;
3531 r = radeon_bo_pin(rdev->ih.ring_obj,
3532 RADEON_GEM_DOMAIN_GTT,
3533 &rdev->ih.gpu_addr);
3534 if (r) {
3535 radeon_bo_unreserve(rdev->ih.ring_obj);
3536 DRM_ERROR("radeon: failed to pin ih ring buffer (%d).\n", r);
3537 return r;
3538 }
3539 r = radeon_bo_kmap(rdev->ih.ring_obj,
3540 (void **)__UNVOLATILE(&rdev->ih.ring));
3541 radeon_bo_unreserve(rdev->ih.ring_obj);
3542 if (r) {
3543 DRM_ERROR("radeon: failed to map ih ring buffer (%d).\n", r);
3544 return r;
3545 }
3546 }
3547 return 0;
3548 }
3549
r600_ih_ring_fini(struct radeon_device * rdev)3550 void r600_ih_ring_fini(struct radeon_device *rdev)
3551 {
3552 int r;
3553 if (rdev->ih.ring_obj) {
3554 r = radeon_bo_reserve(rdev->ih.ring_obj, false);
3555 if (likely(r == 0)) {
3556 radeon_bo_kunmap(rdev->ih.ring_obj);
3557 radeon_bo_unpin(rdev->ih.ring_obj);
3558 radeon_bo_unreserve(rdev->ih.ring_obj);
3559 }
3560 radeon_bo_unref(&rdev->ih.ring_obj);
3561 rdev->ih.ring = NULL;
3562 rdev->ih.ring_obj = NULL;
3563 }
3564 }
3565
r600_rlc_stop(struct radeon_device * rdev)3566 void r600_rlc_stop(struct radeon_device *rdev)
3567 {
3568
3569 if ((rdev->family >= CHIP_RV770) &&
3570 (rdev->family <= CHIP_RV740)) {
3571 /* r7xx asics need to soft reset RLC before halting */
3572 WREG32(SRBM_SOFT_RESET, SOFT_RESET_RLC);
3573 RREG32(SRBM_SOFT_RESET);
3574 mdelay(15);
3575 WREG32(SRBM_SOFT_RESET, 0);
3576 RREG32(SRBM_SOFT_RESET);
3577 }
3578
3579 WREG32(RLC_CNTL, 0);
3580 }
3581
r600_rlc_start(struct radeon_device * rdev)3582 static void r600_rlc_start(struct radeon_device *rdev)
3583 {
3584 WREG32(RLC_CNTL, RLC_ENABLE);
3585 }
3586
r600_rlc_resume(struct radeon_device * rdev)3587 static int r600_rlc_resume(struct radeon_device *rdev)
3588 {
3589 u32 i;
3590 const __be32 *fw_data;
3591
3592 if (!rdev->rlc_fw)
3593 return -EINVAL;
3594
3595 r600_rlc_stop(rdev);
3596
3597 WREG32(RLC_HB_CNTL, 0);
3598
3599 WREG32(RLC_HB_BASE, 0);
3600 WREG32(RLC_HB_RPTR, 0);
3601 WREG32(RLC_HB_WPTR, 0);
3602 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
3603 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
3604 WREG32(RLC_MC_CNTL, 0);
3605 WREG32(RLC_UCODE_CNTL, 0);
3606
3607 fw_data = (const __be32 *)rdev->rlc_fw->data;
3608 if (rdev->family >= CHIP_RV770) {
3609 for (i = 0; i < R700_RLC_UCODE_SIZE; i++) {
3610 WREG32(RLC_UCODE_ADDR, i);
3611 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3612 }
3613 } else {
3614 for (i = 0; i < R600_RLC_UCODE_SIZE; i++) {
3615 WREG32(RLC_UCODE_ADDR, i);
3616 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3617 }
3618 }
3619 WREG32(RLC_UCODE_ADDR, 0);
3620
3621 r600_rlc_start(rdev);
3622
3623 return 0;
3624 }
3625
r600_enable_interrupts(struct radeon_device * rdev)3626 static void r600_enable_interrupts(struct radeon_device *rdev)
3627 {
3628 u32 ih_cntl = RREG32(IH_CNTL);
3629 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
3630
3631 ih_cntl |= ENABLE_INTR;
3632 ih_rb_cntl |= IH_RB_ENABLE;
3633 WREG32(IH_CNTL, ih_cntl);
3634 WREG32(IH_RB_CNTL, ih_rb_cntl);
3635 rdev->ih.enabled = true;
3636 }
3637
r600_disable_interrupts(struct radeon_device * rdev)3638 void r600_disable_interrupts(struct radeon_device *rdev)
3639 {
3640 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
3641 u32 ih_cntl = RREG32(IH_CNTL);
3642
3643 ih_rb_cntl &= ~IH_RB_ENABLE;
3644 ih_cntl &= ~ENABLE_INTR;
3645 WREG32(IH_RB_CNTL, ih_rb_cntl);
3646 WREG32(IH_CNTL, ih_cntl);
3647 /* set rptr, wptr to 0 */
3648 WREG32(IH_RB_RPTR, 0);
3649 WREG32(IH_RB_WPTR, 0);
3650 rdev->ih.enabled = false;
3651 rdev->ih.rptr = 0;
3652 }
3653
r600_disable_interrupt_state(struct radeon_device * rdev)3654 static void r600_disable_interrupt_state(struct radeon_device *rdev)
3655 {
3656 u32 tmp;
3657
3658 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
3659 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
3660 WREG32(DMA_CNTL, tmp);
3661 WREG32(GRBM_INT_CNTL, 0);
3662 WREG32(DxMODE_INT_MASK, 0);
3663 WREG32(D1GRPH_INTERRUPT_CONTROL, 0);
3664 WREG32(D2GRPH_INTERRUPT_CONTROL, 0);
3665 if (ASIC_IS_DCE3(rdev)) {
3666 WREG32(DCE3_DACA_AUTODETECT_INT_CONTROL, 0);
3667 WREG32(DCE3_DACB_AUTODETECT_INT_CONTROL, 0);
3668 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3669 WREG32(DC_HPD1_INT_CONTROL, tmp);
3670 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3671 WREG32(DC_HPD2_INT_CONTROL, tmp);
3672 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3673 WREG32(DC_HPD3_INT_CONTROL, tmp);
3674 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3675 WREG32(DC_HPD4_INT_CONTROL, tmp);
3676 if (ASIC_IS_DCE32(rdev)) {
3677 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3678 WREG32(DC_HPD5_INT_CONTROL, tmp);
3679 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3680 WREG32(DC_HPD6_INT_CONTROL, tmp);
3681 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3682 WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, tmp);
3683 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3684 WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, tmp);
3685 } else {
3686 tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3687 WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
3688 tmp = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3689 WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, tmp);
3690 }
3691 } else {
3692 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
3693 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
3694 tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
3695 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
3696 tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
3697 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
3698 tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
3699 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
3700 tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3701 WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
3702 tmp = RREG32(HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3703 WREG32(HDMI1_AUDIO_PACKET_CONTROL, tmp);
3704 }
3705 }
3706
r600_irq_init(struct radeon_device * rdev)3707 int r600_irq_init(struct radeon_device *rdev)
3708 {
3709 int ret = 0;
3710 int rb_bufsz;
3711 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
3712
3713 /* allocate ring */
3714 ret = r600_ih_ring_alloc(rdev);
3715 if (ret)
3716 return ret;
3717
3718 /* disable irqs */
3719 r600_disable_interrupts(rdev);
3720
3721 /* init rlc */
3722 if (rdev->family >= CHIP_CEDAR)
3723 ret = evergreen_rlc_resume(rdev);
3724 else
3725 ret = r600_rlc_resume(rdev);
3726 if (ret) {
3727 r600_ih_ring_fini(rdev);
3728 return ret;
3729 }
3730
3731 /* setup interrupt control */
3732 /* set dummy read address to dummy page address */
3733 WREG32(INTERRUPT_CNTL2, rdev->dummy_page.addr >> 8);
3734 interrupt_cntl = RREG32(INTERRUPT_CNTL);
3735 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
3736 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
3737 */
3738 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
3739 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
3740 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
3741 WREG32(INTERRUPT_CNTL, interrupt_cntl);
3742
3743 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
3744 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
3745
3746 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
3747 IH_WPTR_OVERFLOW_CLEAR |
3748 (rb_bufsz << 1));
3749
3750 if (rdev->wb.enabled)
3751 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
3752
3753 /* set the writeback address whether it's enabled or not */
3754 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
3755 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
3756
3757 WREG32(IH_RB_CNTL, ih_rb_cntl);
3758
3759 /* set rptr, wptr to 0 */
3760 WREG32(IH_RB_RPTR, 0);
3761 WREG32(IH_RB_WPTR, 0);
3762
3763 /* Default settings for IH_CNTL (disabled at first) */
3764 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10);
3765 /* RPTR_REARM only works if msi's are enabled */
3766 if (rdev->msi_enabled)
3767 ih_cntl |= RPTR_REARM;
3768 WREG32(IH_CNTL, ih_cntl);
3769
3770 /* force the active interrupt state to all disabled */
3771 if (rdev->family >= CHIP_CEDAR)
3772 evergreen_disable_interrupt_state(rdev);
3773 else
3774 r600_disable_interrupt_state(rdev);
3775
3776 /* at this point everything should be setup correctly to enable master */
3777 pci_set_master(rdev->pdev);
3778
3779 /* enable irqs */
3780 r600_enable_interrupts(rdev);
3781
3782 return ret;
3783 }
3784
r600_irq_suspend(struct radeon_device * rdev)3785 void r600_irq_suspend(struct radeon_device *rdev)
3786 {
3787 r600_irq_disable(rdev);
3788 r600_rlc_stop(rdev);
3789 }
3790
r600_irq_fini(struct radeon_device * rdev)3791 void r600_irq_fini(struct radeon_device *rdev)
3792 {
3793 r600_irq_suspend(rdev);
3794 r600_ih_ring_fini(rdev);
3795 }
3796
r600_irq_set(struct radeon_device * rdev)3797 int r600_irq_set(struct radeon_device *rdev)
3798 {
3799 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
3800 u32 mode_int = 0;
3801 u32 hpd1, hpd2, hpd3, hpd4 = 0, hpd5 = 0, hpd6 = 0;
3802 u32 grbm_int_cntl = 0;
3803 u32 hdmi0, hdmi1;
3804 u32 dma_cntl;
3805 u32 thermal_int = 0;
3806
3807 if (!rdev->irq.installed) {
3808 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
3809 return -EINVAL;
3810 }
3811 /* don't enable anything if the ih is disabled */
3812 if (!rdev->ih.enabled) {
3813 r600_disable_interrupts(rdev);
3814 /* force the active interrupt state to all disabled */
3815 r600_disable_interrupt_state(rdev);
3816 return 0;
3817 }
3818
3819 if (ASIC_IS_DCE3(rdev)) {
3820 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
3821 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
3822 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
3823 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
3824 if (ASIC_IS_DCE32(rdev)) {
3825 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
3826 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
3827 hdmi0 = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
3828 hdmi1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
3829 } else {
3830 hdmi0 = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3831 hdmi1 = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3832 }
3833 } else {
3834 hpd1 = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & ~DC_HPDx_INT_EN;
3835 hpd2 = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & ~DC_HPDx_INT_EN;
3836 hpd3 = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & ~DC_HPDx_INT_EN;
3837 hdmi0 = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3838 hdmi1 = RREG32(HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3839 }
3840
3841 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
3842
3843 if ((rdev->family > CHIP_R600) && (rdev->family < CHIP_RV770)) {
3844 thermal_int = RREG32(CG_THERMAL_INT) &
3845 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
3846 } else if (rdev->family >= CHIP_RV770) {
3847 thermal_int = RREG32(RV770_CG_THERMAL_INT) &
3848 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
3849 }
3850 if (rdev->irq.dpm_thermal) {
3851 DRM_DEBUG("dpm thermal\n");
3852 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
3853 }
3854
3855 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
3856 DRM_DEBUG("r600_irq_set: sw int\n");
3857 cp_int_cntl |= RB_INT_ENABLE;
3858 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
3859 }
3860
3861 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
3862 DRM_DEBUG("r600_irq_set: sw int dma\n");
3863 dma_cntl |= TRAP_ENABLE;
3864 }
3865
3866 if (rdev->irq.crtc_vblank_int[0] ||
3867 atomic_read(&rdev->irq.pflip[0])) {
3868 DRM_DEBUG("r600_irq_set: vblank 0\n");
3869 mode_int |= D1MODE_VBLANK_INT_MASK;
3870 }
3871 if (rdev->irq.crtc_vblank_int[1] ||
3872 atomic_read(&rdev->irq.pflip[1])) {
3873 DRM_DEBUG("r600_irq_set: vblank 1\n");
3874 mode_int |= D2MODE_VBLANK_INT_MASK;
3875 }
3876 if (rdev->irq.hpd[0]) {
3877 DRM_DEBUG("r600_irq_set: hpd 1\n");
3878 hpd1 |= DC_HPDx_INT_EN;
3879 }
3880 if (rdev->irq.hpd[1]) {
3881 DRM_DEBUG("r600_irq_set: hpd 2\n");
3882 hpd2 |= DC_HPDx_INT_EN;
3883 }
3884 if (rdev->irq.hpd[2]) {
3885 DRM_DEBUG("r600_irq_set: hpd 3\n");
3886 hpd3 |= DC_HPDx_INT_EN;
3887 }
3888 if (rdev->irq.hpd[3]) {
3889 DRM_DEBUG("r600_irq_set: hpd 4\n");
3890 hpd4 |= DC_HPDx_INT_EN;
3891 }
3892 if (rdev->irq.hpd[4]) {
3893 DRM_DEBUG("r600_irq_set: hpd 5\n");
3894 hpd5 |= DC_HPDx_INT_EN;
3895 }
3896 if (rdev->irq.hpd[5]) {
3897 DRM_DEBUG("r600_irq_set: hpd 6\n");
3898 hpd6 |= DC_HPDx_INT_EN;
3899 }
3900 if (rdev->irq.afmt[0]) {
3901 DRM_DEBUG("r600_irq_set: hdmi 0\n");
3902 hdmi0 |= HDMI0_AZ_FORMAT_WTRIG_MASK;
3903 }
3904 if (rdev->irq.afmt[1]) {
3905 DRM_DEBUG("r600_irq_set: hdmi 0\n");
3906 hdmi1 |= HDMI0_AZ_FORMAT_WTRIG_MASK;
3907 }
3908
3909 WREG32(CP_INT_CNTL, cp_int_cntl);
3910 WREG32(DMA_CNTL, dma_cntl);
3911 WREG32(DxMODE_INT_MASK, mode_int);
3912 WREG32(D1GRPH_INTERRUPT_CONTROL, DxGRPH_PFLIP_INT_MASK);
3913 WREG32(D2GRPH_INTERRUPT_CONTROL, DxGRPH_PFLIP_INT_MASK);
3914 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
3915 if (ASIC_IS_DCE3(rdev)) {
3916 WREG32(DC_HPD1_INT_CONTROL, hpd1);
3917 WREG32(DC_HPD2_INT_CONTROL, hpd2);
3918 WREG32(DC_HPD3_INT_CONTROL, hpd3);
3919 WREG32(DC_HPD4_INT_CONTROL, hpd4);
3920 if (ASIC_IS_DCE32(rdev)) {
3921 WREG32(DC_HPD5_INT_CONTROL, hpd5);
3922 WREG32(DC_HPD6_INT_CONTROL, hpd6);
3923 WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, hdmi0);
3924 WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, hdmi1);
3925 } else {
3926 WREG32(HDMI0_AUDIO_PACKET_CONTROL, hdmi0);
3927 WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, hdmi1);
3928 }
3929 } else {
3930 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, hpd1);
3931 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, hpd2);
3932 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, hpd3);
3933 WREG32(HDMI0_AUDIO_PACKET_CONTROL, hdmi0);
3934 WREG32(HDMI1_AUDIO_PACKET_CONTROL, hdmi1);
3935 }
3936 if ((rdev->family > CHIP_R600) && (rdev->family < CHIP_RV770)) {
3937 WREG32(CG_THERMAL_INT, thermal_int);
3938 } else if (rdev->family >= CHIP_RV770) {
3939 WREG32(RV770_CG_THERMAL_INT, thermal_int);
3940 }
3941
3942 /* posting read */
3943 RREG32(R_000E50_SRBM_STATUS);
3944
3945 return 0;
3946 }
3947
r600_irq_ack(struct radeon_device * rdev)3948 static void r600_irq_ack(struct radeon_device *rdev)
3949 {
3950 u32 tmp;
3951
3952 if (ASIC_IS_DCE3(rdev)) {
3953 rdev->irq.stat_regs.r600.disp_int = RREG32(DCE3_DISP_INTERRUPT_STATUS);
3954 rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE);
3955 rdev->irq.stat_regs.r600.disp_int_cont2 = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE2);
3956 if (ASIC_IS_DCE32(rdev)) {
3957 rdev->irq.stat_regs.r600.hdmi0_status = RREG32(AFMT_STATUS + DCE3_HDMI_OFFSET0);
3958 rdev->irq.stat_regs.r600.hdmi1_status = RREG32(AFMT_STATUS + DCE3_HDMI_OFFSET1);
3959 } else {
3960 rdev->irq.stat_regs.r600.hdmi0_status = RREG32(HDMI0_STATUS);
3961 rdev->irq.stat_regs.r600.hdmi1_status = RREG32(DCE3_HDMI1_STATUS);
3962 }
3963 } else {
3964 rdev->irq.stat_regs.r600.disp_int = RREG32(DISP_INTERRUPT_STATUS);
3965 rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
3966 rdev->irq.stat_regs.r600.disp_int_cont2 = 0;
3967 rdev->irq.stat_regs.r600.hdmi0_status = RREG32(HDMI0_STATUS);
3968 rdev->irq.stat_regs.r600.hdmi1_status = RREG32(HDMI1_STATUS);
3969 }
3970 rdev->irq.stat_regs.r600.d1grph_int = RREG32(D1GRPH_INTERRUPT_STATUS);
3971 rdev->irq.stat_regs.r600.d2grph_int = RREG32(D2GRPH_INTERRUPT_STATUS);
3972
3973 if (rdev->irq.stat_regs.r600.d1grph_int & DxGRPH_PFLIP_INT_OCCURRED)
3974 WREG32(D1GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
3975 if (rdev->irq.stat_regs.r600.d2grph_int & DxGRPH_PFLIP_INT_OCCURRED)
3976 WREG32(D2GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
3977 if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT)
3978 WREG32(D1MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
3979 if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT)
3980 WREG32(D1MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
3981 if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT)
3982 WREG32(D2MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
3983 if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT)
3984 WREG32(D2MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
3985 if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
3986 if (ASIC_IS_DCE3(rdev)) {
3987 tmp = RREG32(DC_HPD1_INT_CONTROL);
3988 tmp |= DC_HPDx_INT_ACK;
3989 WREG32(DC_HPD1_INT_CONTROL, tmp);
3990 } else {
3991 tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
3992 tmp |= DC_HPDx_INT_ACK;
3993 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
3994 }
3995 }
3996 if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
3997 if (ASIC_IS_DCE3(rdev)) {
3998 tmp = RREG32(DC_HPD2_INT_CONTROL);
3999 tmp |= DC_HPDx_INT_ACK;
4000 WREG32(DC_HPD2_INT_CONTROL, tmp);
4001 } else {
4002 tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
4003 tmp |= DC_HPDx_INT_ACK;
4004 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
4005 }
4006 }
4007 if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
4008 if (ASIC_IS_DCE3(rdev)) {
4009 tmp = RREG32(DC_HPD3_INT_CONTROL);
4010 tmp |= DC_HPDx_INT_ACK;
4011 WREG32(DC_HPD3_INT_CONTROL, tmp);
4012 } else {
4013 tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
4014 tmp |= DC_HPDx_INT_ACK;
4015 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
4016 }
4017 }
4018 if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
4019 tmp = RREG32(DC_HPD4_INT_CONTROL);
4020 tmp |= DC_HPDx_INT_ACK;
4021 WREG32(DC_HPD4_INT_CONTROL, tmp);
4022 }
4023 if (ASIC_IS_DCE32(rdev)) {
4024 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
4025 tmp = RREG32(DC_HPD5_INT_CONTROL);
4026 tmp |= DC_HPDx_INT_ACK;
4027 WREG32(DC_HPD5_INT_CONTROL, tmp);
4028 }
4029 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
4030 tmp = RREG32(DC_HPD6_INT_CONTROL);
4031 tmp |= DC_HPDx_INT_ACK;
4032 WREG32(DC_HPD6_INT_CONTROL, tmp);
4033 }
4034 if (rdev->irq.stat_regs.r600.hdmi0_status & AFMT_AZ_FORMAT_WTRIG) {
4035 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0);
4036 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4037 WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, tmp);
4038 }
4039 if (rdev->irq.stat_regs.r600.hdmi1_status & AFMT_AZ_FORMAT_WTRIG) {
4040 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1);
4041 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4042 WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, tmp);
4043 }
4044 } else {
4045 if (rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG) {
4046 tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL);
4047 tmp |= HDMI0_AZ_FORMAT_WTRIG_ACK;
4048 WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
4049 }
4050 if (rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG) {
4051 if (ASIC_IS_DCE3(rdev)) {
4052 tmp = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL);
4053 tmp |= HDMI0_AZ_FORMAT_WTRIG_ACK;
4054 WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, tmp);
4055 } else {
4056 tmp = RREG32(HDMI1_AUDIO_PACKET_CONTROL);
4057 tmp |= HDMI0_AZ_FORMAT_WTRIG_ACK;
4058 WREG32(HDMI1_AUDIO_PACKET_CONTROL, tmp);
4059 }
4060 }
4061 }
4062 }
4063
r600_irq_disable(struct radeon_device * rdev)4064 void r600_irq_disable(struct radeon_device *rdev)
4065 {
4066 r600_disable_interrupts(rdev);
4067 /* Wait and acknowledge irq */
4068 mdelay(1);
4069 r600_irq_ack(rdev);
4070 r600_disable_interrupt_state(rdev);
4071 }
4072
r600_get_ih_wptr(struct radeon_device * rdev)4073 static u32 r600_get_ih_wptr(struct radeon_device *rdev)
4074 {
4075 u32 wptr, tmp;
4076
4077 if (rdev->wb.enabled)
4078 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4079 else
4080 wptr = RREG32(IH_RB_WPTR);
4081
4082 if (wptr & RB_OVERFLOW) {
4083 wptr &= ~RB_OVERFLOW;
4084 /* When a ring buffer overflow happen start parsing interrupt
4085 * from the last not overwritten vector (wptr + 16). Hopefully
4086 * this should allow us to catchup.
4087 */
4088 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4089 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
4090 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4091 tmp = RREG32(IH_RB_CNTL);
4092 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4093 WREG32(IH_RB_CNTL, tmp);
4094 }
4095 return (wptr & rdev->ih.ptr_mask);
4096 }
4097
4098 /* r600 IV Ring
4099 * Each IV ring entry is 128 bits:
4100 * [7:0] - interrupt source id
4101 * [31:8] - reserved
4102 * [59:32] - interrupt source data
4103 * [127:60] - reserved
4104 *
4105 * The basic interrupt vector entries
4106 * are decoded as follows:
4107 * src_id src_data description
4108 * 1 0 D1 Vblank
4109 * 1 1 D1 Vline
4110 * 5 0 D2 Vblank
4111 * 5 1 D2 Vline
4112 * 19 0 FP Hot plug detection A
4113 * 19 1 FP Hot plug detection B
4114 * 19 2 DAC A auto-detection
4115 * 19 3 DAC B auto-detection
4116 * 21 4 HDMI block A
4117 * 21 5 HDMI block B
4118 * 176 - CP_INT RB
4119 * 177 - CP_INT IB1
4120 * 178 - CP_INT IB2
4121 * 181 - EOP Interrupt
4122 * 233 - GUI Idle
4123 *
4124 * Note, these are based on r600 and may need to be
4125 * adjusted or added to on newer asics
4126 */
4127
r600_irq_process(struct radeon_device * rdev)4128 int r600_irq_process(struct radeon_device *rdev)
4129 {
4130 u32 wptr;
4131 u32 rptr;
4132 u32 src_id, src_data;
4133 u32 ring_index;
4134 bool queue_hotplug = false;
4135 bool queue_hdmi = false;
4136 bool queue_thermal = false;
4137
4138 if (!rdev->ih.enabled || rdev->shutdown)
4139 return IRQ_NONE;
4140
4141 /* No MSIs, need a dummy read to flush PCI DMAs */
4142 if (!rdev->msi_enabled)
4143 RREG32(IH_RB_WPTR);
4144
4145 wptr = r600_get_ih_wptr(rdev);
4146
4147 restart_ih:
4148 /* is somebody else already processing irqs? */
4149 if (atomic_xchg(&rdev->ih.lock, 1))
4150 return IRQ_NONE;
4151
4152 rptr = rdev->ih.rptr;
4153 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4154
4155 /* Order reading of wptr vs. reading of IH ring data */
4156 rmb();
4157
4158 /* display interrupts */
4159 r600_irq_ack(rdev);
4160
4161 while (rptr != wptr) {
4162 /* wptr/rptr are in bytes! */
4163 ring_index = rptr / 4;
4164 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4165 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4166
4167 switch (src_id) {
4168 case 1: /* D1 vblank/vline */
4169 switch (src_data) {
4170 case 0: /* D1 vblank */
4171 if (!(rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT))
4172 DRM_DEBUG("IH: D1 vblank - IH event w/o asserted irq bit?\n");
4173
4174 if (rdev->irq.crtc_vblank_int[0]) {
4175 drm_handle_vblank(rdev->ddev, 0);
4176 #ifdef __NetBSD__
4177 spin_lock(&rdev->irq.vblank_lock);
4178 rdev->pm.vblank_sync = true;
4179 DRM_SPIN_WAKEUP_ONE(&rdev->irq.vblank_queue, &rdev->irq.vblank_lock);
4180 spin_unlock(&rdev->irq.vblank_lock);
4181 #else
4182 rdev->pm.vblank_sync = true;
4183 wake_up(&rdev->irq.vblank_queue);
4184 #endif
4185 }
4186 if (atomic_read(&rdev->irq.pflip[0]))
4187 radeon_crtc_handle_vblank(rdev, 0);
4188 rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4189 DRM_DEBUG("IH: D1 vblank\n");
4190
4191 break;
4192 case 1: /* D1 vline */
4193 if (!(rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT))
4194 DRM_DEBUG("IH: D1 vline - IH event w/o asserted irq bit?\n");
4195
4196 rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4197 DRM_DEBUG("IH: D1 vline\n");
4198
4199 break;
4200 default:
4201 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4202 break;
4203 }
4204 break;
4205 case 5: /* D2 vblank/vline */
4206 switch (src_data) {
4207 case 0: /* D2 vblank */
4208 if (!(rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT))
4209 DRM_DEBUG("IH: D2 vblank - IH event w/o asserted irq bit?\n");
4210
4211 if (rdev->irq.crtc_vblank_int[1]) {
4212 drm_handle_vblank(rdev->ddev, 1);
4213 #ifdef __NetBSD__
4214 spin_lock(&rdev->irq.vblank_lock);
4215 rdev->pm.vblank_sync = true;
4216 DRM_SPIN_WAKEUP_ONE(&rdev->irq.vblank_queue, &rdev->irq.vblank_lock);
4217 spin_unlock(&rdev->irq.vblank_lock);
4218 #else
4219 rdev->pm.vblank_sync = true;
4220 wake_up(&rdev->irq.vblank_queue);
4221 #endif
4222 }
4223 if (atomic_read(&rdev->irq.pflip[1]))
4224 radeon_crtc_handle_vblank(rdev, 1);
4225 rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VBLANK_INTERRUPT;
4226 DRM_DEBUG("IH: D2 vblank\n");
4227
4228 break;
4229 case 1: /* D1 vline */
4230 if (!(rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT))
4231 DRM_DEBUG("IH: D2 vline - IH event w/o asserted irq bit?\n");
4232
4233 rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VLINE_INTERRUPT;
4234 DRM_DEBUG("IH: D2 vline\n");
4235
4236 break;
4237 default:
4238 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4239 break;
4240 }
4241 break;
4242 case 9: /* D1 pflip */
4243 DRM_DEBUG("IH: D1 flip\n");
4244 if (radeon_use_pflipirq > 0)
4245 radeon_crtc_handle_flip(rdev, 0);
4246 break;
4247 case 11: /* D2 pflip */
4248 DRM_DEBUG("IH: D2 flip\n");
4249 if (radeon_use_pflipirq > 0)
4250 radeon_crtc_handle_flip(rdev, 1);
4251 break;
4252 case 19: /* HPD/DAC hotplug */
4253 switch (src_data) {
4254 case 0:
4255 if (!(rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT))
4256 DRM_DEBUG("IH: HPD1 - IH event w/o asserted irq bit?\n");
4257
4258 rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD1_INTERRUPT;
4259 queue_hotplug = true;
4260 DRM_DEBUG("IH: HPD1\n");
4261 break;
4262 case 1:
4263 if (!(rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT))
4264 DRM_DEBUG("IH: HPD2 - IH event w/o asserted irq bit?\n");
4265
4266 rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD2_INTERRUPT;
4267 queue_hotplug = true;
4268 DRM_DEBUG("IH: HPD2\n");
4269 break;
4270 case 4:
4271 if (!(rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT))
4272 DRM_DEBUG("IH: HPD3 - IH event w/o asserted irq bit?\n");
4273
4274 rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD3_INTERRUPT;
4275 queue_hotplug = true;
4276 DRM_DEBUG("IH: HPD3\n");
4277 break;
4278 case 5:
4279 if (!(rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT))
4280 DRM_DEBUG("IH: HPD4 - IH event w/o asserted irq bit?\n");
4281
4282 rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD4_INTERRUPT;
4283 queue_hotplug = true;
4284 DRM_DEBUG("IH: HPD4\n");
4285 break;
4286 case 10:
4287 if (!(rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT))
4288 DRM_DEBUG("IH: HPD5 - IH event w/o asserted irq bit?\n");
4289
4290 rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD5_INTERRUPT;
4291 queue_hotplug = true;
4292 DRM_DEBUG("IH: HPD5\n");
4293 break;
4294 case 12:
4295 if (!(rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT))
4296 DRM_DEBUG("IH: HPD6 - IH event w/o asserted irq bit?\n");
4297
4298 rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD6_INTERRUPT;
4299 queue_hotplug = true;
4300 DRM_DEBUG("IH: HPD6\n");
4301
4302 break;
4303 default:
4304 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4305 break;
4306 }
4307 break;
4308 case 21: /* hdmi */
4309 switch (src_data) {
4310 case 4:
4311 if (!(rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG))
4312 DRM_DEBUG("IH: HDMI0 - IH event w/o asserted irq bit?\n");
4313
4314 rdev->irq.stat_regs.r600.hdmi0_status &= ~HDMI0_AZ_FORMAT_WTRIG;
4315 queue_hdmi = true;
4316 DRM_DEBUG("IH: HDMI0\n");
4317
4318 break;
4319 case 5:
4320 if (!(rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG))
4321 DRM_DEBUG("IH: HDMI1 - IH event w/o asserted irq bit?\n");
4322
4323 rdev->irq.stat_regs.r600.hdmi1_status &= ~HDMI0_AZ_FORMAT_WTRIG;
4324 queue_hdmi = true;
4325 DRM_DEBUG("IH: HDMI1\n");
4326
4327 break;
4328 default:
4329 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
4330 break;
4331 }
4332 break;
4333 case 124: /* UVD */
4334 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4335 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4336 break;
4337 case 176: /* CP_INT in ring buffer */
4338 case 177: /* CP_INT in IB1 */
4339 case 178: /* CP_INT in IB2 */
4340 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4341 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4342 break;
4343 case 181: /* CP EOP event */
4344 DRM_DEBUG("IH: CP EOP\n");
4345 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4346 break;
4347 case 224: /* DMA trap event */
4348 DRM_DEBUG("IH: DMA trap\n");
4349 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4350 break;
4351 case 230: /* thermal low to high */
4352 DRM_DEBUG("IH: thermal low to high\n");
4353 rdev->pm.dpm.thermal.high_to_low = false;
4354 queue_thermal = true;
4355 break;
4356 case 231: /* thermal high to low */
4357 DRM_DEBUG("IH: thermal high to low\n");
4358 rdev->pm.dpm.thermal.high_to_low = true;
4359 queue_thermal = true;
4360 break;
4361 case 233: /* GUI IDLE */
4362 DRM_DEBUG("IH: GUI idle\n");
4363 break;
4364 default:
4365 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4366 break;
4367 }
4368
4369 /* wptr/rptr are in bytes! */
4370 rptr += 16;
4371 rptr &= rdev->ih.ptr_mask;
4372 WREG32(IH_RB_RPTR, rptr);
4373 }
4374 if (queue_hotplug)
4375 schedule_delayed_work(&rdev->hotplug_work, 0);
4376 if (queue_hdmi)
4377 schedule_work(&rdev->audio_work);
4378 if (queue_thermal && rdev->pm.dpm_enabled)
4379 schedule_work(&rdev->pm.dpm.thermal.work);
4380 rdev->ih.rptr = rptr;
4381 atomic_set(&rdev->ih.lock, 0);
4382
4383 /* make sure wptr hasn't changed while processing */
4384 wptr = r600_get_ih_wptr(rdev);
4385 if (wptr != rptr)
4386 goto restart_ih;
4387
4388 return IRQ_HANDLED;
4389 }
4390
4391 /*
4392 * Debugfs info
4393 */
4394 #if defined(CONFIG_DEBUG_FS)
4395
r600_debugfs_mc_info(struct seq_file * m,void * data)4396 static int r600_debugfs_mc_info(struct seq_file *m, void *data)
4397 {
4398 struct drm_info_node *node = (struct drm_info_node *) m->private;
4399 struct drm_device *dev = node->minor->dev;
4400 struct radeon_device *rdev = dev->dev_private;
4401
4402 DREG32_SYS(m, rdev, R_000E50_SRBM_STATUS);
4403 DREG32_SYS(m, rdev, VM_L2_STATUS);
4404 return 0;
4405 }
4406
4407 static struct drm_info_list r600_mc_info_list[] = {
4408 {"r600_mc_info", r600_debugfs_mc_info, 0, NULL},
4409 };
4410 #endif
4411
r600_debugfs_mc_info_init(struct radeon_device * rdev)4412 int r600_debugfs_mc_info_init(struct radeon_device *rdev)
4413 {
4414 #if defined(CONFIG_DEBUG_FS)
4415 return radeon_debugfs_add_files(rdev, r600_mc_info_list, ARRAY_SIZE(r600_mc_info_list));
4416 #else
4417 return 0;
4418 #endif
4419 }
4420
4421 #ifdef __NetBSD__
4422 # define __iomem volatile
4423 # define readl fake_readl
4424 #endif
4425
4426 /**
4427 * r600_mmio_hdp_flush - flush Host Data Path cache via MMIO
4428 * rdev: radeon device structure
4429 *
4430 * Some R6XX/R7XX don't seem to take into account HDP flushes performed
4431 * through the ring buffer. This leads to corruption in rendering, see
4432 * http://bugzilla.kernel.org/show_bug.cgi?id=15186 . To avoid this, we
4433 * directly perform the HDP flush by writing the register through MMIO.
4434 */
r600_mmio_hdp_flush(struct radeon_device * rdev)4435 void r600_mmio_hdp_flush(struct radeon_device *rdev)
4436 {
4437 /* r7xx hw bug. write to HDP_DEBUG1 followed by fb read
4438 * rather than write to HDP_REG_COHERENCY_FLUSH_CNTL.
4439 * This seems to cause problems on some AGP cards. Just use the old
4440 * method for them.
4441 */
4442 if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
4443 rdev->vram_scratch.ptr && !(rdev->flags & RADEON_IS_AGP)) {
4444 void __iomem *ptr = rdev->vram_scratch.ptr;
4445
4446 WREG32(HDP_DEBUG1, 0);
4447 (void)readl(ptr);
4448 } else
4449 WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4450 }
4451
4452 #ifdef __NetBSD__
4453 # undef __iomem
4454 # undef readl
4455 #endif
4456
r600_set_pcie_lanes(struct radeon_device * rdev,int lanes)4457 void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes)
4458 {
4459 u32 link_width_cntl, mask;
4460
4461 if (rdev->flags & RADEON_IS_IGP)
4462 return;
4463
4464 if (!(rdev->flags & RADEON_IS_PCIE))
4465 return;
4466
4467 /* x2 cards have a special sequence */
4468 if (ASIC_IS_X2(rdev))
4469 return;
4470
4471 radeon_gui_idle(rdev);
4472
4473 switch (lanes) {
4474 case 0:
4475 mask = RADEON_PCIE_LC_LINK_WIDTH_X0;
4476 break;
4477 case 1:
4478 mask = RADEON_PCIE_LC_LINK_WIDTH_X1;
4479 break;
4480 case 2:
4481 mask = RADEON_PCIE_LC_LINK_WIDTH_X2;
4482 break;
4483 case 4:
4484 mask = RADEON_PCIE_LC_LINK_WIDTH_X4;
4485 break;
4486 case 8:
4487 mask = RADEON_PCIE_LC_LINK_WIDTH_X8;
4488 break;
4489 case 12:
4490 /* not actually supported */
4491 mask = RADEON_PCIE_LC_LINK_WIDTH_X12;
4492 break;
4493 case 16:
4494 mask = RADEON_PCIE_LC_LINK_WIDTH_X16;
4495 break;
4496 default:
4497 DRM_ERROR("invalid pcie lane request: %d\n", lanes);
4498 return;
4499 }
4500
4501 link_width_cntl = RREG32_PCIE_PORT(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
4502 link_width_cntl &= ~RADEON_PCIE_LC_LINK_WIDTH_MASK;
4503 link_width_cntl |= mask << RADEON_PCIE_LC_LINK_WIDTH_SHIFT;
4504 link_width_cntl |= (RADEON_PCIE_LC_RECONFIG_NOW |
4505 R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE);
4506
4507 WREG32_PCIE_PORT(RADEON_PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4508 }
4509
r600_get_pcie_lanes(struct radeon_device * rdev)4510 int r600_get_pcie_lanes(struct radeon_device *rdev)
4511 {
4512 u32 link_width_cntl;
4513
4514 if (rdev->flags & RADEON_IS_IGP)
4515 return 0;
4516
4517 if (!(rdev->flags & RADEON_IS_PCIE))
4518 return 0;
4519
4520 /* x2 cards have a special sequence */
4521 if (ASIC_IS_X2(rdev))
4522 return 0;
4523
4524 radeon_gui_idle(rdev);
4525
4526 link_width_cntl = RREG32_PCIE_PORT(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
4527
4528 switch ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) >> RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT) {
4529 case RADEON_PCIE_LC_LINK_WIDTH_X1:
4530 return 1;
4531 case RADEON_PCIE_LC_LINK_WIDTH_X2:
4532 return 2;
4533 case RADEON_PCIE_LC_LINK_WIDTH_X4:
4534 return 4;
4535 case RADEON_PCIE_LC_LINK_WIDTH_X8:
4536 return 8;
4537 case RADEON_PCIE_LC_LINK_WIDTH_X12:
4538 /* not actually supported */
4539 return 12;
4540 case RADEON_PCIE_LC_LINK_WIDTH_X0:
4541 case RADEON_PCIE_LC_LINK_WIDTH_X16:
4542 default:
4543 return 16;
4544 }
4545 }
4546
r600_pcie_gen2_enable(struct radeon_device * rdev)4547 static void r600_pcie_gen2_enable(struct radeon_device *rdev)
4548 {
4549 #ifndef __NetBSD__ /* XXX radeon pcie */
4550 u32 link_width_cntl, lanes, speed_cntl, training_cntl, tmp;
4551 u16 link_cntl2;
4552
4553 if (radeon_pcie_gen2 == 0)
4554 return;
4555
4556 if (rdev->flags & RADEON_IS_IGP)
4557 return;
4558
4559 if (!(rdev->flags & RADEON_IS_PCIE))
4560 return;
4561
4562 /* x2 cards have a special sequence */
4563 if (ASIC_IS_X2(rdev))
4564 return;
4565
4566 /* only RV6xx+ chips are supported */
4567 if (rdev->family <= CHIP_R600)
4568 return;
4569
4570 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
4571 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
4572 return;
4573
4574 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
4575 if (speed_cntl & LC_CURRENT_DATA_RATE) {
4576 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
4577 return;
4578 }
4579
4580 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
4581
4582 /* 55 nm r6xx asics */
4583 if ((rdev->family == CHIP_RV670) ||
4584 (rdev->family == CHIP_RV620) ||
4585 (rdev->family == CHIP_RV635)) {
4586 /* advertise upconfig capability */
4587 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
4588 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
4589 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4590 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
4591 if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
4592 lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
4593 link_width_cntl &= ~(LC_LINK_WIDTH_MASK |
4594 LC_RECONFIG_ARC_MISSING_ESCAPE);
4595 link_width_cntl |= lanes | LC_RECONFIG_NOW | LC_RENEGOTIATE_EN;
4596 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4597 } else {
4598 link_width_cntl |= LC_UPCONFIGURE_DIS;
4599 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4600 }
4601 }
4602
4603 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
4604 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
4605 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
4606
4607 /* 55 nm r6xx asics */
4608 if ((rdev->family == CHIP_RV670) ||
4609 (rdev->family == CHIP_RV620) ||
4610 (rdev->family == CHIP_RV635)) {
4611 WREG32(MM_CFGREGS_CNTL, 0x8);
4612 link_cntl2 = RREG32(0x4088);
4613 WREG32(MM_CFGREGS_CNTL, 0);
4614 /* not supported yet */
4615 if (link_cntl2 & SELECTABLE_DEEMPHASIS)
4616 return;
4617 }
4618
4619 speed_cntl &= ~LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK;
4620 speed_cntl |= (0x3 << LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT);
4621 speed_cntl &= ~LC_VOLTAGE_TIMER_SEL_MASK;
4622 speed_cntl &= ~LC_FORCE_DIS_HW_SPEED_CHANGE;
4623 speed_cntl |= LC_FORCE_EN_HW_SPEED_CHANGE;
4624 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
4625
4626 tmp = RREG32(0x541c);
4627 WREG32(0x541c, tmp | 0x8);
4628 WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
4629 link_cntl2 = RREG16(0x4088);
4630 link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
4631 link_cntl2 |= 0x2;
4632 WREG16(0x4088, link_cntl2);
4633 WREG32(MM_CFGREGS_CNTL, 0);
4634
4635 if ((rdev->family == CHIP_RV670) ||
4636 (rdev->family == CHIP_RV620) ||
4637 (rdev->family == CHIP_RV635)) {
4638 training_cntl = RREG32_PCIE_PORT(PCIE_LC_TRAINING_CNTL);
4639 training_cntl &= ~LC_POINT_7_PLUS_EN;
4640 WREG32_PCIE_PORT(PCIE_LC_TRAINING_CNTL, training_cntl);
4641 } else {
4642 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
4643 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
4644 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
4645 }
4646
4647 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
4648 speed_cntl |= LC_GEN2_EN_STRAP;
4649 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
4650
4651 } else {
4652 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
4653 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
4654 if (1)
4655 link_width_cntl |= LC_UPCONFIGURE_DIS;
4656 else
4657 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
4658 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4659 }
4660 #endif
4661 }
4662
4663 /**
4664 * r600_get_gpu_clock_counter - return GPU clock counter snapshot
4665 *
4666 * @rdev: radeon_device pointer
4667 *
4668 * Fetches a GPU clock counter snapshot (R6xx-cayman).
4669 * Returns the 64 bit clock counter snapshot.
4670 */
r600_get_gpu_clock_counter(struct radeon_device * rdev)4671 uint64_t r600_get_gpu_clock_counter(struct radeon_device *rdev)
4672 {
4673 uint64_t clock;
4674
4675 mutex_lock(&rdev->gpu_clock_mutex);
4676 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
4677 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
4678 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4679 mutex_unlock(&rdev->gpu_clock_mutex);
4680 return clock;
4681 }
4682