1 /* $NetBSD: radeon_evergreen.c,v 1.5 2021/12/18 23:45:43 riastradh Exp $ */
2
3 /*
4 * Copyright 2010 Advanced Micro Devices, Inc.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Alex Deucher
25 */
26
27 #include <sys/cdefs.h>
28 __KERNEL_RCSID(0, "$NetBSD: radeon_evergreen.c,v 1.5 2021/12/18 23:45:43 riastradh Exp $");
29
30 #include <linux/firmware.h>
31 #include <linux/pci.h>
32 #include <linux/slab.h>
33
34 #include <drm/drm_vblank.h>
35 #include <drm/radeon_drm.h>
36
37 #include "atom.h"
38 #include "avivod.h"
39 #include "evergreen_blit_shaders.h"
40 #include "evergreen_reg.h"
41 #include "evergreend.h"
42 #include "radeon.h"
43 #include "radeon_asic.h"
44 #include "radeon_audio.h"
45 #include "radeon_ucode.h"
46
47 #define DC_HPDx_CONTROL(x) (DC_HPD1_CONTROL + (x * 0xc))
48 #define DC_HPDx_INT_CONTROL(x) (DC_HPD1_INT_CONTROL + (x * 0xc))
49 #define DC_HPDx_INT_STATUS_REG(x) (DC_HPD1_INT_STATUS + (x * 0xc))
50
51 /*
52 * Indirect registers accessor
53 */
eg_cg_rreg(struct radeon_device * rdev,u32 reg)54 u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
55 {
56 unsigned long flags;
57 u32 r;
58
59 spin_lock_irqsave(&rdev->cg_idx_lock, flags);
60 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
61 r = RREG32(EVERGREEN_CG_IND_DATA);
62 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
63 return r;
64 }
65
eg_cg_wreg(struct radeon_device * rdev,u32 reg,u32 v)66 void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
67 {
68 unsigned long flags;
69
70 spin_lock_irqsave(&rdev->cg_idx_lock, flags);
71 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
72 WREG32(EVERGREEN_CG_IND_DATA, (v));
73 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
74 }
75
eg_pif_phy0_rreg(struct radeon_device * rdev,u32 reg)76 u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
77 {
78 unsigned long flags;
79 u32 r;
80
81 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
82 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
83 r = RREG32(EVERGREEN_PIF_PHY0_DATA);
84 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
85 return r;
86 }
87
eg_pif_phy0_wreg(struct radeon_device * rdev,u32 reg,u32 v)88 void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
89 {
90 unsigned long flags;
91
92 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
93 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
94 WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
95 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
96 }
97
eg_pif_phy1_rreg(struct radeon_device * rdev,u32 reg)98 u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
99 {
100 unsigned long flags;
101 u32 r;
102
103 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
104 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
105 r = RREG32(EVERGREEN_PIF_PHY1_DATA);
106 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
107 return r;
108 }
109
eg_pif_phy1_wreg(struct radeon_device * rdev,u32 reg,u32 v)110 void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
111 {
112 unsigned long flags;
113
114 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
115 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
116 WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
117 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
118 }
119
120 static const u32 crtc_offsets[6] =
121 {
122 EVERGREEN_CRTC0_REGISTER_OFFSET,
123 EVERGREEN_CRTC1_REGISTER_OFFSET,
124 EVERGREEN_CRTC2_REGISTER_OFFSET,
125 EVERGREEN_CRTC3_REGISTER_OFFSET,
126 EVERGREEN_CRTC4_REGISTER_OFFSET,
127 EVERGREEN_CRTC5_REGISTER_OFFSET
128 };
129
130 #include "clearstate_evergreen.h"
131
132 static const u32 sumo_rlc_save_restore_register_list[] =
133 {
134 0x98fc,
135 0x9830,
136 0x9834,
137 0x9838,
138 0x9870,
139 0x9874,
140 0x8a14,
141 0x8b24,
142 0x8bcc,
143 0x8b10,
144 0x8d00,
145 0x8d04,
146 0x8c00,
147 0x8c04,
148 0x8c08,
149 0x8c0c,
150 0x8d8c,
151 0x8c20,
152 0x8c24,
153 0x8c28,
154 0x8c18,
155 0x8c1c,
156 0x8cf0,
157 0x8e2c,
158 0x8e38,
159 0x8c30,
160 0x9508,
161 0x9688,
162 0x9608,
163 0x960c,
164 0x9610,
165 0x9614,
166 0x88c4,
167 0x88d4,
168 0xa008,
169 0x900c,
170 0x9100,
171 0x913c,
172 0x98f8,
173 0x98f4,
174 0x9b7c,
175 0x3f8c,
176 0x8950,
177 0x8954,
178 0x8a18,
179 0x8b28,
180 0x9144,
181 0x9148,
182 0x914c,
183 0x3f90,
184 0x3f94,
185 0x915c,
186 0x9160,
187 0x9178,
188 0x917c,
189 0x9180,
190 0x918c,
191 0x9190,
192 0x9194,
193 0x9198,
194 0x919c,
195 0x91a8,
196 0x91ac,
197 0x91b0,
198 0x91b4,
199 0x91b8,
200 0x91c4,
201 0x91c8,
202 0x91cc,
203 0x91d0,
204 0x91d4,
205 0x91e0,
206 0x91e4,
207 0x91ec,
208 0x91f0,
209 0x91f4,
210 0x9200,
211 0x9204,
212 0x929c,
213 0x9150,
214 0x802c,
215 };
216
217 static void evergreen_gpu_init(struct radeon_device *rdev);
218 void evergreen_fini(struct radeon_device *rdev);
219 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
220 void evergreen_program_aspm(struct radeon_device *rdev);
221 extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
222 int ring, u32 cp_int_cntl);
223 extern void cayman_vm_decode_fault(struct radeon_device *rdev,
224 u32 status, u32 addr);
225 void cik_init_cp_pg_table(struct radeon_device *rdev);
226
227 extern u32 si_get_csb_size(struct radeon_device *rdev);
228 extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
229 extern u32 cik_get_csb_size(struct radeon_device *rdev);
230 extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
231 extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
232
233 static const u32 evergreen_golden_registers[] =
234 {
235 0x3f90, 0xffff0000, 0xff000000,
236 0x9148, 0xffff0000, 0xff000000,
237 0x3f94, 0xffff0000, 0xff000000,
238 0x914c, 0xffff0000, 0xff000000,
239 0x9b7c, 0xffffffff, 0x00000000,
240 0x8a14, 0xffffffff, 0x00000007,
241 0x8b10, 0xffffffff, 0x00000000,
242 0x960c, 0xffffffff, 0x54763210,
243 0x88c4, 0xffffffff, 0x000000c2,
244 0x88d4, 0xffffffff, 0x00000010,
245 0x8974, 0xffffffff, 0x00000000,
246 0xc78, 0x00000080, 0x00000080,
247 0x5eb4, 0xffffffff, 0x00000002,
248 0x5e78, 0xffffffff, 0x001000f0,
249 0x6104, 0x01000300, 0x00000000,
250 0x5bc0, 0x00300000, 0x00000000,
251 0x7030, 0xffffffff, 0x00000011,
252 0x7c30, 0xffffffff, 0x00000011,
253 0x10830, 0xffffffff, 0x00000011,
254 0x11430, 0xffffffff, 0x00000011,
255 0x12030, 0xffffffff, 0x00000011,
256 0x12c30, 0xffffffff, 0x00000011,
257 0xd02c, 0xffffffff, 0x08421000,
258 0x240c, 0xffffffff, 0x00000380,
259 0x8b24, 0xffffffff, 0x00ff0fff,
260 0x28a4c, 0x06000000, 0x06000000,
261 0x10c, 0x00000001, 0x00000001,
262 0x8d00, 0xffffffff, 0x100e4848,
263 0x8d04, 0xffffffff, 0x00164745,
264 0x8c00, 0xffffffff, 0xe4000003,
265 0x8c04, 0xffffffff, 0x40600060,
266 0x8c08, 0xffffffff, 0x001c001c,
267 0x8cf0, 0xffffffff, 0x08e00620,
268 0x8c20, 0xffffffff, 0x00800080,
269 0x8c24, 0xffffffff, 0x00800080,
270 0x8c18, 0xffffffff, 0x20202078,
271 0x8c1c, 0xffffffff, 0x00001010,
272 0x28350, 0xffffffff, 0x00000000,
273 0xa008, 0xffffffff, 0x00010000,
274 0x5c4, 0xffffffff, 0x00000001,
275 0x9508, 0xffffffff, 0x00000002,
276 0x913c, 0x0000000f, 0x0000000a
277 };
278
279 static const u32 evergreen_golden_registers2[] =
280 {
281 0x2f4c, 0xffffffff, 0x00000000,
282 0x54f4, 0xffffffff, 0x00000000,
283 0x54f0, 0xffffffff, 0x00000000,
284 0x5498, 0xffffffff, 0x00000000,
285 0x549c, 0xffffffff, 0x00000000,
286 0x5494, 0xffffffff, 0x00000000,
287 0x53cc, 0xffffffff, 0x00000000,
288 0x53c8, 0xffffffff, 0x00000000,
289 0x53c4, 0xffffffff, 0x00000000,
290 0x53c0, 0xffffffff, 0x00000000,
291 0x53bc, 0xffffffff, 0x00000000,
292 0x53b8, 0xffffffff, 0x00000000,
293 0x53b4, 0xffffffff, 0x00000000,
294 0x53b0, 0xffffffff, 0x00000000
295 };
296
297 static const u32 cypress_mgcg_init[] =
298 {
299 0x802c, 0xffffffff, 0xc0000000,
300 0x5448, 0xffffffff, 0x00000100,
301 0x55e4, 0xffffffff, 0x00000100,
302 0x160c, 0xffffffff, 0x00000100,
303 0x5644, 0xffffffff, 0x00000100,
304 0xc164, 0xffffffff, 0x00000100,
305 0x8a18, 0xffffffff, 0x00000100,
306 0x897c, 0xffffffff, 0x06000100,
307 0x8b28, 0xffffffff, 0x00000100,
308 0x9144, 0xffffffff, 0x00000100,
309 0x9a60, 0xffffffff, 0x00000100,
310 0x9868, 0xffffffff, 0x00000100,
311 0x8d58, 0xffffffff, 0x00000100,
312 0x9510, 0xffffffff, 0x00000100,
313 0x949c, 0xffffffff, 0x00000100,
314 0x9654, 0xffffffff, 0x00000100,
315 0x9030, 0xffffffff, 0x00000100,
316 0x9034, 0xffffffff, 0x00000100,
317 0x9038, 0xffffffff, 0x00000100,
318 0x903c, 0xffffffff, 0x00000100,
319 0x9040, 0xffffffff, 0x00000100,
320 0xa200, 0xffffffff, 0x00000100,
321 0xa204, 0xffffffff, 0x00000100,
322 0xa208, 0xffffffff, 0x00000100,
323 0xa20c, 0xffffffff, 0x00000100,
324 0x971c, 0xffffffff, 0x00000100,
325 0x977c, 0xffffffff, 0x00000100,
326 0x3f80, 0xffffffff, 0x00000100,
327 0xa210, 0xffffffff, 0x00000100,
328 0xa214, 0xffffffff, 0x00000100,
329 0x4d8, 0xffffffff, 0x00000100,
330 0x9784, 0xffffffff, 0x00000100,
331 0x9698, 0xffffffff, 0x00000100,
332 0x4d4, 0xffffffff, 0x00000200,
333 0x30cc, 0xffffffff, 0x00000100,
334 0xd0c0, 0xffffffff, 0xff000100,
335 0x802c, 0xffffffff, 0x40000000,
336 0x915c, 0xffffffff, 0x00010000,
337 0x9160, 0xffffffff, 0x00030002,
338 0x9178, 0xffffffff, 0x00070000,
339 0x917c, 0xffffffff, 0x00030002,
340 0x9180, 0xffffffff, 0x00050004,
341 0x918c, 0xffffffff, 0x00010006,
342 0x9190, 0xffffffff, 0x00090008,
343 0x9194, 0xffffffff, 0x00070000,
344 0x9198, 0xffffffff, 0x00030002,
345 0x919c, 0xffffffff, 0x00050004,
346 0x91a8, 0xffffffff, 0x00010006,
347 0x91ac, 0xffffffff, 0x00090008,
348 0x91b0, 0xffffffff, 0x00070000,
349 0x91b4, 0xffffffff, 0x00030002,
350 0x91b8, 0xffffffff, 0x00050004,
351 0x91c4, 0xffffffff, 0x00010006,
352 0x91c8, 0xffffffff, 0x00090008,
353 0x91cc, 0xffffffff, 0x00070000,
354 0x91d0, 0xffffffff, 0x00030002,
355 0x91d4, 0xffffffff, 0x00050004,
356 0x91e0, 0xffffffff, 0x00010006,
357 0x91e4, 0xffffffff, 0x00090008,
358 0x91e8, 0xffffffff, 0x00000000,
359 0x91ec, 0xffffffff, 0x00070000,
360 0x91f0, 0xffffffff, 0x00030002,
361 0x91f4, 0xffffffff, 0x00050004,
362 0x9200, 0xffffffff, 0x00010006,
363 0x9204, 0xffffffff, 0x00090008,
364 0x9208, 0xffffffff, 0x00070000,
365 0x920c, 0xffffffff, 0x00030002,
366 0x9210, 0xffffffff, 0x00050004,
367 0x921c, 0xffffffff, 0x00010006,
368 0x9220, 0xffffffff, 0x00090008,
369 0x9224, 0xffffffff, 0x00070000,
370 0x9228, 0xffffffff, 0x00030002,
371 0x922c, 0xffffffff, 0x00050004,
372 0x9238, 0xffffffff, 0x00010006,
373 0x923c, 0xffffffff, 0x00090008,
374 0x9240, 0xffffffff, 0x00070000,
375 0x9244, 0xffffffff, 0x00030002,
376 0x9248, 0xffffffff, 0x00050004,
377 0x9254, 0xffffffff, 0x00010006,
378 0x9258, 0xffffffff, 0x00090008,
379 0x925c, 0xffffffff, 0x00070000,
380 0x9260, 0xffffffff, 0x00030002,
381 0x9264, 0xffffffff, 0x00050004,
382 0x9270, 0xffffffff, 0x00010006,
383 0x9274, 0xffffffff, 0x00090008,
384 0x9278, 0xffffffff, 0x00070000,
385 0x927c, 0xffffffff, 0x00030002,
386 0x9280, 0xffffffff, 0x00050004,
387 0x928c, 0xffffffff, 0x00010006,
388 0x9290, 0xffffffff, 0x00090008,
389 0x9294, 0xffffffff, 0x00000000,
390 0x929c, 0xffffffff, 0x00000001,
391 0x802c, 0xffffffff, 0x40010000,
392 0x915c, 0xffffffff, 0x00010000,
393 0x9160, 0xffffffff, 0x00030002,
394 0x9178, 0xffffffff, 0x00070000,
395 0x917c, 0xffffffff, 0x00030002,
396 0x9180, 0xffffffff, 0x00050004,
397 0x918c, 0xffffffff, 0x00010006,
398 0x9190, 0xffffffff, 0x00090008,
399 0x9194, 0xffffffff, 0x00070000,
400 0x9198, 0xffffffff, 0x00030002,
401 0x919c, 0xffffffff, 0x00050004,
402 0x91a8, 0xffffffff, 0x00010006,
403 0x91ac, 0xffffffff, 0x00090008,
404 0x91b0, 0xffffffff, 0x00070000,
405 0x91b4, 0xffffffff, 0x00030002,
406 0x91b8, 0xffffffff, 0x00050004,
407 0x91c4, 0xffffffff, 0x00010006,
408 0x91c8, 0xffffffff, 0x00090008,
409 0x91cc, 0xffffffff, 0x00070000,
410 0x91d0, 0xffffffff, 0x00030002,
411 0x91d4, 0xffffffff, 0x00050004,
412 0x91e0, 0xffffffff, 0x00010006,
413 0x91e4, 0xffffffff, 0x00090008,
414 0x91e8, 0xffffffff, 0x00000000,
415 0x91ec, 0xffffffff, 0x00070000,
416 0x91f0, 0xffffffff, 0x00030002,
417 0x91f4, 0xffffffff, 0x00050004,
418 0x9200, 0xffffffff, 0x00010006,
419 0x9204, 0xffffffff, 0x00090008,
420 0x9208, 0xffffffff, 0x00070000,
421 0x920c, 0xffffffff, 0x00030002,
422 0x9210, 0xffffffff, 0x00050004,
423 0x921c, 0xffffffff, 0x00010006,
424 0x9220, 0xffffffff, 0x00090008,
425 0x9224, 0xffffffff, 0x00070000,
426 0x9228, 0xffffffff, 0x00030002,
427 0x922c, 0xffffffff, 0x00050004,
428 0x9238, 0xffffffff, 0x00010006,
429 0x923c, 0xffffffff, 0x00090008,
430 0x9240, 0xffffffff, 0x00070000,
431 0x9244, 0xffffffff, 0x00030002,
432 0x9248, 0xffffffff, 0x00050004,
433 0x9254, 0xffffffff, 0x00010006,
434 0x9258, 0xffffffff, 0x00090008,
435 0x925c, 0xffffffff, 0x00070000,
436 0x9260, 0xffffffff, 0x00030002,
437 0x9264, 0xffffffff, 0x00050004,
438 0x9270, 0xffffffff, 0x00010006,
439 0x9274, 0xffffffff, 0x00090008,
440 0x9278, 0xffffffff, 0x00070000,
441 0x927c, 0xffffffff, 0x00030002,
442 0x9280, 0xffffffff, 0x00050004,
443 0x928c, 0xffffffff, 0x00010006,
444 0x9290, 0xffffffff, 0x00090008,
445 0x9294, 0xffffffff, 0x00000000,
446 0x929c, 0xffffffff, 0x00000001,
447 0x802c, 0xffffffff, 0xc0000000
448 };
449
450 static const u32 redwood_mgcg_init[] =
451 {
452 0x802c, 0xffffffff, 0xc0000000,
453 0x5448, 0xffffffff, 0x00000100,
454 0x55e4, 0xffffffff, 0x00000100,
455 0x160c, 0xffffffff, 0x00000100,
456 0x5644, 0xffffffff, 0x00000100,
457 0xc164, 0xffffffff, 0x00000100,
458 0x8a18, 0xffffffff, 0x00000100,
459 0x897c, 0xffffffff, 0x06000100,
460 0x8b28, 0xffffffff, 0x00000100,
461 0x9144, 0xffffffff, 0x00000100,
462 0x9a60, 0xffffffff, 0x00000100,
463 0x9868, 0xffffffff, 0x00000100,
464 0x8d58, 0xffffffff, 0x00000100,
465 0x9510, 0xffffffff, 0x00000100,
466 0x949c, 0xffffffff, 0x00000100,
467 0x9654, 0xffffffff, 0x00000100,
468 0x9030, 0xffffffff, 0x00000100,
469 0x9034, 0xffffffff, 0x00000100,
470 0x9038, 0xffffffff, 0x00000100,
471 0x903c, 0xffffffff, 0x00000100,
472 0x9040, 0xffffffff, 0x00000100,
473 0xa200, 0xffffffff, 0x00000100,
474 0xa204, 0xffffffff, 0x00000100,
475 0xa208, 0xffffffff, 0x00000100,
476 0xa20c, 0xffffffff, 0x00000100,
477 0x971c, 0xffffffff, 0x00000100,
478 0x977c, 0xffffffff, 0x00000100,
479 0x3f80, 0xffffffff, 0x00000100,
480 0xa210, 0xffffffff, 0x00000100,
481 0xa214, 0xffffffff, 0x00000100,
482 0x4d8, 0xffffffff, 0x00000100,
483 0x9784, 0xffffffff, 0x00000100,
484 0x9698, 0xffffffff, 0x00000100,
485 0x4d4, 0xffffffff, 0x00000200,
486 0x30cc, 0xffffffff, 0x00000100,
487 0xd0c0, 0xffffffff, 0xff000100,
488 0x802c, 0xffffffff, 0x40000000,
489 0x915c, 0xffffffff, 0x00010000,
490 0x9160, 0xffffffff, 0x00030002,
491 0x9178, 0xffffffff, 0x00070000,
492 0x917c, 0xffffffff, 0x00030002,
493 0x9180, 0xffffffff, 0x00050004,
494 0x918c, 0xffffffff, 0x00010006,
495 0x9190, 0xffffffff, 0x00090008,
496 0x9194, 0xffffffff, 0x00070000,
497 0x9198, 0xffffffff, 0x00030002,
498 0x919c, 0xffffffff, 0x00050004,
499 0x91a8, 0xffffffff, 0x00010006,
500 0x91ac, 0xffffffff, 0x00090008,
501 0x91b0, 0xffffffff, 0x00070000,
502 0x91b4, 0xffffffff, 0x00030002,
503 0x91b8, 0xffffffff, 0x00050004,
504 0x91c4, 0xffffffff, 0x00010006,
505 0x91c8, 0xffffffff, 0x00090008,
506 0x91cc, 0xffffffff, 0x00070000,
507 0x91d0, 0xffffffff, 0x00030002,
508 0x91d4, 0xffffffff, 0x00050004,
509 0x91e0, 0xffffffff, 0x00010006,
510 0x91e4, 0xffffffff, 0x00090008,
511 0x91e8, 0xffffffff, 0x00000000,
512 0x91ec, 0xffffffff, 0x00070000,
513 0x91f0, 0xffffffff, 0x00030002,
514 0x91f4, 0xffffffff, 0x00050004,
515 0x9200, 0xffffffff, 0x00010006,
516 0x9204, 0xffffffff, 0x00090008,
517 0x9294, 0xffffffff, 0x00000000,
518 0x929c, 0xffffffff, 0x00000001,
519 0x802c, 0xffffffff, 0xc0000000
520 };
521
522 static const u32 cedar_golden_registers[] =
523 {
524 0x3f90, 0xffff0000, 0xff000000,
525 0x9148, 0xffff0000, 0xff000000,
526 0x3f94, 0xffff0000, 0xff000000,
527 0x914c, 0xffff0000, 0xff000000,
528 0x9b7c, 0xffffffff, 0x00000000,
529 0x8a14, 0xffffffff, 0x00000007,
530 0x8b10, 0xffffffff, 0x00000000,
531 0x960c, 0xffffffff, 0x54763210,
532 0x88c4, 0xffffffff, 0x000000c2,
533 0x88d4, 0xffffffff, 0x00000000,
534 0x8974, 0xffffffff, 0x00000000,
535 0xc78, 0x00000080, 0x00000080,
536 0x5eb4, 0xffffffff, 0x00000002,
537 0x5e78, 0xffffffff, 0x001000f0,
538 0x6104, 0x01000300, 0x00000000,
539 0x5bc0, 0x00300000, 0x00000000,
540 0x7030, 0xffffffff, 0x00000011,
541 0x7c30, 0xffffffff, 0x00000011,
542 0x10830, 0xffffffff, 0x00000011,
543 0x11430, 0xffffffff, 0x00000011,
544 0xd02c, 0xffffffff, 0x08421000,
545 0x240c, 0xffffffff, 0x00000380,
546 0x8b24, 0xffffffff, 0x00ff0fff,
547 0x28a4c, 0x06000000, 0x06000000,
548 0x10c, 0x00000001, 0x00000001,
549 0x8d00, 0xffffffff, 0x100e4848,
550 0x8d04, 0xffffffff, 0x00164745,
551 0x8c00, 0xffffffff, 0xe4000003,
552 0x8c04, 0xffffffff, 0x40600060,
553 0x8c08, 0xffffffff, 0x001c001c,
554 0x8cf0, 0xffffffff, 0x08e00410,
555 0x8c20, 0xffffffff, 0x00800080,
556 0x8c24, 0xffffffff, 0x00800080,
557 0x8c18, 0xffffffff, 0x20202078,
558 0x8c1c, 0xffffffff, 0x00001010,
559 0x28350, 0xffffffff, 0x00000000,
560 0xa008, 0xffffffff, 0x00010000,
561 0x5c4, 0xffffffff, 0x00000001,
562 0x9508, 0xffffffff, 0x00000002
563 };
564
565 static const u32 cedar_mgcg_init[] =
566 {
567 0x802c, 0xffffffff, 0xc0000000,
568 0x5448, 0xffffffff, 0x00000100,
569 0x55e4, 0xffffffff, 0x00000100,
570 0x160c, 0xffffffff, 0x00000100,
571 0x5644, 0xffffffff, 0x00000100,
572 0xc164, 0xffffffff, 0x00000100,
573 0x8a18, 0xffffffff, 0x00000100,
574 0x897c, 0xffffffff, 0x06000100,
575 0x8b28, 0xffffffff, 0x00000100,
576 0x9144, 0xffffffff, 0x00000100,
577 0x9a60, 0xffffffff, 0x00000100,
578 0x9868, 0xffffffff, 0x00000100,
579 0x8d58, 0xffffffff, 0x00000100,
580 0x9510, 0xffffffff, 0x00000100,
581 0x949c, 0xffffffff, 0x00000100,
582 0x9654, 0xffffffff, 0x00000100,
583 0x9030, 0xffffffff, 0x00000100,
584 0x9034, 0xffffffff, 0x00000100,
585 0x9038, 0xffffffff, 0x00000100,
586 0x903c, 0xffffffff, 0x00000100,
587 0x9040, 0xffffffff, 0x00000100,
588 0xa200, 0xffffffff, 0x00000100,
589 0xa204, 0xffffffff, 0x00000100,
590 0xa208, 0xffffffff, 0x00000100,
591 0xa20c, 0xffffffff, 0x00000100,
592 0x971c, 0xffffffff, 0x00000100,
593 0x977c, 0xffffffff, 0x00000100,
594 0x3f80, 0xffffffff, 0x00000100,
595 0xa210, 0xffffffff, 0x00000100,
596 0xa214, 0xffffffff, 0x00000100,
597 0x4d8, 0xffffffff, 0x00000100,
598 0x9784, 0xffffffff, 0x00000100,
599 0x9698, 0xffffffff, 0x00000100,
600 0x4d4, 0xffffffff, 0x00000200,
601 0x30cc, 0xffffffff, 0x00000100,
602 0xd0c0, 0xffffffff, 0xff000100,
603 0x802c, 0xffffffff, 0x40000000,
604 0x915c, 0xffffffff, 0x00010000,
605 0x9178, 0xffffffff, 0x00050000,
606 0x917c, 0xffffffff, 0x00030002,
607 0x918c, 0xffffffff, 0x00010004,
608 0x9190, 0xffffffff, 0x00070006,
609 0x9194, 0xffffffff, 0x00050000,
610 0x9198, 0xffffffff, 0x00030002,
611 0x91a8, 0xffffffff, 0x00010004,
612 0x91ac, 0xffffffff, 0x00070006,
613 0x91e8, 0xffffffff, 0x00000000,
614 0x9294, 0xffffffff, 0x00000000,
615 0x929c, 0xffffffff, 0x00000001,
616 0x802c, 0xffffffff, 0xc0000000
617 };
618
619 static const u32 juniper_mgcg_init[] =
620 {
621 0x802c, 0xffffffff, 0xc0000000,
622 0x5448, 0xffffffff, 0x00000100,
623 0x55e4, 0xffffffff, 0x00000100,
624 0x160c, 0xffffffff, 0x00000100,
625 0x5644, 0xffffffff, 0x00000100,
626 0xc164, 0xffffffff, 0x00000100,
627 0x8a18, 0xffffffff, 0x00000100,
628 0x897c, 0xffffffff, 0x06000100,
629 0x8b28, 0xffffffff, 0x00000100,
630 0x9144, 0xffffffff, 0x00000100,
631 0x9a60, 0xffffffff, 0x00000100,
632 0x9868, 0xffffffff, 0x00000100,
633 0x8d58, 0xffffffff, 0x00000100,
634 0x9510, 0xffffffff, 0x00000100,
635 0x949c, 0xffffffff, 0x00000100,
636 0x9654, 0xffffffff, 0x00000100,
637 0x9030, 0xffffffff, 0x00000100,
638 0x9034, 0xffffffff, 0x00000100,
639 0x9038, 0xffffffff, 0x00000100,
640 0x903c, 0xffffffff, 0x00000100,
641 0x9040, 0xffffffff, 0x00000100,
642 0xa200, 0xffffffff, 0x00000100,
643 0xa204, 0xffffffff, 0x00000100,
644 0xa208, 0xffffffff, 0x00000100,
645 0xa20c, 0xffffffff, 0x00000100,
646 0x971c, 0xffffffff, 0x00000100,
647 0xd0c0, 0xffffffff, 0xff000100,
648 0x802c, 0xffffffff, 0x40000000,
649 0x915c, 0xffffffff, 0x00010000,
650 0x9160, 0xffffffff, 0x00030002,
651 0x9178, 0xffffffff, 0x00070000,
652 0x917c, 0xffffffff, 0x00030002,
653 0x9180, 0xffffffff, 0x00050004,
654 0x918c, 0xffffffff, 0x00010006,
655 0x9190, 0xffffffff, 0x00090008,
656 0x9194, 0xffffffff, 0x00070000,
657 0x9198, 0xffffffff, 0x00030002,
658 0x919c, 0xffffffff, 0x00050004,
659 0x91a8, 0xffffffff, 0x00010006,
660 0x91ac, 0xffffffff, 0x00090008,
661 0x91b0, 0xffffffff, 0x00070000,
662 0x91b4, 0xffffffff, 0x00030002,
663 0x91b8, 0xffffffff, 0x00050004,
664 0x91c4, 0xffffffff, 0x00010006,
665 0x91c8, 0xffffffff, 0x00090008,
666 0x91cc, 0xffffffff, 0x00070000,
667 0x91d0, 0xffffffff, 0x00030002,
668 0x91d4, 0xffffffff, 0x00050004,
669 0x91e0, 0xffffffff, 0x00010006,
670 0x91e4, 0xffffffff, 0x00090008,
671 0x91e8, 0xffffffff, 0x00000000,
672 0x91ec, 0xffffffff, 0x00070000,
673 0x91f0, 0xffffffff, 0x00030002,
674 0x91f4, 0xffffffff, 0x00050004,
675 0x9200, 0xffffffff, 0x00010006,
676 0x9204, 0xffffffff, 0x00090008,
677 0x9208, 0xffffffff, 0x00070000,
678 0x920c, 0xffffffff, 0x00030002,
679 0x9210, 0xffffffff, 0x00050004,
680 0x921c, 0xffffffff, 0x00010006,
681 0x9220, 0xffffffff, 0x00090008,
682 0x9224, 0xffffffff, 0x00070000,
683 0x9228, 0xffffffff, 0x00030002,
684 0x922c, 0xffffffff, 0x00050004,
685 0x9238, 0xffffffff, 0x00010006,
686 0x923c, 0xffffffff, 0x00090008,
687 0x9240, 0xffffffff, 0x00070000,
688 0x9244, 0xffffffff, 0x00030002,
689 0x9248, 0xffffffff, 0x00050004,
690 0x9254, 0xffffffff, 0x00010006,
691 0x9258, 0xffffffff, 0x00090008,
692 0x925c, 0xffffffff, 0x00070000,
693 0x9260, 0xffffffff, 0x00030002,
694 0x9264, 0xffffffff, 0x00050004,
695 0x9270, 0xffffffff, 0x00010006,
696 0x9274, 0xffffffff, 0x00090008,
697 0x9278, 0xffffffff, 0x00070000,
698 0x927c, 0xffffffff, 0x00030002,
699 0x9280, 0xffffffff, 0x00050004,
700 0x928c, 0xffffffff, 0x00010006,
701 0x9290, 0xffffffff, 0x00090008,
702 0x9294, 0xffffffff, 0x00000000,
703 0x929c, 0xffffffff, 0x00000001,
704 0x802c, 0xffffffff, 0xc0000000,
705 0x977c, 0xffffffff, 0x00000100,
706 0x3f80, 0xffffffff, 0x00000100,
707 0xa210, 0xffffffff, 0x00000100,
708 0xa214, 0xffffffff, 0x00000100,
709 0x4d8, 0xffffffff, 0x00000100,
710 0x9784, 0xffffffff, 0x00000100,
711 0x9698, 0xffffffff, 0x00000100,
712 0x4d4, 0xffffffff, 0x00000200,
713 0x30cc, 0xffffffff, 0x00000100,
714 0x802c, 0xffffffff, 0xc0000000
715 };
716
717 static const u32 supersumo_golden_registers[] =
718 {
719 0x5eb4, 0xffffffff, 0x00000002,
720 0x5c4, 0xffffffff, 0x00000001,
721 0x7030, 0xffffffff, 0x00000011,
722 0x7c30, 0xffffffff, 0x00000011,
723 0x6104, 0x01000300, 0x00000000,
724 0x5bc0, 0x00300000, 0x00000000,
725 0x8c04, 0xffffffff, 0x40600060,
726 0x8c08, 0xffffffff, 0x001c001c,
727 0x8c20, 0xffffffff, 0x00800080,
728 0x8c24, 0xffffffff, 0x00800080,
729 0x8c18, 0xffffffff, 0x20202078,
730 0x8c1c, 0xffffffff, 0x00001010,
731 0x918c, 0xffffffff, 0x00010006,
732 0x91a8, 0xffffffff, 0x00010006,
733 0x91c4, 0xffffffff, 0x00010006,
734 0x91e0, 0xffffffff, 0x00010006,
735 0x9200, 0xffffffff, 0x00010006,
736 0x9150, 0xffffffff, 0x6e944040,
737 0x917c, 0xffffffff, 0x00030002,
738 0x9180, 0xffffffff, 0x00050004,
739 0x9198, 0xffffffff, 0x00030002,
740 0x919c, 0xffffffff, 0x00050004,
741 0x91b4, 0xffffffff, 0x00030002,
742 0x91b8, 0xffffffff, 0x00050004,
743 0x91d0, 0xffffffff, 0x00030002,
744 0x91d4, 0xffffffff, 0x00050004,
745 0x91f0, 0xffffffff, 0x00030002,
746 0x91f4, 0xffffffff, 0x00050004,
747 0x915c, 0xffffffff, 0x00010000,
748 0x9160, 0xffffffff, 0x00030002,
749 0x3f90, 0xffff0000, 0xff000000,
750 0x9178, 0xffffffff, 0x00070000,
751 0x9194, 0xffffffff, 0x00070000,
752 0x91b0, 0xffffffff, 0x00070000,
753 0x91cc, 0xffffffff, 0x00070000,
754 0x91ec, 0xffffffff, 0x00070000,
755 0x9148, 0xffff0000, 0xff000000,
756 0x9190, 0xffffffff, 0x00090008,
757 0x91ac, 0xffffffff, 0x00090008,
758 0x91c8, 0xffffffff, 0x00090008,
759 0x91e4, 0xffffffff, 0x00090008,
760 0x9204, 0xffffffff, 0x00090008,
761 0x3f94, 0xffff0000, 0xff000000,
762 0x914c, 0xffff0000, 0xff000000,
763 0x929c, 0xffffffff, 0x00000001,
764 0x8a18, 0xffffffff, 0x00000100,
765 0x8b28, 0xffffffff, 0x00000100,
766 0x9144, 0xffffffff, 0x00000100,
767 0x5644, 0xffffffff, 0x00000100,
768 0x9b7c, 0xffffffff, 0x00000000,
769 0x8030, 0xffffffff, 0x0000100a,
770 0x8a14, 0xffffffff, 0x00000007,
771 0x8b24, 0xffffffff, 0x00ff0fff,
772 0x8b10, 0xffffffff, 0x00000000,
773 0x28a4c, 0x06000000, 0x06000000,
774 0x4d8, 0xffffffff, 0x00000100,
775 0x913c, 0xffff000f, 0x0100000a,
776 0x960c, 0xffffffff, 0x54763210,
777 0x88c4, 0xffffffff, 0x000000c2,
778 0x88d4, 0xffffffff, 0x00000010,
779 0x8974, 0xffffffff, 0x00000000,
780 0xc78, 0x00000080, 0x00000080,
781 0x5e78, 0xffffffff, 0x001000f0,
782 0xd02c, 0xffffffff, 0x08421000,
783 0xa008, 0xffffffff, 0x00010000,
784 0x8d00, 0xffffffff, 0x100e4848,
785 0x8d04, 0xffffffff, 0x00164745,
786 0x8c00, 0xffffffff, 0xe4000003,
787 0x8cf0, 0x1fffffff, 0x08e00620,
788 0x28350, 0xffffffff, 0x00000000,
789 0x9508, 0xffffffff, 0x00000002
790 };
791
792 static const u32 sumo_golden_registers[] =
793 {
794 0x900c, 0x00ffffff, 0x0017071f,
795 0x8c18, 0xffffffff, 0x10101060,
796 0x8c1c, 0xffffffff, 0x00001010,
797 0x8c30, 0x0000000f, 0x00000005,
798 0x9688, 0x0000000f, 0x00000007
799 };
800
801 static const u32 wrestler_golden_registers[] =
802 {
803 0x5eb4, 0xffffffff, 0x00000002,
804 0x5c4, 0xffffffff, 0x00000001,
805 0x7030, 0xffffffff, 0x00000011,
806 0x7c30, 0xffffffff, 0x00000011,
807 0x6104, 0x01000300, 0x00000000,
808 0x5bc0, 0x00300000, 0x00000000,
809 0x918c, 0xffffffff, 0x00010006,
810 0x91a8, 0xffffffff, 0x00010006,
811 0x9150, 0xffffffff, 0x6e944040,
812 0x917c, 0xffffffff, 0x00030002,
813 0x9198, 0xffffffff, 0x00030002,
814 0x915c, 0xffffffff, 0x00010000,
815 0x3f90, 0xffff0000, 0xff000000,
816 0x9178, 0xffffffff, 0x00070000,
817 0x9194, 0xffffffff, 0x00070000,
818 0x9148, 0xffff0000, 0xff000000,
819 0x9190, 0xffffffff, 0x00090008,
820 0x91ac, 0xffffffff, 0x00090008,
821 0x3f94, 0xffff0000, 0xff000000,
822 0x914c, 0xffff0000, 0xff000000,
823 0x929c, 0xffffffff, 0x00000001,
824 0x8a18, 0xffffffff, 0x00000100,
825 0x8b28, 0xffffffff, 0x00000100,
826 0x9144, 0xffffffff, 0x00000100,
827 0x9b7c, 0xffffffff, 0x00000000,
828 0x8030, 0xffffffff, 0x0000100a,
829 0x8a14, 0xffffffff, 0x00000001,
830 0x8b24, 0xffffffff, 0x00ff0fff,
831 0x8b10, 0xffffffff, 0x00000000,
832 0x28a4c, 0x06000000, 0x06000000,
833 0x4d8, 0xffffffff, 0x00000100,
834 0x913c, 0xffff000f, 0x0100000a,
835 0x960c, 0xffffffff, 0x54763210,
836 0x88c4, 0xffffffff, 0x000000c2,
837 0x88d4, 0xffffffff, 0x00000010,
838 0x8974, 0xffffffff, 0x00000000,
839 0xc78, 0x00000080, 0x00000080,
840 0x5e78, 0xffffffff, 0x001000f0,
841 0xd02c, 0xffffffff, 0x08421000,
842 0xa008, 0xffffffff, 0x00010000,
843 0x8d00, 0xffffffff, 0x100e4848,
844 0x8d04, 0xffffffff, 0x00164745,
845 0x8c00, 0xffffffff, 0xe4000003,
846 0x8cf0, 0x1fffffff, 0x08e00410,
847 0x28350, 0xffffffff, 0x00000000,
848 0x9508, 0xffffffff, 0x00000002,
849 0x900c, 0xffffffff, 0x0017071f,
850 0x8c18, 0xffffffff, 0x10101060,
851 0x8c1c, 0xffffffff, 0x00001010
852 };
853
854 static const u32 barts_golden_registers[] =
855 {
856 0x5eb4, 0xffffffff, 0x00000002,
857 0x5e78, 0x8f311ff1, 0x001000f0,
858 0x3f90, 0xffff0000, 0xff000000,
859 0x9148, 0xffff0000, 0xff000000,
860 0x3f94, 0xffff0000, 0xff000000,
861 0x914c, 0xffff0000, 0xff000000,
862 0xc78, 0x00000080, 0x00000080,
863 0xbd4, 0x70073777, 0x00010001,
864 0xd02c, 0xbfffff1f, 0x08421000,
865 0xd0b8, 0x03773777, 0x02011003,
866 0x5bc0, 0x00200000, 0x50100000,
867 0x98f8, 0x33773777, 0x02011003,
868 0x98fc, 0xffffffff, 0x76543210,
869 0x7030, 0x31000311, 0x00000011,
870 0x2f48, 0x00000007, 0x02011003,
871 0x6b28, 0x00000010, 0x00000012,
872 0x7728, 0x00000010, 0x00000012,
873 0x10328, 0x00000010, 0x00000012,
874 0x10f28, 0x00000010, 0x00000012,
875 0x11b28, 0x00000010, 0x00000012,
876 0x12728, 0x00000010, 0x00000012,
877 0x240c, 0x000007ff, 0x00000380,
878 0x8a14, 0xf000001f, 0x00000007,
879 0x8b24, 0x3fff3fff, 0x00ff0fff,
880 0x8b10, 0x0000ff0f, 0x00000000,
881 0x28a4c, 0x07ffffff, 0x06000000,
882 0x10c, 0x00000001, 0x00010003,
883 0xa02c, 0xffffffff, 0x0000009b,
884 0x913c, 0x0000000f, 0x0100000a,
885 0x8d00, 0xffff7f7f, 0x100e4848,
886 0x8d04, 0x00ffffff, 0x00164745,
887 0x8c00, 0xfffc0003, 0xe4000003,
888 0x8c04, 0xf8ff00ff, 0x40600060,
889 0x8c08, 0x00ff00ff, 0x001c001c,
890 0x8cf0, 0x1fff1fff, 0x08e00620,
891 0x8c20, 0x0fff0fff, 0x00800080,
892 0x8c24, 0x0fff0fff, 0x00800080,
893 0x8c18, 0xffffffff, 0x20202078,
894 0x8c1c, 0x0000ffff, 0x00001010,
895 0x28350, 0x00000f01, 0x00000000,
896 0x9508, 0x3700001f, 0x00000002,
897 0x960c, 0xffffffff, 0x54763210,
898 0x88c4, 0x001f3ae3, 0x000000c2,
899 0x88d4, 0x0000001f, 0x00000010,
900 0x8974, 0xffffffff, 0x00000000
901 };
902
903 static const u32 turks_golden_registers[] =
904 {
905 0x5eb4, 0xffffffff, 0x00000002,
906 0x5e78, 0x8f311ff1, 0x001000f0,
907 0x8c8, 0x00003000, 0x00001070,
908 0x8cc, 0x000fffff, 0x00040035,
909 0x3f90, 0xffff0000, 0xfff00000,
910 0x9148, 0xffff0000, 0xfff00000,
911 0x3f94, 0xffff0000, 0xfff00000,
912 0x914c, 0xffff0000, 0xfff00000,
913 0xc78, 0x00000080, 0x00000080,
914 0xbd4, 0x00073007, 0x00010002,
915 0xd02c, 0xbfffff1f, 0x08421000,
916 0xd0b8, 0x03773777, 0x02010002,
917 0x5bc0, 0x00200000, 0x50100000,
918 0x98f8, 0x33773777, 0x00010002,
919 0x98fc, 0xffffffff, 0x33221100,
920 0x7030, 0x31000311, 0x00000011,
921 0x2f48, 0x33773777, 0x00010002,
922 0x6b28, 0x00000010, 0x00000012,
923 0x7728, 0x00000010, 0x00000012,
924 0x10328, 0x00000010, 0x00000012,
925 0x10f28, 0x00000010, 0x00000012,
926 0x11b28, 0x00000010, 0x00000012,
927 0x12728, 0x00000010, 0x00000012,
928 0x240c, 0x000007ff, 0x00000380,
929 0x8a14, 0xf000001f, 0x00000007,
930 0x8b24, 0x3fff3fff, 0x00ff0fff,
931 0x8b10, 0x0000ff0f, 0x00000000,
932 0x28a4c, 0x07ffffff, 0x06000000,
933 0x10c, 0x00000001, 0x00010003,
934 0xa02c, 0xffffffff, 0x0000009b,
935 0x913c, 0x0000000f, 0x0100000a,
936 0x8d00, 0xffff7f7f, 0x100e4848,
937 0x8d04, 0x00ffffff, 0x00164745,
938 0x8c00, 0xfffc0003, 0xe4000003,
939 0x8c04, 0xf8ff00ff, 0x40600060,
940 0x8c08, 0x00ff00ff, 0x001c001c,
941 0x8cf0, 0x1fff1fff, 0x08e00410,
942 0x8c20, 0x0fff0fff, 0x00800080,
943 0x8c24, 0x0fff0fff, 0x00800080,
944 0x8c18, 0xffffffff, 0x20202078,
945 0x8c1c, 0x0000ffff, 0x00001010,
946 0x28350, 0x00000f01, 0x00000000,
947 0x9508, 0x3700001f, 0x00000002,
948 0x960c, 0xffffffff, 0x54763210,
949 0x88c4, 0x001f3ae3, 0x000000c2,
950 0x88d4, 0x0000001f, 0x00000010,
951 0x8974, 0xffffffff, 0x00000000
952 };
953
954 static const u32 caicos_golden_registers[] =
955 {
956 0x5eb4, 0xffffffff, 0x00000002,
957 0x5e78, 0x8f311ff1, 0x001000f0,
958 0x8c8, 0x00003420, 0x00001450,
959 0x8cc, 0x000fffff, 0x00040035,
960 0x3f90, 0xffff0000, 0xfffc0000,
961 0x9148, 0xffff0000, 0xfffc0000,
962 0x3f94, 0xffff0000, 0xfffc0000,
963 0x914c, 0xffff0000, 0xfffc0000,
964 0xc78, 0x00000080, 0x00000080,
965 0xbd4, 0x00073007, 0x00010001,
966 0xd02c, 0xbfffff1f, 0x08421000,
967 0xd0b8, 0x03773777, 0x02010001,
968 0x5bc0, 0x00200000, 0x50100000,
969 0x98f8, 0x33773777, 0x02010001,
970 0x98fc, 0xffffffff, 0x33221100,
971 0x7030, 0x31000311, 0x00000011,
972 0x2f48, 0x33773777, 0x02010001,
973 0x6b28, 0x00000010, 0x00000012,
974 0x7728, 0x00000010, 0x00000012,
975 0x10328, 0x00000010, 0x00000012,
976 0x10f28, 0x00000010, 0x00000012,
977 0x11b28, 0x00000010, 0x00000012,
978 0x12728, 0x00000010, 0x00000012,
979 0x240c, 0x000007ff, 0x00000380,
980 0x8a14, 0xf000001f, 0x00000001,
981 0x8b24, 0x3fff3fff, 0x00ff0fff,
982 0x8b10, 0x0000ff0f, 0x00000000,
983 0x28a4c, 0x07ffffff, 0x06000000,
984 0x10c, 0x00000001, 0x00010003,
985 0xa02c, 0xffffffff, 0x0000009b,
986 0x913c, 0x0000000f, 0x0100000a,
987 0x8d00, 0xffff7f7f, 0x100e4848,
988 0x8d04, 0x00ffffff, 0x00164745,
989 0x8c00, 0xfffc0003, 0xe4000003,
990 0x8c04, 0xf8ff00ff, 0x40600060,
991 0x8c08, 0x00ff00ff, 0x001c001c,
992 0x8cf0, 0x1fff1fff, 0x08e00410,
993 0x8c20, 0x0fff0fff, 0x00800080,
994 0x8c24, 0x0fff0fff, 0x00800080,
995 0x8c18, 0xffffffff, 0x20202078,
996 0x8c1c, 0x0000ffff, 0x00001010,
997 0x28350, 0x00000f01, 0x00000000,
998 0x9508, 0x3700001f, 0x00000002,
999 0x960c, 0xffffffff, 0x54763210,
1000 0x88c4, 0x001f3ae3, 0x000000c2,
1001 0x88d4, 0x0000001f, 0x00000010,
1002 0x8974, 0xffffffff, 0x00000000
1003 };
1004
evergreen_init_golden_registers(struct radeon_device * rdev)1005 static void evergreen_init_golden_registers(struct radeon_device *rdev)
1006 {
1007 switch (rdev->family) {
1008 case CHIP_CYPRESS:
1009 case CHIP_HEMLOCK:
1010 radeon_program_register_sequence(rdev,
1011 evergreen_golden_registers,
1012 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1013 radeon_program_register_sequence(rdev,
1014 evergreen_golden_registers2,
1015 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1016 radeon_program_register_sequence(rdev,
1017 cypress_mgcg_init,
1018 (const u32)ARRAY_SIZE(cypress_mgcg_init));
1019 break;
1020 case CHIP_JUNIPER:
1021 radeon_program_register_sequence(rdev,
1022 evergreen_golden_registers,
1023 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1024 radeon_program_register_sequence(rdev,
1025 evergreen_golden_registers2,
1026 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1027 radeon_program_register_sequence(rdev,
1028 juniper_mgcg_init,
1029 (const u32)ARRAY_SIZE(juniper_mgcg_init));
1030 break;
1031 case CHIP_REDWOOD:
1032 radeon_program_register_sequence(rdev,
1033 evergreen_golden_registers,
1034 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1035 radeon_program_register_sequence(rdev,
1036 evergreen_golden_registers2,
1037 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1038 radeon_program_register_sequence(rdev,
1039 redwood_mgcg_init,
1040 (const u32)ARRAY_SIZE(redwood_mgcg_init));
1041 break;
1042 case CHIP_CEDAR:
1043 radeon_program_register_sequence(rdev,
1044 cedar_golden_registers,
1045 (const u32)ARRAY_SIZE(cedar_golden_registers));
1046 radeon_program_register_sequence(rdev,
1047 evergreen_golden_registers2,
1048 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1049 radeon_program_register_sequence(rdev,
1050 cedar_mgcg_init,
1051 (const u32)ARRAY_SIZE(cedar_mgcg_init));
1052 break;
1053 case CHIP_PALM:
1054 radeon_program_register_sequence(rdev,
1055 wrestler_golden_registers,
1056 (const u32)ARRAY_SIZE(wrestler_golden_registers));
1057 break;
1058 case CHIP_SUMO:
1059 radeon_program_register_sequence(rdev,
1060 supersumo_golden_registers,
1061 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1062 break;
1063 case CHIP_SUMO2:
1064 radeon_program_register_sequence(rdev,
1065 supersumo_golden_registers,
1066 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1067 radeon_program_register_sequence(rdev,
1068 sumo_golden_registers,
1069 (const u32)ARRAY_SIZE(sumo_golden_registers));
1070 break;
1071 case CHIP_BARTS:
1072 radeon_program_register_sequence(rdev,
1073 barts_golden_registers,
1074 (const u32)ARRAY_SIZE(barts_golden_registers));
1075 break;
1076 case CHIP_TURKS:
1077 radeon_program_register_sequence(rdev,
1078 turks_golden_registers,
1079 (const u32)ARRAY_SIZE(turks_golden_registers));
1080 break;
1081 case CHIP_CAICOS:
1082 radeon_program_register_sequence(rdev,
1083 caicos_golden_registers,
1084 (const u32)ARRAY_SIZE(caicos_golden_registers));
1085 break;
1086 default:
1087 break;
1088 }
1089 }
1090
1091 /**
1092 * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1093 *
1094 * @rdev: radeon_device pointer
1095 * @reg: register offset in bytes
1096 * @val: register value
1097 *
1098 * Returns 0 for success or -EINVAL for an invalid register
1099 *
1100 */
evergreen_get_allowed_info_register(struct radeon_device * rdev,u32 reg,u32 * val)1101 int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1102 u32 reg, u32 *val)
1103 {
1104 switch (reg) {
1105 case GRBM_STATUS:
1106 case GRBM_STATUS_SE0:
1107 case GRBM_STATUS_SE1:
1108 case SRBM_STATUS:
1109 case SRBM_STATUS2:
1110 case DMA_STATUS_REG:
1111 case UVD_STATUS:
1112 *val = RREG32(reg);
1113 return 0;
1114 default:
1115 return -EINVAL;
1116 }
1117 }
1118
evergreen_tiling_fields(unsigned tiling_flags,unsigned * bankw,unsigned * bankh,unsigned * mtaspect,unsigned * tile_split)1119 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1120 unsigned *bankh, unsigned *mtaspect,
1121 unsigned *tile_split)
1122 {
1123 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1124 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1125 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1126 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1127 switch (*bankw) {
1128 default:
1129 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1130 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1131 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1132 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1133 }
1134 switch (*bankh) {
1135 default:
1136 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1137 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1138 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1139 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1140 }
1141 switch (*mtaspect) {
1142 default:
1143 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1144 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1145 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1146 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1147 }
1148 }
1149
sumo_set_uvd_clock(struct radeon_device * rdev,u32 clock,u32 cntl_reg,u32 status_reg)1150 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1151 u32 cntl_reg, u32 status_reg)
1152 {
1153 int r, i;
1154 struct atom_clock_dividers dividers;
1155
1156 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1157 clock, false, ÷rs);
1158 if (r)
1159 return r;
1160
1161 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1162
1163 for (i = 0; i < 100; i++) {
1164 if (RREG32(status_reg) & DCLK_STATUS)
1165 break;
1166 mdelay(10);
1167 }
1168 if (i == 100)
1169 return -ETIMEDOUT;
1170
1171 return 0;
1172 }
1173
sumo_set_uvd_clocks(struct radeon_device * rdev,u32 vclk,u32 dclk)1174 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1175 {
1176 int r = 0;
1177 u32 cg_scratch = RREG32(CG_SCRATCH1);
1178
1179 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1180 if (r)
1181 goto done;
1182 cg_scratch &= 0xffff0000;
1183 cg_scratch |= vclk / 100; /* Mhz */
1184
1185 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1186 if (r)
1187 goto done;
1188 cg_scratch &= 0x0000ffff;
1189 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1190
1191 done:
1192 WREG32(CG_SCRATCH1, cg_scratch);
1193
1194 return r;
1195 }
1196
evergreen_set_uvd_clocks(struct radeon_device * rdev,u32 vclk,u32 dclk)1197 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1198 {
1199 /* start off with something large */
1200 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1201 int r;
1202
1203 /* bypass vclk and dclk with bclk */
1204 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1205 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1206 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1207
1208 /* put PLL in bypass mode */
1209 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1210
1211 if (!vclk || !dclk) {
1212 /* keep the Bypass mode, put PLL to sleep */
1213 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1214 return 0;
1215 }
1216
1217 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1218 16384, 0x03FFFFFF, 0, 128, 5,
1219 &fb_div, &vclk_div, &dclk_div);
1220 if (r)
1221 return r;
1222
1223 /* set VCO_MODE to 1 */
1224 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1225
1226 /* toggle UPLL_SLEEP to 1 then back to 0 */
1227 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1228 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1229
1230 /* deassert UPLL_RESET */
1231 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1232
1233 mdelay(1);
1234
1235 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1236 if (r)
1237 return r;
1238
1239 /* assert UPLL_RESET again */
1240 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1241
1242 /* disable spread spectrum. */
1243 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1244
1245 /* set feedback divider */
1246 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1247
1248 /* set ref divider to 0 */
1249 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1250
1251 if (fb_div < 307200)
1252 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1253 else
1254 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1255
1256 /* set PDIV_A and PDIV_B */
1257 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1258 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1259 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1260
1261 /* give the PLL some time to settle */
1262 mdelay(15);
1263
1264 /* deassert PLL_RESET */
1265 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1266
1267 mdelay(15);
1268
1269 /* switch from bypass mode to normal mode */
1270 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1271
1272 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1273 if (r)
1274 return r;
1275
1276 /* switch VCLK and DCLK selection */
1277 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1278 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1279 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1280
1281 mdelay(100);
1282
1283 return 0;
1284 }
1285
evergreen_fix_pci_max_read_req_size(struct radeon_device * rdev)1286 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1287 {
1288 #ifndef __NetBSD__ /* XXX radeon pcie */
1289 int readrq;
1290 u16 v;
1291
1292 readrq = pcie_get_readrq(rdev->pdev);
1293 v = ffs(readrq) - 8;
1294 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1295 * to avoid hangs or perfomance issues
1296 */
1297 if ((v == 0) || (v == 6) || (v == 7))
1298 pcie_set_readrq(rdev->pdev, 512);
1299 #endif
1300 }
1301
dce4_program_fmt(struct drm_encoder * encoder)1302 void dce4_program_fmt(struct drm_encoder *encoder)
1303 {
1304 struct drm_device *dev = encoder->dev;
1305 struct radeon_device *rdev = dev->dev_private;
1306 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1307 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1308 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1309 int bpc = 0;
1310 u32 tmp = 0;
1311 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1312
1313 if (connector) {
1314 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1315 bpc = radeon_get_monitor_bpc(connector);
1316 dither = radeon_connector->dither;
1317 }
1318
1319 /* LVDS/eDP FMT is set up by atom */
1320 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1321 return;
1322
1323 /* not needed for analog */
1324 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1325 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1326 return;
1327
1328 if (bpc == 0)
1329 return;
1330
1331 switch (bpc) {
1332 case 6:
1333 if (dither == RADEON_FMT_DITHER_ENABLE)
1334 /* XXX sort out optimal dither settings */
1335 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1336 FMT_SPATIAL_DITHER_EN);
1337 else
1338 tmp |= FMT_TRUNCATE_EN;
1339 break;
1340 case 8:
1341 if (dither == RADEON_FMT_DITHER_ENABLE)
1342 /* XXX sort out optimal dither settings */
1343 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1344 FMT_RGB_RANDOM_ENABLE |
1345 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1346 else
1347 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1348 break;
1349 case 10:
1350 default:
1351 /* not needed */
1352 break;
1353 }
1354
1355 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1356 }
1357
dce4_is_in_vblank(struct radeon_device * rdev,int crtc)1358 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1359 {
1360 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1361 return true;
1362 else
1363 return false;
1364 }
1365
dce4_is_counter_moving(struct radeon_device * rdev,int crtc)1366 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1367 {
1368 u32 pos1, pos2;
1369
1370 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1371 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1372
1373 if (pos1 != pos2)
1374 return true;
1375 else
1376 return false;
1377 }
1378
1379 /**
1380 * dce4_wait_for_vblank - vblank wait asic callback.
1381 *
1382 * @rdev: radeon_device pointer
1383 * @crtc: crtc to wait for vblank on
1384 *
1385 * Wait for vblank on the requested crtc (evergreen+).
1386 */
dce4_wait_for_vblank(struct radeon_device * rdev,int crtc)1387 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1388 {
1389 unsigned i = 0;
1390
1391 if (crtc >= rdev->num_crtc)
1392 return;
1393
1394 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1395 return;
1396
1397 /* depending on when we hit vblank, we may be close to active; if so,
1398 * wait for another frame.
1399 */
1400 while (dce4_is_in_vblank(rdev, crtc)) {
1401 if (i++ % 100 == 0) {
1402 if (!dce4_is_counter_moving(rdev, crtc))
1403 break;
1404 }
1405 }
1406
1407 while (!dce4_is_in_vblank(rdev, crtc)) {
1408 if (i++ % 100 == 0) {
1409 if (!dce4_is_counter_moving(rdev, crtc))
1410 break;
1411 }
1412 }
1413 }
1414
1415 /**
1416 * evergreen_page_flip - pageflip callback.
1417 *
1418 * @rdev: radeon_device pointer
1419 * @crtc_id: crtc to cleanup pageflip on
1420 * @crtc_base: new address of the crtc (GPU MC address)
1421 *
1422 * Triggers the actual pageflip by updating the primary
1423 * surface base address (evergreen+).
1424 */
evergreen_page_flip(struct radeon_device * rdev,int crtc_id,u64 crtc_base,bool async)1425 void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base,
1426 bool async)
1427 {
1428 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1429
1430 /* update the scanout addresses */
1431 WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset,
1432 async ? EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
1433 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1434 upper_32_bits(crtc_base));
1435 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1436 (u32)crtc_base);
1437 /* post the write */
1438 RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset);
1439 }
1440
1441 /**
1442 * evergreen_page_flip_pending - check if page flip is still pending
1443 *
1444 * @rdev: radeon_device pointer
1445 * @crtc_id: crtc to check
1446 *
1447 * Returns the current update pending status.
1448 */
evergreen_page_flip_pending(struct radeon_device * rdev,int crtc_id)1449 bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1450 {
1451 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1452
1453 /* Return current update_pending status: */
1454 return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1455 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1456 }
1457
1458 /* get temperature in millidegrees */
evergreen_get_temp(struct radeon_device * rdev)1459 int evergreen_get_temp(struct radeon_device *rdev)
1460 {
1461 u32 temp, toffset;
1462 int actual_temp = 0;
1463
1464 if (rdev->family == CHIP_JUNIPER) {
1465 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1466 TOFFSET_SHIFT;
1467 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1468 TS0_ADC_DOUT_SHIFT;
1469
1470 if (toffset & 0x100)
1471 actual_temp = temp / 2 - (0x200 - toffset);
1472 else
1473 actual_temp = temp / 2 + toffset;
1474
1475 actual_temp = actual_temp * 1000;
1476
1477 } else {
1478 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1479 ASIC_T_SHIFT;
1480
1481 if (temp & 0x400)
1482 actual_temp = -256;
1483 else if (temp & 0x200)
1484 actual_temp = 255;
1485 else if (temp & 0x100) {
1486 actual_temp = temp & 0x1ff;
1487 actual_temp |= ~0x1ff;
1488 } else
1489 actual_temp = temp & 0xff;
1490
1491 actual_temp = (actual_temp * 1000) / 2;
1492 }
1493
1494 return actual_temp;
1495 }
1496
sumo_get_temp(struct radeon_device * rdev)1497 int sumo_get_temp(struct radeon_device *rdev)
1498 {
1499 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1500 int actual_temp = temp - 49;
1501
1502 return actual_temp * 1000;
1503 }
1504
1505 /**
1506 * sumo_pm_init_profile - Initialize power profiles callback.
1507 *
1508 * @rdev: radeon_device pointer
1509 *
1510 * Initialize the power states used in profile mode
1511 * (sumo, trinity, SI).
1512 * Used for profile mode only.
1513 */
sumo_pm_init_profile(struct radeon_device * rdev)1514 void sumo_pm_init_profile(struct radeon_device *rdev)
1515 {
1516 int idx;
1517
1518 /* default */
1519 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1520 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1521 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1522 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1523
1524 /* low,mid sh/mh */
1525 if (rdev->flags & RADEON_IS_MOBILITY)
1526 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1527 else
1528 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1529
1530 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1531 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1532 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1533 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1534
1535 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1536 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1537 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1538 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1539
1540 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1541 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1542 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1543 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1544
1545 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1546 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1547 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1548 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1549
1550 /* high sh/mh */
1551 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1552 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1553 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1554 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1555 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1556 rdev->pm.power_state[idx].num_clock_modes - 1;
1557
1558 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1559 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1560 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1561 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1562 rdev->pm.power_state[idx].num_clock_modes - 1;
1563 }
1564
1565 /**
1566 * btc_pm_init_profile - Initialize power profiles callback.
1567 *
1568 * @rdev: radeon_device pointer
1569 *
1570 * Initialize the power states used in profile mode
1571 * (BTC, cayman).
1572 * Used for profile mode only.
1573 */
btc_pm_init_profile(struct radeon_device * rdev)1574 void btc_pm_init_profile(struct radeon_device *rdev)
1575 {
1576 int idx;
1577
1578 /* default */
1579 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1580 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1581 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1582 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1583 /* starting with BTC, there is one state that is used for both
1584 * MH and SH. Difference is that we always use the high clock index for
1585 * mclk.
1586 */
1587 if (rdev->flags & RADEON_IS_MOBILITY)
1588 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1589 else
1590 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1591 /* low sh */
1592 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1593 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1594 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1595 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1596 /* mid sh */
1597 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1598 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1599 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1600 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1601 /* high sh */
1602 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1603 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1604 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1605 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1606 /* low mh */
1607 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1608 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1609 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1610 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1611 /* mid mh */
1612 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1613 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1614 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1615 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1616 /* high mh */
1617 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1618 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1619 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1620 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1621 }
1622
1623 /**
1624 * evergreen_pm_misc - set additional pm hw parameters callback.
1625 *
1626 * @rdev: radeon_device pointer
1627 *
1628 * Set non-clock parameters associated with a power state
1629 * (voltage, etc.) (evergreen+).
1630 */
evergreen_pm_misc(struct radeon_device * rdev)1631 void evergreen_pm_misc(struct radeon_device *rdev)
1632 {
1633 int req_ps_idx = rdev->pm.requested_power_state_index;
1634 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1635 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1636 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1637
1638 if (voltage->type == VOLTAGE_SW) {
1639 /* 0xff0x are flags rather then an actual voltage */
1640 if ((voltage->voltage & 0xff00) == 0xff00)
1641 return;
1642 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1643 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1644 rdev->pm.current_vddc = voltage->voltage;
1645 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1646 }
1647
1648 /* starting with BTC, there is one state that is used for both
1649 * MH and SH. Difference is that we always use the high clock index for
1650 * mclk and vddci.
1651 */
1652 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1653 (rdev->family >= CHIP_BARTS) &&
1654 rdev->pm.active_crtc_count &&
1655 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1656 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1657 voltage = &rdev->pm.power_state[req_ps_idx].
1658 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1659
1660 /* 0xff0x are flags rather then an actual voltage */
1661 if ((voltage->vddci & 0xff00) == 0xff00)
1662 return;
1663 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1664 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1665 rdev->pm.current_vddci = voltage->vddci;
1666 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1667 }
1668 }
1669 }
1670
1671 /**
1672 * evergreen_pm_prepare - pre-power state change callback.
1673 *
1674 * @rdev: radeon_device pointer
1675 *
1676 * Prepare for a power state change (evergreen+).
1677 */
evergreen_pm_prepare(struct radeon_device * rdev)1678 void evergreen_pm_prepare(struct radeon_device *rdev)
1679 {
1680 struct drm_device *ddev = rdev->ddev;
1681 struct drm_crtc *crtc;
1682 struct radeon_crtc *radeon_crtc;
1683 u32 tmp;
1684
1685 /* disable any active CRTCs */
1686 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1687 radeon_crtc = to_radeon_crtc(crtc);
1688 if (radeon_crtc->enabled) {
1689 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1690 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1691 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1692 }
1693 }
1694 }
1695
1696 /**
1697 * evergreen_pm_finish - post-power state change callback.
1698 *
1699 * @rdev: radeon_device pointer
1700 *
1701 * Clean up after a power state change (evergreen+).
1702 */
evergreen_pm_finish(struct radeon_device * rdev)1703 void evergreen_pm_finish(struct radeon_device *rdev)
1704 {
1705 struct drm_device *ddev = rdev->ddev;
1706 struct drm_crtc *crtc;
1707 struct radeon_crtc *radeon_crtc;
1708 u32 tmp;
1709
1710 /* enable any active CRTCs */
1711 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1712 radeon_crtc = to_radeon_crtc(crtc);
1713 if (radeon_crtc->enabled) {
1714 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1715 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1716 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1717 }
1718 }
1719 }
1720
1721 /**
1722 * evergreen_hpd_sense - hpd sense callback.
1723 *
1724 * @rdev: radeon_device pointer
1725 * @hpd: hpd (hotplug detect) pin
1726 *
1727 * Checks if a digital monitor is connected (evergreen+).
1728 * Returns true if connected, false if not connected.
1729 */
evergreen_hpd_sense(struct radeon_device * rdev,enum radeon_hpd_id hpd)1730 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1731 {
1732 if (hpd == RADEON_HPD_NONE)
1733 return false;
1734
1735 return !!(RREG32(DC_HPDx_INT_STATUS_REG(hpd)) & DC_HPDx_SENSE);
1736 }
1737
1738 /**
1739 * evergreen_hpd_set_polarity - hpd set polarity callback.
1740 *
1741 * @rdev: radeon_device pointer
1742 * @hpd: hpd (hotplug detect) pin
1743 *
1744 * Set the polarity of the hpd pin (evergreen+).
1745 */
evergreen_hpd_set_polarity(struct radeon_device * rdev,enum radeon_hpd_id hpd)1746 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1747 enum radeon_hpd_id hpd)
1748 {
1749 bool connected = evergreen_hpd_sense(rdev, hpd);
1750
1751 if (hpd == RADEON_HPD_NONE)
1752 return;
1753
1754 if (connected)
1755 WREG32_AND(DC_HPDx_INT_CONTROL(hpd), ~DC_HPDx_INT_POLARITY);
1756 else
1757 WREG32_OR(DC_HPDx_INT_CONTROL(hpd), DC_HPDx_INT_POLARITY);
1758 }
1759
1760 /**
1761 * evergreen_hpd_init - hpd setup callback.
1762 *
1763 * @rdev: radeon_device pointer
1764 *
1765 * Setup the hpd pins used by the card (evergreen+).
1766 * Enable the pin, set the polarity, and enable the hpd interrupts.
1767 */
evergreen_hpd_init(struct radeon_device * rdev)1768 void evergreen_hpd_init(struct radeon_device *rdev)
1769 {
1770 struct drm_device *dev = rdev->ddev;
1771 struct drm_connector *connector;
1772 unsigned enabled = 0;
1773 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1774 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1775
1776 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1777 enum radeon_hpd_id hpd =
1778 to_radeon_connector(connector)->hpd.hpd;
1779
1780 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1781 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1782 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1783 * aux dp channel on imac and help (but not completely fix)
1784 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1785 * also avoid interrupt storms during dpms.
1786 */
1787 continue;
1788 }
1789
1790 if (hpd == RADEON_HPD_NONE)
1791 continue;
1792
1793 WREG32(DC_HPDx_CONTROL(hpd), tmp);
1794 enabled |= 1 << hpd;
1795
1796 radeon_hpd_set_polarity(rdev, hpd);
1797 }
1798 radeon_irq_kms_enable_hpd(rdev, enabled);
1799 }
1800
1801 /**
1802 * evergreen_hpd_fini - hpd tear down callback.
1803 *
1804 * @rdev: radeon_device pointer
1805 *
1806 * Tear down the hpd pins used by the card (evergreen+).
1807 * Disable the hpd interrupts.
1808 */
evergreen_hpd_fini(struct radeon_device * rdev)1809 void evergreen_hpd_fini(struct radeon_device *rdev)
1810 {
1811 struct drm_device *dev = rdev->ddev;
1812 struct drm_connector *connector;
1813 unsigned disabled = 0;
1814
1815 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1816 enum radeon_hpd_id hpd =
1817 to_radeon_connector(connector)->hpd.hpd;
1818
1819 if (hpd == RADEON_HPD_NONE)
1820 continue;
1821
1822 WREG32(DC_HPDx_CONTROL(hpd), 0);
1823 disabled |= 1 << hpd;
1824 }
1825 radeon_irq_kms_disable_hpd(rdev, disabled);
1826 }
1827
1828 /* watermark setup */
1829
evergreen_line_buffer_adjust(struct radeon_device * rdev,struct radeon_crtc * radeon_crtc,struct drm_display_mode * mode,struct drm_display_mode * other_mode)1830 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1831 struct radeon_crtc *radeon_crtc,
1832 struct drm_display_mode *mode,
1833 struct drm_display_mode *other_mode)
1834 {
1835 u32 tmp, buffer_alloc, i;
1836 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1837 /*
1838 * Line Buffer Setup
1839 * There are 3 line buffers, each one shared by 2 display controllers.
1840 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1841 * the display controllers. The paritioning is done via one of four
1842 * preset allocations specified in bits 2:0:
1843 * first display controller
1844 * 0 - first half of lb (3840 * 2)
1845 * 1 - first 3/4 of lb (5760 * 2)
1846 * 2 - whole lb (7680 * 2), other crtc must be disabled
1847 * 3 - first 1/4 of lb (1920 * 2)
1848 * second display controller
1849 * 4 - second half of lb (3840 * 2)
1850 * 5 - second 3/4 of lb (5760 * 2)
1851 * 6 - whole lb (7680 * 2), other crtc must be disabled
1852 * 7 - last 1/4 of lb (1920 * 2)
1853 */
1854 /* this can get tricky if we have two large displays on a paired group
1855 * of crtcs. Ideally for multiple large displays we'd assign them to
1856 * non-linked crtcs for maximum line buffer allocation.
1857 */
1858 if (radeon_crtc->base.enabled && mode) {
1859 if (other_mode) {
1860 tmp = 0; /* 1/2 */
1861 buffer_alloc = 1;
1862 } else {
1863 tmp = 2; /* whole */
1864 buffer_alloc = 2;
1865 }
1866 } else {
1867 tmp = 0;
1868 buffer_alloc = 0;
1869 }
1870
1871 /* second controller of the pair uses second half of the lb */
1872 if (radeon_crtc->crtc_id % 2)
1873 tmp += 4;
1874 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1875
1876 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1877 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1878 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1879 for (i = 0; i < rdev->usec_timeout; i++) {
1880 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1881 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1882 break;
1883 udelay(1);
1884 }
1885 }
1886
1887 if (radeon_crtc->base.enabled && mode) {
1888 switch (tmp) {
1889 case 0:
1890 case 4:
1891 default:
1892 if (ASIC_IS_DCE5(rdev))
1893 return 4096 * 2;
1894 else
1895 return 3840 * 2;
1896 case 1:
1897 case 5:
1898 if (ASIC_IS_DCE5(rdev))
1899 return 6144 * 2;
1900 else
1901 return 5760 * 2;
1902 case 2:
1903 case 6:
1904 if (ASIC_IS_DCE5(rdev))
1905 return 8192 * 2;
1906 else
1907 return 7680 * 2;
1908 case 3:
1909 case 7:
1910 if (ASIC_IS_DCE5(rdev))
1911 return 2048 * 2;
1912 else
1913 return 1920 * 2;
1914 }
1915 }
1916
1917 /* controller not enabled, so no lb used */
1918 return 0;
1919 }
1920
evergreen_get_number_of_dram_channels(struct radeon_device * rdev)1921 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1922 {
1923 u32 tmp = RREG32(MC_SHARED_CHMAP);
1924
1925 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1926 case 0:
1927 default:
1928 return 1;
1929 case 1:
1930 return 2;
1931 case 2:
1932 return 4;
1933 case 3:
1934 return 8;
1935 }
1936 }
1937
1938 struct evergreen_wm_params {
1939 u32 dram_channels; /* number of dram channels */
1940 u32 yclk; /* bandwidth per dram data pin in kHz */
1941 u32 sclk; /* engine clock in kHz */
1942 u32 disp_clk; /* display clock in kHz */
1943 u32 src_width; /* viewport width */
1944 u32 active_time; /* active display time in ns */
1945 u32 blank_time; /* blank time in ns */
1946 bool interlaced; /* mode is interlaced */
1947 fixed20_12 vsc; /* vertical scale ratio */
1948 u32 num_heads; /* number of active crtcs */
1949 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1950 u32 lb_size; /* line buffer allocated to pipe */
1951 u32 vtaps; /* vertical scaler taps */
1952 };
1953
evergreen_dram_bandwidth(struct evergreen_wm_params * wm)1954 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1955 {
1956 /* Calculate DRAM Bandwidth and the part allocated to display. */
1957 fixed20_12 dram_efficiency; /* 0.7 */
1958 fixed20_12 yclk, dram_channels, bandwidth;
1959 fixed20_12 a;
1960
1961 a.full = dfixed_const(1000);
1962 yclk.full = dfixed_const(wm->yclk);
1963 yclk.full = dfixed_div(yclk, a);
1964 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1965 a.full = dfixed_const(10);
1966 dram_efficiency.full = dfixed_const(7);
1967 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1968 bandwidth.full = dfixed_mul(dram_channels, yclk);
1969 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1970
1971 return dfixed_trunc(bandwidth);
1972 }
1973
evergreen_dram_bandwidth_for_display(struct evergreen_wm_params * wm)1974 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1975 {
1976 /* Calculate DRAM Bandwidth and the part allocated to display. */
1977 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1978 fixed20_12 yclk, dram_channels, bandwidth;
1979 fixed20_12 a;
1980
1981 a.full = dfixed_const(1000);
1982 yclk.full = dfixed_const(wm->yclk);
1983 yclk.full = dfixed_div(yclk, a);
1984 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1985 a.full = dfixed_const(10);
1986 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1987 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1988 bandwidth.full = dfixed_mul(dram_channels, yclk);
1989 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1990
1991 return dfixed_trunc(bandwidth);
1992 }
1993
evergreen_data_return_bandwidth(struct evergreen_wm_params * wm)1994 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1995 {
1996 /* Calculate the display Data return Bandwidth */
1997 fixed20_12 return_efficiency; /* 0.8 */
1998 fixed20_12 sclk, bandwidth;
1999 fixed20_12 a;
2000
2001 a.full = dfixed_const(1000);
2002 sclk.full = dfixed_const(wm->sclk);
2003 sclk.full = dfixed_div(sclk, a);
2004 a.full = dfixed_const(10);
2005 return_efficiency.full = dfixed_const(8);
2006 return_efficiency.full = dfixed_div(return_efficiency, a);
2007 a.full = dfixed_const(32);
2008 bandwidth.full = dfixed_mul(a, sclk);
2009 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2010
2011 return dfixed_trunc(bandwidth);
2012 }
2013
evergreen_dmif_request_bandwidth(struct evergreen_wm_params * wm)2014 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2015 {
2016 /* Calculate the DMIF Request Bandwidth */
2017 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2018 fixed20_12 disp_clk, bandwidth;
2019 fixed20_12 a;
2020
2021 a.full = dfixed_const(1000);
2022 disp_clk.full = dfixed_const(wm->disp_clk);
2023 disp_clk.full = dfixed_div(disp_clk, a);
2024 a.full = dfixed_const(10);
2025 disp_clk_request_efficiency.full = dfixed_const(8);
2026 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2027 a.full = dfixed_const(32);
2028 bandwidth.full = dfixed_mul(a, disp_clk);
2029 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2030
2031 return dfixed_trunc(bandwidth);
2032 }
2033
evergreen_available_bandwidth(struct evergreen_wm_params * wm)2034 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2035 {
2036 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2037 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2038 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2039 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2040
2041 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2042 }
2043
evergreen_average_bandwidth(struct evergreen_wm_params * wm)2044 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2045 {
2046 /* Calculate the display mode Average Bandwidth
2047 * DisplayMode should contain the source and destination dimensions,
2048 * timing, etc.
2049 */
2050 fixed20_12 bpp;
2051 fixed20_12 line_time;
2052 fixed20_12 src_width;
2053 fixed20_12 bandwidth;
2054 fixed20_12 a;
2055
2056 a.full = dfixed_const(1000);
2057 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2058 line_time.full = dfixed_div(line_time, a);
2059 bpp.full = dfixed_const(wm->bytes_per_pixel);
2060 src_width.full = dfixed_const(wm->src_width);
2061 bandwidth.full = dfixed_mul(src_width, bpp);
2062 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2063 bandwidth.full = dfixed_div(bandwidth, line_time);
2064
2065 return dfixed_trunc(bandwidth);
2066 }
2067
evergreen_latency_watermark(struct evergreen_wm_params * wm)2068 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2069 {
2070 /* First calcualte the latency in ns */
2071 u32 mc_latency = 2000; /* 2000 ns. */
2072 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2073 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2074 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2075 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2076 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2077 (wm->num_heads * cursor_line_pair_return_time);
2078 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2079 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2080 fixed20_12 a, b, c;
2081
2082 if (wm->num_heads == 0)
2083 return 0;
2084
2085 a.full = dfixed_const(2);
2086 b.full = dfixed_const(1);
2087 if ((wm->vsc.full > a.full) ||
2088 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2089 (wm->vtaps >= 5) ||
2090 ((wm->vsc.full >= a.full) && wm->interlaced))
2091 max_src_lines_per_dst_line = 4;
2092 else
2093 max_src_lines_per_dst_line = 2;
2094
2095 a.full = dfixed_const(available_bandwidth);
2096 b.full = dfixed_const(wm->num_heads);
2097 a.full = dfixed_div(a, b);
2098
2099 lb_fill_bw = min(dfixed_trunc(a), wm->disp_clk * wm->bytes_per_pixel / 1000);
2100
2101 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2102 b.full = dfixed_const(1000);
2103 c.full = dfixed_const(lb_fill_bw);
2104 b.full = dfixed_div(c, b);
2105 a.full = dfixed_div(a, b);
2106 line_fill_time = dfixed_trunc(a);
2107
2108 if (line_fill_time < wm->active_time)
2109 return latency;
2110 else
2111 return latency + (line_fill_time - wm->active_time);
2112
2113 }
2114
evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params * wm)2115 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2116 {
2117 if (evergreen_average_bandwidth(wm) <=
2118 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2119 return true;
2120 else
2121 return false;
2122 };
2123
evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params * wm)2124 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2125 {
2126 if (evergreen_average_bandwidth(wm) <=
2127 (evergreen_available_bandwidth(wm) / wm->num_heads))
2128 return true;
2129 else
2130 return false;
2131 };
2132
evergreen_check_latency_hiding(struct evergreen_wm_params * wm)2133 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2134 {
2135 u32 lb_partitions = wm->lb_size / wm->src_width;
2136 u32 line_time = wm->active_time + wm->blank_time;
2137 u32 latency_tolerant_lines;
2138 u32 latency_hiding;
2139 fixed20_12 a;
2140
2141 a.full = dfixed_const(1);
2142 if (wm->vsc.full > a.full)
2143 latency_tolerant_lines = 1;
2144 else {
2145 if (lb_partitions <= (wm->vtaps + 1))
2146 latency_tolerant_lines = 1;
2147 else
2148 latency_tolerant_lines = 2;
2149 }
2150
2151 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2152
2153 if (evergreen_latency_watermark(wm) <= latency_hiding)
2154 return true;
2155 else
2156 return false;
2157 }
2158
evergreen_program_watermarks(struct radeon_device * rdev,struct radeon_crtc * radeon_crtc,u32 lb_size,u32 num_heads)2159 static void evergreen_program_watermarks(struct radeon_device *rdev,
2160 struct radeon_crtc *radeon_crtc,
2161 u32 lb_size, u32 num_heads)
2162 {
2163 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2164 struct evergreen_wm_params wm_low, wm_high;
2165 u32 dram_channels;
2166 u32 active_time;
2167 u32 line_time = 0;
2168 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2169 u32 priority_a_mark = 0, priority_b_mark = 0;
2170 u32 priority_a_cnt = PRIORITY_OFF;
2171 u32 priority_b_cnt = PRIORITY_OFF;
2172 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2173 u32 tmp, arb_control3;
2174 fixed20_12 a, b, c;
2175
2176 if (radeon_crtc->base.enabled && num_heads && mode) {
2177 active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
2178 (u32)mode->clock);
2179 line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
2180 (u32)mode->clock);
2181 line_time = min(line_time, (u32)65535);
2182 priority_a_cnt = 0;
2183 priority_b_cnt = 0;
2184 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2185
2186 /* watermark for high clocks */
2187 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2188 wm_high.yclk =
2189 radeon_dpm_get_mclk(rdev, false) * 10;
2190 wm_high.sclk =
2191 radeon_dpm_get_sclk(rdev, false) * 10;
2192 } else {
2193 wm_high.yclk = rdev->pm.current_mclk * 10;
2194 wm_high.sclk = rdev->pm.current_sclk * 10;
2195 }
2196
2197 wm_high.disp_clk = mode->clock;
2198 wm_high.src_width = mode->crtc_hdisplay;
2199 wm_high.active_time = active_time;
2200 wm_high.blank_time = line_time - wm_high.active_time;
2201 wm_high.interlaced = false;
2202 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2203 wm_high.interlaced = true;
2204 wm_high.vsc = radeon_crtc->vsc;
2205 wm_high.vtaps = 1;
2206 if (radeon_crtc->rmx_type != RMX_OFF)
2207 wm_high.vtaps = 2;
2208 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2209 wm_high.lb_size = lb_size;
2210 wm_high.dram_channels = dram_channels;
2211 wm_high.num_heads = num_heads;
2212
2213 /* watermark for low clocks */
2214 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2215 wm_low.yclk =
2216 radeon_dpm_get_mclk(rdev, true) * 10;
2217 wm_low.sclk =
2218 radeon_dpm_get_sclk(rdev, true) * 10;
2219 } else {
2220 wm_low.yclk = rdev->pm.current_mclk * 10;
2221 wm_low.sclk = rdev->pm.current_sclk * 10;
2222 }
2223
2224 wm_low.disp_clk = mode->clock;
2225 wm_low.src_width = mode->crtc_hdisplay;
2226 wm_low.active_time = active_time;
2227 wm_low.blank_time = line_time - wm_low.active_time;
2228 wm_low.interlaced = false;
2229 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2230 wm_low.interlaced = true;
2231 wm_low.vsc = radeon_crtc->vsc;
2232 wm_low.vtaps = 1;
2233 if (radeon_crtc->rmx_type != RMX_OFF)
2234 wm_low.vtaps = 2;
2235 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2236 wm_low.lb_size = lb_size;
2237 wm_low.dram_channels = dram_channels;
2238 wm_low.num_heads = num_heads;
2239
2240 /* set for high clocks */
2241 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2242 /* set for low clocks */
2243 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2244
2245 /* possibly force display priority to high */
2246 /* should really do this at mode validation time... */
2247 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2248 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2249 !evergreen_check_latency_hiding(&wm_high) ||
2250 (rdev->disp_priority == 2)) {
2251 DRM_DEBUG_KMS("force priority a to high\n");
2252 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2253 }
2254 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2255 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2256 !evergreen_check_latency_hiding(&wm_low) ||
2257 (rdev->disp_priority == 2)) {
2258 DRM_DEBUG_KMS("force priority b to high\n");
2259 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2260 }
2261
2262 a.full = dfixed_const(1000);
2263 b.full = dfixed_const(mode->clock);
2264 b.full = dfixed_div(b, a);
2265 c.full = dfixed_const(latency_watermark_a);
2266 c.full = dfixed_mul(c, b);
2267 c.full = dfixed_mul(c, radeon_crtc->hsc);
2268 c.full = dfixed_div(c, a);
2269 a.full = dfixed_const(16);
2270 c.full = dfixed_div(c, a);
2271 priority_a_mark = dfixed_trunc(c);
2272 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2273
2274 a.full = dfixed_const(1000);
2275 b.full = dfixed_const(mode->clock);
2276 b.full = dfixed_div(b, a);
2277 c.full = dfixed_const(latency_watermark_b);
2278 c.full = dfixed_mul(c, b);
2279 c.full = dfixed_mul(c, radeon_crtc->hsc);
2280 c.full = dfixed_div(c, a);
2281 a.full = dfixed_const(16);
2282 c.full = dfixed_div(c, a);
2283 priority_b_mark = dfixed_trunc(c);
2284 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2285
2286 /* Save number of lines the linebuffer leads before the scanout */
2287 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2288 }
2289
2290 /* select wm A */
2291 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2292 tmp = arb_control3;
2293 tmp &= ~LATENCY_WATERMARK_MASK(3);
2294 tmp |= LATENCY_WATERMARK_MASK(1);
2295 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2296 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2297 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2298 LATENCY_HIGH_WATERMARK(line_time)));
2299 /* select wm B */
2300 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2301 tmp &= ~LATENCY_WATERMARK_MASK(3);
2302 tmp |= LATENCY_WATERMARK_MASK(2);
2303 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2304 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2305 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2306 LATENCY_HIGH_WATERMARK(line_time)));
2307 /* restore original selection */
2308 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2309
2310 /* write the priority marks */
2311 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2312 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2313
2314 /* save values for DPM */
2315 radeon_crtc->line_time = line_time;
2316 radeon_crtc->wm_high = latency_watermark_a;
2317 radeon_crtc->wm_low = latency_watermark_b;
2318 }
2319
2320 /**
2321 * evergreen_bandwidth_update - update display watermarks callback.
2322 *
2323 * @rdev: radeon_device pointer
2324 *
2325 * Update the display watermarks based on the requested mode(s)
2326 * (evergreen+).
2327 */
evergreen_bandwidth_update(struct radeon_device * rdev)2328 void evergreen_bandwidth_update(struct radeon_device *rdev)
2329 {
2330 struct drm_display_mode *mode0 = NULL;
2331 struct drm_display_mode *mode1 = NULL;
2332 u32 num_heads = 0, lb_size;
2333 int i;
2334
2335 if (!rdev->mode_info.mode_config_initialized)
2336 return;
2337
2338 radeon_update_display_priority(rdev);
2339
2340 for (i = 0; i < rdev->num_crtc; i++) {
2341 if (rdev->mode_info.crtcs[i]->base.enabled)
2342 num_heads++;
2343 }
2344 for (i = 0; i < rdev->num_crtc; i += 2) {
2345 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2346 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2347 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2348 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2349 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2350 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2351 }
2352 }
2353
2354 /**
2355 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2356 *
2357 * @rdev: radeon_device pointer
2358 *
2359 * Wait for the MC (memory controller) to be idle.
2360 * (evergreen+).
2361 * Returns 0 if the MC is idle, -1 if not.
2362 */
evergreen_mc_wait_for_idle(struct radeon_device * rdev)2363 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2364 {
2365 unsigned i;
2366 u32 tmp;
2367
2368 for (i = 0; i < rdev->usec_timeout; i++) {
2369 /* read MC_STATUS */
2370 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2371 if (!tmp)
2372 return 0;
2373 udelay(1);
2374 }
2375 return -1;
2376 }
2377
2378 /*
2379 * GART
2380 */
evergreen_pcie_gart_tlb_flush(struct radeon_device * rdev)2381 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2382 {
2383 unsigned i;
2384 u32 tmp;
2385
2386 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2387
2388 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2389 for (i = 0; i < rdev->usec_timeout; i++) {
2390 /* read MC_STATUS */
2391 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2392 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2393 if (tmp == 2) {
2394 pr_warn("[drm] r600 flush TLB failed\n");
2395 return;
2396 }
2397 if (tmp) {
2398 return;
2399 }
2400 udelay(1);
2401 }
2402 }
2403
evergreen_pcie_gart_enable(struct radeon_device * rdev)2404 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2405 {
2406 u32 tmp;
2407 int r;
2408
2409 if (rdev->gart.robj == NULL) {
2410 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2411 return -EINVAL;
2412 }
2413 r = radeon_gart_table_vram_pin(rdev);
2414 if (r)
2415 return r;
2416 /* Setup L2 cache */
2417 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2418 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2419 EFFECTIVE_L2_QUEUE_SIZE(7));
2420 WREG32(VM_L2_CNTL2, 0);
2421 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2422 /* Setup TLB control */
2423 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2424 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2425 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2426 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2427 if (rdev->flags & RADEON_IS_IGP) {
2428 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2429 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2430 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2431 } else {
2432 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2433 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2434 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2435 if ((rdev->family == CHIP_JUNIPER) ||
2436 (rdev->family == CHIP_CYPRESS) ||
2437 (rdev->family == CHIP_HEMLOCK) ||
2438 (rdev->family == CHIP_BARTS))
2439 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2440 }
2441 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2442 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2443 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2444 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2445 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2446 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2447 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2448 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2449 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2450 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2451 (u32)(rdev->dummy_page.addr >> 12));
2452 WREG32(VM_CONTEXT1_CNTL, 0);
2453
2454 evergreen_pcie_gart_tlb_flush(rdev);
2455 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2456 (unsigned)(rdev->mc.gtt_size >> 20),
2457 (unsigned long long)rdev->gart.table_addr);
2458 rdev->gart.ready = true;
2459 return 0;
2460 }
2461
evergreen_pcie_gart_disable(struct radeon_device * rdev)2462 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2463 {
2464 u32 tmp;
2465
2466 /* Disable all tables */
2467 WREG32(VM_CONTEXT0_CNTL, 0);
2468 WREG32(VM_CONTEXT1_CNTL, 0);
2469
2470 /* Setup L2 cache */
2471 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2472 EFFECTIVE_L2_QUEUE_SIZE(7));
2473 WREG32(VM_L2_CNTL2, 0);
2474 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2475 /* Setup TLB control */
2476 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2477 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2478 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2479 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2480 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2481 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2482 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2483 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2484 radeon_gart_table_vram_unpin(rdev);
2485 }
2486
evergreen_pcie_gart_fini(struct radeon_device * rdev)2487 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2488 {
2489 evergreen_pcie_gart_disable(rdev);
2490 radeon_gart_table_vram_free(rdev);
2491 radeon_gart_fini(rdev);
2492 }
2493
2494
evergreen_agp_enable(struct radeon_device * rdev)2495 static void evergreen_agp_enable(struct radeon_device *rdev)
2496 {
2497 u32 tmp;
2498
2499 /* Setup L2 cache */
2500 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2501 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2502 EFFECTIVE_L2_QUEUE_SIZE(7));
2503 WREG32(VM_L2_CNTL2, 0);
2504 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2505 /* Setup TLB control */
2506 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2507 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2508 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2509 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2510 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2511 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2512 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2513 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2514 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2515 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2516 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2517 WREG32(VM_CONTEXT0_CNTL, 0);
2518 WREG32(VM_CONTEXT1_CNTL, 0);
2519 }
2520
2521 static const unsigned ni_dig_offsets[] =
2522 {
2523 NI_DIG0_REGISTER_OFFSET,
2524 NI_DIG1_REGISTER_OFFSET,
2525 NI_DIG2_REGISTER_OFFSET,
2526 NI_DIG3_REGISTER_OFFSET,
2527 NI_DIG4_REGISTER_OFFSET,
2528 NI_DIG5_REGISTER_OFFSET
2529 };
2530
2531 static const unsigned ni_tx_offsets[] =
2532 {
2533 NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1,
2534 NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1,
2535 NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1,
2536 NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1,
2537 NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1,
2538 NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1
2539 };
2540
2541 static const unsigned evergreen_dp_offsets[] =
2542 {
2543 EVERGREEN_DP0_REGISTER_OFFSET,
2544 EVERGREEN_DP1_REGISTER_OFFSET,
2545 EVERGREEN_DP2_REGISTER_OFFSET,
2546 EVERGREEN_DP3_REGISTER_OFFSET,
2547 EVERGREEN_DP4_REGISTER_OFFSET,
2548 EVERGREEN_DP5_REGISTER_OFFSET
2549 };
2550
2551 static const unsigned evergreen_disp_int_status[] =
2552 {
2553 DISP_INTERRUPT_STATUS,
2554 DISP_INTERRUPT_STATUS_CONTINUE,
2555 DISP_INTERRUPT_STATUS_CONTINUE2,
2556 DISP_INTERRUPT_STATUS_CONTINUE3,
2557 DISP_INTERRUPT_STATUS_CONTINUE4,
2558 DISP_INTERRUPT_STATUS_CONTINUE5
2559 };
2560
2561 /*
2562 * Assumption is that EVERGREEN_CRTC_MASTER_EN enable for requested crtc
2563 * We go from crtc to connector and it is not relible since it
2564 * should be an opposite direction .If crtc is enable then
2565 * find the dig_fe which selects this crtc and insure that it enable.
2566 * if such dig_fe is found then find dig_be which selects found dig_be and
2567 * insure that it enable and in DP_SST mode.
2568 * if UNIPHY_PLL_CONTROL1.enable then we should disconnect timing
2569 * from dp symbols clocks .
2570 */
evergreen_is_dp_sst_stream_enabled(struct radeon_device * rdev,unsigned crtc_id,unsigned * ret_dig_fe)2571 static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev,
2572 unsigned crtc_id, unsigned *ret_dig_fe)
2573 {
2574 unsigned i;
2575 unsigned dig_fe;
2576 unsigned dig_be;
2577 unsigned dig_en_be;
2578 unsigned uniphy_pll;
2579 unsigned digs_fe_selected;
2580 unsigned dig_be_mode;
2581 unsigned dig_fe_mask;
2582 bool is_enabled = false;
2583 bool found_crtc = false;
2584
2585 /* loop through all running dig_fe to find selected crtc */
2586 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2587 dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]);
2588 if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON &&
2589 crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) {
2590 /* found running pipe */
2591 found_crtc = true;
2592 dig_fe_mask = 1 << i;
2593 dig_fe = i;
2594 break;
2595 }
2596 }
2597
2598 if (found_crtc) {
2599 /* loop through all running dig_be to find selected dig_fe */
2600 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2601 dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]);
2602 /* if dig_fe_selected by dig_be? */
2603 digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be);
2604 dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be);
2605 if (dig_fe_mask & digs_fe_selected &&
2606 /* if dig_be in sst mode? */
2607 dig_be_mode == NI_DIG_BE_DPSST) {
2608 dig_en_be = RREG32(NI_DIG_BE_EN_CNTL +
2609 ni_dig_offsets[i]);
2610 uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 +
2611 ni_tx_offsets[i]);
2612 /* dig_be enable and tx is running */
2613 if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE &&
2614 dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON &&
2615 uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) {
2616 is_enabled = true;
2617 *ret_dig_fe = dig_fe;
2618 break;
2619 }
2620 }
2621 }
2622 }
2623
2624 return is_enabled;
2625 }
2626
2627 /*
2628 * Blank dig when in dp sst mode
2629 * Dig ignores crtc timing
2630 */
evergreen_blank_dp_output(struct radeon_device * rdev,unsigned dig_fe)2631 static void evergreen_blank_dp_output(struct radeon_device *rdev,
2632 unsigned dig_fe)
2633 {
2634 unsigned stream_ctrl;
2635 unsigned fifo_ctrl;
2636 unsigned counter = 0;
2637
2638 if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) {
2639 DRM_ERROR("invalid dig_fe %d\n", dig_fe);
2640 return;
2641 }
2642
2643 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2644 evergreen_dp_offsets[dig_fe]);
2645 if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) {
2646 DRM_ERROR("dig %d , should be enable\n", dig_fe);
2647 return;
2648 }
2649
2650 stream_ctrl &=~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE;
2651 WREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2652 evergreen_dp_offsets[dig_fe], stream_ctrl);
2653
2654 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2655 evergreen_dp_offsets[dig_fe]);
2656 while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) {
2657 msleep(1);
2658 counter++;
2659 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2660 evergreen_dp_offsets[dig_fe]);
2661 }
2662 if (counter >= 32 )
2663 DRM_ERROR("counter exceeds %d\n", counter);
2664
2665 fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]);
2666 fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET;
2667 WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl);
2668
2669 }
2670
evergreen_mc_stop(struct radeon_device * rdev,struct evergreen_mc_save * save)2671 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2672 {
2673 u32 crtc_enabled, tmp, frame_count, blackout;
2674 int i, j;
2675 unsigned dig_fe;
2676
2677 if (!ASIC_IS_NODCE(rdev)) {
2678 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2679 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2680
2681 /* disable VGA render */
2682 WREG32(VGA_RENDER_CONTROL, 0);
2683 }
2684 /* blank the display controllers */
2685 for (i = 0; i < rdev->num_crtc; i++) {
2686 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2687 if (crtc_enabled) {
2688 save->crtc_enabled[i] = true;
2689 if (ASIC_IS_DCE6(rdev)) {
2690 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2691 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2692 radeon_wait_for_vblank(rdev, i);
2693 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2694 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2695 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2696 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2697 }
2698 } else {
2699 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2700 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2701 radeon_wait_for_vblank(rdev, i);
2702 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2703 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2704 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2705 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2706 }
2707 }
2708 /* wait for the next frame */
2709 frame_count = radeon_get_vblank_counter(rdev, i);
2710 for (j = 0; j < rdev->usec_timeout; j++) {
2711 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2712 break;
2713 udelay(1);
2714 }
2715 /*we should disable dig if it drives dp sst*/
2716 /*but we are in radeon_device_init and the topology is unknown*/
2717 /*and it is available after radeon_modeset_init*/
2718 /*the following method radeon_atom_encoder_dpms_dig*/
2719 /*does the job if we initialize it properly*/
2720 /*for now we do it this manually*/
2721 /**/
2722 if (ASIC_IS_DCE5(rdev) &&
2723 evergreen_is_dp_sst_stream_enabled(rdev, i ,&dig_fe))
2724 evergreen_blank_dp_output(rdev, dig_fe);
2725 /*we could remove 6 lines below*/
2726 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2727 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2728 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2729 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2730 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2731 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2732 save->crtc_enabled[i] = false;
2733 /* ***** */
2734 } else {
2735 save->crtc_enabled[i] = false;
2736 }
2737 }
2738
2739 radeon_mc_wait_for_idle(rdev);
2740
2741 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2742 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2743 /* Block CPU access */
2744 WREG32(BIF_FB_EN, 0);
2745 /* blackout the MC */
2746 blackout &= ~BLACKOUT_MODE_MASK;
2747 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2748 }
2749 /* wait for the MC to settle */
2750 udelay(100);
2751
2752 /* lock double buffered regs */
2753 for (i = 0; i < rdev->num_crtc; i++) {
2754 if (save->crtc_enabled[i]) {
2755 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2756 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2757 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2758 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2759 }
2760 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2761 if (!(tmp & 1)) {
2762 tmp |= 1;
2763 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2764 }
2765 }
2766 }
2767 }
2768
evergreen_mc_resume(struct radeon_device * rdev,struct evergreen_mc_save * save)2769 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2770 {
2771 u32 tmp, frame_count;
2772 int i, j;
2773
2774 /* update crtc base addresses */
2775 for (i = 0; i < rdev->num_crtc; i++) {
2776 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2777 upper_32_bits(rdev->mc.vram_start));
2778 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2779 upper_32_bits(rdev->mc.vram_start));
2780 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2781 (u32)rdev->mc.vram_start);
2782 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2783 (u32)rdev->mc.vram_start);
2784 }
2785
2786 if (!ASIC_IS_NODCE(rdev)) {
2787 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2788 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2789 }
2790
2791 /* unlock regs and wait for update */
2792 for (i = 0; i < rdev->num_crtc; i++) {
2793 if (save->crtc_enabled[i]) {
2794 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2795 if ((tmp & 0x7) != 0) {
2796 tmp &= ~0x7;
2797 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2798 }
2799 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2800 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2801 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2802 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2803 }
2804 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2805 if (tmp & 1) {
2806 tmp &= ~1;
2807 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2808 }
2809 for (j = 0; j < rdev->usec_timeout; j++) {
2810 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2811 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2812 break;
2813 udelay(1);
2814 }
2815 }
2816 }
2817
2818 /* unblackout the MC */
2819 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2820 tmp &= ~BLACKOUT_MODE_MASK;
2821 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2822 /* allow CPU access */
2823 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2824
2825 for (i = 0; i < rdev->num_crtc; i++) {
2826 if (save->crtc_enabled[i]) {
2827 if (ASIC_IS_DCE6(rdev)) {
2828 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2829 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2830 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2831 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2832 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2833 } else {
2834 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2835 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2836 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2837 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2838 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2839 }
2840 /* wait for the next frame */
2841 frame_count = radeon_get_vblank_counter(rdev, i);
2842 for (j = 0; j < rdev->usec_timeout; j++) {
2843 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2844 break;
2845 udelay(1);
2846 }
2847 }
2848 }
2849 if (!ASIC_IS_NODCE(rdev)) {
2850 /* Unlock vga access */
2851 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2852 mdelay(1);
2853 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2854 }
2855 }
2856
evergreen_mc_program(struct radeon_device * rdev)2857 void evergreen_mc_program(struct radeon_device *rdev)
2858 {
2859 struct evergreen_mc_save save;
2860 u32 tmp;
2861 int i, j;
2862
2863 /* Initialize HDP */
2864 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2865 WREG32((0x2c14 + j), 0x00000000);
2866 WREG32((0x2c18 + j), 0x00000000);
2867 WREG32((0x2c1c + j), 0x00000000);
2868 WREG32((0x2c20 + j), 0x00000000);
2869 WREG32((0x2c24 + j), 0x00000000);
2870 }
2871 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2872
2873 evergreen_mc_stop(rdev, &save);
2874 if (evergreen_mc_wait_for_idle(rdev)) {
2875 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2876 }
2877 /* Lockout access through VGA aperture*/
2878 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2879 /* Update configuration */
2880 if (rdev->flags & RADEON_IS_AGP) {
2881 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2882 /* VRAM before AGP */
2883 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2884 rdev->mc.vram_start >> 12);
2885 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2886 rdev->mc.gtt_end >> 12);
2887 } else {
2888 /* VRAM after AGP */
2889 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2890 rdev->mc.gtt_start >> 12);
2891 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2892 rdev->mc.vram_end >> 12);
2893 }
2894 } else {
2895 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2896 rdev->mc.vram_start >> 12);
2897 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2898 rdev->mc.vram_end >> 12);
2899 }
2900 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2901 /* llano/ontario only */
2902 if ((rdev->family == CHIP_PALM) ||
2903 (rdev->family == CHIP_SUMO) ||
2904 (rdev->family == CHIP_SUMO2)) {
2905 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2906 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2907 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2908 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2909 }
2910 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2911 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2912 WREG32(MC_VM_FB_LOCATION, tmp);
2913 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2914 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2915 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2916 if (rdev->flags & RADEON_IS_AGP) {
2917 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2918 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2919 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2920 } else {
2921 WREG32(MC_VM_AGP_BASE, 0);
2922 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2923 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2924 }
2925 if (evergreen_mc_wait_for_idle(rdev)) {
2926 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2927 }
2928 evergreen_mc_resume(rdev, &save);
2929 /* we need to own VRAM, so turn off the VGA renderer here
2930 * to stop it overwriting our objects */
2931 rv515_vga_render_disable(rdev);
2932 }
2933
2934 /*
2935 * CP.
2936 */
evergreen_ring_ib_execute(struct radeon_device * rdev,struct radeon_ib * ib)2937 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2938 {
2939 struct radeon_ring *ring = &rdev->ring[ib->ring];
2940 u32 next_rptr;
2941
2942 /* set to DX10/11 mode */
2943 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2944 radeon_ring_write(ring, 1);
2945
2946 if (ring->rptr_save_reg) {
2947 next_rptr = ring->wptr + 3 + 4;
2948 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2949 radeon_ring_write(ring, ((ring->rptr_save_reg -
2950 PACKET3_SET_CONFIG_REG_START) >> 2));
2951 radeon_ring_write(ring, next_rptr);
2952 } else if (rdev->wb.enabled) {
2953 next_rptr = ring->wptr + 5 + 4;
2954 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2955 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2956 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2957 radeon_ring_write(ring, next_rptr);
2958 radeon_ring_write(ring, 0);
2959 }
2960
2961 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2962 radeon_ring_write(ring,
2963 #ifdef __BIG_ENDIAN
2964 (2 << 0) |
2965 #endif
2966 (ib->gpu_addr & 0xFFFFFFFC));
2967 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2968 radeon_ring_write(ring, ib->length_dw);
2969 }
2970
2971
evergreen_cp_load_microcode(struct radeon_device * rdev)2972 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2973 {
2974 const __be32 *fw_data;
2975 int i;
2976
2977 if (!rdev->me_fw || !rdev->pfp_fw)
2978 return -EINVAL;
2979
2980 r700_cp_stop(rdev);
2981 WREG32(CP_RB_CNTL,
2982 #ifdef __BIG_ENDIAN
2983 BUF_SWAP_32BIT |
2984 #endif
2985 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2986
2987 fw_data = (const __be32 *)rdev->pfp_fw->data;
2988 WREG32(CP_PFP_UCODE_ADDR, 0);
2989 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2990 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2991 WREG32(CP_PFP_UCODE_ADDR, 0);
2992
2993 fw_data = (const __be32 *)rdev->me_fw->data;
2994 WREG32(CP_ME_RAM_WADDR, 0);
2995 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2996 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2997
2998 WREG32(CP_PFP_UCODE_ADDR, 0);
2999 WREG32(CP_ME_RAM_WADDR, 0);
3000 WREG32(CP_ME_RAM_RADDR, 0);
3001 return 0;
3002 }
3003
evergreen_cp_start(struct radeon_device * rdev)3004 static int evergreen_cp_start(struct radeon_device *rdev)
3005 {
3006 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3007 int r, i;
3008 uint32_t cp_me;
3009
3010 r = radeon_ring_lock(rdev, ring, 7);
3011 if (r) {
3012 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3013 return r;
3014 }
3015 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3016 radeon_ring_write(ring, 0x1);
3017 radeon_ring_write(ring, 0x0);
3018 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
3019 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3020 radeon_ring_write(ring, 0);
3021 radeon_ring_write(ring, 0);
3022 radeon_ring_unlock_commit(rdev, ring, false);
3023
3024 cp_me = 0xff;
3025 WREG32(CP_ME_CNTL, cp_me);
3026
3027 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
3028 if (r) {
3029 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3030 return r;
3031 }
3032
3033 /* setup clear context state */
3034 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3035 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3036
3037 for (i = 0; i < evergreen_default_size; i++)
3038 radeon_ring_write(ring, evergreen_default_state[i]);
3039
3040 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3041 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3042
3043 /* set clear context state */
3044 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3045 radeon_ring_write(ring, 0);
3046
3047 /* SQ_VTX_BASE_VTX_LOC */
3048 radeon_ring_write(ring, 0xc0026f00);
3049 radeon_ring_write(ring, 0x00000000);
3050 radeon_ring_write(ring, 0x00000000);
3051 radeon_ring_write(ring, 0x00000000);
3052
3053 /* Clear consts */
3054 radeon_ring_write(ring, 0xc0036f00);
3055 radeon_ring_write(ring, 0x00000bc4);
3056 radeon_ring_write(ring, 0xffffffff);
3057 radeon_ring_write(ring, 0xffffffff);
3058 radeon_ring_write(ring, 0xffffffff);
3059
3060 radeon_ring_write(ring, 0xc0026900);
3061 radeon_ring_write(ring, 0x00000316);
3062 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3063 radeon_ring_write(ring, 0x00000010); /* */
3064
3065 radeon_ring_unlock_commit(rdev, ring, false);
3066
3067 return 0;
3068 }
3069
evergreen_cp_resume(struct radeon_device * rdev)3070 static int evergreen_cp_resume(struct radeon_device *rdev)
3071 {
3072 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3073 u32 tmp;
3074 u32 rb_bufsz;
3075 int r;
3076
3077 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3078 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3079 SOFT_RESET_PA |
3080 SOFT_RESET_SH |
3081 SOFT_RESET_VGT |
3082 SOFT_RESET_SPI |
3083 SOFT_RESET_SX));
3084 RREG32(GRBM_SOFT_RESET);
3085 mdelay(15);
3086 WREG32(GRBM_SOFT_RESET, 0);
3087 RREG32(GRBM_SOFT_RESET);
3088
3089 /* Set ring buffer size */
3090 rb_bufsz = order_base_2(ring->ring_size / 8);
3091 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3092 #ifdef __BIG_ENDIAN
3093 tmp |= BUF_SWAP_32BIT;
3094 #endif
3095 WREG32(CP_RB_CNTL, tmp);
3096 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3097 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3098
3099 /* Set the write pointer delay */
3100 WREG32(CP_RB_WPTR_DELAY, 0);
3101
3102 /* Initialize the ring buffer's read and write pointers */
3103 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3104 WREG32(CP_RB_RPTR_WR, 0);
3105 ring->wptr = 0;
3106 WREG32(CP_RB_WPTR, ring->wptr);
3107
3108 /* set the wb address whether it's enabled or not */
3109 WREG32(CP_RB_RPTR_ADDR,
3110 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
3111 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3112 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3113
3114 if (rdev->wb.enabled)
3115 WREG32(SCRATCH_UMSK, 0xff);
3116 else {
3117 tmp |= RB_NO_UPDATE;
3118 WREG32(SCRATCH_UMSK, 0);
3119 }
3120
3121 mdelay(1);
3122 WREG32(CP_RB_CNTL, tmp);
3123
3124 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3125 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3126
3127 evergreen_cp_start(rdev);
3128 ring->ready = true;
3129 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3130 if (r) {
3131 ring->ready = false;
3132 return r;
3133 }
3134 return 0;
3135 }
3136
3137 /*
3138 * Core functions
3139 */
evergreen_gpu_init(struct radeon_device * rdev)3140 static void evergreen_gpu_init(struct radeon_device *rdev)
3141 {
3142 u32 gb_addr_config;
3143 u32 mc_shared_chmap __unused, mc_arb_ramcfg;
3144 u32 sx_debug_1;
3145 u32 smx_dc_ctl0;
3146 u32 sq_config;
3147 u32 sq_lds_resource_mgmt;
3148 u32 sq_gpr_resource_mgmt_1;
3149 u32 sq_gpr_resource_mgmt_2;
3150 u32 sq_gpr_resource_mgmt_3;
3151 u32 sq_thread_resource_mgmt;
3152 u32 sq_thread_resource_mgmt_2;
3153 u32 sq_stack_resource_mgmt_1;
3154 u32 sq_stack_resource_mgmt_2;
3155 u32 sq_stack_resource_mgmt_3;
3156 u32 vgt_cache_invalidation;
3157 u32 hdp_host_path_cntl, tmp;
3158 u32 disabled_rb_mask;
3159 int i, j, ps_thread_count;
3160
3161 switch (rdev->family) {
3162 case CHIP_CYPRESS:
3163 case CHIP_HEMLOCK:
3164 rdev->config.evergreen.num_ses = 2;
3165 rdev->config.evergreen.max_pipes = 4;
3166 rdev->config.evergreen.max_tile_pipes = 8;
3167 rdev->config.evergreen.max_simds = 10;
3168 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3169 rdev->config.evergreen.max_gprs = 256;
3170 rdev->config.evergreen.max_threads = 248;
3171 rdev->config.evergreen.max_gs_threads = 32;
3172 rdev->config.evergreen.max_stack_entries = 512;
3173 rdev->config.evergreen.sx_num_of_sets = 4;
3174 rdev->config.evergreen.sx_max_export_size = 256;
3175 rdev->config.evergreen.sx_max_export_pos_size = 64;
3176 rdev->config.evergreen.sx_max_export_smx_size = 192;
3177 rdev->config.evergreen.max_hw_contexts = 8;
3178 rdev->config.evergreen.sq_num_cf_insts = 2;
3179
3180 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3181 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3182 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3183 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3184 break;
3185 case CHIP_JUNIPER:
3186 rdev->config.evergreen.num_ses = 1;
3187 rdev->config.evergreen.max_pipes = 4;
3188 rdev->config.evergreen.max_tile_pipes = 4;
3189 rdev->config.evergreen.max_simds = 10;
3190 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3191 rdev->config.evergreen.max_gprs = 256;
3192 rdev->config.evergreen.max_threads = 248;
3193 rdev->config.evergreen.max_gs_threads = 32;
3194 rdev->config.evergreen.max_stack_entries = 512;
3195 rdev->config.evergreen.sx_num_of_sets = 4;
3196 rdev->config.evergreen.sx_max_export_size = 256;
3197 rdev->config.evergreen.sx_max_export_pos_size = 64;
3198 rdev->config.evergreen.sx_max_export_smx_size = 192;
3199 rdev->config.evergreen.max_hw_contexts = 8;
3200 rdev->config.evergreen.sq_num_cf_insts = 2;
3201
3202 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3203 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3204 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3205 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3206 break;
3207 case CHIP_REDWOOD:
3208 rdev->config.evergreen.num_ses = 1;
3209 rdev->config.evergreen.max_pipes = 4;
3210 rdev->config.evergreen.max_tile_pipes = 4;
3211 rdev->config.evergreen.max_simds = 5;
3212 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3213 rdev->config.evergreen.max_gprs = 256;
3214 rdev->config.evergreen.max_threads = 248;
3215 rdev->config.evergreen.max_gs_threads = 32;
3216 rdev->config.evergreen.max_stack_entries = 256;
3217 rdev->config.evergreen.sx_num_of_sets = 4;
3218 rdev->config.evergreen.sx_max_export_size = 256;
3219 rdev->config.evergreen.sx_max_export_pos_size = 64;
3220 rdev->config.evergreen.sx_max_export_smx_size = 192;
3221 rdev->config.evergreen.max_hw_contexts = 8;
3222 rdev->config.evergreen.sq_num_cf_insts = 2;
3223
3224 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3225 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3226 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3227 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3228 break;
3229 case CHIP_CEDAR:
3230 default:
3231 rdev->config.evergreen.num_ses = 1;
3232 rdev->config.evergreen.max_pipes = 2;
3233 rdev->config.evergreen.max_tile_pipes = 2;
3234 rdev->config.evergreen.max_simds = 2;
3235 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3236 rdev->config.evergreen.max_gprs = 256;
3237 rdev->config.evergreen.max_threads = 192;
3238 rdev->config.evergreen.max_gs_threads = 16;
3239 rdev->config.evergreen.max_stack_entries = 256;
3240 rdev->config.evergreen.sx_num_of_sets = 4;
3241 rdev->config.evergreen.sx_max_export_size = 128;
3242 rdev->config.evergreen.sx_max_export_pos_size = 32;
3243 rdev->config.evergreen.sx_max_export_smx_size = 96;
3244 rdev->config.evergreen.max_hw_contexts = 4;
3245 rdev->config.evergreen.sq_num_cf_insts = 1;
3246
3247 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3248 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3249 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3250 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3251 break;
3252 case CHIP_PALM:
3253 rdev->config.evergreen.num_ses = 1;
3254 rdev->config.evergreen.max_pipes = 2;
3255 rdev->config.evergreen.max_tile_pipes = 2;
3256 rdev->config.evergreen.max_simds = 2;
3257 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3258 rdev->config.evergreen.max_gprs = 256;
3259 rdev->config.evergreen.max_threads = 192;
3260 rdev->config.evergreen.max_gs_threads = 16;
3261 rdev->config.evergreen.max_stack_entries = 256;
3262 rdev->config.evergreen.sx_num_of_sets = 4;
3263 rdev->config.evergreen.sx_max_export_size = 128;
3264 rdev->config.evergreen.sx_max_export_pos_size = 32;
3265 rdev->config.evergreen.sx_max_export_smx_size = 96;
3266 rdev->config.evergreen.max_hw_contexts = 4;
3267 rdev->config.evergreen.sq_num_cf_insts = 1;
3268
3269 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3270 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3271 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3272 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3273 break;
3274 case CHIP_SUMO:
3275 rdev->config.evergreen.num_ses = 1;
3276 rdev->config.evergreen.max_pipes = 4;
3277 rdev->config.evergreen.max_tile_pipes = 4;
3278 if (rdev->pdev->device == 0x9648)
3279 rdev->config.evergreen.max_simds = 3;
3280 else if ((rdev->pdev->device == 0x9647) ||
3281 (rdev->pdev->device == 0x964a))
3282 rdev->config.evergreen.max_simds = 4;
3283 else
3284 rdev->config.evergreen.max_simds = 5;
3285 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3286 rdev->config.evergreen.max_gprs = 256;
3287 rdev->config.evergreen.max_threads = 248;
3288 rdev->config.evergreen.max_gs_threads = 32;
3289 rdev->config.evergreen.max_stack_entries = 256;
3290 rdev->config.evergreen.sx_num_of_sets = 4;
3291 rdev->config.evergreen.sx_max_export_size = 256;
3292 rdev->config.evergreen.sx_max_export_pos_size = 64;
3293 rdev->config.evergreen.sx_max_export_smx_size = 192;
3294 rdev->config.evergreen.max_hw_contexts = 8;
3295 rdev->config.evergreen.sq_num_cf_insts = 2;
3296
3297 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3298 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3299 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3300 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3301 break;
3302 case CHIP_SUMO2:
3303 rdev->config.evergreen.num_ses = 1;
3304 rdev->config.evergreen.max_pipes = 4;
3305 rdev->config.evergreen.max_tile_pipes = 4;
3306 rdev->config.evergreen.max_simds = 2;
3307 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3308 rdev->config.evergreen.max_gprs = 256;
3309 rdev->config.evergreen.max_threads = 248;
3310 rdev->config.evergreen.max_gs_threads = 32;
3311 rdev->config.evergreen.max_stack_entries = 512;
3312 rdev->config.evergreen.sx_num_of_sets = 4;
3313 rdev->config.evergreen.sx_max_export_size = 256;
3314 rdev->config.evergreen.sx_max_export_pos_size = 64;
3315 rdev->config.evergreen.sx_max_export_smx_size = 192;
3316 rdev->config.evergreen.max_hw_contexts = 4;
3317 rdev->config.evergreen.sq_num_cf_insts = 2;
3318
3319 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3320 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3321 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3322 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3323 break;
3324 case CHIP_BARTS:
3325 rdev->config.evergreen.num_ses = 2;
3326 rdev->config.evergreen.max_pipes = 4;
3327 rdev->config.evergreen.max_tile_pipes = 8;
3328 rdev->config.evergreen.max_simds = 7;
3329 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3330 rdev->config.evergreen.max_gprs = 256;
3331 rdev->config.evergreen.max_threads = 248;
3332 rdev->config.evergreen.max_gs_threads = 32;
3333 rdev->config.evergreen.max_stack_entries = 512;
3334 rdev->config.evergreen.sx_num_of_sets = 4;
3335 rdev->config.evergreen.sx_max_export_size = 256;
3336 rdev->config.evergreen.sx_max_export_pos_size = 64;
3337 rdev->config.evergreen.sx_max_export_smx_size = 192;
3338 rdev->config.evergreen.max_hw_contexts = 8;
3339 rdev->config.evergreen.sq_num_cf_insts = 2;
3340
3341 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3342 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3343 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3344 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3345 break;
3346 case CHIP_TURKS:
3347 rdev->config.evergreen.num_ses = 1;
3348 rdev->config.evergreen.max_pipes = 4;
3349 rdev->config.evergreen.max_tile_pipes = 4;
3350 rdev->config.evergreen.max_simds = 6;
3351 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3352 rdev->config.evergreen.max_gprs = 256;
3353 rdev->config.evergreen.max_threads = 248;
3354 rdev->config.evergreen.max_gs_threads = 32;
3355 rdev->config.evergreen.max_stack_entries = 256;
3356 rdev->config.evergreen.sx_num_of_sets = 4;
3357 rdev->config.evergreen.sx_max_export_size = 256;
3358 rdev->config.evergreen.sx_max_export_pos_size = 64;
3359 rdev->config.evergreen.sx_max_export_smx_size = 192;
3360 rdev->config.evergreen.max_hw_contexts = 8;
3361 rdev->config.evergreen.sq_num_cf_insts = 2;
3362
3363 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3364 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3365 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3366 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3367 break;
3368 case CHIP_CAICOS:
3369 rdev->config.evergreen.num_ses = 1;
3370 rdev->config.evergreen.max_pipes = 2;
3371 rdev->config.evergreen.max_tile_pipes = 2;
3372 rdev->config.evergreen.max_simds = 2;
3373 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3374 rdev->config.evergreen.max_gprs = 256;
3375 rdev->config.evergreen.max_threads = 192;
3376 rdev->config.evergreen.max_gs_threads = 16;
3377 rdev->config.evergreen.max_stack_entries = 256;
3378 rdev->config.evergreen.sx_num_of_sets = 4;
3379 rdev->config.evergreen.sx_max_export_size = 128;
3380 rdev->config.evergreen.sx_max_export_pos_size = 32;
3381 rdev->config.evergreen.sx_max_export_smx_size = 96;
3382 rdev->config.evergreen.max_hw_contexts = 4;
3383 rdev->config.evergreen.sq_num_cf_insts = 1;
3384
3385 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3386 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3387 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3388 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3389 break;
3390 }
3391
3392 /* Initialize HDP */
3393 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3394 WREG32((0x2c14 + j), 0x00000000);
3395 WREG32((0x2c18 + j), 0x00000000);
3396 WREG32((0x2c1c + j), 0x00000000);
3397 WREG32((0x2c20 + j), 0x00000000);
3398 WREG32((0x2c24 + j), 0x00000000);
3399 }
3400
3401 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3402 WREG32(SRBM_INT_CNTL, 0x1);
3403 WREG32(SRBM_INT_ACK, 0x1);
3404
3405 evergreen_fix_pci_max_read_req_size(rdev);
3406
3407 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3408 if ((rdev->family == CHIP_PALM) ||
3409 (rdev->family == CHIP_SUMO) ||
3410 (rdev->family == CHIP_SUMO2))
3411 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3412 else
3413 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3414
3415 /* setup tiling info dword. gb_addr_config is not adequate since it does
3416 * not have bank info, so create a custom tiling dword.
3417 * bits 3:0 num_pipes
3418 * bits 7:4 num_banks
3419 * bits 11:8 group_size
3420 * bits 15:12 row_size
3421 */
3422 rdev->config.evergreen.tile_config = 0;
3423 switch (rdev->config.evergreen.max_tile_pipes) {
3424 case 1:
3425 default:
3426 rdev->config.evergreen.tile_config |= (0 << 0);
3427 break;
3428 case 2:
3429 rdev->config.evergreen.tile_config |= (1 << 0);
3430 break;
3431 case 4:
3432 rdev->config.evergreen.tile_config |= (2 << 0);
3433 break;
3434 case 8:
3435 rdev->config.evergreen.tile_config |= (3 << 0);
3436 break;
3437 }
3438 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3439 if (rdev->flags & RADEON_IS_IGP)
3440 rdev->config.evergreen.tile_config |= 1 << 4;
3441 else {
3442 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3443 case 0: /* four banks */
3444 rdev->config.evergreen.tile_config |= 0 << 4;
3445 break;
3446 case 1: /* eight banks */
3447 rdev->config.evergreen.tile_config |= 1 << 4;
3448 break;
3449 case 2: /* sixteen banks */
3450 default:
3451 rdev->config.evergreen.tile_config |= 2 << 4;
3452 break;
3453 }
3454 }
3455 rdev->config.evergreen.tile_config |= 0 << 8;
3456 rdev->config.evergreen.tile_config |=
3457 ((gb_addr_config & 0x30000000) >> 28) << 12;
3458
3459 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3460 u32 efuse_straps_4;
3461 u32 efuse_straps_3;
3462
3463 efuse_straps_4 = RREG32_RCU(0x204);
3464 efuse_straps_3 = RREG32_RCU(0x203);
3465 tmp = (((efuse_straps_4 & 0xf) << 4) |
3466 ((efuse_straps_3 & 0xf0000000) >> 28));
3467 } else {
3468 tmp = 0;
3469 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3470 u32 rb_disable_bitmap;
3471
3472 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3473 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3474 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3475 tmp <<= 4;
3476 tmp |= rb_disable_bitmap;
3477 }
3478 }
3479 /* enabled rb are just the one not disabled :) */
3480 disabled_rb_mask = tmp;
3481 tmp = 0;
3482 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3483 tmp |= (1 << i);
3484 /* if all the backends are disabled, fix it up here */
3485 if ((disabled_rb_mask & tmp) == tmp) {
3486 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3487 disabled_rb_mask &= ~(1 << i);
3488 }
3489
3490 for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3491 u32 simd_disable_bitmap;
3492
3493 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3494 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3495 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3496 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3497 tmp <<= 16;
3498 tmp |= simd_disable_bitmap;
3499 }
3500 rdev->config.evergreen.active_simds = hweight32(~tmp);
3501
3502 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3503 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3504
3505 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3506 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3507 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3508 WREG32(DMA_TILING_CONFIG, gb_addr_config);
3509 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3510 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3511 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3512
3513 if ((rdev->config.evergreen.max_backends == 1) &&
3514 (rdev->flags & RADEON_IS_IGP)) {
3515 if ((disabled_rb_mask & 3) == 1) {
3516 /* RB0 disabled, RB1 enabled */
3517 tmp = 0x11111111;
3518 } else {
3519 /* RB1 disabled, RB0 enabled */
3520 tmp = 0x00000000;
3521 }
3522 } else {
3523 tmp = gb_addr_config & NUM_PIPES_MASK;
3524 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3525 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3526 }
3527 rdev->config.evergreen.backend_map = tmp;
3528 WREG32(GB_BACKEND_MAP, tmp);
3529
3530 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3531 WREG32(CGTS_TCC_DISABLE, 0);
3532 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3533 WREG32(CGTS_USER_TCC_DISABLE, 0);
3534
3535 /* set HW defaults for 3D engine */
3536 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3537 ROQ_IB2_START(0x2b)));
3538
3539 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3540
3541 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3542 SYNC_GRADIENT |
3543 SYNC_WALKER |
3544 SYNC_ALIGNER));
3545
3546 sx_debug_1 = RREG32(SX_DEBUG_1);
3547 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3548 WREG32(SX_DEBUG_1, sx_debug_1);
3549
3550
3551 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3552 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3553 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3554 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3555
3556 if (rdev->family <= CHIP_SUMO2)
3557 WREG32(SMX_SAR_CTL0, 0x00010000);
3558
3559 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3560 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3561 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3562
3563 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3564 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3565 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3566
3567 WREG32(VGT_NUM_INSTANCES, 1);
3568 WREG32(SPI_CONFIG_CNTL, 0);
3569 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3570 WREG32(CP_PERFMON_CNTL, 0);
3571
3572 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3573 FETCH_FIFO_HIWATER(0x4) |
3574 DONE_FIFO_HIWATER(0xe0) |
3575 ALU_UPDATE_FIFO_HIWATER(0x8)));
3576
3577 sq_config = RREG32(SQ_CONFIG);
3578 sq_config &= ~(PS_PRIO(3) |
3579 VS_PRIO(3) |
3580 GS_PRIO(3) |
3581 ES_PRIO(3));
3582 sq_config |= (VC_ENABLE |
3583 EXPORT_SRC_C |
3584 PS_PRIO(0) |
3585 VS_PRIO(1) |
3586 GS_PRIO(2) |
3587 ES_PRIO(3));
3588
3589 switch (rdev->family) {
3590 case CHIP_CEDAR:
3591 case CHIP_PALM:
3592 case CHIP_SUMO:
3593 case CHIP_SUMO2:
3594 case CHIP_CAICOS:
3595 /* no vertex cache */
3596 sq_config &= ~VC_ENABLE;
3597 break;
3598 default:
3599 break;
3600 }
3601
3602 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3603
3604 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3605 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3606 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3607 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3608 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3609 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3610 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3611
3612 switch (rdev->family) {
3613 case CHIP_CEDAR:
3614 case CHIP_PALM:
3615 case CHIP_SUMO:
3616 case CHIP_SUMO2:
3617 ps_thread_count = 96;
3618 break;
3619 default:
3620 ps_thread_count = 128;
3621 break;
3622 }
3623
3624 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3625 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3626 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3627 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3628 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3629 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3630
3631 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3632 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3633 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3634 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3635 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3636 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3637
3638 WREG32(SQ_CONFIG, sq_config);
3639 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3640 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3641 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3642 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3643 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3644 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3645 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3646 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3647 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3648 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3649
3650 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3651 FORCE_EOV_MAX_REZ_CNT(255)));
3652
3653 switch (rdev->family) {
3654 case CHIP_CEDAR:
3655 case CHIP_PALM:
3656 case CHIP_SUMO:
3657 case CHIP_SUMO2:
3658 case CHIP_CAICOS:
3659 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3660 break;
3661 default:
3662 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3663 break;
3664 }
3665 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3666 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3667
3668 WREG32(VGT_GS_VERTEX_REUSE, 16);
3669 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3670 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3671
3672 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3673 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3674
3675 WREG32(CB_PERF_CTR0_SEL_0, 0);
3676 WREG32(CB_PERF_CTR0_SEL_1, 0);
3677 WREG32(CB_PERF_CTR1_SEL_0, 0);
3678 WREG32(CB_PERF_CTR1_SEL_1, 0);
3679 WREG32(CB_PERF_CTR2_SEL_0, 0);
3680 WREG32(CB_PERF_CTR2_SEL_1, 0);
3681 WREG32(CB_PERF_CTR3_SEL_0, 0);
3682 WREG32(CB_PERF_CTR3_SEL_1, 0);
3683
3684 /* clear render buffer base addresses */
3685 WREG32(CB_COLOR0_BASE, 0);
3686 WREG32(CB_COLOR1_BASE, 0);
3687 WREG32(CB_COLOR2_BASE, 0);
3688 WREG32(CB_COLOR3_BASE, 0);
3689 WREG32(CB_COLOR4_BASE, 0);
3690 WREG32(CB_COLOR5_BASE, 0);
3691 WREG32(CB_COLOR6_BASE, 0);
3692 WREG32(CB_COLOR7_BASE, 0);
3693 WREG32(CB_COLOR8_BASE, 0);
3694 WREG32(CB_COLOR9_BASE, 0);
3695 WREG32(CB_COLOR10_BASE, 0);
3696 WREG32(CB_COLOR11_BASE, 0);
3697
3698 /* set the shader const cache sizes to 0 */
3699 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3700 WREG32(i, 0);
3701 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3702 WREG32(i, 0);
3703
3704 tmp = RREG32(HDP_MISC_CNTL);
3705 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3706 WREG32(HDP_MISC_CNTL, tmp);
3707
3708 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3709 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3710
3711 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3712
3713 udelay(50);
3714
3715 }
3716
evergreen_mc_init(struct radeon_device * rdev)3717 int evergreen_mc_init(struct radeon_device *rdev)
3718 {
3719 u32 tmp;
3720 int chansize, numchan;
3721
3722 /* Get VRAM informations */
3723 rdev->mc.vram_is_ddr = true;
3724 if ((rdev->family == CHIP_PALM) ||
3725 (rdev->family == CHIP_SUMO) ||
3726 (rdev->family == CHIP_SUMO2))
3727 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3728 else
3729 tmp = RREG32(MC_ARB_RAMCFG);
3730 if (tmp & CHANSIZE_OVERRIDE) {
3731 chansize = 16;
3732 } else if (tmp & CHANSIZE_MASK) {
3733 chansize = 64;
3734 } else {
3735 chansize = 32;
3736 }
3737 tmp = RREG32(MC_SHARED_CHMAP);
3738 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3739 case 0:
3740 default:
3741 numchan = 1;
3742 break;
3743 case 1:
3744 numchan = 2;
3745 break;
3746 case 2:
3747 numchan = 4;
3748 break;
3749 case 3:
3750 numchan = 8;
3751 break;
3752 }
3753 rdev->mc.vram_width = numchan * chansize;
3754 /* Could aper size report 0 ? */
3755 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3756 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3757 /* Setup GPU memory space */
3758 if ((rdev->family == CHIP_PALM) ||
3759 (rdev->family == CHIP_SUMO) ||
3760 (rdev->family == CHIP_SUMO2)) {
3761 /* size in bytes on fusion */
3762 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3763 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3764 } else {
3765 /* size in MB on evergreen/cayman/tn */
3766 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3767 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3768 }
3769 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3770 r700_vram_gtt_location(rdev, &rdev->mc);
3771 radeon_update_bandwidth_info(rdev);
3772
3773 return 0;
3774 }
3775
evergreen_print_gpu_status_regs(struct radeon_device * rdev)3776 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3777 {
3778 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
3779 RREG32(GRBM_STATUS));
3780 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
3781 RREG32(GRBM_STATUS_SE0));
3782 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
3783 RREG32(GRBM_STATUS_SE1));
3784 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
3785 RREG32(SRBM_STATUS));
3786 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3787 RREG32(SRBM_STATUS2));
3788 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3789 RREG32(CP_STALLED_STAT1));
3790 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3791 RREG32(CP_STALLED_STAT2));
3792 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3793 RREG32(CP_BUSY_STAT));
3794 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3795 RREG32(CP_STAT));
3796 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3797 RREG32(DMA_STATUS_REG));
3798 if (rdev->family >= CHIP_CAYMAN) {
3799 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3800 RREG32(DMA_STATUS_REG + 0x800));
3801 }
3802 }
3803
evergreen_is_display_hung(struct radeon_device * rdev)3804 bool evergreen_is_display_hung(struct radeon_device *rdev)
3805 {
3806 u32 crtc_hung = 0;
3807 u32 crtc_status[6];
3808 u32 i, j, tmp;
3809
3810 for (i = 0; i < rdev->num_crtc; i++) {
3811 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3812 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3813 crtc_hung |= (1 << i);
3814 }
3815 }
3816
3817 for (j = 0; j < 10; j++) {
3818 for (i = 0; i < rdev->num_crtc; i++) {
3819 if (crtc_hung & (1 << i)) {
3820 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3821 if (tmp != crtc_status[i])
3822 crtc_hung &= ~(1 << i);
3823 }
3824 }
3825 if (crtc_hung == 0)
3826 return false;
3827 udelay(100);
3828 }
3829
3830 return true;
3831 }
3832
evergreen_gpu_check_soft_reset(struct radeon_device * rdev)3833 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3834 {
3835 u32 reset_mask = 0;
3836 u32 tmp;
3837
3838 /* GRBM_STATUS */
3839 tmp = RREG32(GRBM_STATUS);
3840 if (tmp & (PA_BUSY | SC_BUSY |
3841 SH_BUSY | SX_BUSY |
3842 TA_BUSY | VGT_BUSY |
3843 DB_BUSY | CB_BUSY |
3844 SPI_BUSY | VGT_BUSY_NO_DMA))
3845 reset_mask |= RADEON_RESET_GFX;
3846
3847 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3848 CP_BUSY | CP_COHERENCY_BUSY))
3849 reset_mask |= RADEON_RESET_CP;
3850
3851 if (tmp & GRBM_EE_BUSY)
3852 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3853
3854 /* DMA_STATUS_REG */
3855 tmp = RREG32(DMA_STATUS_REG);
3856 if (!(tmp & DMA_IDLE))
3857 reset_mask |= RADEON_RESET_DMA;
3858
3859 /* SRBM_STATUS2 */
3860 tmp = RREG32(SRBM_STATUS2);
3861 if (tmp & DMA_BUSY)
3862 reset_mask |= RADEON_RESET_DMA;
3863
3864 /* SRBM_STATUS */
3865 tmp = RREG32(SRBM_STATUS);
3866 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3867 reset_mask |= RADEON_RESET_RLC;
3868
3869 if (tmp & IH_BUSY)
3870 reset_mask |= RADEON_RESET_IH;
3871
3872 if (tmp & SEM_BUSY)
3873 reset_mask |= RADEON_RESET_SEM;
3874
3875 if (tmp & GRBM_RQ_PENDING)
3876 reset_mask |= RADEON_RESET_GRBM;
3877
3878 if (tmp & VMC_BUSY)
3879 reset_mask |= RADEON_RESET_VMC;
3880
3881 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3882 MCC_BUSY | MCD_BUSY))
3883 reset_mask |= RADEON_RESET_MC;
3884
3885 if (evergreen_is_display_hung(rdev))
3886 reset_mask |= RADEON_RESET_DISPLAY;
3887
3888 /* VM_L2_STATUS */
3889 tmp = RREG32(VM_L2_STATUS);
3890 if (tmp & L2_BUSY)
3891 reset_mask |= RADEON_RESET_VMC;
3892
3893 /* Skip MC reset as it's mostly likely not hung, just busy */
3894 if (reset_mask & RADEON_RESET_MC) {
3895 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3896 reset_mask &= ~RADEON_RESET_MC;
3897 }
3898
3899 return reset_mask;
3900 }
3901
evergreen_gpu_soft_reset(struct radeon_device * rdev,u32 reset_mask)3902 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3903 {
3904 struct evergreen_mc_save save;
3905 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3906 u32 tmp;
3907
3908 if (reset_mask == 0)
3909 return;
3910
3911 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3912
3913 evergreen_print_gpu_status_regs(rdev);
3914
3915 /* Disable CP parsing/prefetching */
3916 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3917
3918 if (reset_mask & RADEON_RESET_DMA) {
3919 /* Disable DMA */
3920 tmp = RREG32(DMA_RB_CNTL);
3921 tmp &= ~DMA_RB_ENABLE;
3922 WREG32(DMA_RB_CNTL, tmp);
3923 }
3924
3925 udelay(50);
3926
3927 evergreen_mc_stop(rdev, &save);
3928 if (evergreen_mc_wait_for_idle(rdev)) {
3929 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3930 }
3931
3932 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3933 grbm_soft_reset |= SOFT_RESET_DB |
3934 SOFT_RESET_CB |
3935 SOFT_RESET_PA |
3936 SOFT_RESET_SC |
3937 SOFT_RESET_SPI |
3938 SOFT_RESET_SX |
3939 SOFT_RESET_SH |
3940 SOFT_RESET_TC |
3941 SOFT_RESET_TA |
3942 SOFT_RESET_VC |
3943 SOFT_RESET_VGT;
3944 }
3945
3946 if (reset_mask & RADEON_RESET_CP) {
3947 grbm_soft_reset |= SOFT_RESET_CP |
3948 SOFT_RESET_VGT;
3949
3950 srbm_soft_reset |= SOFT_RESET_GRBM;
3951 }
3952
3953 if (reset_mask & RADEON_RESET_DMA)
3954 srbm_soft_reset |= SOFT_RESET_DMA;
3955
3956 if (reset_mask & RADEON_RESET_DISPLAY)
3957 srbm_soft_reset |= SOFT_RESET_DC;
3958
3959 if (reset_mask & RADEON_RESET_RLC)
3960 srbm_soft_reset |= SOFT_RESET_RLC;
3961
3962 if (reset_mask & RADEON_RESET_SEM)
3963 srbm_soft_reset |= SOFT_RESET_SEM;
3964
3965 if (reset_mask & RADEON_RESET_IH)
3966 srbm_soft_reset |= SOFT_RESET_IH;
3967
3968 if (reset_mask & RADEON_RESET_GRBM)
3969 srbm_soft_reset |= SOFT_RESET_GRBM;
3970
3971 if (reset_mask & RADEON_RESET_VMC)
3972 srbm_soft_reset |= SOFT_RESET_VMC;
3973
3974 if (!(rdev->flags & RADEON_IS_IGP)) {
3975 if (reset_mask & RADEON_RESET_MC)
3976 srbm_soft_reset |= SOFT_RESET_MC;
3977 }
3978
3979 if (grbm_soft_reset) {
3980 tmp = RREG32(GRBM_SOFT_RESET);
3981 tmp |= grbm_soft_reset;
3982 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3983 WREG32(GRBM_SOFT_RESET, tmp);
3984 tmp = RREG32(GRBM_SOFT_RESET);
3985
3986 udelay(50);
3987
3988 tmp &= ~grbm_soft_reset;
3989 WREG32(GRBM_SOFT_RESET, tmp);
3990 tmp = RREG32(GRBM_SOFT_RESET);
3991 }
3992
3993 if (srbm_soft_reset) {
3994 tmp = RREG32(SRBM_SOFT_RESET);
3995 tmp |= srbm_soft_reset;
3996 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3997 WREG32(SRBM_SOFT_RESET, tmp);
3998 tmp = RREG32(SRBM_SOFT_RESET);
3999
4000 udelay(50);
4001
4002 tmp &= ~srbm_soft_reset;
4003 WREG32(SRBM_SOFT_RESET, tmp);
4004 tmp = RREG32(SRBM_SOFT_RESET);
4005 }
4006
4007 /* Wait a little for things to settle down */
4008 udelay(50);
4009
4010 evergreen_mc_resume(rdev, &save);
4011 udelay(50);
4012
4013 evergreen_print_gpu_status_regs(rdev);
4014 }
4015
evergreen_gpu_pci_config_reset(struct radeon_device * rdev)4016 void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
4017 {
4018 struct evergreen_mc_save save;
4019 u32 tmp, i;
4020
4021 dev_info(rdev->dev, "GPU pci config reset\n");
4022
4023 /* disable dpm? */
4024
4025 /* Disable CP parsing/prefetching */
4026 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
4027 udelay(50);
4028 /* Disable DMA */
4029 tmp = RREG32(DMA_RB_CNTL);
4030 tmp &= ~DMA_RB_ENABLE;
4031 WREG32(DMA_RB_CNTL, tmp);
4032 /* XXX other engines? */
4033
4034 /* halt the rlc */
4035 r600_rlc_stop(rdev);
4036
4037 udelay(50);
4038
4039 /* set mclk/sclk to bypass */
4040 rv770_set_clk_bypass_mode(rdev);
4041 /* disable BM */
4042 pci_clear_master(rdev->pdev);
4043 /* disable mem access */
4044 evergreen_mc_stop(rdev, &save);
4045 if (evergreen_mc_wait_for_idle(rdev)) {
4046 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4047 }
4048 /* reset */
4049 radeon_pci_config_reset(rdev);
4050 /* wait for asic to come out of reset */
4051 for (i = 0; i < rdev->usec_timeout; i++) {
4052 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4053 break;
4054 udelay(1);
4055 }
4056 }
4057
evergreen_asic_reset(struct radeon_device * rdev,bool hard)4058 int evergreen_asic_reset(struct radeon_device *rdev, bool hard)
4059 {
4060 u32 reset_mask;
4061
4062 if (hard) {
4063 evergreen_gpu_pci_config_reset(rdev);
4064 return 0;
4065 }
4066
4067 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4068
4069 if (reset_mask)
4070 r600_set_bios_scratch_engine_hung(rdev, true);
4071
4072 /* try soft reset */
4073 evergreen_gpu_soft_reset(rdev, reset_mask);
4074
4075 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4076
4077 /* try pci config reset */
4078 if (reset_mask && radeon_hard_reset)
4079 evergreen_gpu_pci_config_reset(rdev);
4080
4081 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4082
4083 if (!reset_mask)
4084 r600_set_bios_scratch_engine_hung(rdev, false);
4085
4086 return 0;
4087 }
4088
4089 /**
4090 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
4091 *
4092 * @rdev: radeon_device pointer
4093 * @ring: radeon_ring structure holding ring information
4094 *
4095 * Check if the GFX engine is locked up.
4096 * Returns true if the engine appears to be locked up, false if not.
4097 */
evergreen_gfx_is_lockup(struct radeon_device * rdev,struct radeon_ring * ring)4098 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4099 {
4100 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4101
4102 if (!(reset_mask & (RADEON_RESET_GFX |
4103 RADEON_RESET_COMPUTE |
4104 RADEON_RESET_CP))) {
4105 radeon_ring_lockup_update(rdev, ring);
4106 return false;
4107 }
4108 return radeon_ring_test_lockup(rdev, ring);
4109 }
4110
4111 /*
4112 * RLC
4113 */
4114 #define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
4115 #define RLC_CLEAR_STATE_END_MARKER 0x00000001
4116
sumo_rlc_fini(struct radeon_device * rdev)4117 void sumo_rlc_fini(struct radeon_device *rdev)
4118 {
4119 int r;
4120
4121 /* save restore block */
4122 if (rdev->rlc.save_restore_obj) {
4123 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4124 if (unlikely(r != 0))
4125 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4126 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4127 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4128
4129 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4130 rdev->rlc.save_restore_obj = NULL;
4131 }
4132
4133 /* clear state block */
4134 if (rdev->rlc.clear_state_obj) {
4135 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4136 if (unlikely(r != 0))
4137 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4138 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4139 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4140
4141 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4142 rdev->rlc.clear_state_obj = NULL;
4143 }
4144
4145 /* clear state block */
4146 if (rdev->rlc.cp_table_obj) {
4147 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4148 if (unlikely(r != 0))
4149 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4150 radeon_bo_unpin(rdev->rlc.cp_table_obj);
4151 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4152
4153 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4154 rdev->rlc.cp_table_obj = NULL;
4155 }
4156 }
4157
4158 #define CP_ME_TABLE_SIZE 96
4159
sumo_rlc_init(struct radeon_device * rdev)4160 int sumo_rlc_init(struct radeon_device *rdev)
4161 {
4162 const u32 *src_ptr;
4163 volatile u32 *dst_ptr;
4164 u32 dws, data, i, j, k, reg_num;
4165 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4166 u64 reg_list_mc_addr;
4167 const struct cs_section_def *cs_data;
4168 int r;
4169
4170 src_ptr = rdev->rlc.reg_list;
4171 dws = rdev->rlc.reg_list_size;
4172 if (rdev->family >= CHIP_BONAIRE) {
4173 dws += (5 * 16) + 48 + 48 + 64;
4174 }
4175 cs_data = rdev->rlc.cs_data;
4176
4177 if (src_ptr) {
4178 /* save restore block */
4179 if (rdev->rlc.save_restore_obj == NULL) {
4180 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4181 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4182 NULL, &rdev->rlc.save_restore_obj);
4183 if (r) {
4184 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4185 return r;
4186 }
4187 }
4188
4189 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4190 if (unlikely(r != 0)) {
4191 sumo_rlc_fini(rdev);
4192 return r;
4193 }
4194 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4195 &rdev->rlc.save_restore_gpu_addr);
4196 if (r) {
4197 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4198 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4199 sumo_rlc_fini(rdev);
4200 return r;
4201 }
4202
4203 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)__UNVOLATILE(&rdev->rlc.sr_ptr));
4204 if (r) {
4205 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4206 sumo_rlc_fini(rdev);
4207 return r;
4208 }
4209 /* write the sr buffer */
4210 dst_ptr = rdev->rlc.sr_ptr;
4211 if (rdev->family >= CHIP_TAHITI) {
4212 /* SI */
4213 for (i = 0; i < rdev->rlc.reg_list_size; i++)
4214 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4215 } else {
4216 /* ON/LN/TN */
4217 /* format:
4218 * dw0: (reg2 << 16) | reg1
4219 * dw1: reg1 save space
4220 * dw2: reg2 save space
4221 */
4222 for (i = 0; i < dws; i++) {
4223 data = src_ptr[i] >> 2;
4224 i++;
4225 if (i < dws)
4226 data |= (src_ptr[i] >> 2) << 16;
4227 j = (((i - 1) * 3) / 2);
4228 dst_ptr[j] = cpu_to_le32(data);
4229 }
4230 j = ((i * 3) / 2);
4231 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4232 }
4233 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4234 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4235 }
4236
4237 if (cs_data) {
4238 /* clear state block */
4239 if (rdev->family >= CHIP_BONAIRE) {
4240 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4241 } else if (rdev->family >= CHIP_TAHITI) {
4242 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4243 dws = rdev->rlc.clear_state_size + (256 / 4);
4244 } else {
4245 reg_list_num = 0;
4246 dws = 0;
4247 for (i = 0; cs_data[i].section != NULL; i++) {
4248 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4249 reg_list_num++;
4250 dws += cs_data[i].section[j].reg_count;
4251 }
4252 }
4253 reg_list_blk_index = (3 * reg_list_num + 2);
4254 dws += reg_list_blk_index;
4255 rdev->rlc.clear_state_size = dws;
4256 }
4257
4258 if (rdev->rlc.clear_state_obj == NULL) {
4259 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4260 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4261 NULL, &rdev->rlc.clear_state_obj);
4262 if (r) {
4263 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4264 sumo_rlc_fini(rdev);
4265 return r;
4266 }
4267 }
4268 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4269 if (unlikely(r != 0)) {
4270 sumo_rlc_fini(rdev);
4271 return r;
4272 }
4273 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4274 &rdev->rlc.clear_state_gpu_addr);
4275 if (r) {
4276 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4277 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4278 sumo_rlc_fini(rdev);
4279 return r;
4280 }
4281
4282 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)__UNVOLATILE(&rdev->rlc.cs_ptr));
4283 if (r) {
4284 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4285 sumo_rlc_fini(rdev);
4286 return r;
4287 }
4288 /* set up the cs buffer */
4289 dst_ptr = rdev->rlc.cs_ptr;
4290 if (rdev->family >= CHIP_BONAIRE) {
4291 cik_get_csb_buffer(rdev, dst_ptr);
4292 } else if (rdev->family >= CHIP_TAHITI) {
4293 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4294 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4295 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4296 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4297 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4298 } else {
4299 reg_list_hdr_blk_index = 0;
4300 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4301 data = upper_32_bits(reg_list_mc_addr);
4302 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4303 reg_list_hdr_blk_index++;
4304 for (i = 0; cs_data[i].section != NULL; i++) {
4305 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4306 reg_num = cs_data[i].section[j].reg_count;
4307 data = reg_list_mc_addr & 0xffffffff;
4308 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4309 reg_list_hdr_blk_index++;
4310
4311 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4312 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4313 reg_list_hdr_blk_index++;
4314
4315 data = 0x08000000 | (reg_num * 4);
4316 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4317 reg_list_hdr_blk_index++;
4318
4319 for (k = 0; k < reg_num; k++) {
4320 data = cs_data[i].section[j].extent[k];
4321 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4322 }
4323 reg_list_mc_addr += reg_num * 4;
4324 reg_list_blk_index += reg_num;
4325 }
4326 }
4327 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4328 }
4329 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4330 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4331 }
4332
4333 if (rdev->rlc.cp_table_size) {
4334 if (rdev->rlc.cp_table_obj == NULL) {
4335 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4336 PAGE_SIZE, true,
4337 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4338 NULL, &rdev->rlc.cp_table_obj);
4339 if (r) {
4340 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4341 sumo_rlc_fini(rdev);
4342 return r;
4343 }
4344 }
4345
4346 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4347 if (unlikely(r != 0)) {
4348 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4349 sumo_rlc_fini(rdev);
4350 return r;
4351 }
4352 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4353 &rdev->rlc.cp_table_gpu_addr);
4354 if (r) {
4355 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4356 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4357 sumo_rlc_fini(rdev);
4358 return r;
4359 }
4360 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)__UNVOLATILE(&rdev->rlc.cp_table_ptr));
4361 if (r) {
4362 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4363 sumo_rlc_fini(rdev);
4364 return r;
4365 }
4366
4367 cik_init_cp_pg_table(rdev);
4368
4369 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4370 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4371
4372 }
4373
4374 return 0;
4375 }
4376
evergreen_rlc_start(struct radeon_device * rdev)4377 static void evergreen_rlc_start(struct radeon_device *rdev)
4378 {
4379 u32 mask = RLC_ENABLE;
4380
4381 if (rdev->flags & RADEON_IS_IGP) {
4382 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4383 }
4384
4385 WREG32(RLC_CNTL, mask);
4386 }
4387
evergreen_rlc_resume(struct radeon_device * rdev)4388 int evergreen_rlc_resume(struct radeon_device *rdev)
4389 {
4390 u32 i;
4391 const __be32 *fw_data;
4392
4393 if (!rdev->rlc_fw)
4394 return -EINVAL;
4395
4396 r600_rlc_stop(rdev);
4397
4398 WREG32(RLC_HB_CNTL, 0);
4399
4400 if (rdev->flags & RADEON_IS_IGP) {
4401 if (rdev->family == CHIP_ARUBA) {
4402 u32 always_on_bitmap =
4403 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4404 /* find out the number of active simds */
4405 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4406 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4407 tmp = hweight32(~tmp);
4408 if (tmp == rdev->config.cayman.max_simds_per_se) {
4409 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4410 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4411 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4412 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4413 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4414 }
4415 } else {
4416 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4417 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4418 }
4419 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4420 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4421 } else {
4422 WREG32(RLC_HB_BASE, 0);
4423 WREG32(RLC_HB_RPTR, 0);
4424 WREG32(RLC_HB_WPTR, 0);
4425 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4426 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4427 }
4428 WREG32(RLC_MC_CNTL, 0);
4429 WREG32(RLC_UCODE_CNTL, 0);
4430
4431 fw_data = (const __be32 *)rdev->rlc_fw->data;
4432 if (rdev->family >= CHIP_ARUBA) {
4433 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4434 WREG32(RLC_UCODE_ADDR, i);
4435 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4436 }
4437 } else if (rdev->family >= CHIP_CAYMAN) {
4438 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4439 WREG32(RLC_UCODE_ADDR, i);
4440 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4441 }
4442 } else {
4443 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4444 WREG32(RLC_UCODE_ADDR, i);
4445 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4446 }
4447 }
4448 WREG32(RLC_UCODE_ADDR, 0);
4449
4450 evergreen_rlc_start(rdev);
4451
4452 return 0;
4453 }
4454
4455 /* Interrupts */
4456
evergreen_get_vblank_counter(struct radeon_device * rdev,int crtc)4457 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4458 {
4459 if (crtc >= rdev->num_crtc)
4460 return 0;
4461 else
4462 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4463 }
4464
evergreen_disable_interrupt_state(struct radeon_device * rdev)4465 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4466 {
4467 int i;
4468 u32 tmp;
4469
4470 if (rdev->family >= CHIP_CAYMAN) {
4471 cayman_cp_int_cntl_setup(rdev, 0,
4472 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4473 cayman_cp_int_cntl_setup(rdev, 1, 0);
4474 cayman_cp_int_cntl_setup(rdev, 2, 0);
4475 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4476 WREG32(CAYMAN_DMA1_CNTL, tmp);
4477 } else
4478 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4479 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4480 WREG32(DMA_CNTL, tmp);
4481 WREG32(GRBM_INT_CNTL, 0);
4482 WREG32(SRBM_INT_CNTL, 0);
4483 for (i = 0; i < rdev->num_crtc; i++)
4484 WREG32(INT_MASK + crtc_offsets[i], 0);
4485 for (i = 0; i < rdev->num_crtc; i++)
4486 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], 0);
4487
4488 /* only one DAC on DCE5 */
4489 if (!ASIC_IS_DCE5(rdev))
4490 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4491 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4492
4493 for (i = 0; i < 6; i++)
4494 WREG32_AND(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_POLARITY);
4495 }
4496
4497 /* Note that the order we write back regs here is important */
evergreen_irq_set(struct radeon_device * rdev)4498 int evergreen_irq_set(struct radeon_device *rdev)
4499 {
4500 int i;
4501 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4502 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4503 u32 grbm_int_cntl = 0;
4504 u32 dma_cntl, dma_cntl1 = 0;
4505 u32 thermal_int = 0;
4506
4507 if (!rdev->irq.installed) {
4508 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4509 return -EINVAL;
4510 }
4511 /* don't enable anything if the ih is disabled */
4512 if (!rdev->ih.enabled) {
4513 r600_disable_interrupts(rdev);
4514 /* force the active interrupt state to all disabled */
4515 evergreen_disable_interrupt_state(rdev);
4516 return 0;
4517 }
4518
4519 if (rdev->family == CHIP_ARUBA)
4520 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4521 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4522 else
4523 thermal_int = RREG32(CG_THERMAL_INT) &
4524 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4525
4526 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4527
4528 if (rdev->family >= CHIP_CAYMAN) {
4529 /* enable CP interrupts on all rings */
4530 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4531 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4532 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4533 }
4534 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4535 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4536 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4537 }
4538 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4539 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4540 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4541 }
4542 } else {
4543 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4544 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4545 cp_int_cntl |= RB_INT_ENABLE;
4546 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4547 }
4548 }
4549
4550 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4551 DRM_DEBUG("r600_irq_set: sw int dma\n");
4552 dma_cntl |= TRAP_ENABLE;
4553 }
4554
4555 if (rdev->family >= CHIP_CAYMAN) {
4556 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4557 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4558 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4559 dma_cntl1 |= TRAP_ENABLE;
4560 }
4561 }
4562
4563 if (rdev->irq.dpm_thermal) {
4564 DRM_DEBUG("dpm thermal\n");
4565 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4566 }
4567
4568 if (rdev->family >= CHIP_CAYMAN) {
4569 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4570 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4571 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4572 } else
4573 WREG32(CP_INT_CNTL, cp_int_cntl);
4574
4575 WREG32(DMA_CNTL, dma_cntl);
4576
4577 if (rdev->family >= CHIP_CAYMAN)
4578 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4579
4580 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4581
4582 for (i = 0; i < rdev->num_crtc; i++) {
4583 radeon_irq_kms_set_irq_n_enabled(
4584 rdev, INT_MASK + crtc_offsets[i],
4585 VBLANK_INT_MASK,
4586 rdev->irq.crtc_vblank_int[i] ||
4587 atomic_read(&rdev->irq.pflip[i]), "vblank", i);
4588 }
4589
4590 for (i = 0; i < rdev->num_crtc; i++)
4591 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], GRPH_PFLIP_INT_MASK);
4592
4593 for (i = 0; i < 6; i++) {
4594 radeon_irq_kms_set_irq_n_enabled(
4595 rdev, DC_HPDx_INT_CONTROL(i),
4596 DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN,
4597 rdev->irq.hpd[i], "HPD", i);
4598 }
4599
4600 if (rdev->family == CHIP_ARUBA)
4601 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4602 else
4603 WREG32(CG_THERMAL_INT, thermal_int);
4604
4605 for (i = 0; i < 6; i++) {
4606 radeon_irq_kms_set_irq_n_enabled(
4607 rdev, AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4608 AFMT_AZ_FORMAT_WTRIG_MASK,
4609 rdev->irq.afmt[i], "HDMI", i);
4610 }
4611
4612 /* posting read */
4613 RREG32(SRBM_STATUS);
4614
4615 return 0;
4616 }
4617
4618 /* Note that the order we write back regs here is important */
evergreen_irq_ack(struct radeon_device * rdev)4619 static void evergreen_irq_ack(struct radeon_device *rdev)
4620 {
4621 int i, j;
4622 u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int;
4623 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4624 u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4625
4626 for (i = 0; i < 6; i++) {
4627 disp_int[i] = RREG32(evergreen_disp_int_status[i]);
4628 afmt_status[i] = RREG32(AFMT_STATUS + crtc_offsets[i]);
4629 if (i < rdev->num_crtc)
4630 grph_int[i] = RREG32(GRPH_INT_STATUS + crtc_offsets[i]);
4631 }
4632
4633 /* We write back each interrupt register in pairs of two */
4634 for (i = 0; i < rdev->num_crtc; i += 2) {
4635 for (j = i; j < (i + 2); j++) {
4636 if (grph_int[j] & GRPH_PFLIP_INT_OCCURRED)
4637 WREG32(GRPH_INT_STATUS + crtc_offsets[j],
4638 GRPH_PFLIP_INT_CLEAR);
4639 }
4640
4641 for (j = i; j < (i + 2); j++) {
4642 if (disp_int[j] & LB_D1_VBLANK_INTERRUPT)
4643 WREG32(VBLANK_STATUS + crtc_offsets[j],
4644 VBLANK_ACK);
4645 if (disp_int[j] & LB_D1_VLINE_INTERRUPT)
4646 WREG32(VLINE_STATUS + crtc_offsets[j],
4647 VLINE_ACK);
4648 }
4649 }
4650
4651 for (i = 0; i < 6; i++) {
4652 if (disp_int[i] & DC_HPD1_INTERRUPT)
4653 WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_ACK);
4654 }
4655
4656 for (i = 0; i < 6; i++) {
4657 if (disp_int[i] & DC_HPD1_RX_INTERRUPT)
4658 WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_RX_INT_ACK);
4659 }
4660
4661 for (i = 0; i < 6; i++) {
4662 if (afmt_status[i] & AFMT_AZ_FORMAT_WTRIG)
4663 WREG32_OR(AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4664 AFMT_AZ_FORMAT_WTRIG_ACK);
4665 }
4666 }
4667
evergreen_irq_disable(struct radeon_device * rdev)4668 static void evergreen_irq_disable(struct radeon_device *rdev)
4669 {
4670 r600_disable_interrupts(rdev);
4671 /* Wait and acknowledge irq */
4672 mdelay(1);
4673 evergreen_irq_ack(rdev);
4674 evergreen_disable_interrupt_state(rdev);
4675 }
4676
evergreen_irq_suspend(struct radeon_device * rdev)4677 void evergreen_irq_suspend(struct radeon_device *rdev)
4678 {
4679 evergreen_irq_disable(rdev);
4680 r600_rlc_stop(rdev);
4681 }
4682
evergreen_get_ih_wptr(struct radeon_device * rdev)4683 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4684 {
4685 u32 wptr, tmp;
4686
4687 if (rdev->wb.enabled)
4688 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4689 else
4690 wptr = RREG32(IH_RB_WPTR);
4691
4692 if (wptr & RB_OVERFLOW) {
4693 wptr &= ~RB_OVERFLOW;
4694 /* When a ring buffer overflow happen start parsing interrupt
4695 * from the last not overwritten vector (wptr + 16). Hopefully
4696 * this should allow us to catchup.
4697 */
4698 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4699 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
4700 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4701 tmp = RREG32(IH_RB_CNTL);
4702 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4703 WREG32(IH_RB_CNTL, tmp);
4704 }
4705 return (wptr & rdev->ih.ptr_mask);
4706 }
4707
evergreen_irq_process(struct radeon_device * rdev)4708 int evergreen_irq_process(struct radeon_device *rdev)
4709 {
4710 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4711 u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4712 u32 crtc_idx, hpd_idx, afmt_idx;
4713 u32 mask;
4714 u32 wptr;
4715 u32 rptr;
4716 u32 src_id, src_data;
4717 u32 ring_index;
4718 bool queue_hotplug = false;
4719 bool queue_hdmi = false;
4720 bool queue_dp = false;
4721 bool queue_thermal = false;
4722 u32 status, addr;
4723 const char *event_name;
4724
4725 if (!rdev->ih.enabled || rdev->shutdown)
4726 return IRQ_NONE;
4727
4728 wptr = evergreen_get_ih_wptr(rdev);
4729
4730 restart_ih:
4731 /* is somebody else already processing irqs? */
4732 if (atomic_xchg(&rdev->ih.lock, 1))
4733 return IRQ_NONE;
4734
4735 rptr = rdev->ih.rptr;
4736 DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4737
4738 /* Order reading of wptr vs. reading of IH ring data */
4739 rmb();
4740
4741 /* display interrupts */
4742 evergreen_irq_ack(rdev);
4743
4744 while (rptr != wptr) {
4745 /* wptr/rptr are in bytes! */
4746 ring_index = rptr / 4;
4747 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4748 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4749
4750 switch (src_id) {
4751 case 1: /* D1 vblank/vline */
4752 case 2: /* D2 vblank/vline */
4753 case 3: /* D3 vblank/vline */
4754 case 4: /* D4 vblank/vline */
4755 case 5: /* D5 vblank/vline */
4756 case 6: /* D6 vblank/vline */
4757 crtc_idx = src_id - 1;
4758
4759 if (src_data == 0) { /* vblank */
4760 mask = LB_D1_VBLANK_INTERRUPT;
4761 event_name = "vblank";
4762
4763 if (rdev->irq.crtc_vblank_int[crtc_idx]) {
4764 drm_handle_vblank(rdev->ddev, crtc_idx);
4765 #ifdef __NetBSD__
4766 spin_lock(&rdev->irq.vblank_lock);
4767 rdev->pm.vblank_sync = true;
4768 DRM_SPIN_WAKEUP_ONE(&rdev->irq.vblank_queue, &rdev->irq.vblank_lock);
4769 spin_unlock(&rdev->irq.vblank_lock);
4770 #else
4771 rdev->pm.vblank_sync = true;
4772 wake_up(&rdev->irq.vblank_queue);
4773 #endif
4774 }
4775 if (atomic_read(&rdev->irq.pflip[crtc_idx])) {
4776 radeon_crtc_handle_vblank(rdev,
4777 crtc_idx);
4778 }
4779
4780 } else if (src_data == 1) { /* vline */
4781 mask = LB_D1_VLINE_INTERRUPT;
4782 event_name = "vline";
4783 } else {
4784 DRM_DEBUG("Unhandled interrupt: %d %d\n",
4785 src_id, src_data);
4786 break;
4787 }
4788
4789 if (!(disp_int[crtc_idx] & mask)) {
4790 DRM_DEBUG("IH: D%d %s - IH event w/o asserted irq bit?\n",
4791 crtc_idx + 1, event_name);
4792 }
4793
4794 disp_int[crtc_idx] &= ~mask;
4795 DRM_DEBUG("IH: D%d %s\n", crtc_idx + 1, event_name);
4796
4797 break;
4798 case 8: /* D1 page flip */
4799 case 10: /* D2 page flip */
4800 case 12: /* D3 page flip */
4801 case 14: /* D4 page flip */
4802 case 16: /* D5 page flip */
4803 case 18: /* D6 page flip */
4804 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
4805 if (radeon_use_pflipirq > 0)
4806 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
4807 break;
4808 case 42: /* HPD hotplug */
4809 if (src_data <= 5) {
4810 hpd_idx = src_data;
4811 mask = DC_HPD1_INTERRUPT;
4812 queue_hotplug = true;
4813 event_name = "HPD";
4814
4815 } else if (src_data <= 11) {
4816 hpd_idx = src_data - 6;
4817 mask = DC_HPD1_RX_INTERRUPT;
4818 queue_dp = true;
4819 event_name = "HPD_RX";
4820
4821 } else {
4822 DRM_DEBUG("Unhandled interrupt: %d %d\n",
4823 src_id, src_data);
4824 break;
4825 }
4826
4827 if (!(disp_int[hpd_idx] & mask))
4828 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4829
4830 disp_int[hpd_idx] &= ~mask;
4831 DRM_DEBUG("IH: %s%d\n", event_name, hpd_idx + 1);
4832
4833 break;
4834 case 44: /* hdmi */
4835 afmt_idx = src_data;
4836 if (!(afmt_status[afmt_idx] & AFMT_AZ_FORMAT_WTRIG))
4837 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4838
4839 if (afmt_idx > 5) {
4840 DRM_ERROR("Unhandled interrupt: %d %d\n",
4841 src_id, src_data);
4842 break;
4843 }
4844 afmt_status[afmt_idx] &= ~AFMT_AZ_FORMAT_WTRIG;
4845 queue_hdmi = true;
4846 DRM_DEBUG("IH: HDMI%d\n", afmt_idx + 1);
4847 break;
4848 case 96:
4849 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
4850 WREG32(SRBM_INT_ACK, 0x1);
4851 break;
4852 case 124: /* UVD */
4853 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4854 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4855 break;
4856 case 146:
4857 case 147:
4858 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
4859 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
4860 /* reset addr and status */
4861 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4862 if (addr == 0x0 && status == 0x0)
4863 break;
4864 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4865 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4866 addr);
4867 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4868 status);
4869 cayman_vm_decode_fault(rdev, status, addr);
4870 break;
4871 case 176: /* CP_INT in ring buffer */
4872 case 177: /* CP_INT in IB1 */
4873 case 178: /* CP_INT in IB2 */
4874 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4875 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4876 break;
4877 case 181: /* CP EOP event */
4878 DRM_DEBUG("IH: CP EOP\n");
4879 if (rdev->family >= CHIP_CAYMAN) {
4880 switch (src_data) {
4881 case 0:
4882 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4883 break;
4884 case 1:
4885 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4886 break;
4887 case 2:
4888 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4889 break;
4890 }
4891 } else
4892 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4893 break;
4894 case 224: /* DMA trap event */
4895 DRM_DEBUG("IH: DMA trap\n");
4896 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4897 break;
4898 case 230: /* thermal low to high */
4899 DRM_DEBUG("IH: thermal low to high\n");
4900 rdev->pm.dpm.thermal.high_to_low = false;
4901 queue_thermal = true;
4902 break;
4903 case 231: /* thermal high to low */
4904 DRM_DEBUG("IH: thermal high to low\n");
4905 rdev->pm.dpm.thermal.high_to_low = true;
4906 queue_thermal = true;
4907 break;
4908 case 233: /* GUI IDLE */
4909 DRM_DEBUG("IH: GUI idle\n");
4910 break;
4911 case 244: /* DMA trap event */
4912 if (rdev->family >= CHIP_CAYMAN) {
4913 DRM_DEBUG("IH: DMA1 trap\n");
4914 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4915 }
4916 break;
4917 default:
4918 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4919 break;
4920 }
4921
4922 /* wptr/rptr are in bytes! */
4923 rptr += 16;
4924 rptr &= rdev->ih.ptr_mask;
4925 WREG32(IH_RB_RPTR, rptr);
4926 }
4927 if (queue_dp)
4928 schedule_work(&rdev->dp_work);
4929 if (queue_hotplug)
4930 schedule_delayed_work(&rdev->hotplug_work, 0);
4931 if (queue_hdmi)
4932 schedule_work(&rdev->audio_work);
4933 if (queue_thermal && rdev->pm.dpm_enabled)
4934 schedule_work(&rdev->pm.dpm.thermal.work);
4935 rdev->ih.rptr = rptr;
4936 atomic_set(&rdev->ih.lock, 0);
4937
4938 /* make sure wptr hasn't changed while processing */
4939 wptr = evergreen_get_ih_wptr(rdev);
4940 if (wptr != rptr)
4941 goto restart_ih;
4942
4943 return IRQ_HANDLED;
4944 }
4945
evergreen_uvd_init(struct radeon_device * rdev)4946 static void evergreen_uvd_init(struct radeon_device *rdev)
4947 {
4948 int r;
4949
4950 if (!rdev->has_uvd)
4951 return;
4952
4953 r = radeon_uvd_init(rdev);
4954 if (r) {
4955 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
4956 /*
4957 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
4958 * to early fails uvd_v2_2_resume() and thus nothing happens
4959 * there. So it is pointless to try to go through that code
4960 * hence why we disable uvd here.
4961 */
4962 rdev->has_uvd = false;
4963 return;
4964 }
4965 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
4966 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
4967 }
4968
evergreen_uvd_start(struct radeon_device * rdev)4969 static void evergreen_uvd_start(struct radeon_device *rdev)
4970 {
4971 int r;
4972
4973 if (!rdev->has_uvd)
4974 return;
4975
4976 r = uvd_v2_2_resume(rdev);
4977 if (r) {
4978 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
4979 goto error;
4980 }
4981 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
4982 if (r) {
4983 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
4984 goto error;
4985 }
4986 return;
4987
4988 error:
4989 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
4990 }
4991
evergreen_uvd_resume(struct radeon_device * rdev)4992 static void evergreen_uvd_resume(struct radeon_device *rdev)
4993 {
4994 struct radeon_ring *ring;
4995 int r;
4996
4997 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
4998 return;
4999
5000 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5001 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
5002 if (r) {
5003 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
5004 return;
5005 }
5006 r = uvd_v1_0_init(rdev);
5007 if (r) {
5008 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
5009 return;
5010 }
5011 }
5012
evergreen_startup(struct radeon_device * rdev)5013 static int evergreen_startup(struct radeon_device *rdev)
5014 {
5015 struct radeon_ring *ring;
5016 int r;
5017
5018 /* enable pcie gen2 link */
5019 evergreen_pcie_gen2_enable(rdev);
5020 /* enable aspm */
5021 evergreen_program_aspm(rdev);
5022
5023 /* scratch needs to be initialized before MC */
5024 r = r600_vram_scratch_init(rdev);
5025 if (r)
5026 return r;
5027
5028 evergreen_mc_program(rdev);
5029
5030 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5031 r = ni_mc_load_microcode(rdev);
5032 if (r) {
5033 DRM_ERROR("Failed to load MC firmware!\n");
5034 return r;
5035 }
5036 }
5037
5038 if (rdev->flags & RADEON_IS_AGP) {
5039 evergreen_agp_enable(rdev);
5040 } else {
5041 r = evergreen_pcie_gart_enable(rdev);
5042 if (r)
5043 return r;
5044 }
5045 evergreen_gpu_init(rdev);
5046
5047 /* allocate rlc buffers */
5048 if (rdev->flags & RADEON_IS_IGP) {
5049 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5050 rdev->rlc.reg_list_size =
5051 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5052 rdev->rlc.cs_data = evergreen_cs_data;
5053 r = sumo_rlc_init(rdev);
5054 if (r) {
5055 DRM_ERROR("Failed to init rlc BOs!\n");
5056 return r;
5057 }
5058 }
5059
5060 /* allocate wb buffer */
5061 r = radeon_wb_init(rdev);
5062 if (r)
5063 return r;
5064
5065 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5066 if (r) {
5067 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5068 return r;
5069 }
5070
5071 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5072 if (r) {
5073 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5074 return r;
5075 }
5076
5077 evergreen_uvd_start(rdev);
5078
5079 /* Enable IRQ */
5080 if (!rdev->irq.installed) {
5081 r = radeon_irq_kms_init(rdev);
5082 if (r)
5083 return r;
5084 }
5085
5086 r = r600_irq_init(rdev);
5087 if (r) {
5088 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5089 radeon_irq_kms_fini(rdev);
5090 return r;
5091 }
5092 evergreen_irq_set(rdev);
5093
5094 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5095 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5096 RADEON_CP_PACKET2);
5097 if (r)
5098 return r;
5099
5100 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5101 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5102 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5103 if (r)
5104 return r;
5105
5106 r = evergreen_cp_load_microcode(rdev);
5107 if (r)
5108 return r;
5109 r = evergreen_cp_resume(rdev);
5110 if (r)
5111 return r;
5112 r = r600_dma_resume(rdev);
5113 if (r)
5114 return r;
5115
5116 evergreen_uvd_resume(rdev);
5117
5118 r = radeon_ib_pool_init(rdev);
5119 if (r) {
5120 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5121 return r;
5122 }
5123
5124 r = radeon_audio_init(rdev);
5125 if (r) {
5126 DRM_ERROR("radeon: audio init failed\n");
5127 return r;
5128 }
5129
5130 return 0;
5131 }
5132
evergreen_resume(struct radeon_device * rdev)5133 int evergreen_resume(struct radeon_device *rdev)
5134 {
5135 int r;
5136
5137 /* reset the asic, the gfx blocks are often in a bad state
5138 * after the driver is unloaded or after a resume
5139 */
5140 if (radeon_asic_reset(rdev))
5141 dev_warn(rdev->dev, "GPU reset failed !\n");
5142 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5143 * posting will perform necessary task to bring back GPU into good
5144 * shape.
5145 */
5146 /* post card */
5147 atom_asic_init(rdev->mode_info.atom_context);
5148
5149 /* init golden registers */
5150 evergreen_init_golden_registers(rdev);
5151
5152 if (rdev->pm.pm_method == PM_METHOD_DPM)
5153 radeon_pm_resume(rdev);
5154
5155 rdev->accel_working = true;
5156 r = evergreen_startup(rdev);
5157 if (r) {
5158 DRM_ERROR("evergreen startup failed on resume\n");
5159 rdev->accel_working = false;
5160 return r;
5161 }
5162
5163 return r;
5164
5165 }
5166
evergreen_suspend(struct radeon_device * rdev)5167 int evergreen_suspend(struct radeon_device *rdev)
5168 {
5169 radeon_pm_suspend(rdev);
5170 radeon_audio_fini(rdev);
5171 if (rdev->has_uvd) {
5172 uvd_v1_0_fini(rdev);
5173 radeon_uvd_suspend(rdev);
5174 }
5175 r700_cp_stop(rdev);
5176 r600_dma_stop(rdev);
5177 evergreen_irq_suspend(rdev);
5178 radeon_wb_disable(rdev);
5179 evergreen_pcie_gart_disable(rdev);
5180
5181 return 0;
5182 }
5183
5184 /* Plan is to move initialization in that function and use
5185 * helper function so that radeon_device_init pretty much
5186 * do nothing more than calling asic specific function. This
5187 * should also allow to remove a bunch of callback function
5188 * like vram_info.
5189 */
evergreen_init(struct radeon_device * rdev)5190 int evergreen_init(struct radeon_device *rdev)
5191 {
5192 int r;
5193
5194 /* Read BIOS */
5195 if (!radeon_get_bios(rdev)) {
5196 if (ASIC_IS_AVIVO(rdev))
5197 return -EINVAL;
5198 }
5199 /* Must be an ATOMBIOS */
5200 if (!rdev->is_atom_bios) {
5201 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5202 return -EINVAL;
5203 }
5204 r = radeon_atombios_init(rdev);
5205 if (r)
5206 return r;
5207 /* reset the asic, the gfx blocks are often in a bad state
5208 * after the driver is unloaded or after a resume
5209 */
5210 if (radeon_asic_reset(rdev))
5211 dev_warn(rdev->dev, "GPU reset failed !\n");
5212 /* Post card if necessary */
5213 if (!radeon_card_posted(rdev)) {
5214 if (!rdev->bios) {
5215 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5216 return -EINVAL;
5217 }
5218 DRM_INFO("GPU not posted. posting now...\n");
5219 atom_asic_init(rdev->mode_info.atom_context);
5220 }
5221 /* init golden registers */
5222 evergreen_init_golden_registers(rdev);
5223 /* Initialize scratch registers */
5224 r600_scratch_init(rdev);
5225 /* Initialize surface registers */
5226 radeon_surface_init(rdev);
5227 /* Initialize clocks */
5228 radeon_get_clock_info(rdev->ddev);
5229 /* Fence driver */
5230 r = radeon_fence_driver_init(rdev);
5231 if (r)
5232 return r;
5233 /* initialize AGP */
5234 if (rdev->flags & RADEON_IS_AGP) {
5235 r = radeon_agp_init(rdev);
5236 if (r)
5237 radeon_agp_disable(rdev);
5238 }
5239 /* initialize memory controller */
5240 r = evergreen_mc_init(rdev);
5241 if (r)
5242 return r;
5243 /* Memory manager */
5244 r = radeon_bo_init(rdev);
5245 if (r)
5246 return r;
5247
5248 if (ASIC_IS_DCE5(rdev)) {
5249 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5250 r = ni_init_microcode(rdev);
5251 if (r) {
5252 DRM_ERROR("Failed to load firmware!\n");
5253 return r;
5254 }
5255 }
5256 } else {
5257 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5258 r = r600_init_microcode(rdev);
5259 if (r) {
5260 DRM_ERROR("Failed to load firmware!\n");
5261 return r;
5262 }
5263 }
5264 }
5265
5266 /* Initialize power management */
5267 radeon_pm_init(rdev);
5268
5269 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5270 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5271
5272 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5273 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5274
5275 evergreen_uvd_init(rdev);
5276
5277 rdev->ih.ring_obj = NULL;
5278 r600_ih_ring_init(rdev, 64 * 1024);
5279
5280 r = r600_pcie_gart_init(rdev);
5281 if (r)
5282 return r;
5283
5284 rdev->accel_working = true;
5285 r = evergreen_startup(rdev);
5286 if (r) {
5287 dev_err(rdev->dev, "disabling GPU acceleration\n");
5288 r700_cp_fini(rdev);
5289 r600_dma_fini(rdev);
5290 r600_irq_fini(rdev);
5291 if (rdev->flags & RADEON_IS_IGP)
5292 sumo_rlc_fini(rdev);
5293 radeon_wb_fini(rdev);
5294 radeon_ib_pool_fini(rdev);
5295 radeon_irq_kms_fini(rdev);
5296 evergreen_pcie_gart_fini(rdev);
5297 rdev->accel_working = false;
5298 }
5299
5300 /* Don't start up if the MC ucode is missing on BTC parts.
5301 * The default clocks and voltages before the MC ucode
5302 * is loaded are not suffient for advanced operations.
5303 */
5304 if (ASIC_IS_DCE5(rdev)) {
5305 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5306 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5307 return -EINVAL;
5308 }
5309 }
5310
5311 return 0;
5312 }
5313
evergreen_fini(struct radeon_device * rdev)5314 void evergreen_fini(struct radeon_device *rdev)
5315 {
5316 radeon_pm_fini(rdev);
5317 radeon_audio_fini(rdev);
5318 r700_cp_fini(rdev);
5319 r600_dma_fini(rdev);
5320 r600_irq_fini(rdev);
5321 if (rdev->flags & RADEON_IS_IGP)
5322 sumo_rlc_fini(rdev);
5323 radeon_wb_fini(rdev);
5324 radeon_ib_pool_fini(rdev);
5325 radeon_irq_kms_fini(rdev);
5326 uvd_v1_0_fini(rdev);
5327 radeon_uvd_fini(rdev);
5328 evergreen_pcie_gart_fini(rdev);
5329 r600_vram_scratch_fini(rdev);
5330 radeon_gem_fini(rdev);
5331 radeon_fence_driver_fini(rdev);
5332 radeon_agp_fini(rdev);
5333 radeon_bo_fini(rdev);
5334 radeon_atombios_fini(rdev);
5335 kfree(rdev->bios);
5336 rdev->bios = NULL;
5337 }
5338
evergreen_pcie_gen2_enable(struct radeon_device * rdev)5339 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5340 {
5341 #ifndef __NetBSD__ /* XXX radeon pcie */
5342 u32 link_width_cntl, speed_cntl;
5343
5344 if (radeon_pcie_gen2 == 0)
5345 return;
5346
5347 if (rdev->flags & RADEON_IS_IGP)
5348 return;
5349
5350 if (!(rdev->flags & RADEON_IS_PCIE))
5351 return;
5352
5353 /* x2 cards have a special sequence */
5354 if (ASIC_IS_X2(rdev))
5355 return;
5356
5357 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5358 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5359 return;
5360
5361 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5362 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5363 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5364 return;
5365 }
5366
5367 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5368
5369 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5370 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5371
5372 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5373 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5374 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5375
5376 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5377 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5378 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5379
5380 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5381 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5382 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5383
5384 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5385 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5386 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5387
5388 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5389 speed_cntl |= LC_GEN2_EN_STRAP;
5390 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5391
5392 } else {
5393 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5394 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5395 if (1)
5396 link_width_cntl |= LC_UPCONFIGURE_DIS;
5397 else
5398 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5399 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5400 }
5401 #endif
5402 }
5403
evergreen_program_aspm(struct radeon_device * rdev)5404 void evergreen_program_aspm(struct radeon_device *rdev)
5405 {
5406 u32 data, orig;
5407 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5408 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5409 /* fusion_platform = true
5410 * if the system is a fusion system
5411 * (APU or DGPU in a fusion system).
5412 * todo: check if the system is a fusion platform.
5413 */
5414 bool fusion_platform = false;
5415
5416 if (radeon_aspm == 0)
5417 return;
5418
5419 if (!(rdev->flags & RADEON_IS_PCIE))
5420 return;
5421
5422 switch (rdev->family) {
5423 case CHIP_CYPRESS:
5424 case CHIP_HEMLOCK:
5425 case CHIP_JUNIPER:
5426 case CHIP_REDWOOD:
5427 case CHIP_CEDAR:
5428 case CHIP_SUMO:
5429 case CHIP_SUMO2:
5430 case CHIP_PALM:
5431 case CHIP_ARUBA:
5432 disable_l0s = true;
5433 break;
5434 default:
5435 disable_l0s = false;
5436 break;
5437 }
5438
5439 if (rdev->flags & RADEON_IS_IGP)
5440 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5441
5442 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5443 if (fusion_platform)
5444 data &= ~MULTI_PIF;
5445 else
5446 data |= MULTI_PIF;
5447 if (data != orig)
5448 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5449
5450 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5451 if (fusion_platform)
5452 data &= ~MULTI_PIF;
5453 else
5454 data |= MULTI_PIF;
5455 if (data != orig)
5456 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5457
5458 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5459 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5460 if (!disable_l0s) {
5461 if (rdev->family >= CHIP_BARTS)
5462 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5463 else
5464 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5465 }
5466
5467 if (!disable_l1) {
5468 if (rdev->family >= CHIP_BARTS)
5469 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5470 else
5471 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5472
5473 if (!disable_plloff_in_l1) {
5474 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5475 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5476 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5477 if (data != orig)
5478 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5479
5480 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5481 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5482 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5483 if (data != orig)
5484 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5485
5486 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5487 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5488 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5489 if (data != orig)
5490 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5491
5492 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5493 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5494 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5495 if (data != orig)
5496 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5497
5498 if (rdev->family >= CHIP_BARTS) {
5499 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5500 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5501 data |= PLL_RAMP_UP_TIME_0(4);
5502 if (data != orig)
5503 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5504
5505 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5506 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5507 data |= PLL_RAMP_UP_TIME_1(4);
5508 if (data != orig)
5509 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5510
5511 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5512 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5513 data |= PLL_RAMP_UP_TIME_0(4);
5514 if (data != orig)
5515 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5516
5517 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5518 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5519 data |= PLL_RAMP_UP_TIME_1(4);
5520 if (data != orig)
5521 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5522 }
5523
5524 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5525 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5526 data |= LC_DYN_LANES_PWR_STATE(3);
5527 if (data != orig)
5528 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5529
5530 if (rdev->family >= CHIP_BARTS) {
5531 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5532 data &= ~LS2_EXIT_TIME_MASK;
5533 data |= LS2_EXIT_TIME(1);
5534 if (data != orig)
5535 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5536
5537 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5538 data &= ~LS2_EXIT_TIME_MASK;
5539 data |= LS2_EXIT_TIME(1);
5540 if (data != orig)
5541 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5542 }
5543 }
5544 }
5545
5546 /* evergreen parts only */
5547 if (rdev->family < CHIP_BARTS)
5548 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5549
5550 if (pcie_lc_cntl != pcie_lc_cntl_old)
5551 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5552 }
5553