1 /*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24 #include <linux/bitops.h>
25 #include <linux/firmware.h>
26 #include <linux/platform_device.h>
27 #include <linux/slab.h>
28 #include <drm/drmP.h>
29 #include "radeon.h"
30 #include "radeon_asic.h"
31 #include <drm/radeon_drm.h>
32 #include "evergreend.h"
33 #include "atom.h"
34 #include "avivod.h"
35 #include "evergreen_reg.h"
36 #include "evergreen_blit_shaders.h"
37 #include "radeon_ucode.h"
38
39 static const u32 crtc_offsets[6] =
40 {
41 EVERGREEN_CRTC0_REGISTER_OFFSET,
42 EVERGREEN_CRTC1_REGISTER_OFFSET,
43 EVERGREEN_CRTC2_REGISTER_OFFSET,
44 EVERGREEN_CRTC3_REGISTER_OFFSET,
45 EVERGREEN_CRTC4_REGISTER_OFFSET,
46 EVERGREEN_CRTC5_REGISTER_OFFSET
47 };
48
49 #include "clearstate_evergreen.h"
50
51 static const u32 sumo_rlc_save_restore_register_list[] =
52 {
53 0x98fc,
54 0x9830,
55 0x9834,
56 0x9838,
57 0x9870,
58 0x9874,
59 0x8a14,
60 0x8b24,
61 0x8bcc,
62 0x8b10,
63 0x8d00,
64 0x8d04,
65 0x8c00,
66 0x8c04,
67 0x8c08,
68 0x8c0c,
69 0x8d8c,
70 0x8c20,
71 0x8c24,
72 0x8c28,
73 0x8c18,
74 0x8c1c,
75 0x8cf0,
76 0x8e2c,
77 0x8e38,
78 0x8c30,
79 0x9508,
80 0x9688,
81 0x9608,
82 0x960c,
83 0x9610,
84 0x9614,
85 0x88c4,
86 0x88d4,
87 0xa008,
88 0x900c,
89 0x9100,
90 0x913c,
91 0x98f8,
92 0x98f4,
93 0x9b7c,
94 0x3f8c,
95 0x8950,
96 0x8954,
97 0x8a18,
98 0x8b28,
99 0x9144,
100 0x9148,
101 0x914c,
102 0x3f90,
103 0x3f94,
104 0x915c,
105 0x9160,
106 0x9178,
107 0x917c,
108 0x9180,
109 0x918c,
110 0x9190,
111 0x9194,
112 0x9198,
113 0x919c,
114 0x91a8,
115 0x91ac,
116 0x91b0,
117 0x91b4,
118 0x91b8,
119 0x91c4,
120 0x91c8,
121 0x91cc,
122 0x91d0,
123 0x91d4,
124 0x91e0,
125 0x91e4,
126 0x91ec,
127 0x91f0,
128 0x91f4,
129 0x9200,
130 0x9204,
131 0x929c,
132 0x9150,
133 0x802c,
134 };
135
136 static void evergreen_gpu_init(struct radeon_device *rdev);
137 void evergreen_fini(struct radeon_device *rdev);
138 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
139 void evergreen_program_aspm(struct radeon_device *rdev);
140 extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
141 int ring, u32 cp_int_cntl);
142 extern void cayman_vm_decode_fault(struct radeon_device *rdev,
143 u32 status, u32 addr);
144 void cik_init_cp_pg_table(struct radeon_device *rdev);
145
146 extern u32 si_get_csb_size(struct radeon_device *rdev);
147 extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
148 extern u32 cik_get_csb_size(struct radeon_device *rdev);
149 extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
150 extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
151
152 static const u32 evergreen_golden_registers[] =
153 {
154 0x3f90, 0xffff0000, 0xff000000,
155 0x9148, 0xffff0000, 0xff000000,
156 0x3f94, 0xffff0000, 0xff000000,
157 0x914c, 0xffff0000, 0xff000000,
158 0x9b7c, 0xffffffff, 0x00000000,
159 0x8a14, 0xffffffff, 0x00000007,
160 0x8b10, 0xffffffff, 0x00000000,
161 0x960c, 0xffffffff, 0x54763210,
162 0x88c4, 0xffffffff, 0x000000c2,
163 0x88d4, 0xffffffff, 0x00000010,
164 0x8974, 0xffffffff, 0x00000000,
165 0xc78, 0x00000080, 0x00000080,
166 0x5eb4, 0xffffffff, 0x00000002,
167 0x5e78, 0xffffffff, 0x001000f0,
168 0x6104, 0x01000300, 0x00000000,
169 0x5bc0, 0x00300000, 0x00000000,
170 0x7030, 0xffffffff, 0x00000011,
171 0x7c30, 0xffffffff, 0x00000011,
172 0x10830, 0xffffffff, 0x00000011,
173 0x11430, 0xffffffff, 0x00000011,
174 0x12030, 0xffffffff, 0x00000011,
175 0x12c30, 0xffffffff, 0x00000011,
176 0xd02c, 0xffffffff, 0x08421000,
177 0x240c, 0xffffffff, 0x00000380,
178 0x8b24, 0xffffffff, 0x00ff0fff,
179 0x28a4c, 0x06000000, 0x06000000,
180 0x10c, 0x00000001, 0x00000001,
181 0x8d00, 0xffffffff, 0x100e4848,
182 0x8d04, 0xffffffff, 0x00164745,
183 0x8c00, 0xffffffff, 0xe4000003,
184 0x8c04, 0xffffffff, 0x40600060,
185 0x8c08, 0xffffffff, 0x001c001c,
186 0x8cf0, 0xffffffff, 0x08e00620,
187 0x8c20, 0xffffffff, 0x00800080,
188 0x8c24, 0xffffffff, 0x00800080,
189 0x8c18, 0xffffffff, 0x20202078,
190 0x8c1c, 0xffffffff, 0x00001010,
191 0x28350, 0xffffffff, 0x00000000,
192 0xa008, 0xffffffff, 0x00010000,
193 0x5cc, 0xffffffff, 0x00000001,
194 0x9508, 0xffffffff, 0x00000002,
195 0x913c, 0x0000000f, 0x0000000a
196 };
197
198 static const u32 evergreen_golden_registers2[] =
199 {
200 0x2f4c, 0xffffffff, 0x00000000,
201 0x54f4, 0xffffffff, 0x00000000,
202 0x54f0, 0xffffffff, 0x00000000,
203 0x5498, 0xffffffff, 0x00000000,
204 0x549c, 0xffffffff, 0x00000000,
205 0x5494, 0xffffffff, 0x00000000,
206 0x53cc, 0xffffffff, 0x00000000,
207 0x53c8, 0xffffffff, 0x00000000,
208 0x53c4, 0xffffffff, 0x00000000,
209 0x53c0, 0xffffffff, 0x00000000,
210 0x53bc, 0xffffffff, 0x00000000,
211 0x53b8, 0xffffffff, 0x00000000,
212 0x53b4, 0xffffffff, 0x00000000,
213 0x53b0, 0xffffffff, 0x00000000
214 };
215
216 static const u32 cypress_mgcg_init[] =
217 {
218 0x802c, 0xffffffff, 0xc0000000,
219 0x5448, 0xffffffff, 0x00000100,
220 0x55e4, 0xffffffff, 0x00000100,
221 0x160c, 0xffffffff, 0x00000100,
222 0x5644, 0xffffffff, 0x00000100,
223 0xc164, 0xffffffff, 0x00000100,
224 0x8a18, 0xffffffff, 0x00000100,
225 0x897c, 0xffffffff, 0x06000100,
226 0x8b28, 0xffffffff, 0x00000100,
227 0x9144, 0xffffffff, 0x00000100,
228 0x9a60, 0xffffffff, 0x00000100,
229 0x9868, 0xffffffff, 0x00000100,
230 0x8d58, 0xffffffff, 0x00000100,
231 0x9510, 0xffffffff, 0x00000100,
232 0x949c, 0xffffffff, 0x00000100,
233 0x9654, 0xffffffff, 0x00000100,
234 0x9030, 0xffffffff, 0x00000100,
235 0x9034, 0xffffffff, 0x00000100,
236 0x9038, 0xffffffff, 0x00000100,
237 0x903c, 0xffffffff, 0x00000100,
238 0x9040, 0xffffffff, 0x00000100,
239 0xa200, 0xffffffff, 0x00000100,
240 0xa204, 0xffffffff, 0x00000100,
241 0xa208, 0xffffffff, 0x00000100,
242 0xa20c, 0xffffffff, 0x00000100,
243 0x971c, 0xffffffff, 0x00000100,
244 0x977c, 0xffffffff, 0x00000100,
245 0x3f80, 0xffffffff, 0x00000100,
246 0xa210, 0xffffffff, 0x00000100,
247 0xa214, 0xffffffff, 0x00000100,
248 0x4d8, 0xffffffff, 0x00000100,
249 0x9784, 0xffffffff, 0x00000100,
250 0x9698, 0xffffffff, 0x00000100,
251 0x4d4, 0xffffffff, 0x00000200,
252 0x30cc, 0xffffffff, 0x00000100,
253 0xd0c0, 0xffffffff, 0xff000100,
254 0x802c, 0xffffffff, 0x40000000,
255 0x915c, 0xffffffff, 0x00010000,
256 0x9160, 0xffffffff, 0x00030002,
257 0x9178, 0xffffffff, 0x00070000,
258 0x917c, 0xffffffff, 0x00030002,
259 0x9180, 0xffffffff, 0x00050004,
260 0x918c, 0xffffffff, 0x00010006,
261 0x9190, 0xffffffff, 0x00090008,
262 0x9194, 0xffffffff, 0x00070000,
263 0x9198, 0xffffffff, 0x00030002,
264 0x919c, 0xffffffff, 0x00050004,
265 0x91a8, 0xffffffff, 0x00010006,
266 0x91ac, 0xffffffff, 0x00090008,
267 0x91b0, 0xffffffff, 0x00070000,
268 0x91b4, 0xffffffff, 0x00030002,
269 0x91b8, 0xffffffff, 0x00050004,
270 0x91c4, 0xffffffff, 0x00010006,
271 0x91c8, 0xffffffff, 0x00090008,
272 0x91cc, 0xffffffff, 0x00070000,
273 0x91d0, 0xffffffff, 0x00030002,
274 0x91d4, 0xffffffff, 0x00050004,
275 0x91e0, 0xffffffff, 0x00010006,
276 0x91e4, 0xffffffff, 0x00090008,
277 0x91e8, 0xffffffff, 0x00000000,
278 0x91ec, 0xffffffff, 0x00070000,
279 0x91f0, 0xffffffff, 0x00030002,
280 0x91f4, 0xffffffff, 0x00050004,
281 0x9200, 0xffffffff, 0x00010006,
282 0x9204, 0xffffffff, 0x00090008,
283 0x9208, 0xffffffff, 0x00070000,
284 0x920c, 0xffffffff, 0x00030002,
285 0x9210, 0xffffffff, 0x00050004,
286 0x921c, 0xffffffff, 0x00010006,
287 0x9220, 0xffffffff, 0x00090008,
288 0x9224, 0xffffffff, 0x00070000,
289 0x9228, 0xffffffff, 0x00030002,
290 0x922c, 0xffffffff, 0x00050004,
291 0x9238, 0xffffffff, 0x00010006,
292 0x923c, 0xffffffff, 0x00090008,
293 0x9240, 0xffffffff, 0x00070000,
294 0x9244, 0xffffffff, 0x00030002,
295 0x9248, 0xffffffff, 0x00050004,
296 0x9254, 0xffffffff, 0x00010006,
297 0x9258, 0xffffffff, 0x00090008,
298 0x925c, 0xffffffff, 0x00070000,
299 0x9260, 0xffffffff, 0x00030002,
300 0x9264, 0xffffffff, 0x00050004,
301 0x9270, 0xffffffff, 0x00010006,
302 0x9274, 0xffffffff, 0x00090008,
303 0x9278, 0xffffffff, 0x00070000,
304 0x927c, 0xffffffff, 0x00030002,
305 0x9280, 0xffffffff, 0x00050004,
306 0x928c, 0xffffffff, 0x00010006,
307 0x9290, 0xffffffff, 0x00090008,
308 0x9294, 0xffffffff, 0x00000000,
309 0x929c, 0xffffffff, 0x00000001,
310 0x802c, 0xffffffff, 0x40010000,
311 0x915c, 0xffffffff, 0x00010000,
312 0x9160, 0xffffffff, 0x00030002,
313 0x9178, 0xffffffff, 0x00070000,
314 0x917c, 0xffffffff, 0x00030002,
315 0x9180, 0xffffffff, 0x00050004,
316 0x918c, 0xffffffff, 0x00010006,
317 0x9190, 0xffffffff, 0x00090008,
318 0x9194, 0xffffffff, 0x00070000,
319 0x9198, 0xffffffff, 0x00030002,
320 0x919c, 0xffffffff, 0x00050004,
321 0x91a8, 0xffffffff, 0x00010006,
322 0x91ac, 0xffffffff, 0x00090008,
323 0x91b0, 0xffffffff, 0x00070000,
324 0x91b4, 0xffffffff, 0x00030002,
325 0x91b8, 0xffffffff, 0x00050004,
326 0x91c4, 0xffffffff, 0x00010006,
327 0x91c8, 0xffffffff, 0x00090008,
328 0x91cc, 0xffffffff, 0x00070000,
329 0x91d0, 0xffffffff, 0x00030002,
330 0x91d4, 0xffffffff, 0x00050004,
331 0x91e0, 0xffffffff, 0x00010006,
332 0x91e4, 0xffffffff, 0x00090008,
333 0x91e8, 0xffffffff, 0x00000000,
334 0x91ec, 0xffffffff, 0x00070000,
335 0x91f0, 0xffffffff, 0x00030002,
336 0x91f4, 0xffffffff, 0x00050004,
337 0x9200, 0xffffffff, 0x00010006,
338 0x9204, 0xffffffff, 0x00090008,
339 0x9208, 0xffffffff, 0x00070000,
340 0x920c, 0xffffffff, 0x00030002,
341 0x9210, 0xffffffff, 0x00050004,
342 0x921c, 0xffffffff, 0x00010006,
343 0x9220, 0xffffffff, 0x00090008,
344 0x9224, 0xffffffff, 0x00070000,
345 0x9228, 0xffffffff, 0x00030002,
346 0x922c, 0xffffffff, 0x00050004,
347 0x9238, 0xffffffff, 0x00010006,
348 0x923c, 0xffffffff, 0x00090008,
349 0x9240, 0xffffffff, 0x00070000,
350 0x9244, 0xffffffff, 0x00030002,
351 0x9248, 0xffffffff, 0x00050004,
352 0x9254, 0xffffffff, 0x00010006,
353 0x9258, 0xffffffff, 0x00090008,
354 0x925c, 0xffffffff, 0x00070000,
355 0x9260, 0xffffffff, 0x00030002,
356 0x9264, 0xffffffff, 0x00050004,
357 0x9270, 0xffffffff, 0x00010006,
358 0x9274, 0xffffffff, 0x00090008,
359 0x9278, 0xffffffff, 0x00070000,
360 0x927c, 0xffffffff, 0x00030002,
361 0x9280, 0xffffffff, 0x00050004,
362 0x928c, 0xffffffff, 0x00010006,
363 0x9290, 0xffffffff, 0x00090008,
364 0x9294, 0xffffffff, 0x00000000,
365 0x929c, 0xffffffff, 0x00000001,
366 0x802c, 0xffffffff, 0xc0000000
367 };
368
369 static const u32 redwood_mgcg_init[] =
370 {
371 0x802c, 0xffffffff, 0xc0000000,
372 0x5448, 0xffffffff, 0x00000100,
373 0x55e4, 0xffffffff, 0x00000100,
374 0x160c, 0xffffffff, 0x00000100,
375 0x5644, 0xffffffff, 0x00000100,
376 0xc164, 0xffffffff, 0x00000100,
377 0x8a18, 0xffffffff, 0x00000100,
378 0x897c, 0xffffffff, 0x06000100,
379 0x8b28, 0xffffffff, 0x00000100,
380 0x9144, 0xffffffff, 0x00000100,
381 0x9a60, 0xffffffff, 0x00000100,
382 0x9868, 0xffffffff, 0x00000100,
383 0x8d58, 0xffffffff, 0x00000100,
384 0x9510, 0xffffffff, 0x00000100,
385 0x949c, 0xffffffff, 0x00000100,
386 0x9654, 0xffffffff, 0x00000100,
387 0x9030, 0xffffffff, 0x00000100,
388 0x9034, 0xffffffff, 0x00000100,
389 0x9038, 0xffffffff, 0x00000100,
390 0x903c, 0xffffffff, 0x00000100,
391 0x9040, 0xffffffff, 0x00000100,
392 0xa200, 0xffffffff, 0x00000100,
393 0xa204, 0xffffffff, 0x00000100,
394 0xa208, 0xffffffff, 0x00000100,
395 0xa20c, 0xffffffff, 0x00000100,
396 0x971c, 0xffffffff, 0x00000100,
397 0x977c, 0xffffffff, 0x00000100,
398 0x3f80, 0xffffffff, 0x00000100,
399 0xa210, 0xffffffff, 0x00000100,
400 0xa214, 0xffffffff, 0x00000100,
401 0x4d8, 0xffffffff, 0x00000100,
402 0x9784, 0xffffffff, 0x00000100,
403 0x9698, 0xffffffff, 0x00000100,
404 0x4d4, 0xffffffff, 0x00000200,
405 0x30cc, 0xffffffff, 0x00000100,
406 0xd0c0, 0xffffffff, 0xff000100,
407 0x802c, 0xffffffff, 0x40000000,
408 0x915c, 0xffffffff, 0x00010000,
409 0x9160, 0xffffffff, 0x00030002,
410 0x9178, 0xffffffff, 0x00070000,
411 0x917c, 0xffffffff, 0x00030002,
412 0x9180, 0xffffffff, 0x00050004,
413 0x918c, 0xffffffff, 0x00010006,
414 0x9190, 0xffffffff, 0x00090008,
415 0x9194, 0xffffffff, 0x00070000,
416 0x9198, 0xffffffff, 0x00030002,
417 0x919c, 0xffffffff, 0x00050004,
418 0x91a8, 0xffffffff, 0x00010006,
419 0x91ac, 0xffffffff, 0x00090008,
420 0x91b0, 0xffffffff, 0x00070000,
421 0x91b4, 0xffffffff, 0x00030002,
422 0x91b8, 0xffffffff, 0x00050004,
423 0x91c4, 0xffffffff, 0x00010006,
424 0x91c8, 0xffffffff, 0x00090008,
425 0x91cc, 0xffffffff, 0x00070000,
426 0x91d0, 0xffffffff, 0x00030002,
427 0x91d4, 0xffffffff, 0x00050004,
428 0x91e0, 0xffffffff, 0x00010006,
429 0x91e4, 0xffffffff, 0x00090008,
430 0x91e8, 0xffffffff, 0x00000000,
431 0x91ec, 0xffffffff, 0x00070000,
432 0x91f0, 0xffffffff, 0x00030002,
433 0x91f4, 0xffffffff, 0x00050004,
434 0x9200, 0xffffffff, 0x00010006,
435 0x9204, 0xffffffff, 0x00090008,
436 0x9294, 0xffffffff, 0x00000000,
437 0x929c, 0xffffffff, 0x00000001,
438 0x802c, 0xffffffff, 0xc0000000
439 };
440
441 static const u32 cedar_golden_registers[] =
442 {
443 0x3f90, 0xffff0000, 0xff000000,
444 0x9148, 0xffff0000, 0xff000000,
445 0x3f94, 0xffff0000, 0xff000000,
446 0x914c, 0xffff0000, 0xff000000,
447 0x9b7c, 0xffffffff, 0x00000000,
448 0x8a14, 0xffffffff, 0x00000007,
449 0x8b10, 0xffffffff, 0x00000000,
450 0x960c, 0xffffffff, 0x54763210,
451 0x88c4, 0xffffffff, 0x000000c2,
452 0x88d4, 0xffffffff, 0x00000000,
453 0x8974, 0xffffffff, 0x00000000,
454 0xc78, 0x00000080, 0x00000080,
455 0x5eb4, 0xffffffff, 0x00000002,
456 0x5e78, 0xffffffff, 0x001000f0,
457 0x6104, 0x01000300, 0x00000000,
458 0x5bc0, 0x00300000, 0x00000000,
459 0x7030, 0xffffffff, 0x00000011,
460 0x7c30, 0xffffffff, 0x00000011,
461 0x10830, 0xffffffff, 0x00000011,
462 0x11430, 0xffffffff, 0x00000011,
463 0xd02c, 0xffffffff, 0x08421000,
464 0x240c, 0xffffffff, 0x00000380,
465 0x8b24, 0xffffffff, 0x00ff0fff,
466 0x28a4c, 0x06000000, 0x06000000,
467 0x10c, 0x00000001, 0x00000001,
468 0x8d00, 0xffffffff, 0x100e4848,
469 0x8d04, 0xffffffff, 0x00164745,
470 0x8c00, 0xffffffff, 0xe4000003,
471 0x8c04, 0xffffffff, 0x40600060,
472 0x8c08, 0xffffffff, 0x001c001c,
473 0x8cf0, 0xffffffff, 0x08e00410,
474 0x8c20, 0xffffffff, 0x00800080,
475 0x8c24, 0xffffffff, 0x00800080,
476 0x8c18, 0xffffffff, 0x20202078,
477 0x8c1c, 0xffffffff, 0x00001010,
478 0x28350, 0xffffffff, 0x00000000,
479 0xa008, 0xffffffff, 0x00010000,
480 0x5cc, 0xffffffff, 0x00000001,
481 0x9508, 0xffffffff, 0x00000002
482 };
483
484 static const u32 cedar_mgcg_init[] =
485 {
486 0x802c, 0xffffffff, 0xc0000000,
487 0x5448, 0xffffffff, 0x00000100,
488 0x55e4, 0xffffffff, 0x00000100,
489 0x160c, 0xffffffff, 0x00000100,
490 0x5644, 0xffffffff, 0x00000100,
491 0xc164, 0xffffffff, 0x00000100,
492 0x8a18, 0xffffffff, 0x00000100,
493 0x897c, 0xffffffff, 0x06000100,
494 0x8b28, 0xffffffff, 0x00000100,
495 0x9144, 0xffffffff, 0x00000100,
496 0x9a60, 0xffffffff, 0x00000100,
497 0x9868, 0xffffffff, 0x00000100,
498 0x8d58, 0xffffffff, 0x00000100,
499 0x9510, 0xffffffff, 0x00000100,
500 0x949c, 0xffffffff, 0x00000100,
501 0x9654, 0xffffffff, 0x00000100,
502 0x9030, 0xffffffff, 0x00000100,
503 0x9034, 0xffffffff, 0x00000100,
504 0x9038, 0xffffffff, 0x00000100,
505 0x903c, 0xffffffff, 0x00000100,
506 0x9040, 0xffffffff, 0x00000100,
507 0xa200, 0xffffffff, 0x00000100,
508 0xa204, 0xffffffff, 0x00000100,
509 0xa208, 0xffffffff, 0x00000100,
510 0xa20c, 0xffffffff, 0x00000100,
511 0x971c, 0xffffffff, 0x00000100,
512 0x977c, 0xffffffff, 0x00000100,
513 0x3f80, 0xffffffff, 0x00000100,
514 0xa210, 0xffffffff, 0x00000100,
515 0xa214, 0xffffffff, 0x00000100,
516 0x4d8, 0xffffffff, 0x00000100,
517 0x9784, 0xffffffff, 0x00000100,
518 0x9698, 0xffffffff, 0x00000100,
519 0x4d4, 0xffffffff, 0x00000200,
520 0x30cc, 0xffffffff, 0x00000100,
521 0xd0c0, 0xffffffff, 0xff000100,
522 0x802c, 0xffffffff, 0x40000000,
523 0x915c, 0xffffffff, 0x00010000,
524 0x9178, 0xffffffff, 0x00050000,
525 0x917c, 0xffffffff, 0x00030002,
526 0x918c, 0xffffffff, 0x00010004,
527 0x9190, 0xffffffff, 0x00070006,
528 0x9194, 0xffffffff, 0x00050000,
529 0x9198, 0xffffffff, 0x00030002,
530 0x91a8, 0xffffffff, 0x00010004,
531 0x91ac, 0xffffffff, 0x00070006,
532 0x91e8, 0xffffffff, 0x00000000,
533 0x9294, 0xffffffff, 0x00000000,
534 0x929c, 0xffffffff, 0x00000001,
535 0x802c, 0xffffffff, 0xc0000000
536 };
537
538 static const u32 juniper_mgcg_init[] =
539 {
540 0x802c, 0xffffffff, 0xc0000000,
541 0x5448, 0xffffffff, 0x00000100,
542 0x55e4, 0xffffffff, 0x00000100,
543 0x160c, 0xffffffff, 0x00000100,
544 0x5644, 0xffffffff, 0x00000100,
545 0xc164, 0xffffffff, 0x00000100,
546 0x8a18, 0xffffffff, 0x00000100,
547 0x897c, 0xffffffff, 0x06000100,
548 0x8b28, 0xffffffff, 0x00000100,
549 0x9144, 0xffffffff, 0x00000100,
550 0x9a60, 0xffffffff, 0x00000100,
551 0x9868, 0xffffffff, 0x00000100,
552 0x8d58, 0xffffffff, 0x00000100,
553 0x9510, 0xffffffff, 0x00000100,
554 0x949c, 0xffffffff, 0x00000100,
555 0x9654, 0xffffffff, 0x00000100,
556 0x9030, 0xffffffff, 0x00000100,
557 0x9034, 0xffffffff, 0x00000100,
558 0x9038, 0xffffffff, 0x00000100,
559 0x903c, 0xffffffff, 0x00000100,
560 0x9040, 0xffffffff, 0x00000100,
561 0xa200, 0xffffffff, 0x00000100,
562 0xa204, 0xffffffff, 0x00000100,
563 0xa208, 0xffffffff, 0x00000100,
564 0xa20c, 0xffffffff, 0x00000100,
565 0x971c, 0xffffffff, 0x00000100,
566 0xd0c0, 0xffffffff, 0xff000100,
567 0x802c, 0xffffffff, 0x40000000,
568 0x915c, 0xffffffff, 0x00010000,
569 0x9160, 0xffffffff, 0x00030002,
570 0x9178, 0xffffffff, 0x00070000,
571 0x917c, 0xffffffff, 0x00030002,
572 0x9180, 0xffffffff, 0x00050004,
573 0x918c, 0xffffffff, 0x00010006,
574 0x9190, 0xffffffff, 0x00090008,
575 0x9194, 0xffffffff, 0x00070000,
576 0x9198, 0xffffffff, 0x00030002,
577 0x919c, 0xffffffff, 0x00050004,
578 0x91a8, 0xffffffff, 0x00010006,
579 0x91ac, 0xffffffff, 0x00090008,
580 0x91b0, 0xffffffff, 0x00070000,
581 0x91b4, 0xffffffff, 0x00030002,
582 0x91b8, 0xffffffff, 0x00050004,
583 0x91c4, 0xffffffff, 0x00010006,
584 0x91c8, 0xffffffff, 0x00090008,
585 0x91cc, 0xffffffff, 0x00070000,
586 0x91d0, 0xffffffff, 0x00030002,
587 0x91d4, 0xffffffff, 0x00050004,
588 0x91e0, 0xffffffff, 0x00010006,
589 0x91e4, 0xffffffff, 0x00090008,
590 0x91e8, 0xffffffff, 0x00000000,
591 0x91ec, 0xffffffff, 0x00070000,
592 0x91f0, 0xffffffff, 0x00030002,
593 0x91f4, 0xffffffff, 0x00050004,
594 0x9200, 0xffffffff, 0x00010006,
595 0x9204, 0xffffffff, 0x00090008,
596 0x9208, 0xffffffff, 0x00070000,
597 0x920c, 0xffffffff, 0x00030002,
598 0x9210, 0xffffffff, 0x00050004,
599 0x921c, 0xffffffff, 0x00010006,
600 0x9220, 0xffffffff, 0x00090008,
601 0x9224, 0xffffffff, 0x00070000,
602 0x9228, 0xffffffff, 0x00030002,
603 0x922c, 0xffffffff, 0x00050004,
604 0x9238, 0xffffffff, 0x00010006,
605 0x923c, 0xffffffff, 0x00090008,
606 0x9240, 0xffffffff, 0x00070000,
607 0x9244, 0xffffffff, 0x00030002,
608 0x9248, 0xffffffff, 0x00050004,
609 0x9254, 0xffffffff, 0x00010006,
610 0x9258, 0xffffffff, 0x00090008,
611 0x925c, 0xffffffff, 0x00070000,
612 0x9260, 0xffffffff, 0x00030002,
613 0x9264, 0xffffffff, 0x00050004,
614 0x9270, 0xffffffff, 0x00010006,
615 0x9274, 0xffffffff, 0x00090008,
616 0x9278, 0xffffffff, 0x00070000,
617 0x927c, 0xffffffff, 0x00030002,
618 0x9280, 0xffffffff, 0x00050004,
619 0x928c, 0xffffffff, 0x00010006,
620 0x9290, 0xffffffff, 0x00090008,
621 0x9294, 0xffffffff, 0x00000000,
622 0x929c, 0xffffffff, 0x00000001,
623 0x802c, 0xffffffff, 0xc0000000,
624 0x977c, 0xffffffff, 0x00000100,
625 0x3f80, 0xffffffff, 0x00000100,
626 0xa210, 0xffffffff, 0x00000100,
627 0xa214, 0xffffffff, 0x00000100,
628 0x4d8, 0xffffffff, 0x00000100,
629 0x9784, 0xffffffff, 0x00000100,
630 0x9698, 0xffffffff, 0x00000100,
631 0x4d4, 0xffffffff, 0x00000200,
632 0x30cc, 0xffffffff, 0x00000100,
633 0x802c, 0xffffffff, 0xc0000000
634 };
635
636 static const u32 supersumo_golden_registers[] =
637 {
638 0x5eb4, 0xffffffff, 0x00000002,
639 0x5cc, 0xffffffff, 0x00000001,
640 0x7030, 0xffffffff, 0x00000011,
641 0x7c30, 0xffffffff, 0x00000011,
642 0x6104, 0x01000300, 0x00000000,
643 0x5bc0, 0x00300000, 0x00000000,
644 0x8c04, 0xffffffff, 0x40600060,
645 0x8c08, 0xffffffff, 0x001c001c,
646 0x8c20, 0xffffffff, 0x00800080,
647 0x8c24, 0xffffffff, 0x00800080,
648 0x8c18, 0xffffffff, 0x20202078,
649 0x8c1c, 0xffffffff, 0x00001010,
650 0x918c, 0xffffffff, 0x00010006,
651 0x91a8, 0xffffffff, 0x00010006,
652 0x91c4, 0xffffffff, 0x00010006,
653 0x91e0, 0xffffffff, 0x00010006,
654 0x9200, 0xffffffff, 0x00010006,
655 0x9150, 0xffffffff, 0x6e944040,
656 0x917c, 0xffffffff, 0x00030002,
657 0x9180, 0xffffffff, 0x00050004,
658 0x9198, 0xffffffff, 0x00030002,
659 0x919c, 0xffffffff, 0x00050004,
660 0x91b4, 0xffffffff, 0x00030002,
661 0x91b8, 0xffffffff, 0x00050004,
662 0x91d0, 0xffffffff, 0x00030002,
663 0x91d4, 0xffffffff, 0x00050004,
664 0x91f0, 0xffffffff, 0x00030002,
665 0x91f4, 0xffffffff, 0x00050004,
666 0x915c, 0xffffffff, 0x00010000,
667 0x9160, 0xffffffff, 0x00030002,
668 0x3f90, 0xffff0000, 0xff000000,
669 0x9178, 0xffffffff, 0x00070000,
670 0x9194, 0xffffffff, 0x00070000,
671 0x91b0, 0xffffffff, 0x00070000,
672 0x91cc, 0xffffffff, 0x00070000,
673 0x91ec, 0xffffffff, 0x00070000,
674 0x9148, 0xffff0000, 0xff000000,
675 0x9190, 0xffffffff, 0x00090008,
676 0x91ac, 0xffffffff, 0x00090008,
677 0x91c8, 0xffffffff, 0x00090008,
678 0x91e4, 0xffffffff, 0x00090008,
679 0x9204, 0xffffffff, 0x00090008,
680 0x3f94, 0xffff0000, 0xff000000,
681 0x914c, 0xffff0000, 0xff000000,
682 0x929c, 0xffffffff, 0x00000001,
683 0x8a18, 0xffffffff, 0x00000100,
684 0x8b28, 0xffffffff, 0x00000100,
685 0x9144, 0xffffffff, 0x00000100,
686 0x5644, 0xffffffff, 0x00000100,
687 0x9b7c, 0xffffffff, 0x00000000,
688 0x8030, 0xffffffff, 0x0000100a,
689 0x8a14, 0xffffffff, 0x00000007,
690 0x8b24, 0xffffffff, 0x00ff0fff,
691 0x8b10, 0xffffffff, 0x00000000,
692 0x28a4c, 0x06000000, 0x06000000,
693 0x4d8, 0xffffffff, 0x00000100,
694 0x913c, 0xffff000f, 0x0100000a,
695 0x960c, 0xffffffff, 0x54763210,
696 0x88c4, 0xffffffff, 0x000000c2,
697 0x88d4, 0xffffffff, 0x00000010,
698 0x8974, 0xffffffff, 0x00000000,
699 0xc78, 0x00000080, 0x00000080,
700 0x5e78, 0xffffffff, 0x001000f0,
701 0xd02c, 0xffffffff, 0x08421000,
702 0xa008, 0xffffffff, 0x00010000,
703 0x8d00, 0xffffffff, 0x100e4848,
704 0x8d04, 0xffffffff, 0x00164745,
705 0x8c00, 0xffffffff, 0xe4000003,
706 0x8cf0, 0x1fffffff, 0x08e00620,
707 0x28350, 0xffffffff, 0x00000000,
708 0x9508, 0xffffffff, 0x00000002
709 };
710
711 static const u32 sumo_golden_registers[] =
712 {
713 0x900c, 0x00ffffff, 0x0017071f,
714 0x8c18, 0xffffffff, 0x10101060,
715 0x8c1c, 0xffffffff, 0x00001010,
716 0x8c30, 0x0000000f, 0x00000005,
717 0x9688, 0x0000000f, 0x00000007
718 };
719
720 static const u32 wrestler_golden_registers[] =
721 {
722 0x5eb4, 0xffffffff, 0x00000002,
723 0x5cc, 0xffffffff, 0x00000001,
724 0x7030, 0xffffffff, 0x00000011,
725 0x7c30, 0xffffffff, 0x00000011,
726 0x6104, 0x01000300, 0x00000000,
727 0x5bc0, 0x00300000, 0x00000000,
728 0x918c, 0xffffffff, 0x00010006,
729 0x91a8, 0xffffffff, 0x00010006,
730 0x9150, 0xffffffff, 0x6e944040,
731 0x917c, 0xffffffff, 0x00030002,
732 0x9198, 0xffffffff, 0x00030002,
733 0x915c, 0xffffffff, 0x00010000,
734 0x3f90, 0xffff0000, 0xff000000,
735 0x9178, 0xffffffff, 0x00070000,
736 0x9194, 0xffffffff, 0x00070000,
737 0x9148, 0xffff0000, 0xff000000,
738 0x9190, 0xffffffff, 0x00090008,
739 0x91ac, 0xffffffff, 0x00090008,
740 0x3f94, 0xffff0000, 0xff000000,
741 0x914c, 0xffff0000, 0xff000000,
742 0x929c, 0xffffffff, 0x00000001,
743 0x8a18, 0xffffffff, 0x00000100,
744 0x8b28, 0xffffffff, 0x00000100,
745 0x9144, 0xffffffff, 0x00000100,
746 0x9b7c, 0xffffffff, 0x00000000,
747 0x8030, 0xffffffff, 0x0000100a,
748 0x8a14, 0xffffffff, 0x00000001,
749 0x8b24, 0xffffffff, 0x00ff0fff,
750 0x8b10, 0xffffffff, 0x00000000,
751 0x28a4c, 0x06000000, 0x06000000,
752 0x4d8, 0xffffffff, 0x00000100,
753 0x913c, 0xffff000f, 0x0100000a,
754 0x960c, 0xffffffff, 0x54763210,
755 0x88c4, 0xffffffff, 0x000000c2,
756 0x88d4, 0xffffffff, 0x00000010,
757 0x8974, 0xffffffff, 0x00000000,
758 0xc78, 0x00000080, 0x00000080,
759 0x5e78, 0xffffffff, 0x001000f0,
760 0xd02c, 0xffffffff, 0x08421000,
761 0xa008, 0xffffffff, 0x00010000,
762 0x8d00, 0xffffffff, 0x100e4848,
763 0x8d04, 0xffffffff, 0x00164745,
764 0x8c00, 0xffffffff, 0xe4000003,
765 0x8cf0, 0x1fffffff, 0x08e00410,
766 0x28350, 0xffffffff, 0x00000000,
767 0x9508, 0xffffffff, 0x00000002,
768 0x900c, 0xffffffff, 0x0017071f,
769 0x8c18, 0xffffffff, 0x10101060,
770 0x8c1c, 0xffffffff, 0x00001010
771 };
772
773 static const u32 barts_golden_registers[] =
774 {
775 0x5eb4, 0xffffffff, 0x00000002,
776 0x5e78, 0x8f311ff1, 0x001000f0,
777 0x3f90, 0xffff0000, 0xff000000,
778 0x9148, 0xffff0000, 0xff000000,
779 0x3f94, 0xffff0000, 0xff000000,
780 0x914c, 0xffff0000, 0xff000000,
781 0xc78, 0x00000080, 0x00000080,
782 0xbd4, 0x70073777, 0x00010001,
783 0xd02c, 0xbfffff1f, 0x08421000,
784 0xd0b8, 0x03773777, 0x02011003,
785 0x5bc0, 0x00200000, 0x50100000,
786 0x98f8, 0x33773777, 0x02011003,
787 0x98fc, 0xffffffff, 0x76543210,
788 0x7030, 0x31000311, 0x00000011,
789 0x2f48, 0x00000007, 0x02011003,
790 0x6b28, 0x00000010, 0x00000012,
791 0x7728, 0x00000010, 0x00000012,
792 0x10328, 0x00000010, 0x00000012,
793 0x10f28, 0x00000010, 0x00000012,
794 0x11b28, 0x00000010, 0x00000012,
795 0x12728, 0x00000010, 0x00000012,
796 0x240c, 0x000007ff, 0x00000380,
797 0x8a14, 0xf000001f, 0x00000007,
798 0x8b24, 0x3fff3fff, 0x00ff0fff,
799 0x8b10, 0x0000ff0f, 0x00000000,
800 0x28a4c, 0x07ffffff, 0x06000000,
801 0x10c, 0x00000001, 0x00010003,
802 0xa02c, 0xffffffff, 0x0000009b,
803 0x913c, 0x0000000f, 0x0100000a,
804 0x8d00, 0xffff7f7f, 0x100e4848,
805 0x8d04, 0x00ffffff, 0x00164745,
806 0x8c00, 0xfffc0003, 0xe4000003,
807 0x8c04, 0xf8ff00ff, 0x40600060,
808 0x8c08, 0x00ff00ff, 0x001c001c,
809 0x8cf0, 0x1fff1fff, 0x08e00620,
810 0x8c20, 0x0fff0fff, 0x00800080,
811 0x8c24, 0x0fff0fff, 0x00800080,
812 0x8c18, 0xffffffff, 0x20202078,
813 0x8c1c, 0x0000ffff, 0x00001010,
814 0x28350, 0x00000f01, 0x00000000,
815 0x9508, 0x3700001f, 0x00000002,
816 0x960c, 0xffffffff, 0x54763210,
817 0x88c4, 0x001f3ae3, 0x000000c2,
818 0x88d4, 0x0000001f, 0x00000010,
819 0x8974, 0xffffffff, 0x00000000
820 };
821
822 static const u32 turks_golden_registers[] =
823 {
824 0x5eb4, 0xffffffff, 0x00000002,
825 0x5e78, 0x8f311ff1, 0x001000f0,
826 0x8c8, 0x00003000, 0x00001070,
827 0x8cc, 0x000fffff, 0x00040035,
828 0x3f90, 0xffff0000, 0xfff00000,
829 0x9148, 0xffff0000, 0xfff00000,
830 0x3f94, 0xffff0000, 0xfff00000,
831 0x914c, 0xffff0000, 0xfff00000,
832 0xc78, 0x00000080, 0x00000080,
833 0xbd4, 0x00073007, 0x00010002,
834 0xd02c, 0xbfffff1f, 0x08421000,
835 0xd0b8, 0x03773777, 0x02010002,
836 0x5bc0, 0x00200000, 0x50100000,
837 0x98f8, 0x33773777, 0x00010002,
838 0x98fc, 0xffffffff, 0x33221100,
839 0x7030, 0x31000311, 0x00000011,
840 0x2f48, 0x33773777, 0x00010002,
841 0x6b28, 0x00000010, 0x00000012,
842 0x7728, 0x00000010, 0x00000012,
843 0x10328, 0x00000010, 0x00000012,
844 0x10f28, 0x00000010, 0x00000012,
845 0x11b28, 0x00000010, 0x00000012,
846 0x12728, 0x00000010, 0x00000012,
847 0x240c, 0x000007ff, 0x00000380,
848 0x8a14, 0xf000001f, 0x00000007,
849 0x8b24, 0x3fff3fff, 0x00ff0fff,
850 0x8b10, 0x0000ff0f, 0x00000000,
851 0x28a4c, 0x07ffffff, 0x06000000,
852 0x10c, 0x00000001, 0x00010003,
853 0xa02c, 0xffffffff, 0x0000009b,
854 0x913c, 0x0000000f, 0x0100000a,
855 0x8d00, 0xffff7f7f, 0x100e4848,
856 0x8d04, 0x00ffffff, 0x00164745,
857 0x8c00, 0xfffc0003, 0xe4000003,
858 0x8c04, 0xf8ff00ff, 0x40600060,
859 0x8c08, 0x00ff00ff, 0x001c001c,
860 0x8cf0, 0x1fff1fff, 0x08e00410,
861 0x8c20, 0x0fff0fff, 0x00800080,
862 0x8c24, 0x0fff0fff, 0x00800080,
863 0x8c18, 0xffffffff, 0x20202078,
864 0x8c1c, 0x0000ffff, 0x00001010,
865 0x28350, 0x00000f01, 0x00000000,
866 0x9508, 0x3700001f, 0x00000002,
867 0x960c, 0xffffffff, 0x54763210,
868 0x88c4, 0x001f3ae3, 0x000000c2,
869 0x88d4, 0x0000001f, 0x00000010,
870 0x8974, 0xffffffff, 0x00000000
871 };
872
873 static const u32 caicos_golden_registers[] =
874 {
875 0x5eb4, 0xffffffff, 0x00000002,
876 0x5e78, 0x8f311ff1, 0x001000f0,
877 0x8c8, 0x00003420, 0x00001450,
878 0x8cc, 0x000fffff, 0x00040035,
879 0x3f90, 0xffff0000, 0xfffc0000,
880 0x9148, 0xffff0000, 0xfffc0000,
881 0x3f94, 0xffff0000, 0xfffc0000,
882 0x914c, 0xffff0000, 0xfffc0000,
883 0xc78, 0x00000080, 0x00000080,
884 0xbd4, 0x00073007, 0x00010001,
885 0xd02c, 0xbfffff1f, 0x08421000,
886 0xd0b8, 0x03773777, 0x02010001,
887 0x5bc0, 0x00200000, 0x50100000,
888 0x98f8, 0x33773777, 0x02010001,
889 0x98fc, 0xffffffff, 0x33221100,
890 0x7030, 0x31000311, 0x00000011,
891 0x2f48, 0x33773777, 0x02010001,
892 0x6b28, 0x00000010, 0x00000012,
893 0x7728, 0x00000010, 0x00000012,
894 0x10328, 0x00000010, 0x00000012,
895 0x10f28, 0x00000010, 0x00000012,
896 0x11b28, 0x00000010, 0x00000012,
897 0x12728, 0x00000010, 0x00000012,
898 0x240c, 0x000007ff, 0x00000380,
899 0x8a14, 0xf000001f, 0x00000001,
900 0x8b24, 0x3fff3fff, 0x00ff0fff,
901 0x8b10, 0x0000ff0f, 0x00000000,
902 0x28a4c, 0x07ffffff, 0x06000000,
903 0x10c, 0x00000001, 0x00010003,
904 0xa02c, 0xffffffff, 0x0000009b,
905 0x913c, 0x0000000f, 0x0100000a,
906 0x8d00, 0xffff7f7f, 0x100e4848,
907 0x8d04, 0x00ffffff, 0x00164745,
908 0x8c00, 0xfffc0003, 0xe4000003,
909 0x8c04, 0xf8ff00ff, 0x40600060,
910 0x8c08, 0x00ff00ff, 0x001c001c,
911 0x8cf0, 0x1fff1fff, 0x08e00410,
912 0x8c20, 0x0fff0fff, 0x00800080,
913 0x8c24, 0x0fff0fff, 0x00800080,
914 0x8c18, 0xffffffff, 0x20202078,
915 0x8c1c, 0x0000ffff, 0x00001010,
916 0x28350, 0x00000f01, 0x00000000,
917 0x9508, 0x3700001f, 0x00000002,
918 0x960c, 0xffffffff, 0x54763210,
919 0x88c4, 0x001f3ae3, 0x000000c2,
920 0x88d4, 0x0000001f, 0x00000010,
921 0x8974, 0xffffffff, 0x00000000
922 };
923
evergreen_init_golden_registers(struct radeon_device * rdev)924 static void evergreen_init_golden_registers(struct radeon_device *rdev)
925 {
926 switch (rdev->family) {
927 case CHIP_CYPRESS:
928 case CHIP_HEMLOCK:
929 radeon_program_register_sequence(rdev,
930 evergreen_golden_registers,
931 (const u32)ARRAY_SIZE(evergreen_golden_registers));
932 radeon_program_register_sequence(rdev,
933 evergreen_golden_registers2,
934 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
935 radeon_program_register_sequence(rdev,
936 cypress_mgcg_init,
937 (const u32)ARRAY_SIZE(cypress_mgcg_init));
938 break;
939 case CHIP_JUNIPER:
940 radeon_program_register_sequence(rdev,
941 evergreen_golden_registers,
942 (const u32)ARRAY_SIZE(evergreen_golden_registers));
943 radeon_program_register_sequence(rdev,
944 evergreen_golden_registers2,
945 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
946 radeon_program_register_sequence(rdev,
947 juniper_mgcg_init,
948 (const u32)ARRAY_SIZE(juniper_mgcg_init));
949 break;
950 case CHIP_REDWOOD:
951 radeon_program_register_sequence(rdev,
952 evergreen_golden_registers,
953 (const u32)ARRAY_SIZE(evergreen_golden_registers));
954 radeon_program_register_sequence(rdev,
955 evergreen_golden_registers2,
956 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
957 radeon_program_register_sequence(rdev,
958 redwood_mgcg_init,
959 (const u32)ARRAY_SIZE(redwood_mgcg_init));
960 break;
961 case CHIP_CEDAR:
962 radeon_program_register_sequence(rdev,
963 cedar_golden_registers,
964 (const u32)ARRAY_SIZE(cedar_golden_registers));
965 radeon_program_register_sequence(rdev,
966 evergreen_golden_registers2,
967 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
968 radeon_program_register_sequence(rdev,
969 cedar_mgcg_init,
970 (const u32)ARRAY_SIZE(cedar_mgcg_init));
971 break;
972 case CHIP_PALM:
973 radeon_program_register_sequence(rdev,
974 wrestler_golden_registers,
975 (const u32)ARRAY_SIZE(wrestler_golden_registers));
976 break;
977 case CHIP_SUMO:
978 radeon_program_register_sequence(rdev,
979 supersumo_golden_registers,
980 (const u32)ARRAY_SIZE(supersumo_golden_registers));
981 break;
982 case CHIP_SUMO2:
983 radeon_program_register_sequence(rdev,
984 supersumo_golden_registers,
985 (const u32)ARRAY_SIZE(supersumo_golden_registers));
986 radeon_program_register_sequence(rdev,
987 sumo_golden_registers,
988 (const u32)ARRAY_SIZE(sumo_golden_registers));
989 break;
990 case CHIP_BARTS:
991 radeon_program_register_sequence(rdev,
992 barts_golden_registers,
993 (const u32)ARRAY_SIZE(barts_golden_registers));
994 break;
995 case CHIP_TURKS:
996 radeon_program_register_sequence(rdev,
997 turks_golden_registers,
998 (const u32)ARRAY_SIZE(turks_golden_registers));
999 break;
1000 case CHIP_CAICOS:
1001 radeon_program_register_sequence(rdev,
1002 caicos_golden_registers,
1003 (const u32)ARRAY_SIZE(caicos_golden_registers));
1004 break;
1005 default:
1006 break;
1007 }
1008 }
1009
evergreen_tiling_fields(unsigned tiling_flags,unsigned * bankw,unsigned * bankh,unsigned * mtaspect,unsigned * tile_split)1010 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1011 unsigned *bankh, unsigned *mtaspect,
1012 unsigned *tile_split)
1013 {
1014 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1015 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1016 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1017 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1018 switch (*bankw) {
1019 default:
1020 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1021 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1022 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1023 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1024 }
1025 switch (*bankh) {
1026 default:
1027 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1028 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1029 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1030 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1031 }
1032 switch (*mtaspect) {
1033 default:
1034 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1035 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1036 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1037 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1038 }
1039 }
1040
sumo_set_uvd_clock(struct radeon_device * rdev,u32 clock,u32 cntl_reg,u32 status_reg)1041 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1042 u32 cntl_reg, u32 status_reg)
1043 {
1044 int r, i;
1045 struct atom_clock_dividers dividers;
1046
1047 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1048 clock, false, ÷rs);
1049 if (r)
1050 return r;
1051
1052 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1053
1054 for (i = 0; i < 100; i++) {
1055 if (RREG32(status_reg) & DCLK_STATUS)
1056 break;
1057 mdelay(10);
1058 }
1059 if (i == 100)
1060 return -ETIMEDOUT;
1061
1062 return 0;
1063 }
1064
sumo_set_uvd_clocks(struct radeon_device * rdev,u32 vclk,u32 dclk)1065 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1066 {
1067 int r = 0;
1068 u32 cg_scratch = RREG32(CG_SCRATCH1);
1069
1070 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1071 if (r)
1072 goto done;
1073 cg_scratch &= 0xffff0000;
1074 cg_scratch |= vclk / 100; /* Mhz */
1075
1076 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1077 if (r)
1078 goto done;
1079 cg_scratch &= 0x0000ffff;
1080 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1081
1082 done:
1083 WREG32(CG_SCRATCH1, cg_scratch);
1084
1085 return r;
1086 }
1087
evergreen_set_uvd_clocks(struct radeon_device * rdev,u32 vclk,u32 dclk)1088 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1089 {
1090 /* start off with something large */
1091 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1092 int r;
1093
1094 /* bypass vclk and dclk with bclk */
1095 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1096 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1097 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1098
1099 /* put PLL in bypass mode */
1100 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1101
1102 if (!vclk || !dclk) {
1103 /* keep the Bypass mode, put PLL to sleep */
1104 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1105 return 0;
1106 }
1107
1108 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1109 16384, 0x03FFFFFF, 0, 128, 5,
1110 &fb_div, &vclk_div, &dclk_div);
1111 if (r)
1112 return r;
1113
1114 /* set VCO_MODE to 1 */
1115 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1116
1117 /* toggle UPLL_SLEEP to 1 then back to 0 */
1118 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1119 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1120
1121 /* deassert UPLL_RESET */
1122 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1123
1124 mdelay(1);
1125
1126 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1127 if (r)
1128 return r;
1129
1130 /* assert UPLL_RESET again */
1131 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1132
1133 /* disable spread spectrum. */
1134 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1135
1136 /* set feedback divider */
1137 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1138
1139 /* set ref divider to 0 */
1140 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1141
1142 if (fb_div < 307200)
1143 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1144 else
1145 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1146
1147 /* set PDIV_A and PDIV_B */
1148 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1149 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1150 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1151
1152 /* give the PLL some time to settle */
1153 mdelay(15);
1154
1155 /* deassert PLL_RESET */
1156 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1157
1158 mdelay(15);
1159
1160 /* switch from bypass mode to normal mode */
1161 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1162
1163 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1164 if (r)
1165 return r;
1166
1167 /* switch VCLK and DCLK selection */
1168 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1169 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1170 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1171
1172 mdelay(100);
1173
1174 return 0;
1175 }
1176
evergreen_fix_pci_max_read_req_size(struct radeon_device * rdev)1177 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1178 {
1179 #ifndef __NetBSD__ /* XXX radeon pcie */
1180 int readrq;
1181 u16 v;
1182
1183 readrq = pcie_get_readrq(rdev->pdev);
1184 v = ffs(readrq) - 8;
1185 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1186 * to avoid hangs or perfomance issues
1187 */
1188 if ((v == 0) || (v == 6) || (v == 7))
1189 pcie_set_readrq(rdev->pdev, 512);
1190 #endif
1191 }
1192
dce4_program_fmt(struct drm_encoder * encoder)1193 void dce4_program_fmt(struct drm_encoder *encoder)
1194 {
1195 struct drm_device *dev = encoder->dev;
1196 struct radeon_device *rdev = dev->dev_private;
1197 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1198 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1199 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1200 int bpc = 0;
1201 u32 tmp = 0;
1202 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1203
1204 if (connector) {
1205 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1206 bpc = radeon_get_monitor_bpc(connector);
1207 dither = radeon_connector->dither;
1208 }
1209
1210 /* LVDS/eDP FMT is set up by atom */
1211 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1212 return;
1213
1214 /* not needed for analog */
1215 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1216 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1217 return;
1218
1219 if (bpc == 0)
1220 return;
1221
1222 switch (bpc) {
1223 case 6:
1224 if (dither == RADEON_FMT_DITHER_ENABLE)
1225 /* XXX sort out optimal dither settings */
1226 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1227 FMT_SPATIAL_DITHER_EN);
1228 else
1229 tmp |= FMT_TRUNCATE_EN;
1230 break;
1231 case 8:
1232 if (dither == RADEON_FMT_DITHER_ENABLE)
1233 /* XXX sort out optimal dither settings */
1234 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1235 FMT_RGB_RANDOM_ENABLE |
1236 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1237 else
1238 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1239 break;
1240 case 10:
1241 default:
1242 /* not needed */
1243 break;
1244 }
1245
1246 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1247 }
1248
dce4_is_in_vblank(struct radeon_device * rdev,int crtc)1249 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1250 {
1251 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1252 return true;
1253 else
1254 return false;
1255 }
1256
dce4_is_counter_moving(struct radeon_device * rdev,int crtc)1257 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1258 {
1259 u32 pos1, pos2;
1260
1261 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1262 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1263
1264 if (pos1 != pos2)
1265 return true;
1266 else
1267 return false;
1268 }
1269
1270 /**
1271 * dce4_wait_for_vblank - vblank wait asic callback.
1272 *
1273 * @rdev: radeon_device pointer
1274 * @crtc: crtc to wait for vblank on
1275 *
1276 * Wait for vblank on the requested crtc (evergreen+).
1277 */
dce4_wait_for_vblank(struct radeon_device * rdev,int crtc)1278 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1279 {
1280 unsigned i = 0;
1281
1282 if (crtc >= rdev->num_crtc)
1283 return;
1284
1285 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1286 return;
1287
1288 /* depending on when we hit vblank, we may be close to active; if so,
1289 * wait for another frame.
1290 */
1291 while (dce4_is_in_vblank(rdev, crtc)) {
1292 if (i++ % 100 == 0) {
1293 if (!dce4_is_counter_moving(rdev, crtc))
1294 break;
1295 }
1296 }
1297
1298 while (!dce4_is_in_vblank(rdev, crtc)) {
1299 if (i++ % 100 == 0) {
1300 if (!dce4_is_counter_moving(rdev, crtc))
1301 break;
1302 }
1303 }
1304 }
1305
1306 /**
1307 * radeon_irq_kms_pflip_irq_get - pre-pageflip callback.
1308 *
1309 * @rdev: radeon_device pointer
1310 * @crtc: crtc to prepare for pageflip on
1311 *
1312 * Pre-pageflip callback (evergreen+).
1313 * Enables the pageflip irq (vblank irq).
1314 */
evergreen_pre_page_flip(struct radeon_device * rdev,int crtc)1315 void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
1316 {
1317 /* enable the pflip int */
1318 radeon_irq_kms_pflip_irq_get(rdev, crtc);
1319 }
1320
1321 /**
1322 * evergreen_post_page_flip - pos-pageflip callback.
1323 *
1324 * @rdev: radeon_device pointer
1325 * @crtc: crtc to cleanup pageflip on
1326 *
1327 * Post-pageflip callback (evergreen+).
1328 * Disables the pageflip irq (vblank irq).
1329 */
evergreen_post_page_flip(struct radeon_device * rdev,int crtc)1330 void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
1331 {
1332 /* disable the pflip int */
1333 radeon_irq_kms_pflip_irq_put(rdev, crtc);
1334 }
1335
1336 /**
1337 * evergreen_page_flip - pageflip callback.
1338 *
1339 * @rdev: radeon_device pointer
1340 * @crtc_id: crtc to cleanup pageflip on
1341 * @crtc_base: new address of the crtc (GPU MC address)
1342 *
1343 * Does the actual pageflip (evergreen+).
1344 * During vblank we take the crtc lock and wait for the update_pending
1345 * bit to go high, when it does, we release the lock, and allow the
1346 * double buffered update to take place.
1347 * Returns the current update pending status.
1348 */
evergreen_page_flip(struct radeon_device * rdev,int crtc_id,u64 crtc_base)1349 u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1350 {
1351 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1352 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
1353 int i;
1354
1355 /* Lock the graphics update lock */
1356 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1357 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1358
1359 /* update the scanout addresses */
1360 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1361 upper_32_bits(crtc_base));
1362 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1363 (u32)crtc_base);
1364
1365 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1366 upper_32_bits(crtc_base));
1367 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1368 (u32)crtc_base);
1369
1370 /* Wait for update_pending to go high. */
1371 for (i = 0; i < rdev->usec_timeout; i++) {
1372 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1373 break;
1374 udelay(1);
1375 }
1376 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1377
1378 /* Unlock the lock, so double-buffering can take place inside vblank */
1379 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1380 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1381
1382 /* Return current update_pending status: */
1383 return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
1384 }
1385
1386 /* get temperature in millidegrees */
evergreen_get_temp(struct radeon_device * rdev)1387 int evergreen_get_temp(struct radeon_device *rdev)
1388 {
1389 u32 temp, toffset;
1390 int actual_temp = 0;
1391
1392 if (rdev->family == CHIP_JUNIPER) {
1393 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1394 TOFFSET_SHIFT;
1395 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1396 TS0_ADC_DOUT_SHIFT;
1397
1398 if (toffset & 0x100)
1399 actual_temp = temp / 2 - (0x200 - toffset);
1400 else
1401 actual_temp = temp / 2 + toffset;
1402
1403 actual_temp = actual_temp * 1000;
1404
1405 } else {
1406 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1407 ASIC_T_SHIFT;
1408
1409 if (temp & 0x400)
1410 actual_temp = -256;
1411 else if (temp & 0x200)
1412 actual_temp = 255;
1413 else if (temp & 0x100) {
1414 actual_temp = temp & 0x1ff;
1415 actual_temp |= ~0x1ff;
1416 } else
1417 actual_temp = temp & 0xff;
1418
1419 actual_temp = (actual_temp * 1000) / 2;
1420 }
1421
1422 return actual_temp;
1423 }
1424
sumo_get_temp(struct radeon_device * rdev)1425 int sumo_get_temp(struct radeon_device *rdev)
1426 {
1427 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1428 int actual_temp = temp - 49;
1429
1430 return actual_temp * 1000;
1431 }
1432
1433 /**
1434 * sumo_pm_init_profile - Initialize power profiles callback.
1435 *
1436 * @rdev: radeon_device pointer
1437 *
1438 * Initialize the power states used in profile mode
1439 * (sumo, trinity, SI).
1440 * Used for profile mode only.
1441 */
sumo_pm_init_profile(struct radeon_device * rdev)1442 void sumo_pm_init_profile(struct radeon_device *rdev)
1443 {
1444 int idx;
1445
1446 /* default */
1447 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1448 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1449 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1450 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1451
1452 /* low,mid sh/mh */
1453 if (rdev->flags & RADEON_IS_MOBILITY)
1454 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1455 else
1456 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1457
1458 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1459 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1460 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1461 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1462
1463 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1464 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1465 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1466 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1467
1468 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1469 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1470 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1471 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1472
1473 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1474 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1475 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1476 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1477
1478 /* high sh/mh */
1479 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1480 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1481 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1482 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1483 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1484 rdev->pm.power_state[idx].num_clock_modes - 1;
1485
1486 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1487 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1488 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1489 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1490 rdev->pm.power_state[idx].num_clock_modes - 1;
1491 }
1492
1493 /**
1494 * btc_pm_init_profile - Initialize power profiles callback.
1495 *
1496 * @rdev: radeon_device pointer
1497 *
1498 * Initialize the power states used in profile mode
1499 * (BTC, cayman).
1500 * Used for profile mode only.
1501 */
btc_pm_init_profile(struct radeon_device * rdev)1502 void btc_pm_init_profile(struct radeon_device *rdev)
1503 {
1504 int idx;
1505
1506 /* default */
1507 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1508 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1509 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1510 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1511 /* starting with BTC, there is one state that is used for both
1512 * MH and SH. Difference is that we always use the high clock index for
1513 * mclk.
1514 */
1515 if (rdev->flags & RADEON_IS_MOBILITY)
1516 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1517 else
1518 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1519 /* low sh */
1520 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1521 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1522 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1523 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1524 /* mid sh */
1525 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1526 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1527 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1528 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1529 /* high sh */
1530 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1531 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1532 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1533 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1534 /* low mh */
1535 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1536 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1537 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1538 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1539 /* mid mh */
1540 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1541 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1542 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1543 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1544 /* high mh */
1545 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1546 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1547 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1548 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1549 }
1550
1551 /**
1552 * evergreen_pm_misc - set additional pm hw parameters callback.
1553 *
1554 * @rdev: radeon_device pointer
1555 *
1556 * Set non-clock parameters associated with a power state
1557 * (voltage, etc.) (evergreen+).
1558 */
evergreen_pm_misc(struct radeon_device * rdev)1559 void evergreen_pm_misc(struct radeon_device *rdev)
1560 {
1561 int req_ps_idx = rdev->pm.requested_power_state_index;
1562 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1563 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1564 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1565
1566 if (voltage->type == VOLTAGE_SW) {
1567 /* 0xff0x are flags rather then an actual voltage */
1568 if ((voltage->voltage & 0xff00) == 0xff00)
1569 return;
1570 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1571 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1572 rdev->pm.current_vddc = voltage->voltage;
1573 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1574 }
1575
1576 /* starting with BTC, there is one state that is used for both
1577 * MH and SH. Difference is that we always use the high clock index for
1578 * mclk and vddci.
1579 */
1580 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1581 (rdev->family >= CHIP_BARTS) &&
1582 rdev->pm.active_crtc_count &&
1583 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1584 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1585 voltage = &rdev->pm.power_state[req_ps_idx].
1586 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1587
1588 /* 0xff0x are flags rather then an actual voltage */
1589 if ((voltage->vddci & 0xff00) == 0xff00)
1590 return;
1591 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1592 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1593 rdev->pm.current_vddci = voltage->vddci;
1594 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1595 }
1596 }
1597 }
1598
1599 /**
1600 * evergreen_pm_prepare - pre-power state change callback.
1601 *
1602 * @rdev: radeon_device pointer
1603 *
1604 * Prepare for a power state change (evergreen+).
1605 */
evergreen_pm_prepare(struct radeon_device * rdev)1606 void evergreen_pm_prepare(struct radeon_device *rdev)
1607 {
1608 struct drm_device *ddev = rdev->ddev;
1609 struct drm_crtc *crtc;
1610 struct radeon_crtc *radeon_crtc;
1611 u32 tmp;
1612
1613 /* disable any active CRTCs */
1614 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1615 radeon_crtc = to_radeon_crtc(crtc);
1616 if (radeon_crtc->enabled) {
1617 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1618 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1619 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1620 }
1621 }
1622 }
1623
1624 /**
1625 * evergreen_pm_finish - post-power state change callback.
1626 *
1627 * @rdev: radeon_device pointer
1628 *
1629 * Clean up after a power state change (evergreen+).
1630 */
evergreen_pm_finish(struct radeon_device * rdev)1631 void evergreen_pm_finish(struct radeon_device *rdev)
1632 {
1633 struct drm_device *ddev = rdev->ddev;
1634 struct drm_crtc *crtc;
1635 struct radeon_crtc *radeon_crtc;
1636 u32 tmp;
1637
1638 /* enable any active CRTCs */
1639 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1640 radeon_crtc = to_radeon_crtc(crtc);
1641 if (radeon_crtc->enabled) {
1642 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1643 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1644 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1645 }
1646 }
1647 }
1648
1649 /**
1650 * evergreen_hpd_sense - hpd sense callback.
1651 *
1652 * @rdev: radeon_device pointer
1653 * @hpd: hpd (hotplug detect) pin
1654 *
1655 * Checks if a digital monitor is connected (evergreen+).
1656 * Returns true if connected, false if not connected.
1657 */
evergreen_hpd_sense(struct radeon_device * rdev,enum radeon_hpd_id hpd)1658 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1659 {
1660 bool connected = false;
1661
1662 switch (hpd) {
1663 case RADEON_HPD_1:
1664 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1665 connected = true;
1666 break;
1667 case RADEON_HPD_2:
1668 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1669 connected = true;
1670 break;
1671 case RADEON_HPD_3:
1672 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1673 connected = true;
1674 break;
1675 case RADEON_HPD_4:
1676 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1677 connected = true;
1678 break;
1679 case RADEON_HPD_5:
1680 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1681 connected = true;
1682 break;
1683 case RADEON_HPD_6:
1684 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1685 connected = true;
1686 break;
1687 default:
1688 break;
1689 }
1690
1691 return connected;
1692 }
1693
1694 /**
1695 * evergreen_hpd_set_polarity - hpd set polarity callback.
1696 *
1697 * @rdev: radeon_device pointer
1698 * @hpd: hpd (hotplug detect) pin
1699 *
1700 * Set the polarity of the hpd pin (evergreen+).
1701 */
evergreen_hpd_set_polarity(struct radeon_device * rdev,enum radeon_hpd_id hpd)1702 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1703 enum radeon_hpd_id hpd)
1704 {
1705 u32 tmp;
1706 bool connected = evergreen_hpd_sense(rdev, hpd);
1707
1708 switch (hpd) {
1709 case RADEON_HPD_1:
1710 tmp = RREG32(DC_HPD1_INT_CONTROL);
1711 if (connected)
1712 tmp &= ~DC_HPDx_INT_POLARITY;
1713 else
1714 tmp |= DC_HPDx_INT_POLARITY;
1715 WREG32(DC_HPD1_INT_CONTROL, tmp);
1716 break;
1717 case RADEON_HPD_2:
1718 tmp = RREG32(DC_HPD2_INT_CONTROL);
1719 if (connected)
1720 tmp &= ~DC_HPDx_INT_POLARITY;
1721 else
1722 tmp |= DC_HPDx_INT_POLARITY;
1723 WREG32(DC_HPD2_INT_CONTROL, tmp);
1724 break;
1725 case RADEON_HPD_3:
1726 tmp = RREG32(DC_HPD3_INT_CONTROL);
1727 if (connected)
1728 tmp &= ~DC_HPDx_INT_POLARITY;
1729 else
1730 tmp |= DC_HPDx_INT_POLARITY;
1731 WREG32(DC_HPD3_INT_CONTROL, tmp);
1732 break;
1733 case RADEON_HPD_4:
1734 tmp = RREG32(DC_HPD4_INT_CONTROL);
1735 if (connected)
1736 tmp &= ~DC_HPDx_INT_POLARITY;
1737 else
1738 tmp |= DC_HPDx_INT_POLARITY;
1739 WREG32(DC_HPD4_INT_CONTROL, tmp);
1740 break;
1741 case RADEON_HPD_5:
1742 tmp = RREG32(DC_HPD5_INT_CONTROL);
1743 if (connected)
1744 tmp &= ~DC_HPDx_INT_POLARITY;
1745 else
1746 tmp |= DC_HPDx_INT_POLARITY;
1747 WREG32(DC_HPD5_INT_CONTROL, tmp);
1748 break;
1749 case RADEON_HPD_6:
1750 tmp = RREG32(DC_HPD6_INT_CONTROL);
1751 if (connected)
1752 tmp &= ~DC_HPDx_INT_POLARITY;
1753 else
1754 tmp |= DC_HPDx_INT_POLARITY;
1755 WREG32(DC_HPD6_INT_CONTROL, tmp);
1756 break;
1757 default:
1758 break;
1759 }
1760 }
1761
1762 /**
1763 * evergreen_hpd_init - hpd setup callback.
1764 *
1765 * @rdev: radeon_device pointer
1766 *
1767 * Setup the hpd pins used by the card (evergreen+).
1768 * Enable the pin, set the polarity, and enable the hpd interrupts.
1769 */
evergreen_hpd_init(struct radeon_device * rdev)1770 void evergreen_hpd_init(struct radeon_device *rdev)
1771 {
1772 struct drm_device *dev = rdev->ddev;
1773 struct drm_connector *connector;
1774 unsigned enabled = 0;
1775 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1776 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1777
1778 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1779 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1780
1781 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1782 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1783 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1784 * aux dp channel on imac and help (but not completely fix)
1785 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1786 * also avoid interrupt storms during dpms.
1787 */
1788 continue;
1789 }
1790 switch (radeon_connector->hpd.hpd) {
1791 case RADEON_HPD_1:
1792 WREG32(DC_HPD1_CONTROL, tmp);
1793 break;
1794 case RADEON_HPD_2:
1795 WREG32(DC_HPD2_CONTROL, tmp);
1796 break;
1797 case RADEON_HPD_3:
1798 WREG32(DC_HPD3_CONTROL, tmp);
1799 break;
1800 case RADEON_HPD_4:
1801 WREG32(DC_HPD4_CONTROL, tmp);
1802 break;
1803 case RADEON_HPD_5:
1804 WREG32(DC_HPD5_CONTROL, tmp);
1805 break;
1806 case RADEON_HPD_6:
1807 WREG32(DC_HPD6_CONTROL, tmp);
1808 break;
1809 default:
1810 break;
1811 }
1812 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
1813 enabled |= 1 << radeon_connector->hpd.hpd;
1814 }
1815 radeon_irq_kms_enable_hpd(rdev, enabled);
1816 }
1817
1818 /**
1819 * evergreen_hpd_fini - hpd tear down callback.
1820 *
1821 * @rdev: radeon_device pointer
1822 *
1823 * Tear down the hpd pins used by the card (evergreen+).
1824 * Disable the hpd interrupts.
1825 */
evergreen_hpd_fini(struct radeon_device * rdev)1826 void evergreen_hpd_fini(struct radeon_device *rdev)
1827 {
1828 struct drm_device *dev = rdev->ddev;
1829 struct drm_connector *connector;
1830 unsigned disabled = 0;
1831
1832 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1833 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1834 switch (radeon_connector->hpd.hpd) {
1835 case RADEON_HPD_1:
1836 WREG32(DC_HPD1_CONTROL, 0);
1837 break;
1838 case RADEON_HPD_2:
1839 WREG32(DC_HPD2_CONTROL, 0);
1840 break;
1841 case RADEON_HPD_3:
1842 WREG32(DC_HPD3_CONTROL, 0);
1843 break;
1844 case RADEON_HPD_4:
1845 WREG32(DC_HPD4_CONTROL, 0);
1846 break;
1847 case RADEON_HPD_5:
1848 WREG32(DC_HPD5_CONTROL, 0);
1849 break;
1850 case RADEON_HPD_6:
1851 WREG32(DC_HPD6_CONTROL, 0);
1852 break;
1853 default:
1854 break;
1855 }
1856 disabled |= 1 << radeon_connector->hpd.hpd;
1857 }
1858 radeon_irq_kms_disable_hpd(rdev, disabled);
1859 }
1860
1861 /* watermark setup */
1862
evergreen_line_buffer_adjust(struct radeon_device * rdev,struct radeon_crtc * radeon_crtc,struct drm_display_mode * mode,struct drm_display_mode * other_mode)1863 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1864 struct radeon_crtc *radeon_crtc,
1865 struct drm_display_mode *mode,
1866 struct drm_display_mode *other_mode)
1867 {
1868 u32 tmp, buffer_alloc, i;
1869 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1870 /*
1871 * Line Buffer Setup
1872 * There are 3 line buffers, each one shared by 2 display controllers.
1873 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1874 * the display controllers. The paritioning is done via one of four
1875 * preset allocations specified in bits 2:0:
1876 * first display controller
1877 * 0 - first half of lb (3840 * 2)
1878 * 1 - first 3/4 of lb (5760 * 2)
1879 * 2 - whole lb (7680 * 2), other crtc must be disabled
1880 * 3 - first 1/4 of lb (1920 * 2)
1881 * second display controller
1882 * 4 - second half of lb (3840 * 2)
1883 * 5 - second 3/4 of lb (5760 * 2)
1884 * 6 - whole lb (7680 * 2), other crtc must be disabled
1885 * 7 - last 1/4 of lb (1920 * 2)
1886 */
1887 /* this can get tricky if we have two large displays on a paired group
1888 * of crtcs. Ideally for multiple large displays we'd assign them to
1889 * non-linked crtcs for maximum line buffer allocation.
1890 */
1891 if (radeon_crtc->base.enabled && mode) {
1892 if (other_mode) {
1893 tmp = 0; /* 1/2 */
1894 buffer_alloc = 1;
1895 } else {
1896 tmp = 2; /* whole */
1897 buffer_alloc = 2;
1898 }
1899 } else {
1900 tmp = 0;
1901 buffer_alloc = 0;
1902 }
1903
1904 /* second controller of the pair uses second half of the lb */
1905 if (radeon_crtc->crtc_id % 2)
1906 tmp += 4;
1907 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1908
1909 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1910 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1911 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1912 for (i = 0; i < rdev->usec_timeout; i++) {
1913 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1914 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1915 break;
1916 udelay(1);
1917 }
1918 }
1919
1920 if (radeon_crtc->base.enabled && mode) {
1921 switch (tmp) {
1922 case 0:
1923 case 4:
1924 default:
1925 if (ASIC_IS_DCE5(rdev))
1926 return 4096 * 2;
1927 else
1928 return 3840 * 2;
1929 case 1:
1930 case 5:
1931 if (ASIC_IS_DCE5(rdev))
1932 return 6144 * 2;
1933 else
1934 return 5760 * 2;
1935 case 2:
1936 case 6:
1937 if (ASIC_IS_DCE5(rdev))
1938 return 8192 * 2;
1939 else
1940 return 7680 * 2;
1941 case 3:
1942 case 7:
1943 if (ASIC_IS_DCE5(rdev))
1944 return 2048 * 2;
1945 else
1946 return 1920 * 2;
1947 }
1948 }
1949
1950 /* controller not enabled, so no lb used */
1951 return 0;
1952 }
1953
evergreen_get_number_of_dram_channels(struct radeon_device * rdev)1954 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1955 {
1956 u32 tmp = RREG32(MC_SHARED_CHMAP);
1957
1958 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1959 case 0:
1960 default:
1961 return 1;
1962 case 1:
1963 return 2;
1964 case 2:
1965 return 4;
1966 case 3:
1967 return 8;
1968 }
1969 }
1970
1971 struct evergreen_wm_params {
1972 u32 dram_channels; /* number of dram channels */
1973 u32 yclk; /* bandwidth per dram data pin in kHz */
1974 u32 sclk; /* engine clock in kHz */
1975 u32 disp_clk; /* display clock in kHz */
1976 u32 src_width; /* viewport width */
1977 u32 active_time; /* active display time in ns */
1978 u32 blank_time; /* blank time in ns */
1979 bool interlaced; /* mode is interlaced */
1980 fixed20_12 vsc; /* vertical scale ratio */
1981 u32 num_heads; /* number of active crtcs */
1982 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1983 u32 lb_size; /* line buffer allocated to pipe */
1984 u32 vtaps; /* vertical scaler taps */
1985 };
1986
evergreen_dram_bandwidth(struct evergreen_wm_params * wm)1987 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1988 {
1989 /* Calculate DRAM Bandwidth and the part allocated to display. */
1990 fixed20_12 dram_efficiency; /* 0.7 */
1991 fixed20_12 yclk, dram_channels, bandwidth;
1992 fixed20_12 a;
1993
1994 a.full = dfixed_const(1000);
1995 yclk.full = dfixed_const(wm->yclk);
1996 yclk.full = dfixed_div(yclk, a);
1997 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1998 a.full = dfixed_const(10);
1999 dram_efficiency.full = dfixed_const(7);
2000 dram_efficiency.full = dfixed_div(dram_efficiency, a);
2001 bandwidth.full = dfixed_mul(dram_channels, yclk);
2002 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
2003
2004 return dfixed_trunc(bandwidth);
2005 }
2006
evergreen_dram_bandwidth_for_display(struct evergreen_wm_params * wm)2007 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2008 {
2009 /* Calculate DRAM Bandwidth and the part allocated to display. */
2010 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2011 fixed20_12 yclk, dram_channels, bandwidth;
2012 fixed20_12 a;
2013
2014 a.full = dfixed_const(1000);
2015 yclk.full = dfixed_const(wm->yclk);
2016 yclk.full = dfixed_div(yclk, a);
2017 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2018 a.full = dfixed_const(10);
2019 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2020 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2021 bandwidth.full = dfixed_mul(dram_channels, yclk);
2022 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2023
2024 return dfixed_trunc(bandwidth);
2025 }
2026
evergreen_data_return_bandwidth(struct evergreen_wm_params * wm)2027 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
2028 {
2029 /* Calculate the display Data return Bandwidth */
2030 fixed20_12 return_efficiency; /* 0.8 */
2031 fixed20_12 sclk, bandwidth;
2032 fixed20_12 a;
2033
2034 a.full = dfixed_const(1000);
2035 sclk.full = dfixed_const(wm->sclk);
2036 sclk.full = dfixed_div(sclk, a);
2037 a.full = dfixed_const(10);
2038 return_efficiency.full = dfixed_const(8);
2039 return_efficiency.full = dfixed_div(return_efficiency, a);
2040 a.full = dfixed_const(32);
2041 bandwidth.full = dfixed_mul(a, sclk);
2042 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2043
2044 return dfixed_trunc(bandwidth);
2045 }
2046
evergreen_dmif_request_bandwidth(struct evergreen_wm_params * wm)2047 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2048 {
2049 /* Calculate the DMIF Request Bandwidth */
2050 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2051 fixed20_12 disp_clk, bandwidth;
2052 fixed20_12 a;
2053
2054 a.full = dfixed_const(1000);
2055 disp_clk.full = dfixed_const(wm->disp_clk);
2056 disp_clk.full = dfixed_div(disp_clk, a);
2057 a.full = dfixed_const(10);
2058 disp_clk_request_efficiency.full = dfixed_const(8);
2059 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2060 a.full = dfixed_const(32);
2061 bandwidth.full = dfixed_mul(a, disp_clk);
2062 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2063
2064 return dfixed_trunc(bandwidth);
2065 }
2066
evergreen_available_bandwidth(struct evergreen_wm_params * wm)2067 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2068 {
2069 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2070 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2071 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2072 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2073
2074 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2075 }
2076
evergreen_average_bandwidth(struct evergreen_wm_params * wm)2077 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2078 {
2079 /* Calculate the display mode Average Bandwidth
2080 * DisplayMode should contain the source and destination dimensions,
2081 * timing, etc.
2082 */
2083 fixed20_12 bpp;
2084 fixed20_12 line_time;
2085 fixed20_12 src_width;
2086 fixed20_12 bandwidth;
2087 fixed20_12 a;
2088
2089 a.full = dfixed_const(1000);
2090 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2091 line_time.full = dfixed_div(line_time, a);
2092 bpp.full = dfixed_const(wm->bytes_per_pixel);
2093 src_width.full = dfixed_const(wm->src_width);
2094 bandwidth.full = dfixed_mul(src_width, bpp);
2095 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2096 bandwidth.full = dfixed_div(bandwidth, line_time);
2097
2098 return dfixed_trunc(bandwidth);
2099 }
2100
evergreen_latency_watermark(struct evergreen_wm_params * wm)2101 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2102 {
2103 /* First calcualte the latency in ns */
2104 u32 mc_latency = 2000; /* 2000 ns. */
2105 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2106 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2107 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2108 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2109 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2110 (wm->num_heads * cursor_line_pair_return_time);
2111 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2112 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2113 fixed20_12 a, b, c;
2114
2115 if (wm->num_heads == 0)
2116 return 0;
2117
2118 a.full = dfixed_const(2);
2119 b.full = dfixed_const(1);
2120 if ((wm->vsc.full > a.full) ||
2121 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2122 (wm->vtaps >= 5) ||
2123 ((wm->vsc.full >= a.full) && wm->interlaced))
2124 max_src_lines_per_dst_line = 4;
2125 else
2126 max_src_lines_per_dst_line = 2;
2127
2128 a.full = dfixed_const(available_bandwidth);
2129 b.full = dfixed_const(wm->num_heads);
2130 a.full = dfixed_div(a, b);
2131
2132 b.full = dfixed_const(1000);
2133 c.full = dfixed_const(wm->disp_clk);
2134 b.full = dfixed_div(c, b);
2135 c.full = dfixed_const(wm->bytes_per_pixel);
2136 b.full = dfixed_mul(b, c);
2137
2138 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2139
2140 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2141 b.full = dfixed_const(1000);
2142 c.full = dfixed_const(lb_fill_bw);
2143 b.full = dfixed_div(c, b);
2144 a.full = dfixed_div(a, b);
2145 line_fill_time = dfixed_trunc(a);
2146
2147 if (line_fill_time < wm->active_time)
2148 return latency;
2149 else
2150 return latency + (line_fill_time - wm->active_time);
2151
2152 }
2153
evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params * wm)2154 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2155 {
2156 if (evergreen_average_bandwidth(wm) <=
2157 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2158 return true;
2159 else
2160 return false;
2161 };
2162
evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params * wm)2163 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2164 {
2165 if (evergreen_average_bandwidth(wm) <=
2166 (evergreen_available_bandwidth(wm) / wm->num_heads))
2167 return true;
2168 else
2169 return false;
2170 };
2171
evergreen_check_latency_hiding(struct evergreen_wm_params * wm)2172 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2173 {
2174 u32 lb_partitions = wm->lb_size / wm->src_width;
2175 u32 line_time = wm->active_time + wm->blank_time;
2176 u32 latency_tolerant_lines;
2177 u32 latency_hiding;
2178 fixed20_12 a;
2179
2180 a.full = dfixed_const(1);
2181 if (wm->vsc.full > a.full)
2182 latency_tolerant_lines = 1;
2183 else {
2184 if (lb_partitions <= (wm->vtaps + 1))
2185 latency_tolerant_lines = 1;
2186 else
2187 latency_tolerant_lines = 2;
2188 }
2189
2190 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2191
2192 if (evergreen_latency_watermark(wm) <= latency_hiding)
2193 return true;
2194 else
2195 return false;
2196 }
2197
evergreen_program_watermarks(struct radeon_device * rdev,struct radeon_crtc * radeon_crtc,u32 lb_size,u32 num_heads)2198 static void evergreen_program_watermarks(struct radeon_device *rdev,
2199 struct radeon_crtc *radeon_crtc,
2200 u32 lb_size, u32 num_heads)
2201 {
2202 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2203 struct evergreen_wm_params wm_low, wm_high;
2204 u32 dram_channels;
2205 u32 pixel_period;
2206 u32 line_time = 0;
2207 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2208 u32 priority_a_mark = 0, priority_b_mark = 0;
2209 u32 priority_a_cnt = PRIORITY_OFF;
2210 u32 priority_b_cnt = PRIORITY_OFF;
2211 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2212 u32 tmp, arb_control3;
2213 fixed20_12 a, b, c;
2214
2215 if (radeon_crtc->base.enabled && num_heads && mode) {
2216 pixel_period = 1000000 / (u32)mode->clock;
2217 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2218 priority_a_cnt = 0;
2219 priority_b_cnt = 0;
2220 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2221
2222 /* watermark for high clocks */
2223 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2224 wm_high.yclk =
2225 radeon_dpm_get_mclk(rdev, false) * 10;
2226 wm_high.sclk =
2227 radeon_dpm_get_sclk(rdev, false) * 10;
2228 } else {
2229 wm_high.yclk = rdev->pm.current_mclk * 10;
2230 wm_high.sclk = rdev->pm.current_sclk * 10;
2231 }
2232
2233 wm_high.disp_clk = mode->clock;
2234 wm_high.src_width = mode->crtc_hdisplay;
2235 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2236 wm_high.blank_time = line_time - wm_high.active_time;
2237 wm_high.interlaced = false;
2238 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2239 wm_high.interlaced = true;
2240 wm_high.vsc = radeon_crtc->vsc;
2241 wm_high.vtaps = 1;
2242 if (radeon_crtc->rmx_type != RMX_OFF)
2243 wm_high.vtaps = 2;
2244 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2245 wm_high.lb_size = lb_size;
2246 wm_high.dram_channels = dram_channels;
2247 wm_high.num_heads = num_heads;
2248
2249 /* watermark for low clocks */
2250 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2251 wm_low.yclk =
2252 radeon_dpm_get_mclk(rdev, true) * 10;
2253 wm_low.sclk =
2254 radeon_dpm_get_sclk(rdev, true) * 10;
2255 } else {
2256 wm_low.yclk = rdev->pm.current_mclk * 10;
2257 wm_low.sclk = rdev->pm.current_sclk * 10;
2258 }
2259
2260 wm_low.disp_clk = mode->clock;
2261 wm_low.src_width = mode->crtc_hdisplay;
2262 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2263 wm_low.blank_time = line_time - wm_low.active_time;
2264 wm_low.interlaced = false;
2265 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2266 wm_low.interlaced = true;
2267 wm_low.vsc = radeon_crtc->vsc;
2268 wm_low.vtaps = 1;
2269 if (radeon_crtc->rmx_type != RMX_OFF)
2270 wm_low.vtaps = 2;
2271 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2272 wm_low.lb_size = lb_size;
2273 wm_low.dram_channels = dram_channels;
2274 wm_low.num_heads = num_heads;
2275
2276 /* set for high clocks */
2277 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2278 /* set for low clocks */
2279 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2280
2281 /* possibly force display priority to high */
2282 /* should really do this at mode validation time... */
2283 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2284 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2285 !evergreen_check_latency_hiding(&wm_high) ||
2286 (rdev->disp_priority == 2)) {
2287 DRM_DEBUG_KMS("force priority a to high\n");
2288 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2289 }
2290 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2291 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2292 !evergreen_check_latency_hiding(&wm_low) ||
2293 (rdev->disp_priority == 2)) {
2294 DRM_DEBUG_KMS("force priority b to high\n");
2295 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2296 }
2297
2298 a.full = dfixed_const(1000);
2299 b.full = dfixed_const(mode->clock);
2300 b.full = dfixed_div(b, a);
2301 c.full = dfixed_const(latency_watermark_a);
2302 c.full = dfixed_mul(c, b);
2303 c.full = dfixed_mul(c, radeon_crtc->hsc);
2304 c.full = dfixed_div(c, a);
2305 a.full = dfixed_const(16);
2306 c.full = dfixed_div(c, a);
2307 priority_a_mark = dfixed_trunc(c);
2308 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2309
2310 a.full = dfixed_const(1000);
2311 b.full = dfixed_const(mode->clock);
2312 b.full = dfixed_div(b, a);
2313 c.full = dfixed_const(latency_watermark_b);
2314 c.full = dfixed_mul(c, b);
2315 c.full = dfixed_mul(c, radeon_crtc->hsc);
2316 c.full = dfixed_div(c, a);
2317 a.full = dfixed_const(16);
2318 c.full = dfixed_div(c, a);
2319 priority_b_mark = dfixed_trunc(c);
2320 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2321 }
2322
2323 /* select wm A */
2324 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2325 tmp = arb_control3;
2326 tmp &= ~LATENCY_WATERMARK_MASK(3);
2327 tmp |= LATENCY_WATERMARK_MASK(1);
2328 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2329 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2330 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2331 LATENCY_HIGH_WATERMARK(line_time)));
2332 /* select wm B */
2333 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2334 tmp &= ~LATENCY_WATERMARK_MASK(3);
2335 tmp |= LATENCY_WATERMARK_MASK(2);
2336 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2337 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2338 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2339 LATENCY_HIGH_WATERMARK(line_time)));
2340 /* restore original selection */
2341 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2342
2343 /* write the priority marks */
2344 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2345 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2346
2347 /* save values for DPM */
2348 radeon_crtc->line_time = line_time;
2349 radeon_crtc->wm_high = latency_watermark_a;
2350 radeon_crtc->wm_low = latency_watermark_b;
2351 }
2352
2353 /**
2354 * evergreen_bandwidth_update - update display watermarks callback.
2355 *
2356 * @rdev: radeon_device pointer
2357 *
2358 * Update the display watermarks based on the requested mode(s)
2359 * (evergreen+).
2360 */
evergreen_bandwidth_update(struct radeon_device * rdev)2361 void evergreen_bandwidth_update(struct radeon_device *rdev)
2362 {
2363 struct drm_display_mode *mode0 = NULL;
2364 struct drm_display_mode *mode1 = NULL;
2365 u32 num_heads = 0, lb_size;
2366 int i;
2367
2368 radeon_update_display_priority(rdev);
2369
2370 for (i = 0; i < rdev->num_crtc; i++) {
2371 if (rdev->mode_info.crtcs[i]->base.enabled)
2372 num_heads++;
2373 }
2374 for (i = 0; i < rdev->num_crtc; i += 2) {
2375 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2376 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2377 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2378 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2379 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2380 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2381 }
2382 }
2383
2384 /**
2385 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2386 *
2387 * @rdev: radeon_device pointer
2388 *
2389 * Wait for the MC (memory controller) to be idle.
2390 * (evergreen+).
2391 * Returns 0 if the MC is idle, -1 if not.
2392 */
evergreen_mc_wait_for_idle(struct radeon_device * rdev)2393 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2394 {
2395 unsigned i;
2396 u32 tmp;
2397
2398 for (i = 0; i < rdev->usec_timeout; i++) {
2399 /* read MC_STATUS */
2400 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2401 if (!tmp)
2402 return 0;
2403 udelay(1);
2404 }
2405 return -1;
2406 }
2407
2408 /*
2409 * GART
2410 */
evergreen_pcie_gart_tlb_flush(struct radeon_device * rdev)2411 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2412 {
2413 unsigned i;
2414 u32 tmp;
2415
2416 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2417
2418 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2419 for (i = 0; i < rdev->usec_timeout; i++) {
2420 /* read MC_STATUS */
2421 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2422 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2423 if (tmp == 2) {
2424 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2425 return;
2426 }
2427 if (tmp) {
2428 return;
2429 }
2430 udelay(1);
2431 }
2432 }
2433
evergreen_pcie_gart_enable(struct radeon_device * rdev)2434 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2435 {
2436 u32 tmp;
2437 int r;
2438
2439 if (rdev->gart.robj == NULL) {
2440 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2441 return -EINVAL;
2442 }
2443 r = radeon_gart_table_vram_pin(rdev);
2444 if (r)
2445 return r;
2446 radeon_gart_restore(rdev);
2447 /* Setup L2 cache */
2448 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2449 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2450 EFFECTIVE_L2_QUEUE_SIZE(7));
2451 WREG32(VM_L2_CNTL2, 0);
2452 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2453 /* Setup TLB control */
2454 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2455 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2456 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2457 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2458 if (rdev->flags & RADEON_IS_IGP) {
2459 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2460 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2461 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2462 } else {
2463 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2464 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2465 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2466 if ((rdev->family == CHIP_JUNIPER) ||
2467 (rdev->family == CHIP_CYPRESS) ||
2468 (rdev->family == CHIP_HEMLOCK) ||
2469 (rdev->family == CHIP_BARTS))
2470 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2471 }
2472 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2473 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2474 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2475 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2476 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2477 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2478 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2479 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2480 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2481 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2482 (u32)(rdev->dummy_page.addr >> 12));
2483 WREG32(VM_CONTEXT1_CNTL, 0);
2484
2485 evergreen_pcie_gart_tlb_flush(rdev);
2486 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2487 (unsigned)(rdev->mc.gtt_size >> 20),
2488 (unsigned long long)rdev->gart.table_addr);
2489 rdev->gart.ready = true;
2490 return 0;
2491 }
2492
evergreen_pcie_gart_disable(struct radeon_device * rdev)2493 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2494 {
2495 u32 tmp;
2496
2497 /* Disable all tables */
2498 WREG32(VM_CONTEXT0_CNTL, 0);
2499 WREG32(VM_CONTEXT1_CNTL, 0);
2500
2501 /* Setup L2 cache */
2502 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2503 EFFECTIVE_L2_QUEUE_SIZE(7));
2504 WREG32(VM_L2_CNTL2, 0);
2505 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2506 /* Setup TLB control */
2507 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2508 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2509 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2510 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2511 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2512 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2513 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2514 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2515 radeon_gart_table_vram_unpin(rdev);
2516 }
2517
evergreen_pcie_gart_fini(struct radeon_device * rdev)2518 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2519 {
2520 evergreen_pcie_gart_disable(rdev);
2521 radeon_gart_table_vram_free(rdev);
2522 radeon_gart_fini(rdev);
2523 }
2524
2525
evergreen_agp_enable(struct radeon_device * rdev)2526 static void evergreen_agp_enable(struct radeon_device *rdev)
2527 {
2528 u32 tmp;
2529
2530 /* Setup L2 cache */
2531 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2532 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2533 EFFECTIVE_L2_QUEUE_SIZE(7));
2534 WREG32(VM_L2_CNTL2, 0);
2535 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2536 /* Setup TLB control */
2537 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2538 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2539 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2540 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2541 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2542 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2543 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2544 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2545 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2546 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2547 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2548 WREG32(VM_CONTEXT0_CNTL, 0);
2549 WREG32(VM_CONTEXT1_CNTL, 0);
2550 }
2551
evergreen_mc_stop(struct radeon_device * rdev,struct evergreen_mc_save * save)2552 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2553 {
2554 u32 crtc_enabled, tmp, frame_count, blackout;
2555 int i, j;
2556
2557 if (!ASIC_IS_NODCE(rdev)) {
2558 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2559 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2560
2561 /* disable VGA render */
2562 WREG32(VGA_RENDER_CONTROL, 0);
2563 }
2564 /* blank the display controllers */
2565 for (i = 0; i < rdev->num_crtc; i++) {
2566 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2567 if (crtc_enabled) {
2568 save->crtc_enabled[i] = true;
2569 if (ASIC_IS_DCE6(rdev)) {
2570 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2571 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2572 radeon_wait_for_vblank(rdev, i);
2573 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2574 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2575 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2576 }
2577 } else {
2578 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2579 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2580 radeon_wait_for_vblank(rdev, i);
2581 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2582 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2583 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2584 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2585 }
2586 }
2587 /* wait for the next frame */
2588 frame_count = radeon_get_vblank_counter(rdev, i);
2589 for (j = 0; j < rdev->usec_timeout; j++) {
2590 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2591 break;
2592 udelay(1);
2593 }
2594
2595 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2596 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2597 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2598 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2599 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2600 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2601 save->crtc_enabled[i] = false;
2602 /* ***** */
2603 } else {
2604 save->crtc_enabled[i] = false;
2605 }
2606 }
2607
2608 radeon_mc_wait_for_idle(rdev);
2609
2610 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2611 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2612 /* Block CPU access */
2613 WREG32(BIF_FB_EN, 0);
2614 /* blackout the MC */
2615 blackout &= ~BLACKOUT_MODE_MASK;
2616 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2617 }
2618 /* wait for the MC to settle */
2619 udelay(100);
2620
2621 /* lock double buffered regs */
2622 for (i = 0; i < rdev->num_crtc; i++) {
2623 if (save->crtc_enabled[i]) {
2624 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2625 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2626 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2627 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2628 }
2629 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2630 if (!(tmp & 1)) {
2631 tmp |= 1;
2632 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2633 }
2634 }
2635 }
2636 }
2637
evergreen_mc_resume(struct radeon_device * rdev,struct evergreen_mc_save * save)2638 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2639 {
2640 u32 tmp, frame_count;
2641 int i, j;
2642
2643 /* update crtc base addresses */
2644 for (i = 0; i < rdev->num_crtc; i++) {
2645 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2646 upper_32_bits(rdev->mc.vram_start));
2647 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2648 upper_32_bits(rdev->mc.vram_start));
2649 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2650 (u32)rdev->mc.vram_start);
2651 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2652 (u32)rdev->mc.vram_start);
2653 }
2654
2655 if (!ASIC_IS_NODCE(rdev)) {
2656 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2657 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2658 }
2659
2660 /* unlock regs and wait for update */
2661 for (i = 0; i < rdev->num_crtc; i++) {
2662 if (save->crtc_enabled[i]) {
2663 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2664 if ((tmp & 0x3) != 0) {
2665 tmp &= ~0x3;
2666 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2667 }
2668 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2669 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2670 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2671 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2672 }
2673 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2674 if (tmp & 1) {
2675 tmp &= ~1;
2676 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2677 }
2678 for (j = 0; j < rdev->usec_timeout; j++) {
2679 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2680 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2681 break;
2682 udelay(1);
2683 }
2684 }
2685 }
2686
2687 /* unblackout the MC */
2688 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2689 tmp &= ~BLACKOUT_MODE_MASK;
2690 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2691 /* allow CPU access */
2692 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2693
2694 for (i = 0; i < rdev->num_crtc; i++) {
2695 if (save->crtc_enabled[i]) {
2696 if (ASIC_IS_DCE6(rdev)) {
2697 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2698 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2699 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2700 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2701 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2702 } else {
2703 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2704 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2705 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2706 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2707 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2708 }
2709 /* wait for the next frame */
2710 frame_count = radeon_get_vblank_counter(rdev, i);
2711 for (j = 0; j < rdev->usec_timeout; j++) {
2712 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2713 break;
2714 udelay(1);
2715 }
2716 }
2717 }
2718 if (!ASIC_IS_NODCE(rdev)) {
2719 /* Unlock vga access */
2720 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2721 mdelay(1);
2722 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2723 }
2724 }
2725
evergreen_mc_program(struct radeon_device * rdev)2726 void evergreen_mc_program(struct radeon_device *rdev)
2727 {
2728 struct evergreen_mc_save save;
2729 u32 tmp;
2730 int i, j;
2731
2732 /* Initialize HDP */
2733 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2734 WREG32((0x2c14 + j), 0x00000000);
2735 WREG32((0x2c18 + j), 0x00000000);
2736 WREG32((0x2c1c + j), 0x00000000);
2737 WREG32((0x2c20 + j), 0x00000000);
2738 WREG32((0x2c24 + j), 0x00000000);
2739 }
2740 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2741
2742 evergreen_mc_stop(rdev, &save);
2743 if (evergreen_mc_wait_for_idle(rdev)) {
2744 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2745 }
2746 /* Lockout access through VGA aperture*/
2747 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2748 /* Update configuration */
2749 if (rdev->flags & RADEON_IS_AGP) {
2750 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2751 /* VRAM before AGP */
2752 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2753 rdev->mc.vram_start >> 12);
2754 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2755 rdev->mc.gtt_end >> 12);
2756 } else {
2757 /* VRAM after AGP */
2758 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2759 rdev->mc.gtt_start >> 12);
2760 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2761 rdev->mc.vram_end >> 12);
2762 }
2763 } else {
2764 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2765 rdev->mc.vram_start >> 12);
2766 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2767 rdev->mc.vram_end >> 12);
2768 }
2769 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2770 /* llano/ontario only */
2771 if ((rdev->family == CHIP_PALM) ||
2772 (rdev->family == CHIP_SUMO) ||
2773 (rdev->family == CHIP_SUMO2)) {
2774 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2775 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2776 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2777 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2778 }
2779 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2780 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2781 WREG32(MC_VM_FB_LOCATION, tmp);
2782 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2783 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2784 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2785 if (rdev->flags & RADEON_IS_AGP) {
2786 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2787 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2788 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2789 } else {
2790 WREG32(MC_VM_AGP_BASE, 0);
2791 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2792 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2793 }
2794 if (evergreen_mc_wait_for_idle(rdev)) {
2795 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2796 }
2797 evergreen_mc_resume(rdev, &save);
2798 /* we need to own VRAM, so turn off the VGA renderer here
2799 * to stop it overwriting our objects */
2800 rv515_vga_render_disable(rdev);
2801 }
2802
2803 /*
2804 * CP.
2805 */
evergreen_ring_ib_execute(struct radeon_device * rdev,struct radeon_ib * ib)2806 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2807 {
2808 struct radeon_ring *ring = &rdev->ring[ib->ring];
2809 u32 next_rptr;
2810
2811 /* set to DX10/11 mode */
2812 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2813 radeon_ring_write(ring, 1);
2814
2815 if (ring->rptr_save_reg) {
2816 next_rptr = ring->wptr + 3 + 4;
2817 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2818 radeon_ring_write(ring, ((ring->rptr_save_reg -
2819 PACKET3_SET_CONFIG_REG_START) >> 2));
2820 radeon_ring_write(ring, next_rptr);
2821 } else if (rdev->wb.enabled) {
2822 next_rptr = ring->wptr + 5 + 4;
2823 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2824 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2825 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2826 radeon_ring_write(ring, next_rptr);
2827 radeon_ring_write(ring, 0);
2828 }
2829
2830 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2831 radeon_ring_write(ring,
2832 #ifdef __BIG_ENDIAN
2833 (2 << 0) |
2834 #endif
2835 (ib->gpu_addr & 0xFFFFFFFC));
2836 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2837 radeon_ring_write(ring, ib->length_dw);
2838 }
2839
2840
evergreen_cp_load_microcode(struct radeon_device * rdev)2841 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2842 {
2843 const __be32 *fw_data;
2844 int i;
2845
2846 if (!rdev->me_fw || !rdev->pfp_fw)
2847 return -EINVAL;
2848
2849 r700_cp_stop(rdev);
2850 WREG32(CP_RB_CNTL,
2851 #ifdef __BIG_ENDIAN
2852 BUF_SWAP_32BIT |
2853 #endif
2854 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2855
2856 fw_data = (const __be32 *)rdev->pfp_fw->data;
2857 WREG32(CP_PFP_UCODE_ADDR, 0);
2858 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2859 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2860 WREG32(CP_PFP_UCODE_ADDR, 0);
2861
2862 fw_data = (const __be32 *)rdev->me_fw->data;
2863 WREG32(CP_ME_RAM_WADDR, 0);
2864 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2865 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2866
2867 WREG32(CP_PFP_UCODE_ADDR, 0);
2868 WREG32(CP_ME_RAM_WADDR, 0);
2869 WREG32(CP_ME_RAM_RADDR, 0);
2870 return 0;
2871 }
2872
evergreen_cp_start(struct radeon_device * rdev)2873 static int evergreen_cp_start(struct radeon_device *rdev)
2874 {
2875 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2876 int r, i;
2877 uint32_t cp_me;
2878
2879 r = radeon_ring_lock(rdev, ring, 7);
2880 if (r) {
2881 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2882 return r;
2883 }
2884 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2885 radeon_ring_write(ring, 0x1);
2886 radeon_ring_write(ring, 0x0);
2887 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2888 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2889 radeon_ring_write(ring, 0);
2890 radeon_ring_write(ring, 0);
2891 radeon_ring_unlock_commit(rdev, ring);
2892
2893 cp_me = 0xff;
2894 WREG32(CP_ME_CNTL, cp_me);
2895
2896 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
2897 if (r) {
2898 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2899 return r;
2900 }
2901
2902 /* setup clear context state */
2903 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2904 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2905
2906 for (i = 0; i < evergreen_default_size; i++)
2907 radeon_ring_write(ring, evergreen_default_state[i]);
2908
2909 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2910 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2911
2912 /* set clear context state */
2913 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2914 radeon_ring_write(ring, 0);
2915
2916 /* SQ_VTX_BASE_VTX_LOC */
2917 radeon_ring_write(ring, 0xc0026f00);
2918 radeon_ring_write(ring, 0x00000000);
2919 radeon_ring_write(ring, 0x00000000);
2920 radeon_ring_write(ring, 0x00000000);
2921
2922 /* Clear consts */
2923 radeon_ring_write(ring, 0xc0036f00);
2924 radeon_ring_write(ring, 0x00000bc4);
2925 radeon_ring_write(ring, 0xffffffff);
2926 radeon_ring_write(ring, 0xffffffff);
2927 radeon_ring_write(ring, 0xffffffff);
2928
2929 radeon_ring_write(ring, 0xc0026900);
2930 radeon_ring_write(ring, 0x00000316);
2931 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2932 radeon_ring_write(ring, 0x00000010); /* */
2933
2934 radeon_ring_unlock_commit(rdev, ring);
2935
2936 return 0;
2937 }
2938
evergreen_cp_resume(struct radeon_device * rdev)2939 static int evergreen_cp_resume(struct radeon_device *rdev)
2940 {
2941 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2942 u32 tmp;
2943 u32 rb_bufsz;
2944 int r;
2945
2946 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2947 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2948 SOFT_RESET_PA |
2949 SOFT_RESET_SH |
2950 SOFT_RESET_VGT |
2951 SOFT_RESET_SPI |
2952 SOFT_RESET_SX));
2953 RREG32(GRBM_SOFT_RESET);
2954 mdelay(15);
2955 WREG32(GRBM_SOFT_RESET, 0);
2956 RREG32(GRBM_SOFT_RESET);
2957
2958 /* Set ring buffer size */
2959 rb_bufsz = order_base_2(ring->ring_size / 8);
2960 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2961 #ifdef __BIG_ENDIAN
2962 tmp |= BUF_SWAP_32BIT;
2963 #endif
2964 WREG32(CP_RB_CNTL, tmp);
2965 WREG32(CP_SEM_WAIT_TIMER, 0x0);
2966 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
2967
2968 /* Set the write pointer delay */
2969 WREG32(CP_RB_WPTR_DELAY, 0);
2970
2971 /* Initialize the ring buffer's read and write pointers */
2972 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2973 WREG32(CP_RB_RPTR_WR, 0);
2974 ring->wptr = 0;
2975 WREG32(CP_RB_WPTR, ring->wptr);
2976
2977 /* set the wb address whether it's enabled or not */
2978 WREG32(CP_RB_RPTR_ADDR,
2979 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
2980 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2981 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2982
2983 if (rdev->wb.enabled)
2984 WREG32(SCRATCH_UMSK, 0xff);
2985 else {
2986 tmp |= RB_NO_UPDATE;
2987 WREG32(SCRATCH_UMSK, 0);
2988 }
2989
2990 mdelay(1);
2991 WREG32(CP_RB_CNTL, tmp);
2992
2993 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
2994 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2995
2996 evergreen_cp_start(rdev);
2997 ring->ready = true;
2998 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
2999 if (r) {
3000 ring->ready = false;
3001 return r;
3002 }
3003 return 0;
3004 }
3005
3006 /*
3007 * Core functions
3008 */
evergreen_gpu_init(struct radeon_device * rdev)3009 static void evergreen_gpu_init(struct radeon_device *rdev)
3010 {
3011 u32 gb_addr_config;
3012 u32 mc_shared_chmap __unused, mc_arb_ramcfg;
3013 u32 sx_debug_1;
3014 u32 smx_dc_ctl0;
3015 u32 sq_config;
3016 u32 sq_lds_resource_mgmt;
3017 u32 sq_gpr_resource_mgmt_1;
3018 u32 sq_gpr_resource_mgmt_2;
3019 u32 sq_gpr_resource_mgmt_3;
3020 u32 sq_thread_resource_mgmt;
3021 u32 sq_thread_resource_mgmt_2;
3022 u32 sq_stack_resource_mgmt_1;
3023 u32 sq_stack_resource_mgmt_2;
3024 u32 sq_stack_resource_mgmt_3;
3025 u32 vgt_cache_invalidation;
3026 u32 hdp_host_path_cntl, tmp;
3027 u32 disabled_rb_mask;
3028 int i, j, ps_thread_count;
3029
3030 switch (rdev->family) {
3031 case CHIP_CYPRESS:
3032 case CHIP_HEMLOCK:
3033 rdev->config.evergreen.num_ses = 2;
3034 rdev->config.evergreen.max_pipes = 4;
3035 rdev->config.evergreen.max_tile_pipes = 8;
3036 rdev->config.evergreen.max_simds = 10;
3037 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3038 rdev->config.evergreen.max_gprs = 256;
3039 rdev->config.evergreen.max_threads = 248;
3040 rdev->config.evergreen.max_gs_threads = 32;
3041 rdev->config.evergreen.max_stack_entries = 512;
3042 rdev->config.evergreen.sx_num_of_sets = 4;
3043 rdev->config.evergreen.sx_max_export_size = 256;
3044 rdev->config.evergreen.sx_max_export_pos_size = 64;
3045 rdev->config.evergreen.sx_max_export_smx_size = 192;
3046 rdev->config.evergreen.max_hw_contexts = 8;
3047 rdev->config.evergreen.sq_num_cf_insts = 2;
3048
3049 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3050 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3051 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3052 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3053 break;
3054 case CHIP_JUNIPER:
3055 rdev->config.evergreen.num_ses = 1;
3056 rdev->config.evergreen.max_pipes = 4;
3057 rdev->config.evergreen.max_tile_pipes = 4;
3058 rdev->config.evergreen.max_simds = 10;
3059 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3060 rdev->config.evergreen.max_gprs = 256;
3061 rdev->config.evergreen.max_threads = 248;
3062 rdev->config.evergreen.max_gs_threads = 32;
3063 rdev->config.evergreen.max_stack_entries = 512;
3064 rdev->config.evergreen.sx_num_of_sets = 4;
3065 rdev->config.evergreen.sx_max_export_size = 256;
3066 rdev->config.evergreen.sx_max_export_pos_size = 64;
3067 rdev->config.evergreen.sx_max_export_smx_size = 192;
3068 rdev->config.evergreen.max_hw_contexts = 8;
3069 rdev->config.evergreen.sq_num_cf_insts = 2;
3070
3071 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3072 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3073 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3074 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3075 break;
3076 case CHIP_REDWOOD:
3077 rdev->config.evergreen.num_ses = 1;
3078 rdev->config.evergreen.max_pipes = 4;
3079 rdev->config.evergreen.max_tile_pipes = 4;
3080 rdev->config.evergreen.max_simds = 5;
3081 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3082 rdev->config.evergreen.max_gprs = 256;
3083 rdev->config.evergreen.max_threads = 248;
3084 rdev->config.evergreen.max_gs_threads = 32;
3085 rdev->config.evergreen.max_stack_entries = 256;
3086 rdev->config.evergreen.sx_num_of_sets = 4;
3087 rdev->config.evergreen.sx_max_export_size = 256;
3088 rdev->config.evergreen.sx_max_export_pos_size = 64;
3089 rdev->config.evergreen.sx_max_export_smx_size = 192;
3090 rdev->config.evergreen.max_hw_contexts = 8;
3091 rdev->config.evergreen.sq_num_cf_insts = 2;
3092
3093 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3094 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3095 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3096 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3097 break;
3098 case CHIP_CEDAR:
3099 default:
3100 rdev->config.evergreen.num_ses = 1;
3101 rdev->config.evergreen.max_pipes = 2;
3102 rdev->config.evergreen.max_tile_pipes = 2;
3103 rdev->config.evergreen.max_simds = 2;
3104 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3105 rdev->config.evergreen.max_gprs = 256;
3106 rdev->config.evergreen.max_threads = 192;
3107 rdev->config.evergreen.max_gs_threads = 16;
3108 rdev->config.evergreen.max_stack_entries = 256;
3109 rdev->config.evergreen.sx_num_of_sets = 4;
3110 rdev->config.evergreen.sx_max_export_size = 128;
3111 rdev->config.evergreen.sx_max_export_pos_size = 32;
3112 rdev->config.evergreen.sx_max_export_smx_size = 96;
3113 rdev->config.evergreen.max_hw_contexts = 4;
3114 rdev->config.evergreen.sq_num_cf_insts = 1;
3115
3116 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3117 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3118 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3119 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3120 break;
3121 case CHIP_PALM:
3122 rdev->config.evergreen.num_ses = 1;
3123 rdev->config.evergreen.max_pipes = 2;
3124 rdev->config.evergreen.max_tile_pipes = 2;
3125 rdev->config.evergreen.max_simds = 2;
3126 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3127 rdev->config.evergreen.max_gprs = 256;
3128 rdev->config.evergreen.max_threads = 192;
3129 rdev->config.evergreen.max_gs_threads = 16;
3130 rdev->config.evergreen.max_stack_entries = 256;
3131 rdev->config.evergreen.sx_num_of_sets = 4;
3132 rdev->config.evergreen.sx_max_export_size = 128;
3133 rdev->config.evergreen.sx_max_export_pos_size = 32;
3134 rdev->config.evergreen.sx_max_export_smx_size = 96;
3135 rdev->config.evergreen.max_hw_contexts = 4;
3136 rdev->config.evergreen.sq_num_cf_insts = 1;
3137
3138 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3139 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3140 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3141 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3142 break;
3143 case CHIP_SUMO:
3144 rdev->config.evergreen.num_ses = 1;
3145 rdev->config.evergreen.max_pipes = 4;
3146 rdev->config.evergreen.max_tile_pipes = 4;
3147 if (rdev->pdev->device == 0x9648)
3148 rdev->config.evergreen.max_simds = 3;
3149 else if ((rdev->pdev->device == 0x9647) ||
3150 (rdev->pdev->device == 0x964a))
3151 rdev->config.evergreen.max_simds = 4;
3152 else
3153 rdev->config.evergreen.max_simds = 5;
3154 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3155 rdev->config.evergreen.max_gprs = 256;
3156 rdev->config.evergreen.max_threads = 248;
3157 rdev->config.evergreen.max_gs_threads = 32;
3158 rdev->config.evergreen.max_stack_entries = 256;
3159 rdev->config.evergreen.sx_num_of_sets = 4;
3160 rdev->config.evergreen.sx_max_export_size = 256;
3161 rdev->config.evergreen.sx_max_export_pos_size = 64;
3162 rdev->config.evergreen.sx_max_export_smx_size = 192;
3163 rdev->config.evergreen.max_hw_contexts = 8;
3164 rdev->config.evergreen.sq_num_cf_insts = 2;
3165
3166 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3167 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3168 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3169 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3170 break;
3171 case CHIP_SUMO2:
3172 rdev->config.evergreen.num_ses = 1;
3173 rdev->config.evergreen.max_pipes = 4;
3174 rdev->config.evergreen.max_tile_pipes = 4;
3175 rdev->config.evergreen.max_simds = 2;
3176 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3177 rdev->config.evergreen.max_gprs = 256;
3178 rdev->config.evergreen.max_threads = 248;
3179 rdev->config.evergreen.max_gs_threads = 32;
3180 rdev->config.evergreen.max_stack_entries = 512;
3181 rdev->config.evergreen.sx_num_of_sets = 4;
3182 rdev->config.evergreen.sx_max_export_size = 256;
3183 rdev->config.evergreen.sx_max_export_pos_size = 64;
3184 rdev->config.evergreen.sx_max_export_smx_size = 192;
3185 rdev->config.evergreen.max_hw_contexts = 4;
3186 rdev->config.evergreen.sq_num_cf_insts = 2;
3187
3188 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3189 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3190 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3191 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3192 break;
3193 case CHIP_BARTS:
3194 rdev->config.evergreen.num_ses = 2;
3195 rdev->config.evergreen.max_pipes = 4;
3196 rdev->config.evergreen.max_tile_pipes = 8;
3197 rdev->config.evergreen.max_simds = 7;
3198 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3199 rdev->config.evergreen.max_gprs = 256;
3200 rdev->config.evergreen.max_threads = 248;
3201 rdev->config.evergreen.max_gs_threads = 32;
3202 rdev->config.evergreen.max_stack_entries = 512;
3203 rdev->config.evergreen.sx_num_of_sets = 4;
3204 rdev->config.evergreen.sx_max_export_size = 256;
3205 rdev->config.evergreen.sx_max_export_pos_size = 64;
3206 rdev->config.evergreen.sx_max_export_smx_size = 192;
3207 rdev->config.evergreen.max_hw_contexts = 8;
3208 rdev->config.evergreen.sq_num_cf_insts = 2;
3209
3210 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3211 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3212 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3213 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3214 break;
3215 case CHIP_TURKS:
3216 rdev->config.evergreen.num_ses = 1;
3217 rdev->config.evergreen.max_pipes = 4;
3218 rdev->config.evergreen.max_tile_pipes = 4;
3219 rdev->config.evergreen.max_simds = 6;
3220 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3221 rdev->config.evergreen.max_gprs = 256;
3222 rdev->config.evergreen.max_threads = 248;
3223 rdev->config.evergreen.max_gs_threads = 32;
3224 rdev->config.evergreen.max_stack_entries = 256;
3225 rdev->config.evergreen.sx_num_of_sets = 4;
3226 rdev->config.evergreen.sx_max_export_size = 256;
3227 rdev->config.evergreen.sx_max_export_pos_size = 64;
3228 rdev->config.evergreen.sx_max_export_smx_size = 192;
3229 rdev->config.evergreen.max_hw_contexts = 8;
3230 rdev->config.evergreen.sq_num_cf_insts = 2;
3231
3232 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3233 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3234 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3235 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3236 break;
3237 case CHIP_CAICOS:
3238 rdev->config.evergreen.num_ses = 1;
3239 rdev->config.evergreen.max_pipes = 2;
3240 rdev->config.evergreen.max_tile_pipes = 2;
3241 rdev->config.evergreen.max_simds = 2;
3242 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3243 rdev->config.evergreen.max_gprs = 256;
3244 rdev->config.evergreen.max_threads = 192;
3245 rdev->config.evergreen.max_gs_threads = 16;
3246 rdev->config.evergreen.max_stack_entries = 256;
3247 rdev->config.evergreen.sx_num_of_sets = 4;
3248 rdev->config.evergreen.sx_max_export_size = 128;
3249 rdev->config.evergreen.sx_max_export_pos_size = 32;
3250 rdev->config.evergreen.sx_max_export_smx_size = 96;
3251 rdev->config.evergreen.max_hw_contexts = 4;
3252 rdev->config.evergreen.sq_num_cf_insts = 1;
3253
3254 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3255 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3256 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3257 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3258 break;
3259 }
3260
3261 /* Initialize HDP */
3262 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3263 WREG32((0x2c14 + j), 0x00000000);
3264 WREG32((0x2c18 + j), 0x00000000);
3265 WREG32((0x2c1c + j), 0x00000000);
3266 WREG32((0x2c20 + j), 0x00000000);
3267 WREG32((0x2c24 + j), 0x00000000);
3268 }
3269
3270 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3271
3272 evergreen_fix_pci_max_read_req_size(rdev);
3273
3274 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3275 if ((rdev->family == CHIP_PALM) ||
3276 (rdev->family == CHIP_SUMO) ||
3277 (rdev->family == CHIP_SUMO2))
3278 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3279 else
3280 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3281
3282 /* setup tiling info dword. gb_addr_config is not adequate since it does
3283 * not have bank info, so create a custom tiling dword.
3284 * bits 3:0 num_pipes
3285 * bits 7:4 num_banks
3286 * bits 11:8 group_size
3287 * bits 15:12 row_size
3288 */
3289 rdev->config.evergreen.tile_config = 0;
3290 switch (rdev->config.evergreen.max_tile_pipes) {
3291 case 1:
3292 default:
3293 rdev->config.evergreen.tile_config |= (0 << 0);
3294 break;
3295 case 2:
3296 rdev->config.evergreen.tile_config |= (1 << 0);
3297 break;
3298 case 4:
3299 rdev->config.evergreen.tile_config |= (2 << 0);
3300 break;
3301 case 8:
3302 rdev->config.evergreen.tile_config |= (3 << 0);
3303 break;
3304 }
3305 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3306 if (rdev->flags & RADEON_IS_IGP)
3307 rdev->config.evergreen.tile_config |= 1 << 4;
3308 else {
3309 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3310 case 0: /* four banks */
3311 rdev->config.evergreen.tile_config |= 0 << 4;
3312 break;
3313 case 1: /* eight banks */
3314 rdev->config.evergreen.tile_config |= 1 << 4;
3315 break;
3316 case 2: /* sixteen banks */
3317 default:
3318 rdev->config.evergreen.tile_config |= 2 << 4;
3319 break;
3320 }
3321 }
3322 rdev->config.evergreen.tile_config |= 0 << 8;
3323 rdev->config.evergreen.tile_config |=
3324 ((gb_addr_config & 0x30000000) >> 28) << 12;
3325
3326 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3327 u32 efuse_straps_4;
3328 u32 efuse_straps_3;
3329
3330 efuse_straps_4 = RREG32_RCU(0x204);
3331 efuse_straps_3 = RREG32_RCU(0x203);
3332 tmp = (((efuse_straps_4 & 0xf) << 4) |
3333 ((efuse_straps_3 & 0xf0000000) >> 28));
3334 } else {
3335 tmp = 0;
3336 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3337 u32 rb_disable_bitmap;
3338
3339 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3340 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3341 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3342 tmp <<= 4;
3343 tmp |= rb_disable_bitmap;
3344 }
3345 }
3346 /* enabled rb are just the one not disabled :) */
3347 disabled_rb_mask = tmp;
3348 tmp = 0;
3349 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3350 tmp |= (1 << i);
3351 /* if all the backends are disabled, fix it up here */
3352 if ((disabled_rb_mask & tmp) == tmp) {
3353 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3354 disabled_rb_mask &= ~(1 << i);
3355 }
3356
3357 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3358 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3359
3360 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3361 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3362 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3363 WREG32(DMA_TILING_CONFIG, gb_addr_config);
3364 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3365 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3366 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3367
3368 if ((rdev->config.evergreen.max_backends == 1) &&
3369 (rdev->flags & RADEON_IS_IGP)) {
3370 if ((disabled_rb_mask & 3) == 1) {
3371 /* RB0 disabled, RB1 enabled */
3372 tmp = 0x11111111;
3373 } else {
3374 /* RB1 disabled, RB0 enabled */
3375 tmp = 0x00000000;
3376 }
3377 } else {
3378 tmp = gb_addr_config & NUM_PIPES_MASK;
3379 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3380 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3381 }
3382 WREG32(GB_BACKEND_MAP, tmp);
3383
3384 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3385 WREG32(CGTS_TCC_DISABLE, 0);
3386 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3387 WREG32(CGTS_USER_TCC_DISABLE, 0);
3388
3389 /* set HW defaults for 3D engine */
3390 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3391 ROQ_IB2_START(0x2b)));
3392
3393 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3394
3395 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3396 SYNC_GRADIENT |
3397 SYNC_WALKER |
3398 SYNC_ALIGNER));
3399
3400 sx_debug_1 = RREG32(SX_DEBUG_1);
3401 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3402 WREG32(SX_DEBUG_1, sx_debug_1);
3403
3404
3405 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3406 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3407 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3408 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3409
3410 if (rdev->family <= CHIP_SUMO2)
3411 WREG32(SMX_SAR_CTL0, 0x00010000);
3412
3413 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3414 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3415 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3416
3417 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3418 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3419 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3420
3421 WREG32(VGT_NUM_INSTANCES, 1);
3422 WREG32(SPI_CONFIG_CNTL, 0);
3423 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3424 WREG32(CP_PERFMON_CNTL, 0);
3425
3426 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3427 FETCH_FIFO_HIWATER(0x4) |
3428 DONE_FIFO_HIWATER(0xe0) |
3429 ALU_UPDATE_FIFO_HIWATER(0x8)));
3430
3431 sq_config = RREG32(SQ_CONFIG);
3432 sq_config &= ~(PS_PRIO(3) |
3433 VS_PRIO(3) |
3434 GS_PRIO(3) |
3435 ES_PRIO(3));
3436 sq_config |= (VC_ENABLE |
3437 EXPORT_SRC_C |
3438 PS_PRIO(0) |
3439 VS_PRIO(1) |
3440 GS_PRIO(2) |
3441 ES_PRIO(3));
3442
3443 switch (rdev->family) {
3444 case CHIP_CEDAR:
3445 case CHIP_PALM:
3446 case CHIP_SUMO:
3447 case CHIP_SUMO2:
3448 case CHIP_CAICOS:
3449 /* no vertex cache */
3450 sq_config &= ~VC_ENABLE;
3451 break;
3452 default:
3453 break;
3454 }
3455
3456 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3457
3458 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3459 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3460 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3461 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3462 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3463 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3464 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3465
3466 switch (rdev->family) {
3467 case CHIP_CEDAR:
3468 case CHIP_PALM:
3469 case CHIP_SUMO:
3470 case CHIP_SUMO2:
3471 ps_thread_count = 96;
3472 break;
3473 default:
3474 ps_thread_count = 128;
3475 break;
3476 }
3477
3478 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3479 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3480 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3481 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3482 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3483 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3484
3485 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3486 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3487 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3488 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3489 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3490 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3491
3492 WREG32(SQ_CONFIG, sq_config);
3493 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3494 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3495 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3496 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3497 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3498 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3499 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3500 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3501 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3502 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3503
3504 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3505 FORCE_EOV_MAX_REZ_CNT(255)));
3506
3507 switch (rdev->family) {
3508 case CHIP_CEDAR:
3509 case CHIP_PALM:
3510 case CHIP_SUMO:
3511 case CHIP_SUMO2:
3512 case CHIP_CAICOS:
3513 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3514 break;
3515 default:
3516 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3517 break;
3518 }
3519 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3520 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3521
3522 WREG32(VGT_GS_VERTEX_REUSE, 16);
3523 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3524 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3525
3526 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3527 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3528
3529 WREG32(CB_PERF_CTR0_SEL_0, 0);
3530 WREG32(CB_PERF_CTR0_SEL_1, 0);
3531 WREG32(CB_PERF_CTR1_SEL_0, 0);
3532 WREG32(CB_PERF_CTR1_SEL_1, 0);
3533 WREG32(CB_PERF_CTR2_SEL_0, 0);
3534 WREG32(CB_PERF_CTR2_SEL_1, 0);
3535 WREG32(CB_PERF_CTR3_SEL_0, 0);
3536 WREG32(CB_PERF_CTR3_SEL_1, 0);
3537
3538 /* clear render buffer base addresses */
3539 WREG32(CB_COLOR0_BASE, 0);
3540 WREG32(CB_COLOR1_BASE, 0);
3541 WREG32(CB_COLOR2_BASE, 0);
3542 WREG32(CB_COLOR3_BASE, 0);
3543 WREG32(CB_COLOR4_BASE, 0);
3544 WREG32(CB_COLOR5_BASE, 0);
3545 WREG32(CB_COLOR6_BASE, 0);
3546 WREG32(CB_COLOR7_BASE, 0);
3547 WREG32(CB_COLOR8_BASE, 0);
3548 WREG32(CB_COLOR9_BASE, 0);
3549 WREG32(CB_COLOR10_BASE, 0);
3550 WREG32(CB_COLOR11_BASE, 0);
3551
3552 /* set the shader const cache sizes to 0 */
3553 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3554 WREG32(i, 0);
3555 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3556 WREG32(i, 0);
3557
3558 tmp = RREG32(HDP_MISC_CNTL);
3559 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3560 WREG32(HDP_MISC_CNTL, tmp);
3561
3562 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3563 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3564
3565 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3566
3567 udelay(50);
3568
3569 }
3570
evergreen_mc_init(struct radeon_device * rdev)3571 int evergreen_mc_init(struct radeon_device *rdev)
3572 {
3573 u32 tmp;
3574 int chansize, numchan;
3575
3576 /* Get VRAM informations */
3577 rdev->mc.vram_is_ddr = true;
3578 if ((rdev->family == CHIP_PALM) ||
3579 (rdev->family == CHIP_SUMO) ||
3580 (rdev->family == CHIP_SUMO2))
3581 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3582 else
3583 tmp = RREG32(MC_ARB_RAMCFG);
3584 if (tmp & CHANSIZE_OVERRIDE) {
3585 chansize = 16;
3586 } else if (tmp & CHANSIZE_MASK) {
3587 chansize = 64;
3588 } else {
3589 chansize = 32;
3590 }
3591 tmp = RREG32(MC_SHARED_CHMAP);
3592 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3593 case 0:
3594 default:
3595 numchan = 1;
3596 break;
3597 case 1:
3598 numchan = 2;
3599 break;
3600 case 2:
3601 numchan = 4;
3602 break;
3603 case 3:
3604 numchan = 8;
3605 break;
3606 }
3607 rdev->mc.vram_width = numchan * chansize;
3608 /* Could aper size report 0 ? */
3609 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3610 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3611 /* Setup GPU memory space */
3612 if ((rdev->family == CHIP_PALM) ||
3613 (rdev->family == CHIP_SUMO) ||
3614 (rdev->family == CHIP_SUMO2)) {
3615 /* size in bytes on fusion */
3616 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3617 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3618 } else {
3619 /* size in MB on evergreen/cayman/tn */
3620 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3621 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3622 }
3623 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3624 r700_vram_gtt_location(rdev, &rdev->mc);
3625 radeon_update_bandwidth_info(rdev);
3626
3627 return 0;
3628 }
3629
evergreen_print_gpu_status_regs(struct radeon_device * rdev)3630 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3631 {
3632 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
3633 RREG32(GRBM_STATUS));
3634 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
3635 RREG32(GRBM_STATUS_SE0));
3636 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
3637 RREG32(GRBM_STATUS_SE1));
3638 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
3639 RREG32(SRBM_STATUS));
3640 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3641 RREG32(SRBM_STATUS2));
3642 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3643 RREG32(CP_STALLED_STAT1));
3644 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3645 RREG32(CP_STALLED_STAT2));
3646 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3647 RREG32(CP_BUSY_STAT));
3648 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3649 RREG32(CP_STAT));
3650 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3651 RREG32(DMA_STATUS_REG));
3652 if (rdev->family >= CHIP_CAYMAN) {
3653 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3654 RREG32(DMA_STATUS_REG + 0x800));
3655 }
3656 }
3657
evergreen_is_display_hung(struct radeon_device * rdev)3658 bool evergreen_is_display_hung(struct radeon_device *rdev)
3659 {
3660 u32 crtc_hung = 0;
3661 u32 crtc_status[6];
3662 u32 i, j, tmp;
3663
3664 for (i = 0; i < rdev->num_crtc; i++) {
3665 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3666 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3667 crtc_hung |= (1 << i);
3668 }
3669 }
3670
3671 for (j = 0; j < 10; j++) {
3672 for (i = 0; i < rdev->num_crtc; i++) {
3673 if (crtc_hung & (1 << i)) {
3674 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3675 if (tmp != crtc_status[i])
3676 crtc_hung &= ~(1 << i);
3677 }
3678 }
3679 if (crtc_hung == 0)
3680 return false;
3681 udelay(100);
3682 }
3683
3684 return true;
3685 }
3686
evergreen_gpu_check_soft_reset(struct radeon_device * rdev)3687 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3688 {
3689 u32 reset_mask = 0;
3690 u32 tmp;
3691
3692 /* GRBM_STATUS */
3693 tmp = RREG32(GRBM_STATUS);
3694 if (tmp & (PA_BUSY | SC_BUSY |
3695 SH_BUSY | SX_BUSY |
3696 TA_BUSY | VGT_BUSY |
3697 DB_BUSY | CB_BUSY |
3698 SPI_BUSY | VGT_BUSY_NO_DMA))
3699 reset_mask |= RADEON_RESET_GFX;
3700
3701 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3702 CP_BUSY | CP_COHERENCY_BUSY))
3703 reset_mask |= RADEON_RESET_CP;
3704
3705 if (tmp & GRBM_EE_BUSY)
3706 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3707
3708 /* DMA_STATUS_REG */
3709 tmp = RREG32(DMA_STATUS_REG);
3710 if (!(tmp & DMA_IDLE))
3711 reset_mask |= RADEON_RESET_DMA;
3712
3713 /* SRBM_STATUS2 */
3714 tmp = RREG32(SRBM_STATUS2);
3715 if (tmp & DMA_BUSY)
3716 reset_mask |= RADEON_RESET_DMA;
3717
3718 /* SRBM_STATUS */
3719 tmp = RREG32(SRBM_STATUS);
3720 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3721 reset_mask |= RADEON_RESET_RLC;
3722
3723 if (tmp & IH_BUSY)
3724 reset_mask |= RADEON_RESET_IH;
3725
3726 if (tmp & SEM_BUSY)
3727 reset_mask |= RADEON_RESET_SEM;
3728
3729 if (tmp & GRBM_RQ_PENDING)
3730 reset_mask |= RADEON_RESET_GRBM;
3731
3732 if (tmp & VMC_BUSY)
3733 reset_mask |= RADEON_RESET_VMC;
3734
3735 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3736 MCC_BUSY | MCD_BUSY))
3737 reset_mask |= RADEON_RESET_MC;
3738
3739 if (evergreen_is_display_hung(rdev))
3740 reset_mask |= RADEON_RESET_DISPLAY;
3741
3742 /* VM_L2_STATUS */
3743 tmp = RREG32(VM_L2_STATUS);
3744 if (tmp & L2_BUSY)
3745 reset_mask |= RADEON_RESET_VMC;
3746
3747 /* Skip MC reset as it's mostly likely not hung, just busy */
3748 if (reset_mask & RADEON_RESET_MC) {
3749 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3750 reset_mask &= ~RADEON_RESET_MC;
3751 }
3752
3753 return reset_mask;
3754 }
3755
evergreen_gpu_soft_reset(struct radeon_device * rdev,u32 reset_mask)3756 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3757 {
3758 struct evergreen_mc_save save;
3759 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3760 u32 tmp;
3761
3762 if (reset_mask == 0)
3763 return;
3764
3765 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3766
3767 evergreen_print_gpu_status_regs(rdev);
3768
3769 /* Disable CP parsing/prefetching */
3770 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3771
3772 if (reset_mask & RADEON_RESET_DMA) {
3773 /* Disable DMA */
3774 tmp = RREG32(DMA_RB_CNTL);
3775 tmp &= ~DMA_RB_ENABLE;
3776 WREG32(DMA_RB_CNTL, tmp);
3777 }
3778
3779 udelay(50);
3780
3781 evergreen_mc_stop(rdev, &save);
3782 if (evergreen_mc_wait_for_idle(rdev)) {
3783 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3784 }
3785
3786 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3787 grbm_soft_reset |= SOFT_RESET_DB |
3788 SOFT_RESET_CB |
3789 SOFT_RESET_PA |
3790 SOFT_RESET_SC |
3791 SOFT_RESET_SPI |
3792 SOFT_RESET_SX |
3793 SOFT_RESET_SH |
3794 SOFT_RESET_TC |
3795 SOFT_RESET_TA |
3796 SOFT_RESET_VC |
3797 SOFT_RESET_VGT;
3798 }
3799
3800 if (reset_mask & RADEON_RESET_CP) {
3801 grbm_soft_reset |= SOFT_RESET_CP |
3802 SOFT_RESET_VGT;
3803
3804 srbm_soft_reset |= SOFT_RESET_GRBM;
3805 }
3806
3807 if (reset_mask & RADEON_RESET_DMA)
3808 srbm_soft_reset |= SOFT_RESET_DMA;
3809
3810 if (reset_mask & RADEON_RESET_DISPLAY)
3811 srbm_soft_reset |= SOFT_RESET_DC;
3812
3813 if (reset_mask & RADEON_RESET_RLC)
3814 srbm_soft_reset |= SOFT_RESET_RLC;
3815
3816 if (reset_mask & RADEON_RESET_SEM)
3817 srbm_soft_reset |= SOFT_RESET_SEM;
3818
3819 if (reset_mask & RADEON_RESET_IH)
3820 srbm_soft_reset |= SOFT_RESET_IH;
3821
3822 if (reset_mask & RADEON_RESET_GRBM)
3823 srbm_soft_reset |= SOFT_RESET_GRBM;
3824
3825 if (reset_mask & RADEON_RESET_VMC)
3826 srbm_soft_reset |= SOFT_RESET_VMC;
3827
3828 if (!(rdev->flags & RADEON_IS_IGP)) {
3829 if (reset_mask & RADEON_RESET_MC)
3830 srbm_soft_reset |= SOFT_RESET_MC;
3831 }
3832
3833 if (grbm_soft_reset) {
3834 tmp = RREG32(GRBM_SOFT_RESET);
3835 tmp |= grbm_soft_reset;
3836 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3837 WREG32(GRBM_SOFT_RESET, tmp);
3838 tmp = RREG32(GRBM_SOFT_RESET);
3839
3840 udelay(50);
3841
3842 tmp &= ~grbm_soft_reset;
3843 WREG32(GRBM_SOFT_RESET, tmp);
3844 tmp = RREG32(GRBM_SOFT_RESET);
3845 }
3846
3847 if (srbm_soft_reset) {
3848 tmp = RREG32(SRBM_SOFT_RESET);
3849 tmp |= srbm_soft_reset;
3850 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3851 WREG32(SRBM_SOFT_RESET, tmp);
3852 tmp = RREG32(SRBM_SOFT_RESET);
3853
3854 udelay(50);
3855
3856 tmp &= ~srbm_soft_reset;
3857 WREG32(SRBM_SOFT_RESET, tmp);
3858 tmp = RREG32(SRBM_SOFT_RESET);
3859 }
3860
3861 /* Wait a little for things to settle down */
3862 udelay(50);
3863
3864 evergreen_mc_resume(rdev, &save);
3865 udelay(50);
3866
3867 evergreen_print_gpu_status_regs(rdev);
3868 }
3869
evergreen_gpu_pci_config_reset(struct radeon_device * rdev)3870 void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
3871 {
3872 struct evergreen_mc_save save;
3873 u32 tmp, i;
3874
3875 dev_info(rdev->dev, "GPU pci config reset\n");
3876
3877 /* disable dpm? */
3878
3879 /* Disable CP parsing/prefetching */
3880 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3881 udelay(50);
3882 /* Disable DMA */
3883 tmp = RREG32(DMA_RB_CNTL);
3884 tmp &= ~DMA_RB_ENABLE;
3885 WREG32(DMA_RB_CNTL, tmp);
3886 /* XXX other engines? */
3887
3888 /* halt the rlc */
3889 r600_rlc_stop(rdev);
3890
3891 udelay(50);
3892
3893 /* set mclk/sclk to bypass */
3894 rv770_set_clk_bypass_mode(rdev);
3895 /* disable BM */
3896 pci_clear_master(rdev->pdev);
3897 /* disable mem access */
3898 evergreen_mc_stop(rdev, &save);
3899 if (evergreen_mc_wait_for_idle(rdev)) {
3900 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
3901 }
3902 /* reset */
3903 radeon_pci_config_reset(rdev);
3904 /* wait for asic to come out of reset */
3905 for (i = 0; i < rdev->usec_timeout; i++) {
3906 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
3907 break;
3908 udelay(1);
3909 }
3910 }
3911
evergreen_asic_reset(struct radeon_device * rdev)3912 int evergreen_asic_reset(struct radeon_device *rdev)
3913 {
3914 u32 reset_mask;
3915
3916 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3917
3918 if (reset_mask)
3919 r600_set_bios_scratch_engine_hung(rdev, true);
3920
3921 /* try soft reset */
3922 evergreen_gpu_soft_reset(rdev, reset_mask);
3923
3924 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3925
3926 /* try pci config reset */
3927 if (reset_mask && radeon_hard_reset)
3928 evergreen_gpu_pci_config_reset(rdev);
3929
3930 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3931
3932 if (!reset_mask)
3933 r600_set_bios_scratch_engine_hung(rdev, false);
3934
3935 return 0;
3936 }
3937
3938 /**
3939 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3940 *
3941 * @rdev: radeon_device pointer
3942 * @ring: radeon_ring structure holding ring information
3943 *
3944 * Check if the GFX engine is locked up.
3945 * Returns true if the engine appears to be locked up, false if not.
3946 */
evergreen_gfx_is_lockup(struct radeon_device * rdev,struct radeon_ring * ring)3947 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3948 {
3949 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3950
3951 if (!(reset_mask & (RADEON_RESET_GFX |
3952 RADEON_RESET_COMPUTE |
3953 RADEON_RESET_CP))) {
3954 radeon_ring_lockup_update(rdev, ring);
3955 return false;
3956 }
3957 return radeon_ring_test_lockup(rdev, ring);
3958 }
3959
3960 /*
3961 * RLC
3962 */
3963 #define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
3964 #define RLC_CLEAR_STATE_END_MARKER 0x00000001
3965
sumo_rlc_fini(struct radeon_device * rdev)3966 void sumo_rlc_fini(struct radeon_device *rdev)
3967 {
3968 int r;
3969
3970 /* save restore block */
3971 if (rdev->rlc.save_restore_obj) {
3972 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3973 if (unlikely(r != 0))
3974 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
3975 radeon_bo_unpin(rdev->rlc.save_restore_obj);
3976 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3977
3978 radeon_bo_unref(&rdev->rlc.save_restore_obj);
3979 rdev->rlc.save_restore_obj = NULL;
3980 }
3981
3982 /* clear state block */
3983 if (rdev->rlc.clear_state_obj) {
3984 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3985 if (unlikely(r != 0))
3986 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
3987 radeon_bo_unpin(rdev->rlc.clear_state_obj);
3988 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3989
3990 radeon_bo_unref(&rdev->rlc.clear_state_obj);
3991 rdev->rlc.clear_state_obj = NULL;
3992 }
3993
3994 /* clear state block */
3995 if (rdev->rlc.cp_table_obj) {
3996 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
3997 if (unlikely(r != 0))
3998 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
3999 radeon_bo_unpin(rdev->rlc.cp_table_obj);
4000 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4001
4002 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4003 rdev->rlc.cp_table_obj = NULL;
4004 }
4005 }
4006
4007 #define CP_ME_TABLE_SIZE 96
4008
sumo_rlc_init(struct radeon_device * rdev)4009 int sumo_rlc_init(struct radeon_device *rdev)
4010 {
4011 const u32 *src_ptr;
4012 volatile u32 *dst_ptr;
4013 u32 dws, data, i, j, k, reg_num;
4014 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4015 u64 reg_list_mc_addr;
4016 const struct cs_section_def *cs_data;
4017 int r;
4018
4019 src_ptr = rdev->rlc.reg_list;
4020 dws = rdev->rlc.reg_list_size;
4021 if (rdev->family >= CHIP_BONAIRE) {
4022 dws += (5 * 16) + 48 + 48 + 64;
4023 }
4024 cs_data = rdev->rlc.cs_data;
4025
4026 if (src_ptr) {
4027 /* save restore block */
4028 if (rdev->rlc.save_restore_obj == NULL) {
4029 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4030 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.save_restore_obj);
4031 if (r) {
4032 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4033 return r;
4034 }
4035 }
4036
4037 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4038 if (unlikely(r != 0)) {
4039 sumo_rlc_fini(rdev);
4040 return r;
4041 }
4042 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4043 &rdev->rlc.save_restore_gpu_addr);
4044 if (r) {
4045 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4046 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4047 sumo_rlc_fini(rdev);
4048 return r;
4049 }
4050
4051 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)__UNVOLATILE(&rdev->rlc.sr_ptr));
4052 if (r) {
4053 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4054 sumo_rlc_fini(rdev);
4055 return r;
4056 }
4057 /* write the sr buffer */
4058 dst_ptr = rdev->rlc.sr_ptr;
4059 if (rdev->family >= CHIP_TAHITI) {
4060 /* SI */
4061 for (i = 0; i < rdev->rlc.reg_list_size; i++)
4062 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4063 } else {
4064 /* ON/LN/TN */
4065 /* format:
4066 * dw0: (reg2 << 16) | reg1
4067 * dw1: reg1 save space
4068 * dw2: reg2 save space
4069 */
4070 for (i = 0; i < dws; i++) {
4071 data = src_ptr[i] >> 2;
4072 i++;
4073 if (i < dws)
4074 data |= (src_ptr[i] >> 2) << 16;
4075 j = (((i - 1) * 3) / 2);
4076 dst_ptr[j] = cpu_to_le32(data);
4077 }
4078 j = ((i * 3) / 2);
4079 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4080 }
4081 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4082 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4083 }
4084
4085 if (cs_data) {
4086 /* clear state block */
4087 if (rdev->family >= CHIP_BONAIRE) {
4088 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4089 } else if (rdev->family >= CHIP_TAHITI) {
4090 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4091 dws = rdev->rlc.clear_state_size + (256 / 4);
4092 } else {
4093 reg_list_num = 0;
4094 dws = 0;
4095 for (i = 0; cs_data[i].section != NULL; i++) {
4096 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4097 reg_list_num++;
4098 dws += cs_data[i].section[j].reg_count;
4099 }
4100 }
4101 reg_list_blk_index = (3 * reg_list_num + 2);
4102 dws += reg_list_blk_index;
4103 rdev->rlc.clear_state_size = dws;
4104 }
4105
4106 if (rdev->rlc.clear_state_obj == NULL) {
4107 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4108 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj);
4109 if (r) {
4110 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4111 sumo_rlc_fini(rdev);
4112 return r;
4113 }
4114 }
4115 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4116 if (unlikely(r != 0)) {
4117 sumo_rlc_fini(rdev);
4118 return r;
4119 }
4120 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4121 &rdev->rlc.clear_state_gpu_addr);
4122 if (r) {
4123 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4124 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4125 sumo_rlc_fini(rdev);
4126 return r;
4127 }
4128
4129 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)__UNVOLATILE(&rdev->rlc.cs_ptr));
4130 if (r) {
4131 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4132 sumo_rlc_fini(rdev);
4133 return r;
4134 }
4135 /* set up the cs buffer */
4136 dst_ptr = rdev->rlc.cs_ptr;
4137 if (rdev->family >= CHIP_BONAIRE) {
4138 cik_get_csb_buffer(rdev, dst_ptr);
4139 } else if (rdev->family >= CHIP_TAHITI) {
4140 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4141 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4142 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4143 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4144 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4145 } else {
4146 reg_list_hdr_blk_index = 0;
4147 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4148 data = upper_32_bits(reg_list_mc_addr);
4149 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4150 reg_list_hdr_blk_index++;
4151 for (i = 0; cs_data[i].section != NULL; i++) {
4152 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4153 reg_num = cs_data[i].section[j].reg_count;
4154 data = reg_list_mc_addr & 0xffffffff;
4155 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4156 reg_list_hdr_blk_index++;
4157
4158 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4159 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4160 reg_list_hdr_blk_index++;
4161
4162 data = 0x08000000 | (reg_num * 4);
4163 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4164 reg_list_hdr_blk_index++;
4165
4166 for (k = 0; k < reg_num; k++) {
4167 data = cs_data[i].section[j].extent[k];
4168 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4169 }
4170 reg_list_mc_addr += reg_num * 4;
4171 reg_list_blk_index += reg_num;
4172 }
4173 }
4174 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4175 }
4176 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4177 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4178 }
4179
4180 if (rdev->rlc.cp_table_size) {
4181 if (rdev->rlc.cp_table_obj == NULL) {
4182 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size, PAGE_SIZE, true,
4183 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.cp_table_obj);
4184 if (r) {
4185 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4186 sumo_rlc_fini(rdev);
4187 return r;
4188 }
4189 }
4190
4191 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4192 if (unlikely(r != 0)) {
4193 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4194 sumo_rlc_fini(rdev);
4195 return r;
4196 }
4197 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4198 &rdev->rlc.cp_table_gpu_addr);
4199 if (r) {
4200 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4201 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4202 sumo_rlc_fini(rdev);
4203 return r;
4204 }
4205 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)__UNVOLATILE(&rdev->rlc.cp_table_ptr));
4206 if (r) {
4207 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4208 sumo_rlc_fini(rdev);
4209 return r;
4210 }
4211
4212 cik_init_cp_pg_table(rdev);
4213
4214 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4215 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4216
4217 }
4218
4219 return 0;
4220 }
4221
evergreen_rlc_start(struct radeon_device * rdev)4222 static void evergreen_rlc_start(struct radeon_device *rdev)
4223 {
4224 u32 mask = RLC_ENABLE;
4225
4226 if (rdev->flags & RADEON_IS_IGP) {
4227 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4228 }
4229
4230 WREG32(RLC_CNTL, mask);
4231 }
4232
evergreen_rlc_resume(struct radeon_device * rdev)4233 int evergreen_rlc_resume(struct radeon_device *rdev)
4234 {
4235 u32 i;
4236 const __be32 *fw_data;
4237
4238 if (!rdev->rlc_fw)
4239 return -EINVAL;
4240
4241 r600_rlc_stop(rdev);
4242
4243 WREG32(RLC_HB_CNTL, 0);
4244
4245 if (rdev->flags & RADEON_IS_IGP) {
4246 if (rdev->family == CHIP_ARUBA) {
4247 u32 always_on_bitmap =
4248 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4249 /* find out the number of active simds */
4250 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4251 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4252 tmp = hweight32(~tmp);
4253 if (tmp == rdev->config.cayman.max_simds_per_se) {
4254 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4255 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4256 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4257 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4258 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4259 }
4260 } else {
4261 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4262 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4263 }
4264 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4265 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4266 } else {
4267 WREG32(RLC_HB_BASE, 0);
4268 WREG32(RLC_HB_RPTR, 0);
4269 WREG32(RLC_HB_WPTR, 0);
4270 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4271 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4272 }
4273 WREG32(RLC_MC_CNTL, 0);
4274 WREG32(RLC_UCODE_CNTL, 0);
4275
4276 fw_data = (const __be32 *)rdev->rlc_fw->data;
4277 if (rdev->family >= CHIP_ARUBA) {
4278 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4279 WREG32(RLC_UCODE_ADDR, i);
4280 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4281 }
4282 } else if (rdev->family >= CHIP_CAYMAN) {
4283 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4284 WREG32(RLC_UCODE_ADDR, i);
4285 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4286 }
4287 } else {
4288 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4289 WREG32(RLC_UCODE_ADDR, i);
4290 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4291 }
4292 }
4293 WREG32(RLC_UCODE_ADDR, 0);
4294
4295 evergreen_rlc_start(rdev);
4296
4297 return 0;
4298 }
4299
4300 /* Interrupts */
4301
evergreen_get_vblank_counter(struct radeon_device * rdev,int crtc)4302 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4303 {
4304 if (crtc >= rdev->num_crtc)
4305 return 0;
4306 else
4307 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4308 }
4309
evergreen_disable_interrupt_state(struct radeon_device * rdev)4310 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4311 {
4312 u32 tmp;
4313
4314 if (rdev->family >= CHIP_CAYMAN) {
4315 cayman_cp_int_cntl_setup(rdev, 0,
4316 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4317 cayman_cp_int_cntl_setup(rdev, 1, 0);
4318 cayman_cp_int_cntl_setup(rdev, 2, 0);
4319 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4320 WREG32(CAYMAN_DMA1_CNTL, tmp);
4321 } else
4322 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4323 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4324 WREG32(DMA_CNTL, tmp);
4325 WREG32(GRBM_INT_CNTL, 0);
4326 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4327 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4328 if (rdev->num_crtc >= 4) {
4329 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4330 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4331 }
4332 if (rdev->num_crtc >= 6) {
4333 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4334 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4335 }
4336
4337 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4338 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4339 if (rdev->num_crtc >= 4) {
4340 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4341 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4342 }
4343 if (rdev->num_crtc >= 6) {
4344 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4345 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4346 }
4347
4348 /* only one DAC on DCE5 */
4349 if (!ASIC_IS_DCE5(rdev))
4350 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4351 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4352
4353 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4354 WREG32(DC_HPD1_INT_CONTROL, tmp);
4355 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4356 WREG32(DC_HPD2_INT_CONTROL, tmp);
4357 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4358 WREG32(DC_HPD3_INT_CONTROL, tmp);
4359 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4360 WREG32(DC_HPD4_INT_CONTROL, tmp);
4361 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4362 WREG32(DC_HPD5_INT_CONTROL, tmp);
4363 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4364 WREG32(DC_HPD6_INT_CONTROL, tmp);
4365
4366 }
4367
evergreen_irq_set(struct radeon_device * rdev)4368 int evergreen_irq_set(struct radeon_device *rdev)
4369 {
4370 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4371 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4372 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4373 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
4374 u32 grbm_int_cntl = 0;
4375 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
4376 u32 dma_cntl, dma_cntl1 = 0;
4377 u32 thermal_int = 0;
4378
4379 if (!rdev->irq.installed) {
4380 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4381 return -EINVAL;
4382 }
4383 /* don't enable anything if the ih is disabled */
4384 if (!rdev->ih.enabled) {
4385 r600_disable_interrupts(rdev);
4386 /* force the active interrupt state to all disabled */
4387 evergreen_disable_interrupt_state(rdev);
4388 return 0;
4389 }
4390
4391 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4392 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4393 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4394 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4395 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4396 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
4397 if (rdev->family == CHIP_ARUBA)
4398 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4399 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4400 else
4401 thermal_int = RREG32(CG_THERMAL_INT) &
4402 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4403
4404 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4405 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4406 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4407 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4408 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4409 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4410
4411 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4412
4413 if (rdev->family >= CHIP_CAYMAN) {
4414 /* enable CP interrupts on all rings */
4415 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4416 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4417 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4418 }
4419 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4420 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4421 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4422 }
4423 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4424 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4425 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4426 }
4427 } else {
4428 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4429 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4430 cp_int_cntl |= RB_INT_ENABLE;
4431 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4432 }
4433 }
4434
4435 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4436 DRM_DEBUG("r600_irq_set: sw int dma\n");
4437 dma_cntl |= TRAP_ENABLE;
4438 }
4439
4440 if (rdev->family >= CHIP_CAYMAN) {
4441 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4442 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4443 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4444 dma_cntl1 |= TRAP_ENABLE;
4445 }
4446 }
4447
4448 if (rdev->irq.dpm_thermal) {
4449 DRM_DEBUG("dpm thermal\n");
4450 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4451 }
4452
4453 if (rdev->irq.crtc_vblank_int[0] ||
4454 atomic_read(&rdev->irq.pflip[0])) {
4455 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4456 crtc1 |= VBLANK_INT_MASK;
4457 }
4458 if (rdev->irq.crtc_vblank_int[1] ||
4459 atomic_read(&rdev->irq.pflip[1])) {
4460 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4461 crtc2 |= VBLANK_INT_MASK;
4462 }
4463 if (rdev->irq.crtc_vblank_int[2] ||
4464 atomic_read(&rdev->irq.pflip[2])) {
4465 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4466 crtc3 |= VBLANK_INT_MASK;
4467 }
4468 if (rdev->irq.crtc_vblank_int[3] ||
4469 atomic_read(&rdev->irq.pflip[3])) {
4470 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4471 crtc4 |= VBLANK_INT_MASK;
4472 }
4473 if (rdev->irq.crtc_vblank_int[4] ||
4474 atomic_read(&rdev->irq.pflip[4])) {
4475 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4476 crtc5 |= VBLANK_INT_MASK;
4477 }
4478 if (rdev->irq.crtc_vblank_int[5] ||
4479 atomic_read(&rdev->irq.pflip[5])) {
4480 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4481 crtc6 |= VBLANK_INT_MASK;
4482 }
4483 if (rdev->irq.hpd[0]) {
4484 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4485 hpd1 |= DC_HPDx_INT_EN;
4486 }
4487 if (rdev->irq.hpd[1]) {
4488 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4489 hpd2 |= DC_HPDx_INT_EN;
4490 }
4491 if (rdev->irq.hpd[2]) {
4492 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4493 hpd3 |= DC_HPDx_INT_EN;
4494 }
4495 if (rdev->irq.hpd[3]) {
4496 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4497 hpd4 |= DC_HPDx_INT_EN;
4498 }
4499 if (rdev->irq.hpd[4]) {
4500 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4501 hpd5 |= DC_HPDx_INT_EN;
4502 }
4503 if (rdev->irq.hpd[5]) {
4504 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4505 hpd6 |= DC_HPDx_INT_EN;
4506 }
4507 if (rdev->irq.afmt[0]) {
4508 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4509 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4510 }
4511 if (rdev->irq.afmt[1]) {
4512 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4513 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4514 }
4515 if (rdev->irq.afmt[2]) {
4516 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4517 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4518 }
4519 if (rdev->irq.afmt[3]) {
4520 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4521 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4522 }
4523 if (rdev->irq.afmt[4]) {
4524 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4525 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4526 }
4527 if (rdev->irq.afmt[5]) {
4528 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4529 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4530 }
4531
4532 if (rdev->family >= CHIP_CAYMAN) {
4533 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4534 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4535 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4536 } else
4537 WREG32(CP_INT_CNTL, cp_int_cntl);
4538
4539 WREG32(DMA_CNTL, dma_cntl);
4540
4541 if (rdev->family >= CHIP_CAYMAN)
4542 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4543
4544 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4545
4546 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4547 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
4548 if (rdev->num_crtc >= 4) {
4549 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4550 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
4551 }
4552 if (rdev->num_crtc >= 6) {
4553 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4554 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4555 }
4556
4557 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
4558 GRPH_PFLIP_INT_MASK);
4559 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
4560 GRPH_PFLIP_INT_MASK);
4561 if (rdev->num_crtc >= 4) {
4562 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
4563 GRPH_PFLIP_INT_MASK);
4564 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
4565 GRPH_PFLIP_INT_MASK);
4566 }
4567 if (rdev->num_crtc >= 6) {
4568 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
4569 GRPH_PFLIP_INT_MASK);
4570 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
4571 GRPH_PFLIP_INT_MASK);
4572 }
4573
4574 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4575 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4576 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4577 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4578 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4579 WREG32(DC_HPD6_INT_CONTROL, hpd6);
4580 if (rdev->family == CHIP_ARUBA)
4581 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4582 else
4583 WREG32(CG_THERMAL_INT, thermal_int);
4584
4585 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4586 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4587 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4588 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4589 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4590 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4591
4592 return 0;
4593 }
4594
evergreen_irq_ack(struct radeon_device * rdev)4595 static void evergreen_irq_ack(struct radeon_device *rdev)
4596 {
4597 u32 tmp;
4598
4599 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4600 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4601 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4602 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4603 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4604 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4605 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4606 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4607 if (rdev->num_crtc >= 4) {
4608 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4609 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4610 }
4611 if (rdev->num_crtc >= 6) {
4612 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4613 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4614 }
4615
4616 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4617 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4618 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4619 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4620 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4621 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4622
4623 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4624 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4625 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4626 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4627 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
4628 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
4629 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
4630 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
4631 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
4632 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
4633 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
4634 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4635
4636 if (rdev->num_crtc >= 4) {
4637 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4638 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4639 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4640 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4641 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4642 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4643 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4644 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4645 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4646 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4647 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4648 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4649 }
4650
4651 if (rdev->num_crtc >= 6) {
4652 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4653 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4654 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4655 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4656 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4657 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4658 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4659 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4660 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4661 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4662 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4663 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4664 }
4665
4666 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4667 tmp = RREG32(DC_HPD1_INT_CONTROL);
4668 tmp |= DC_HPDx_INT_ACK;
4669 WREG32(DC_HPD1_INT_CONTROL, tmp);
4670 }
4671 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4672 tmp = RREG32(DC_HPD2_INT_CONTROL);
4673 tmp |= DC_HPDx_INT_ACK;
4674 WREG32(DC_HPD2_INT_CONTROL, tmp);
4675 }
4676 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4677 tmp = RREG32(DC_HPD3_INT_CONTROL);
4678 tmp |= DC_HPDx_INT_ACK;
4679 WREG32(DC_HPD3_INT_CONTROL, tmp);
4680 }
4681 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4682 tmp = RREG32(DC_HPD4_INT_CONTROL);
4683 tmp |= DC_HPDx_INT_ACK;
4684 WREG32(DC_HPD4_INT_CONTROL, tmp);
4685 }
4686 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4687 tmp = RREG32(DC_HPD5_INT_CONTROL);
4688 tmp |= DC_HPDx_INT_ACK;
4689 WREG32(DC_HPD5_INT_CONTROL, tmp);
4690 }
4691 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4692 tmp = RREG32(DC_HPD5_INT_CONTROL);
4693 tmp |= DC_HPDx_INT_ACK;
4694 WREG32(DC_HPD6_INT_CONTROL, tmp);
4695 }
4696 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4697 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4698 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4699 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4700 }
4701 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4702 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4703 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4704 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4705 }
4706 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4707 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4708 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4709 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4710 }
4711 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4712 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4713 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4714 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4715 }
4716 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4717 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4718 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4719 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4720 }
4721 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4722 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4723 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4724 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4725 }
4726 }
4727
evergreen_irq_disable(struct radeon_device * rdev)4728 static void evergreen_irq_disable(struct radeon_device *rdev)
4729 {
4730 r600_disable_interrupts(rdev);
4731 /* Wait and acknowledge irq */
4732 mdelay(1);
4733 evergreen_irq_ack(rdev);
4734 evergreen_disable_interrupt_state(rdev);
4735 }
4736
evergreen_irq_suspend(struct radeon_device * rdev)4737 void evergreen_irq_suspend(struct radeon_device *rdev)
4738 {
4739 evergreen_irq_disable(rdev);
4740 r600_rlc_stop(rdev);
4741 }
4742
evergreen_get_ih_wptr(struct radeon_device * rdev)4743 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4744 {
4745 u32 wptr, tmp;
4746
4747 if (rdev->wb.enabled)
4748 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4749 else
4750 wptr = RREG32(IH_RB_WPTR);
4751
4752 if (wptr & RB_OVERFLOW) {
4753 /* When a ring buffer overflow happen start parsing interrupt
4754 * from the last not overwritten vector (wptr + 16). Hopefully
4755 * this should allow us to catchup.
4756 */
4757 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4758 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4759 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4760 tmp = RREG32(IH_RB_CNTL);
4761 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4762 WREG32(IH_RB_CNTL, tmp);
4763 }
4764 return (wptr & rdev->ih.ptr_mask);
4765 }
4766
evergreen_irq_process(struct radeon_device * rdev)4767 int evergreen_irq_process(struct radeon_device *rdev)
4768 {
4769 u32 wptr;
4770 u32 rptr;
4771 u32 src_id, src_data;
4772 u32 ring_index;
4773 bool queue_hotplug = false;
4774 bool queue_hdmi = false;
4775 bool queue_thermal = false;
4776 u32 status, addr;
4777
4778 if (!rdev->ih.enabled || rdev->shutdown)
4779 return IRQ_NONE;
4780
4781 wptr = evergreen_get_ih_wptr(rdev);
4782
4783 restart_ih:
4784 /* is somebody else already processing irqs? */
4785 if (atomic_xchg(&rdev->ih.lock, 1))
4786 return IRQ_NONE;
4787
4788 rptr = rdev->ih.rptr;
4789 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4790
4791 /* Order reading of wptr vs. reading of IH ring data */
4792 rmb();
4793
4794 /* display interrupts */
4795 evergreen_irq_ack(rdev);
4796
4797 while (rptr != wptr) {
4798 /* wptr/rptr are in bytes! */
4799 ring_index = rptr / 4;
4800 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4801 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4802
4803 switch (src_id) {
4804 case 1: /* D1 vblank/vline */
4805 switch (src_data) {
4806 case 0: /* D1 vblank */
4807 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
4808 if (rdev->irq.crtc_vblank_int[0]) {
4809 drm_handle_vblank(rdev->ddev, 0);
4810 #ifdef __NetBSD__
4811 spin_lock(&rdev->irq.vblank_lock);
4812 rdev->pm.vblank_sync = true;
4813 DRM_SPIN_WAKEUP_ONE(&rdev->irq.vblank_queue, &rdev->irq.vblank_lock);
4814 spin_unlock(&rdev->irq.vblank_lock);
4815 #else
4816 rdev->pm.vblank_sync = true;
4817 wake_up(&rdev->irq.vblank_queue);
4818 #endif
4819 }
4820 if (atomic_read(&rdev->irq.pflip[0]))
4821 radeon_crtc_handle_flip(rdev, 0);
4822 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4823 DRM_DEBUG("IH: D1 vblank\n");
4824 }
4825 break;
4826 case 1: /* D1 vline */
4827 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4828 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4829 DRM_DEBUG("IH: D1 vline\n");
4830 }
4831 break;
4832 default:
4833 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4834 break;
4835 }
4836 break;
4837 case 2: /* D2 vblank/vline */
4838 switch (src_data) {
4839 case 0: /* D2 vblank */
4840 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
4841 if (rdev->irq.crtc_vblank_int[1]) {
4842 drm_handle_vblank(rdev->ddev, 1);
4843 #ifdef __NetBSD__
4844 spin_lock(&rdev->irq.vblank_lock);
4845 rdev->pm.vblank_sync = true;
4846 DRM_SPIN_WAKEUP_ONE(&rdev->irq.vblank_queue, &rdev->irq.vblank_lock);
4847 spin_unlock(&rdev->irq.vblank_lock);
4848 #else
4849 rdev->pm.vblank_sync = true;
4850 wake_up(&rdev->irq.vblank_queue);
4851 #endif
4852 }
4853 if (atomic_read(&rdev->irq.pflip[1]))
4854 radeon_crtc_handle_flip(rdev, 1);
4855 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
4856 DRM_DEBUG("IH: D2 vblank\n");
4857 }
4858 break;
4859 case 1: /* D2 vline */
4860 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4861 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
4862 DRM_DEBUG("IH: D2 vline\n");
4863 }
4864 break;
4865 default:
4866 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4867 break;
4868 }
4869 break;
4870 case 3: /* D3 vblank/vline */
4871 switch (src_data) {
4872 case 0: /* D3 vblank */
4873 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4874 if (rdev->irq.crtc_vblank_int[2]) {
4875 drm_handle_vblank(rdev->ddev, 2);
4876 #ifdef __NetBSD__
4877 spin_lock(&rdev->irq.vblank_lock);
4878 rdev->pm.vblank_sync = true;
4879 DRM_SPIN_WAKEUP_ONE(&rdev->irq.vblank_queue, &rdev->irq.vblank_lock);
4880 spin_unlock(&rdev->irq.vblank_lock);
4881 #else
4882 rdev->pm.vblank_sync = true;
4883 wake_up(&rdev->irq.vblank_queue);
4884 #endif
4885 }
4886 if (atomic_read(&rdev->irq.pflip[2]))
4887 radeon_crtc_handle_flip(rdev, 2);
4888 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
4889 DRM_DEBUG("IH: D3 vblank\n");
4890 }
4891 break;
4892 case 1: /* D3 vline */
4893 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4894 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
4895 DRM_DEBUG("IH: D3 vline\n");
4896 }
4897 break;
4898 default:
4899 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4900 break;
4901 }
4902 break;
4903 case 4: /* D4 vblank/vline */
4904 switch (src_data) {
4905 case 0: /* D4 vblank */
4906 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4907 if (rdev->irq.crtc_vblank_int[3]) {
4908 drm_handle_vblank(rdev->ddev, 3);
4909 #ifdef __NetBSD__
4910 spin_lock(&rdev->irq.vblank_lock);
4911 rdev->pm.vblank_sync = true;
4912 DRM_SPIN_WAKEUP_ONE(&rdev->irq.vblank_queue, &rdev->irq.vblank_lock);
4913 spin_unlock(&rdev->irq.vblank_lock);
4914 #else
4915 rdev->pm.vblank_sync = true;
4916 wake_up(&rdev->irq.vblank_queue);
4917 #endif
4918 }
4919 if (atomic_read(&rdev->irq.pflip[3]))
4920 radeon_crtc_handle_flip(rdev, 3);
4921 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
4922 DRM_DEBUG("IH: D4 vblank\n");
4923 }
4924 break;
4925 case 1: /* D4 vline */
4926 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4927 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
4928 DRM_DEBUG("IH: D4 vline\n");
4929 }
4930 break;
4931 default:
4932 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4933 break;
4934 }
4935 break;
4936 case 5: /* D5 vblank/vline */
4937 switch (src_data) {
4938 case 0: /* D5 vblank */
4939 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4940 if (rdev->irq.crtc_vblank_int[4]) {
4941 drm_handle_vblank(rdev->ddev, 4);
4942 #ifdef __NetBSD__
4943 spin_lock(&rdev->irq.vblank_lock);
4944 rdev->pm.vblank_sync = true;
4945 DRM_SPIN_WAKEUP_ONE(&rdev->irq.vblank_queue, &rdev->irq.vblank_lock);
4946 spin_unlock(&rdev->irq.vblank_lock);
4947 #else
4948 rdev->pm.vblank_sync = true;
4949 wake_up(&rdev->irq.vblank_queue);
4950 #endif
4951 }
4952 if (atomic_read(&rdev->irq.pflip[4]))
4953 radeon_crtc_handle_flip(rdev, 4);
4954 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
4955 DRM_DEBUG("IH: D5 vblank\n");
4956 }
4957 break;
4958 case 1: /* D5 vline */
4959 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4960 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
4961 DRM_DEBUG("IH: D5 vline\n");
4962 }
4963 break;
4964 default:
4965 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4966 break;
4967 }
4968 break;
4969 case 6: /* D6 vblank/vline */
4970 switch (src_data) {
4971 case 0: /* D6 vblank */
4972 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4973 if (rdev->irq.crtc_vblank_int[5]) {
4974 drm_handle_vblank(rdev->ddev, 5);
4975 #ifdef __NetBSD__
4976 spin_lock(&rdev->irq.vblank_lock);
4977 rdev->pm.vblank_sync = true;
4978 DRM_SPIN_WAKEUP_ONE(&rdev->irq.vblank_queue, &rdev->irq.vblank_lock);
4979 spin_unlock(&rdev->irq.vblank_lock);
4980 #else
4981 rdev->pm.vblank_sync = true;
4982 wake_up(&rdev->irq.vblank_queue);
4983 #endif
4984 }
4985 if (atomic_read(&rdev->irq.pflip[5]))
4986 radeon_crtc_handle_flip(rdev, 5);
4987 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
4988 DRM_DEBUG("IH: D6 vblank\n");
4989 }
4990 break;
4991 case 1: /* D6 vline */
4992 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4993 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
4994 DRM_DEBUG("IH: D6 vline\n");
4995 }
4996 break;
4997 default:
4998 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4999 break;
5000 }
5001 break;
5002 case 8: /* D1 page flip */
5003 case 10: /* D2 page flip */
5004 case 12: /* D3 page flip */
5005 case 14: /* D4 page flip */
5006 case 16: /* D5 page flip */
5007 case 18: /* D6 page flip */
5008 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
5009 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
5010 break;
5011 case 42: /* HPD hotplug */
5012 switch (src_data) {
5013 case 0:
5014 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
5015 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
5016 queue_hotplug = true;
5017 DRM_DEBUG("IH: HPD1\n");
5018 }
5019 break;
5020 case 1:
5021 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
5022 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
5023 queue_hotplug = true;
5024 DRM_DEBUG("IH: HPD2\n");
5025 }
5026 break;
5027 case 2:
5028 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
5029 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
5030 queue_hotplug = true;
5031 DRM_DEBUG("IH: HPD3\n");
5032 }
5033 break;
5034 case 3:
5035 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5036 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
5037 queue_hotplug = true;
5038 DRM_DEBUG("IH: HPD4\n");
5039 }
5040 break;
5041 case 4:
5042 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5043 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
5044 queue_hotplug = true;
5045 DRM_DEBUG("IH: HPD5\n");
5046 }
5047 break;
5048 case 5:
5049 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5050 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
5051 queue_hotplug = true;
5052 DRM_DEBUG("IH: HPD6\n");
5053 }
5054 break;
5055 default:
5056 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5057 break;
5058 }
5059 break;
5060 case 44: /* hdmi */
5061 switch (src_data) {
5062 case 0:
5063 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
5064 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
5065 queue_hdmi = true;
5066 DRM_DEBUG("IH: HDMI0\n");
5067 }
5068 break;
5069 case 1:
5070 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
5071 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
5072 queue_hdmi = true;
5073 DRM_DEBUG("IH: HDMI1\n");
5074 }
5075 break;
5076 case 2:
5077 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
5078 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
5079 queue_hdmi = true;
5080 DRM_DEBUG("IH: HDMI2\n");
5081 }
5082 break;
5083 case 3:
5084 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
5085 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
5086 queue_hdmi = true;
5087 DRM_DEBUG("IH: HDMI3\n");
5088 }
5089 break;
5090 case 4:
5091 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
5092 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
5093 queue_hdmi = true;
5094 DRM_DEBUG("IH: HDMI4\n");
5095 }
5096 break;
5097 case 5:
5098 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
5099 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
5100 queue_hdmi = true;
5101 DRM_DEBUG("IH: HDMI5\n");
5102 }
5103 break;
5104 default:
5105 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
5106 break;
5107 }
5108 case 124: /* UVD */
5109 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
5110 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
5111 break;
5112 case 146:
5113 case 147:
5114 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
5115 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
5116 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5117 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5118 addr);
5119 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5120 status);
5121 cayman_vm_decode_fault(rdev, status, addr);
5122 /* reset addr and status */
5123 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5124 break;
5125 case 176: /* CP_INT in ring buffer */
5126 case 177: /* CP_INT in IB1 */
5127 case 178: /* CP_INT in IB2 */
5128 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
5129 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5130 break;
5131 case 181: /* CP EOP event */
5132 DRM_DEBUG("IH: CP EOP\n");
5133 if (rdev->family >= CHIP_CAYMAN) {
5134 switch (src_data) {
5135 case 0:
5136 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5137 break;
5138 case 1:
5139 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5140 break;
5141 case 2:
5142 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5143 break;
5144 }
5145 } else
5146 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5147 break;
5148 case 224: /* DMA trap event */
5149 DRM_DEBUG("IH: DMA trap\n");
5150 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5151 break;
5152 case 230: /* thermal low to high */
5153 DRM_DEBUG("IH: thermal low to high\n");
5154 rdev->pm.dpm.thermal.high_to_low = false;
5155 queue_thermal = true;
5156 break;
5157 case 231: /* thermal high to low */
5158 DRM_DEBUG("IH: thermal high to low\n");
5159 rdev->pm.dpm.thermal.high_to_low = true;
5160 queue_thermal = true;
5161 break;
5162 case 233: /* GUI IDLE */
5163 DRM_DEBUG("IH: GUI idle\n");
5164 break;
5165 case 244: /* DMA trap event */
5166 if (rdev->family >= CHIP_CAYMAN) {
5167 DRM_DEBUG("IH: DMA1 trap\n");
5168 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5169 }
5170 break;
5171 default:
5172 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5173 break;
5174 }
5175
5176 /* wptr/rptr are in bytes! */
5177 rptr += 16;
5178 rptr &= rdev->ih.ptr_mask;
5179 }
5180 if (queue_hotplug)
5181 schedule_work(&rdev->hotplug_work);
5182 if (queue_hdmi)
5183 schedule_work(&rdev->audio_work);
5184 if (queue_thermal && rdev->pm.dpm_enabled)
5185 schedule_work(&rdev->pm.dpm.thermal.work);
5186 rdev->ih.rptr = rptr;
5187 WREG32(IH_RB_RPTR, rdev->ih.rptr);
5188 atomic_set(&rdev->ih.lock, 0);
5189
5190 /* make sure wptr hasn't changed while processing */
5191 wptr = evergreen_get_ih_wptr(rdev);
5192 if (wptr != rptr)
5193 goto restart_ih;
5194
5195 return IRQ_HANDLED;
5196 }
5197
evergreen_startup(struct radeon_device * rdev)5198 static int evergreen_startup(struct radeon_device *rdev)
5199 {
5200 struct radeon_ring *ring;
5201 int r;
5202
5203 /* enable pcie gen2 link */
5204 evergreen_pcie_gen2_enable(rdev);
5205 /* enable aspm */
5206 evergreen_program_aspm(rdev);
5207
5208 /* scratch needs to be initialized before MC */
5209 r = r600_vram_scratch_init(rdev);
5210 if (r)
5211 return r;
5212
5213 evergreen_mc_program(rdev);
5214
5215 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5216 r = ni_mc_load_microcode(rdev);
5217 if (r) {
5218 DRM_ERROR("Failed to load MC firmware!\n");
5219 return r;
5220 }
5221 }
5222
5223 if (rdev->flags & RADEON_IS_AGP) {
5224 evergreen_agp_enable(rdev);
5225 } else {
5226 r = evergreen_pcie_gart_enable(rdev);
5227 if (r)
5228 return r;
5229 }
5230 evergreen_gpu_init(rdev);
5231
5232 /* allocate rlc buffers */
5233 if (rdev->flags & RADEON_IS_IGP) {
5234 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5235 rdev->rlc.reg_list_size =
5236 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5237 rdev->rlc.cs_data = evergreen_cs_data;
5238 r = sumo_rlc_init(rdev);
5239 if (r) {
5240 DRM_ERROR("Failed to init rlc BOs!\n");
5241 return r;
5242 }
5243 }
5244
5245 /* allocate wb buffer */
5246 r = radeon_wb_init(rdev);
5247 if (r)
5248 return r;
5249
5250 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5251 if (r) {
5252 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5253 return r;
5254 }
5255
5256 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5257 if (r) {
5258 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5259 return r;
5260 }
5261
5262 r = uvd_v2_2_resume(rdev);
5263 if (!r) {
5264 r = radeon_fence_driver_start_ring(rdev,
5265 R600_RING_TYPE_UVD_INDEX);
5266 if (r)
5267 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5268 }
5269
5270 if (r)
5271 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5272
5273 /* Enable IRQ */
5274 if (!rdev->irq.installed) {
5275 r = radeon_irq_kms_init(rdev);
5276 if (r)
5277 return r;
5278 }
5279
5280 r = r600_irq_init(rdev);
5281 if (r) {
5282 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5283 radeon_irq_kms_fini(rdev);
5284 return r;
5285 }
5286 evergreen_irq_set(rdev);
5287
5288 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5289 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5290 RADEON_CP_PACKET2);
5291 if (r)
5292 return r;
5293
5294 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5295 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5296 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5297 if (r)
5298 return r;
5299
5300 r = evergreen_cp_load_microcode(rdev);
5301 if (r)
5302 return r;
5303 r = evergreen_cp_resume(rdev);
5304 if (r)
5305 return r;
5306 r = r600_dma_resume(rdev);
5307 if (r)
5308 return r;
5309
5310 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5311 if (ring->ring_size) {
5312 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
5313 RADEON_CP_PACKET2);
5314 if (!r)
5315 r = uvd_v1_0_init(rdev);
5316
5317 if (r)
5318 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5319 }
5320
5321 r = radeon_ib_pool_init(rdev);
5322 if (r) {
5323 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5324 return r;
5325 }
5326
5327 r = r600_audio_init(rdev);
5328 if (r) {
5329 DRM_ERROR("radeon: audio init failed\n");
5330 return r;
5331 }
5332
5333 return 0;
5334 }
5335
evergreen_resume(struct radeon_device * rdev)5336 int evergreen_resume(struct radeon_device *rdev)
5337 {
5338 int r;
5339
5340 /* reset the asic, the gfx blocks are often in a bad state
5341 * after the driver is unloaded or after a resume
5342 */
5343 if (radeon_asic_reset(rdev))
5344 dev_warn(rdev->dev, "GPU reset failed !\n");
5345 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5346 * posting will perform necessary task to bring back GPU into good
5347 * shape.
5348 */
5349 /* post card */
5350 atom_asic_init(rdev->mode_info.atom_context);
5351
5352 /* init golden registers */
5353 evergreen_init_golden_registers(rdev);
5354
5355 if (rdev->pm.pm_method == PM_METHOD_DPM)
5356 radeon_pm_resume(rdev);
5357
5358 rdev->accel_working = true;
5359 r = evergreen_startup(rdev);
5360 if (r) {
5361 DRM_ERROR("evergreen startup failed on resume\n");
5362 rdev->accel_working = false;
5363 return r;
5364 }
5365
5366 return r;
5367
5368 }
5369
evergreen_suspend(struct radeon_device * rdev)5370 int evergreen_suspend(struct radeon_device *rdev)
5371 {
5372 radeon_pm_suspend(rdev);
5373 r600_audio_fini(rdev);
5374 uvd_v1_0_fini(rdev);
5375 radeon_uvd_suspend(rdev);
5376 r700_cp_stop(rdev);
5377 r600_dma_stop(rdev);
5378 evergreen_irq_suspend(rdev);
5379 radeon_wb_disable(rdev);
5380 evergreen_pcie_gart_disable(rdev);
5381
5382 return 0;
5383 }
5384
5385 /* Plan is to move initialization in that function and use
5386 * helper function so that radeon_device_init pretty much
5387 * do nothing more than calling asic specific function. This
5388 * should also allow to remove a bunch of callback function
5389 * like vram_info.
5390 */
evergreen_init(struct radeon_device * rdev)5391 int evergreen_init(struct radeon_device *rdev)
5392 {
5393 int r;
5394
5395 /* Read BIOS */
5396 if (!radeon_get_bios(rdev)) {
5397 if (ASIC_IS_AVIVO(rdev))
5398 return -EINVAL;
5399 }
5400 /* Must be an ATOMBIOS */
5401 if (!rdev->is_atom_bios) {
5402 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5403 return -EINVAL;
5404 }
5405 r = radeon_atombios_init(rdev);
5406 if (r)
5407 return r;
5408 /* reset the asic, the gfx blocks are often in a bad state
5409 * after the driver is unloaded or after a resume
5410 */
5411 if (radeon_asic_reset(rdev))
5412 dev_warn(rdev->dev, "GPU reset failed !\n");
5413 /* Post card if necessary */
5414 if (!radeon_card_posted(rdev)) {
5415 if (!rdev->bios) {
5416 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5417 return -EINVAL;
5418 }
5419 DRM_INFO("GPU not posted. posting now...\n");
5420 atom_asic_init(rdev->mode_info.atom_context);
5421 }
5422 /* init golden registers */
5423 evergreen_init_golden_registers(rdev);
5424 /* Initialize scratch registers */
5425 r600_scratch_init(rdev);
5426 /* Initialize surface registers */
5427 radeon_surface_init(rdev);
5428 /* Initialize clocks */
5429 radeon_get_clock_info(rdev->ddev);
5430 /* Fence driver */
5431 r = radeon_fence_driver_init(rdev);
5432 if (r)
5433 return r;
5434 /* initialize AGP */
5435 if (rdev->flags & RADEON_IS_AGP) {
5436 r = radeon_agp_init(rdev);
5437 if (r)
5438 radeon_agp_disable(rdev);
5439 }
5440 /* initialize memory controller */
5441 r = evergreen_mc_init(rdev);
5442 if (r)
5443 return r;
5444 /* Memory manager */
5445 r = radeon_bo_init(rdev);
5446 if (r)
5447 return r;
5448
5449 if (ASIC_IS_DCE5(rdev)) {
5450 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5451 r = ni_init_microcode(rdev);
5452 if (r) {
5453 DRM_ERROR("Failed to load firmware!\n");
5454 return r;
5455 }
5456 }
5457 } else {
5458 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5459 r = r600_init_microcode(rdev);
5460 if (r) {
5461 DRM_ERROR("Failed to load firmware!\n");
5462 return r;
5463 }
5464 }
5465 }
5466
5467 /* Initialize power management */
5468 radeon_pm_init(rdev);
5469
5470 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5471 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5472
5473 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5474 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5475
5476 r = radeon_uvd_init(rdev);
5477 if (!r) {
5478 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5479 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5480 4096);
5481 }
5482
5483 rdev->ih.ring_obj = NULL;
5484 r600_ih_ring_init(rdev, 64 * 1024);
5485
5486 r = r600_pcie_gart_init(rdev);
5487 if (r)
5488 return r;
5489
5490 rdev->accel_working = true;
5491 r = evergreen_startup(rdev);
5492 if (r) {
5493 dev_err(rdev->dev, "disabling GPU acceleration\n");
5494 r700_cp_fini(rdev);
5495 r600_dma_fini(rdev);
5496 r600_irq_fini(rdev);
5497 if (rdev->flags & RADEON_IS_IGP)
5498 sumo_rlc_fini(rdev);
5499 radeon_wb_fini(rdev);
5500 radeon_ib_pool_fini(rdev);
5501 radeon_irq_kms_fini(rdev);
5502 evergreen_pcie_gart_fini(rdev);
5503 rdev->accel_working = false;
5504 }
5505
5506 /* Don't start up if the MC ucode is missing on BTC parts.
5507 * The default clocks and voltages before the MC ucode
5508 * is loaded are not suffient for advanced operations.
5509 */
5510 if (ASIC_IS_DCE5(rdev)) {
5511 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5512 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5513 return -EINVAL;
5514 }
5515 }
5516
5517 return 0;
5518 }
5519
evergreen_fini(struct radeon_device * rdev)5520 void evergreen_fini(struct radeon_device *rdev)
5521 {
5522 radeon_pm_fini(rdev);
5523 r600_audio_fini(rdev);
5524 r700_cp_fini(rdev);
5525 r600_dma_fini(rdev);
5526 r600_irq_fini(rdev);
5527 if (rdev->flags & RADEON_IS_IGP)
5528 sumo_rlc_fini(rdev);
5529 radeon_wb_fini(rdev);
5530 radeon_ib_pool_fini(rdev);
5531 radeon_irq_kms_fini(rdev);
5532 uvd_v1_0_fini(rdev);
5533 radeon_uvd_fini(rdev);
5534 evergreen_pcie_gart_fini(rdev);
5535 r600_vram_scratch_fini(rdev);
5536 radeon_gem_fini(rdev);
5537 radeon_fence_driver_fini(rdev);
5538 radeon_agp_fini(rdev);
5539 radeon_bo_fini(rdev);
5540 radeon_atombios_fini(rdev);
5541 kfree(rdev->bios);
5542 rdev->bios = NULL;
5543 }
5544
evergreen_pcie_gen2_enable(struct radeon_device * rdev)5545 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5546 {
5547 #ifndef __NetBSD__ /* XXX radeon pcie */
5548 u32 link_width_cntl, speed_cntl;
5549
5550 if (radeon_pcie_gen2 == 0)
5551 return;
5552
5553 if (rdev->flags & RADEON_IS_IGP)
5554 return;
5555
5556 if (!(rdev->flags & RADEON_IS_PCIE))
5557 return;
5558
5559 /* x2 cards have a special sequence */
5560 if (ASIC_IS_X2(rdev))
5561 return;
5562
5563 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5564 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5565 return;
5566
5567 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5568 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5569 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5570 return;
5571 }
5572
5573 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5574
5575 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5576 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5577
5578 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5579 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5580 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5581
5582 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5583 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5584 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5585
5586 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5587 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5588 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5589
5590 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5591 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5592 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5593
5594 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5595 speed_cntl |= LC_GEN2_EN_STRAP;
5596 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5597
5598 } else {
5599 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5600 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5601 if (1)
5602 link_width_cntl |= LC_UPCONFIGURE_DIS;
5603 else
5604 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5605 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5606 }
5607 #endif
5608 }
5609
evergreen_program_aspm(struct radeon_device * rdev)5610 void evergreen_program_aspm(struct radeon_device *rdev)
5611 {
5612 u32 data, orig;
5613 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5614 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5615 /* fusion_platform = true
5616 * if the system is a fusion system
5617 * (APU or DGPU in a fusion system).
5618 * todo: check if the system is a fusion platform.
5619 */
5620 bool fusion_platform = false;
5621
5622 if (radeon_aspm == 0)
5623 return;
5624
5625 if (!(rdev->flags & RADEON_IS_PCIE))
5626 return;
5627
5628 switch (rdev->family) {
5629 case CHIP_CYPRESS:
5630 case CHIP_HEMLOCK:
5631 case CHIP_JUNIPER:
5632 case CHIP_REDWOOD:
5633 case CHIP_CEDAR:
5634 case CHIP_SUMO:
5635 case CHIP_SUMO2:
5636 case CHIP_PALM:
5637 case CHIP_ARUBA:
5638 disable_l0s = true;
5639 break;
5640 default:
5641 disable_l0s = false;
5642 break;
5643 }
5644
5645 if (rdev->flags & RADEON_IS_IGP)
5646 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5647
5648 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5649 if (fusion_platform)
5650 data &= ~MULTI_PIF;
5651 else
5652 data |= MULTI_PIF;
5653 if (data != orig)
5654 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5655
5656 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5657 if (fusion_platform)
5658 data &= ~MULTI_PIF;
5659 else
5660 data |= MULTI_PIF;
5661 if (data != orig)
5662 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5663
5664 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5665 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5666 if (!disable_l0s) {
5667 if (rdev->family >= CHIP_BARTS)
5668 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5669 else
5670 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5671 }
5672
5673 if (!disable_l1) {
5674 if (rdev->family >= CHIP_BARTS)
5675 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5676 else
5677 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5678
5679 if (!disable_plloff_in_l1) {
5680 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5681 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5682 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5683 if (data != orig)
5684 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5685
5686 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5687 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5688 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5689 if (data != orig)
5690 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5691
5692 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5693 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5694 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5695 if (data != orig)
5696 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5697
5698 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5699 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5700 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5701 if (data != orig)
5702 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5703
5704 if (rdev->family >= CHIP_BARTS) {
5705 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5706 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5707 data |= PLL_RAMP_UP_TIME_0(4);
5708 if (data != orig)
5709 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5710
5711 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5712 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5713 data |= PLL_RAMP_UP_TIME_1(4);
5714 if (data != orig)
5715 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5716
5717 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5718 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5719 data |= PLL_RAMP_UP_TIME_0(4);
5720 if (data != orig)
5721 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5722
5723 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5724 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5725 data |= PLL_RAMP_UP_TIME_1(4);
5726 if (data != orig)
5727 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5728 }
5729
5730 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5731 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5732 data |= LC_DYN_LANES_PWR_STATE(3);
5733 if (data != orig)
5734 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5735
5736 if (rdev->family >= CHIP_BARTS) {
5737 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5738 data &= ~LS2_EXIT_TIME_MASK;
5739 data |= LS2_EXIT_TIME(1);
5740 if (data != orig)
5741 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5742
5743 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5744 data &= ~LS2_EXIT_TIME_MASK;
5745 data |= LS2_EXIT_TIME(1);
5746 if (data != orig)
5747 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5748 }
5749 }
5750 }
5751
5752 /* evergreen parts only */
5753 if (rdev->family < CHIP_BARTS)
5754 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5755
5756 if (pcie_lc_cntl != pcie_lc_cntl_old)
5757 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5758 }
5759