1 /*
2 * Copyright 2016 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24 #include <linux/delay.h>
25 #include <linux/kernel.h>
26 #include <linux/firmware.h>
27 #include <linux/module.h>
28 #include <linux/pci.h>
29
30 #include "amdgpu.h"
31 #include "amdgpu_gfx.h"
32 #include "soc15.h"
33 #include "soc15d.h"
34 #include "amdgpu_atomfirmware.h"
35 #include "amdgpu_pm.h"
36
37 #include "gc/gc_9_0_offset.h"
38 #include "gc/gc_9_0_sh_mask.h"
39
40 #include "vega10_enum.h"
41
42 #include "soc15_common.h"
43 #include "clearstate_gfx9.h"
44 #include "v9_structs.h"
45
46 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h"
47
48 #include "amdgpu_ras.h"
49
50 #include "gfx_v9_4.h"
51 #include "gfx_v9_0.h"
52 #include "gfx_v9_4_2.h"
53
54 #include "asic_reg/pwr/pwr_10_0_offset.h"
55 #include "asic_reg/pwr/pwr_10_0_sh_mask.h"
56 #include "asic_reg/gc/gc_9_0_default.h"
57
58 #define GFX9_NUM_GFX_RINGS 1
59 #define GFX9_MEC_HPD_SIZE 4096
60 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
61 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L
62
63 #define mmGCEA_PROBE_MAP 0x070c
64 #define mmGCEA_PROBE_MAP_BASE_IDX 0
65
66 MODULE_FIRMWARE("amdgpu/vega10_ce.bin");
67 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin");
68 MODULE_FIRMWARE("amdgpu/vega10_me.bin");
69 MODULE_FIRMWARE("amdgpu/vega10_mec.bin");
70 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin");
71 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin");
72
73 MODULE_FIRMWARE("amdgpu/vega12_ce.bin");
74 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin");
75 MODULE_FIRMWARE("amdgpu/vega12_me.bin");
76 MODULE_FIRMWARE("amdgpu/vega12_mec.bin");
77 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin");
78 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin");
79
80 MODULE_FIRMWARE("amdgpu/vega20_ce.bin");
81 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin");
82 MODULE_FIRMWARE("amdgpu/vega20_me.bin");
83 MODULE_FIRMWARE("amdgpu/vega20_mec.bin");
84 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin");
85 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin");
86
87 MODULE_FIRMWARE("amdgpu/raven_ce.bin");
88 MODULE_FIRMWARE("amdgpu/raven_pfp.bin");
89 MODULE_FIRMWARE("amdgpu/raven_me.bin");
90 MODULE_FIRMWARE("amdgpu/raven_mec.bin");
91 MODULE_FIRMWARE("amdgpu/raven_mec2.bin");
92 MODULE_FIRMWARE("amdgpu/raven_rlc.bin");
93
94 MODULE_FIRMWARE("amdgpu/picasso_ce.bin");
95 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin");
96 MODULE_FIRMWARE("amdgpu/picasso_me.bin");
97 MODULE_FIRMWARE("amdgpu/picasso_mec.bin");
98 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin");
99 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin");
100 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin");
101
102 MODULE_FIRMWARE("amdgpu/raven2_ce.bin");
103 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin");
104 MODULE_FIRMWARE("amdgpu/raven2_me.bin");
105 MODULE_FIRMWARE("amdgpu/raven2_mec.bin");
106 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin");
107 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin");
108 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin");
109
110 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin");
111 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin");
112
113 MODULE_FIRMWARE("amdgpu/renoir_ce.bin");
114 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin");
115 MODULE_FIRMWARE("amdgpu/renoir_me.bin");
116 MODULE_FIRMWARE("amdgpu/renoir_mec.bin");
117 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin");
118
119 MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin");
120 MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin");
121 MODULE_FIRMWARE("amdgpu/green_sardine_me.bin");
122 MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin");
123 MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin");
124 MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin");
125
126 MODULE_FIRMWARE("amdgpu/aldebaran_mec.bin");
127 MODULE_FIRMWARE("amdgpu/aldebaran_mec2.bin");
128 MODULE_FIRMWARE("amdgpu/aldebaran_rlc.bin");
129
130 #define mmTCP_CHAN_STEER_0_ARCT 0x0b03
131 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX 0
132 #define mmTCP_CHAN_STEER_1_ARCT 0x0b04
133 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX 0
134 #define mmTCP_CHAN_STEER_2_ARCT 0x0b09
135 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX 0
136 #define mmTCP_CHAN_STEER_3_ARCT 0x0b0a
137 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX 0
138 #define mmTCP_CHAN_STEER_4_ARCT 0x0b0b
139 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX 0
140 #define mmTCP_CHAN_STEER_5_ARCT 0x0b0c
141 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX 0
142
143 enum ta_ras_gfx_subblock {
144 /*CPC*/
145 TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0,
146 TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START,
147 TA_RAS_BLOCK__GFX_CPC_UCODE,
148 TA_RAS_BLOCK__GFX_DC_STATE_ME1,
149 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1,
150 TA_RAS_BLOCK__GFX_DC_RESTORE_ME1,
151 TA_RAS_BLOCK__GFX_DC_STATE_ME2,
152 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2,
153 TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
154 TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
155 /* CPF*/
156 TA_RAS_BLOCK__GFX_CPF_INDEX_START,
157 TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START,
158 TA_RAS_BLOCK__GFX_CPF_ROQ_ME1,
159 TA_RAS_BLOCK__GFX_CPF_TAG,
160 TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG,
161 /* CPG*/
162 TA_RAS_BLOCK__GFX_CPG_INDEX_START,
163 TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START,
164 TA_RAS_BLOCK__GFX_CPG_DMA_TAG,
165 TA_RAS_BLOCK__GFX_CPG_TAG,
166 TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG,
167 /* GDS*/
168 TA_RAS_BLOCK__GFX_GDS_INDEX_START,
169 TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START,
170 TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE,
171 TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM,
172 TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM,
173 TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
174 TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
175 /* SPI*/
176 TA_RAS_BLOCK__GFX_SPI_SR_MEM,
177 /* SQ*/
178 TA_RAS_BLOCK__GFX_SQ_INDEX_START,
179 TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START,
180 TA_RAS_BLOCK__GFX_SQ_LDS_D,
181 TA_RAS_BLOCK__GFX_SQ_LDS_I,
182 TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/
183 TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR,
184 /* SQC (3 ranges)*/
185 TA_RAS_BLOCK__GFX_SQC_INDEX_START,
186 /* SQC range 0*/
187 TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START,
188 TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO =
189 TA_RAS_BLOCK__GFX_SQC_INDEX0_START,
190 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF,
191 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO,
192 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF,
193 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO,
194 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF,
195 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
196 TA_RAS_BLOCK__GFX_SQC_INDEX0_END =
197 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
198 /* SQC range 1*/
199 TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
200 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM =
201 TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
202 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO,
203 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO,
204 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM,
205 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM,
206 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO,
207 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO,
208 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM,
209 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
210 TA_RAS_BLOCK__GFX_SQC_INDEX1_END =
211 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
212 /* SQC range 2*/
213 TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
214 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM =
215 TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
216 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO,
217 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO,
218 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM,
219 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM,
220 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO,
221 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO,
222 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM,
223 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
224 TA_RAS_BLOCK__GFX_SQC_INDEX2_END =
225 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
226 TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END,
227 /* TA*/
228 TA_RAS_BLOCK__GFX_TA_INDEX_START,
229 TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START,
230 TA_RAS_BLOCK__GFX_TA_FS_AFIFO,
231 TA_RAS_BLOCK__GFX_TA_FL_LFIFO,
232 TA_RAS_BLOCK__GFX_TA_FX_LFIFO,
233 TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
234 TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
235 /* TCA*/
236 TA_RAS_BLOCK__GFX_TCA_INDEX_START,
237 TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START,
238 TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
239 TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
240 /* TCC (5 sub-ranges)*/
241 TA_RAS_BLOCK__GFX_TCC_INDEX_START,
242 /* TCC range 0*/
243 TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START,
244 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START,
245 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1,
246 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0,
247 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1,
248 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0,
249 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1,
250 TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG,
251 TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
252 TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
253 /* TCC range 1*/
254 TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
255 TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
256 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
257 TA_RAS_BLOCK__GFX_TCC_INDEX1_END =
258 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
259 /* TCC range 2*/
260 TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
261 TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
262 TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL,
263 TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO,
264 TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN,
265 TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ,
266 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO,
267 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM,
268 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
269 TA_RAS_BLOCK__GFX_TCC_INDEX2_END =
270 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
271 /* TCC range 3*/
272 TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
273 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
274 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
275 TA_RAS_BLOCK__GFX_TCC_INDEX3_END =
276 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
277 /* TCC range 4*/
278 TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
279 TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN =
280 TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
281 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
282 TA_RAS_BLOCK__GFX_TCC_INDEX4_END =
283 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
284 TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END,
285 /* TCI*/
286 TA_RAS_BLOCK__GFX_TCI_WRITE_RAM,
287 /* TCP*/
288 TA_RAS_BLOCK__GFX_TCP_INDEX_START,
289 TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START,
290 TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM,
291 TA_RAS_BLOCK__GFX_TCP_CMD_FIFO,
292 TA_RAS_BLOCK__GFX_TCP_VM_FIFO,
293 TA_RAS_BLOCK__GFX_TCP_DB_RAM,
294 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0,
295 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
296 TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
297 /* TD*/
298 TA_RAS_BLOCK__GFX_TD_INDEX_START,
299 TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START,
300 TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI,
301 TA_RAS_BLOCK__GFX_TD_CS_FIFO,
302 TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO,
303 /* EA (3 sub-ranges)*/
304 TA_RAS_BLOCK__GFX_EA_INDEX_START,
305 /* EA range 0*/
306 TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START,
307 TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START,
308 TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM,
309 TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM,
310 TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM,
311 TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM,
312 TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM,
313 TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM,
314 TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
315 TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
316 /* EA range 1*/
317 TA_RAS_BLOCK__GFX_EA_INDEX1_START,
318 TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START,
319 TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM,
320 TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM,
321 TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM,
322 TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM,
323 TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM,
324 TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
325 TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
326 /* EA range 2*/
327 TA_RAS_BLOCK__GFX_EA_INDEX2_START,
328 TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START,
329 TA_RAS_BLOCK__GFX_EA_MAM_D1MEM,
330 TA_RAS_BLOCK__GFX_EA_MAM_D2MEM,
331 TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
332 TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
333 TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END,
334 /* UTC VM L2 bank*/
335 TA_RAS_BLOCK__UTC_VML2_BANK_CACHE,
336 /* UTC VM walker*/
337 TA_RAS_BLOCK__UTC_VML2_WALKER,
338 /* UTC ATC L2 2MB cache*/
339 TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK,
340 /* UTC ATC L2 4KB cache*/
341 TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK,
342 TA_RAS_BLOCK__GFX_MAX
343 };
344
345 struct ras_gfx_subblock {
346 unsigned char *name;
347 int ta_subblock;
348 int hw_supported_error_type;
349 int sw_supported_error_type;
350 };
351
352 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h) \
353 [AMDGPU_RAS_BLOCK__##subblock] = { \
354 #subblock, \
355 TA_RAS_BLOCK__##subblock, \
356 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)), \
357 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)), \
358 }
359
360 static const struct ras_gfx_subblock ras_gfx_subblocks[] = {
361 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1),
362 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1),
363 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
364 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
365 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
366 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
367 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
368 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
369 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
370 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
371 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1),
372 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0),
373 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1),
374 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1),
375 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
376 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0),
377 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0,
378 0),
379 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0,
380 0),
381 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
382 AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0),
383 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0),
384 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1),
385 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0),
386 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0),
387 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1),
388 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
389 0, 0),
390 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
391 0),
392 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
393 0, 0),
394 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0,
395 0),
396 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
397 0, 0),
398 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
399 0),
400 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
401 1),
402 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
403 0, 0, 0),
404 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
405 0),
406 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
407 0),
408 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
409 0),
410 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
411 0),
412 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
413 0),
414 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
415 0, 0),
416 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
417 0),
418 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
419 0),
420 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
421 0, 0, 0),
422 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
423 0),
424 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
425 0),
426 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
427 0),
428 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
429 0),
430 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
431 0),
432 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
433 0, 0),
434 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
435 0),
436 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1),
437 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
438 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
439 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
440 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
441 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0),
442 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
443 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1),
444 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0,
445 1),
446 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0,
447 1),
448 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0,
449 1),
450 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0,
451 0),
452 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0,
453 0),
454 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
455 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
456 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0),
457 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0),
458 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0),
459 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0),
460 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
461 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0),
462 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0),
463 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
464 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0),
465 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0,
466 0),
467 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
468 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0,
469 0),
470 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0,
471 0, 0),
472 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0,
473 0),
474 AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
475 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1),
476 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0),
477 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
478 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
479 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
480 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0),
481 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0),
482 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1),
483 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0),
484 AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
485 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1),
486 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
487 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
488 AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
489 AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
490 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
491 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
492 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
493 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
494 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
495 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
496 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
497 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0),
498 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
499 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
500 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0),
501 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0),
502 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0),
503 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0),
504 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0),
505 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0),
506 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0),
507 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0),
508 };
509
510 static const struct soc15_reg_golden golden_settings_gc_9_0[] =
511 {
512 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
513 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000),
514 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
515 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
516 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
517 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
518 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
519 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
520 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
521 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87),
522 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f),
523 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
524 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
525 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
526 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
527 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
528 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff),
529 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
530 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
531 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
532 };
533
534 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] =
535 {
536 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107),
537 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
538 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
539 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
540 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
541 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
542 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042),
543 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
544 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000),
545 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
546 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
547 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
548 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
549 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
550 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
551 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107),
552 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800),
553 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080)
554 };
555
556 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] =
557 {
558 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080),
559 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
560 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
561 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042),
562 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042),
563 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400),
564 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000),
565 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000),
566 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107),
567 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000),
568 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000)
569 };
570
571 static const struct soc15_reg_golden golden_settings_gc_9_1[] =
572 {
573 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
574 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
575 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
576 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
577 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
578 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
579 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
580 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
581 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
582 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
583 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
584 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
585 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
586 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
587 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
588 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
589 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
590 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
591 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
592 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff),
593 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080),
594 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
595 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
596 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
597 };
598
599 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] =
600 {
601 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
602 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042),
603 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042),
604 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000),
605 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000),
606 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
607 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800)
608 };
609
610 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] =
611 {
612 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000),
613 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
614 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
615 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080),
616 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080),
617 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080),
618 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041),
619 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041),
620 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
621 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
622 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080),
623 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080),
624 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080),
625 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080),
626 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080),
627 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
628 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010),
629 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
630 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
631 };
632
633 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] =
634 {
635 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
636 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
637 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
638 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042),
639 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042),
640 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
641 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
642 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
643 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
644 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
645 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
646 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc),
647 };
648
649 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] =
650 {
651 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff),
652 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000),
653 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382)
654 };
655
656 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] =
657 {
658 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
659 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
660 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
661 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
662 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
663 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
664 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
665 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
666 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87),
667 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f),
668 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
669 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
670 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
671 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
672 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
673 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff)
674 };
675
676 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] =
677 {
678 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080),
679 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
680 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
681 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041),
682 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041),
683 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
684 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107),
685 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
686 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410),
687 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
688 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
689 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
690 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
691 };
692
693 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] =
694 {
695 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
696 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000),
697 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e),
698 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca),
699 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098),
700 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3),
701 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1),
702 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135),
703 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000),
704 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00),
705 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000)
706 };
707
708 static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = {
709 {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)},
710 {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)},
711 };
712
713 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] =
714 {
715 mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
716 mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
717 mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
718 mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
719 mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
720 mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
721 mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
722 mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
723 };
724
725 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] =
726 {
727 mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0,
728 mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0,
729 mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0,
730 mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0,
731 mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0,
732 mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0,
733 mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0,
734 mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0,
735 };
736
gfx_v9_0_rlcg_rw(struct amdgpu_device * adev,u32 offset,u32 v,u32 flag)737 static void gfx_v9_0_rlcg_rw(struct amdgpu_device *adev, u32 offset, u32 v, u32 flag)
738 {
739 static void *scratch_reg0;
740 static void *scratch_reg1;
741 static void *scratch_reg2;
742 static void *scratch_reg3;
743 static void *spare_int;
744 static uint32_t grbm_cntl;
745 static uint32_t grbm_idx;
746
747 scratch_reg0 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] + mmSCRATCH_REG0)*4;
748 scratch_reg1 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG1)*4;
749 scratch_reg2 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG2)*4;
750 scratch_reg3 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG3)*4;
751 spare_int = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] + mmRLC_SPARE_INT)*4;
752
753 grbm_cntl = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] + mmGRBM_GFX_CNTL;
754 grbm_idx = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] + mmGRBM_GFX_INDEX;
755
756 if (amdgpu_sriov_runtime(adev)) {
757 pr_err("shouldn't call rlcg write register during runtime\n");
758 return;
759 }
760
761 if (offset == grbm_cntl || offset == grbm_idx) {
762 if (offset == grbm_cntl)
763 writel(v, scratch_reg2);
764 else if (offset == grbm_idx)
765 writel(v, scratch_reg3);
766
767 writel(v, ((void __iomem *)adev->rmmio) + (offset * 4));
768 } else {
769 uint32_t i = 0;
770 uint32_t retries = 50000;
771
772 writel(v, scratch_reg0);
773 writel(offset | 0x80000000, scratch_reg1);
774 writel(1, spare_int);
775 for (i = 0; i < retries; i++) {
776 u32 tmp;
777
778 tmp = readl(scratch_reg1);
779 if (!(tmp & 0x80000000))
780 break;
781
782 udelay(10);
783 }
784 if (i >= retries)
785 pr_err("timeout: rlcg program reg:0x%05x failed !\n", offset);
786 }
787
788 }
789
gfx_v9_0_rlcg_wreg(struct amdgpu_device * adev,u32 offset,u32 v,u32 flag)790 static void gfx_v9_0_rlcg_wreg(struct amdgpu_device *adev, u32 offset, u32 v, u32 flag)
791 {
792 if (amdgpu_sriov_fullaccess(adev)) {
793 gfx_v9_0_rlcg_rw(adev, offset, v, flag);
794
795 return;
796 }
797
798 if (flag & AMDGPU_REGS_NO_KIQ)
799 WREG32_NO_KIQ(offset, v);
800 else
801 WREG32(offset, v);
802 }
803
804 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042
805 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041
806 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042
807 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041
808
809 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev);
810 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev);
811 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev);
812 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev);
813 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
814 struct amdgpu_cu_info *cu_info);
815 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
816 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring);
817 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring);
818 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
819 void *ras_error_status);
820 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
821 void *inject_if);
822 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev);
823
gfx_v9_0_kiq_set_resources(struct amdgpu_ring * kiq_ring,uint64_t queue_mask)824 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring,
825 uint64_t queue_mask)
826 {
827 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
828 amdgpu_ring_write(kiq_ring,
829 PACKET3_SET_RESOURCES_VMID_MASK(0) |
830 /* vmid_mask:0* queue_type:0 (KIQ) */
831 PACKET3_SET_RESOURCES_QUEUE_TYPE(0));
832 amdgpu_ring_write(kiq_ring,
833 lower_32_bits(queue_mask)); /* queue mask lo */
834 amdgpu_ring_write(kiq_ring,
835 upper_32_bits(queue_mask)); /* queue mask hi */
836 amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */
837 amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */
838 amdgpu_ring_write(kiq_ring, 0); /* oac mask */
839 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
840 }
841
gfx_v9_0_kiq_map_queues(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring)842 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring,
843 struct amdgpu_ring *ring)
844 {
845 struct amdgpu_device *adev = kiq_ring->adev;
846 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
847 uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
848 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
849
850 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
851 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
852 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
853 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
854 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
855 PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
856 PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
857 PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) |
858 /*queue_type: normal compute queue */
859 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) |
860 /* alloc format: all_on_one_pipe */
861 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) |
862 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
863 /* num_queues: must be 1 */
864 PACKET3_MAP_QUEUES_NUM_QUEUES(1));
865 amdgpu_ring_write(kiq_ring,
866 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
867 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
868 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
869 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
870 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
871 }
872
gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring,enum amdgpu_unmap_queues_action action,u64 gpu_addr,u64 seq)873 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
874 struct amdgpu_ring *ring,
875 enum amdgpu_unmap_queues_action action,
876 u64 gpu_addr, u64 seq)
877 {
878 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
879
880 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
881 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
882 PACKET3_UNMAP_QUEUES_ACTION(action) |
883 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
884 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
885 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
886 amdgpu_ring_write(kiq_ring,
887 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
888
889 if (action == PREEMPT_QUEUES_NO_UNMAP) {
890 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr));
891 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr));
892 amdgpu_ring_write(kiq_ring, seq);
893 } else {
894 amdgpu_ring_write(kiq_ring, 0);
895 amdgpu_ring_write(kiq_ring, 0);
896 amdgpu_ring_write(kiq_ring, 0);
897 }
898 }
899
gfx_v9_0_kiq_query_status(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring,u64 addr,u64 seq)900 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring,
901 struct amdgpu_ring *ring,
902 u64 addr,
903 u64 seq)
904 {
905 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
906
907 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
908 amdgpu_ring_write(kiq_ring,
909 PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
910 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
911 PACKET3_QUERY_STATUS_COMMAND(2));
912 /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
913 amdgpu_ring_write(kiq_ring,
914 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
915 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
916 amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
917 amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
918 amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
919 amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
920 }
921
gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring * kiq_ring,uint16_t pasid,uint32_t flush_type,bool all_hub)922 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
923 uint16_t pasid, uint32_t flush_type,
924 bool all_hub)
925 {
926 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
927 amdgpu_ring_write(kiq_ring,
928 PACKET3_INVALIDATE_TLBS_DST_SEL(1) |
929 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
930 PACKET3_INVALIDATE_TLBS_PASID(pasid) |
931 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
932 }
933
934 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = {
935 .kiq_set_resources = gfx_v9_0_kiq_set_resources,
936 .kiq_map_queues = gfx_v9_0_kiq_map_queues,
937 .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues,
938 .kiq_query_status = gfx_v9_0_kiq_query_status,
939 .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs,
940 .set_resources_size = 8,
941 .map_queues_size = 7,
942 .unmap_queues_size = 6,
943 .query_status_size = 7,
944 .invalidate_tlbs_size = 2,
945 };
946
gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device * adev)947 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
948 {
949 adev->gfx.kiq.pmf = &gfx_v9_0_kiq_pm4_funcs;
950 }
951
gfx_v9_0_init_golden_registers(struct amdgpu_device * adev)952 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
953 {
954 switch (adev->asic_type) {
955 case CHIP_VEGA10:
956 soc15_program_register_sequence(adev,
957 golden_settings_gc_9_0,
958 ARRAY_SIZE(golden_settings_gc_9_0));
959 soc15_program_register_sequence(adev,
960 golden_settings_gc_9_0_vg10,
961 ARRAY_SIZE(golden_settings_gc_9_0_vg10));
962 break;
963 case CHIP_VEGA12:
964 soc15_program_register_sequence(adev,
965 golden_settings_gc_9_2_1,
966 ARRAY_SIZE(golden_settings_gc_9_2_1));
967 soc15_program_register_sequence(adev,
968 golden_settings_gc_9_2_1_vg12,
969 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12));
970 break;
971 case CHIP_VEGA20:
972 soc15_program_register_sequence(adev,
973 golden_settings_gc_9_0,
974 ARRAY_SIZE(golden_settings_gc_9_0));
975 soc15_program_register_sequence(adev,
976 golden_settings_gc_9_0_vg20,
977 ARRAY_SIZE(golden_settings_gc_9_0_vg20));
978 break;
979 case CHIP_ARCTURUS:
980 soc15_program_register_sequence(adev,
981 golden_settings_gc_9_4_1_arct,
982 ARRAY_SIZE(golden_settings_gc_9_4_1_arct));
983 break;
984 case CHIP_RAVEN:
985 soc15_program_register_sequence(adev, golden_settings_gc_9_1,
986 ARRAY_SIZE(golden_settings_gc_9_1));
987 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
988 soc15_program_register_sequence(adev,
989 golden_settings_gc_9_1_rv2,
990 ARRAY_SIZE(golden_settings_gc_9_1_rv2));
991 else
992 soc15_program_register_sequence(adev,
993 golden_settings_gc_9_1_rv1,
994 ARRAY_SIZE(golden_settings_gc_9_1_rv1));
995 break;
996 case CHIP_RENOIR:
997 soc15_program_register_sequence(adev,
998 golden_settings_gc_9_1_rn,
999 ARRAY_SIZE(golden_settings_gc_9_1_rn));
1000 return; /* for renoir, don't need common goldensetting */
1001 case CHIP_ALDEBARAN:
1002 gfx_v9_4_2_init_golden_registers(adev,
1003 adev->smuio.funcs->get_die_id(adev));
1004 break;
1005 default:
1006 break;
1007 }
1008
1009 if ((adev->asic_type != CHIP_ARCTURUS) &&
1010 (adev->asic_type != CHIP_ALDEBARAN))
1011 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common,
1012 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common));
1013 }
1014
gfx_v9_0_scratch_init(struct amdgpu_device * adev)1015 static void gfx_v9_0_scratch_init(struct amdgpu_device *adev)
1016 {
1017 adev->gfx.scratch.num_reg = 8;
1018 adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
1019 adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1;
1020 }
1021
gfx_v9_0_write_data_to_reg(struct amdgpu_ring * ring,int eng_sel,bool wc,uint32_t reg,uint32_t val)1022 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
1023 bool wc, uint32_t reg, uint32_t val)
1024 {
1025 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
1026 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
1027 WRITE_DATA_DST_SEL(0) |
1028 (wc ? WR_CONFIRM : 0));
1029 amdgpu_ring_write(ring, reg);
1030 amdgpu_ring_write(ring, 0);
1031 amdgpu_ring_write(ring, val);
1032 }
1033
gfx_v9_0_wait_reg_mem(struct amdgpu_ring * ring,int eng_sel,int mem_space,int opt,uint32_t addr0,uint32_t addr1,uint32_t ref,uint32_t mask,uint32_t inv)1034 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
1035 int mem_space, int opt, uint32_t addr0,
1036 uint32_t addr1, uint32_t ref, uint32_t mask,
1037 uint32_t inv)
1038 {
1039 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
1040 amdgpu_ring_write(ring,
1041 /* memory (1) or register (0) */
1042 (WAIT_REG_MEM_MEM_SPACE(mem_space) |
1043 WAIT_REG_MEM_OPERATION(opt) | /* wait */
1044 WAIT_REG_MEM_FUNCTION(3) | /* equal */
1045 WAIT_REG_MEM_ENGINE(eng_sel)));
1046
1047 if (mem_space)
1048 BUG_ON(addr0 & 0x3); /* Dword align */
1049 amdgpu_ring_write(ring, addr0);
1050 amdgpu_ring_write(ring, addr1);
1051 amdgpu_ring_write(ring, ref);
1052 amdgpu_ring_write(ring, mask);
1053 amdgpu_ring_write(ring, inv); /* poll interval */
1054 }
1055
gfx_v9_0_ring_test_ring(struct amdgpu_ring * ring)1056 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring)
1057 {
1058 struct amdgpu_device *adev = ring->adev;
1059 uint32_t scratch;
1060 uint32_t tmp = 0;
1061 unsigned i;
1062 int r;
1063
1064 r = amdgpu_gfx_scratch_get(adev, &scratch);
1065 if (r)
1066 return r;
1067
1068 WREG32(scratch, 0xCAFEDEAD);
1069 r = amdgpu_ring_alloc(ring, 3);
1070 if (r)
1071 goto error_free_scratch;
1072
1073 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
1074 amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START));
1075 amdgpu_ring_write(ring, 0xDEADBEEF);
1076 amdgpu_ring_commit(ring);
1077
1078 for (i = 0; i < adev->usec_timeout; i++) {
1079 tmp = RREG32(scratch);
1080 if (tmp == 0xDEADBEEF)
1081 break;
1082 udelay(1);
1083 }
1084
1085 if (i >= adev->usec_timeout)
1086 r = -ETIMEDOUT;
1087
1088 error_free_scratch:
1089 amdgpu_gfx_scratch_free(adev, scratch);
1090 return r;
1091 }
1092
gfx_v9_0_ring_test_ib(struct amdgpu_ring * ring,long timeout)1093 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1094 {
1095 struct amdgpu_device *adev = ring->adev;
1096 struct amdgpu_ib ib;
1097 struct dma_fence *f = NULL;
1098
1099 unsigned index;
1100 uint64_t gpu_addr;
1101 uint32_t tmp;
1102 long r;
1103
1104 r = amdgpu_device_wb_get(adev, &index);
1105 if (r)
1106 return r;
1107
1108 gpu_addr = adev->wb.gpu_addr + (index * 4);
1109 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
1110 memset(&ib, 0, sizeof(ib));
1111 r = amdgpu_ib_get(adev, NULL, 16,
1112 AMDGPU_IB_POOL_DIRECT, &ib);
1113 if (r)
1114 goto err1;
1115
1116 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
1117 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
1118 ib.ptr[2] = lower_32_bits(gpu_addr);
1119 ib.ptr[3] = upper_32_bits(gpu_addr);
1120 ib.ptr[4] = 0xDEADBEEF;
1121 ib.length_dw = 5;
1122
1123 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1124 if (r)
1125 goto err2;
1126
1127 r = dma_fence_wait_timeout(f, false, timeout);
1128 if (r == 0) {
1129 r = -ETIMEDOUT;
1130 goto err2;
1131 } else if (r < 0) {
1132 goto err2;
1133 }
1134
1135 tmp = adev->wb.wb[index];
1136 if (tmp == 0xDEADBEEF)
1137 r = 0;
1138 else
1139 r = -EINVAL;
1140
1141 err2:
1142 amdgpu_ib_free(adev, &ib, NULL);
1143 dma_fence_put(f);
1144 err1:
1145 amdgpu_device_wb_free(adev, index);
1146 return r;
1147 }
1148
1149
gfx_v9_0_free_microcode(struct amdgpu_device * adev)1150 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev)
1151 {
1152 release_firmware(adev->gfx.pfp_fw);
1153 adev->gfx.pfp_fw = NULL;
1154 release_firmware(adev->gfx.me_fw);
1155 adev->gfx.me_fw = NULL;
1156 release_firmware(adev->gfx.ce_fw);
1157 adev->gfx.ce_fw = NULL;
1158 release_firmware(adev->gfx.rlc_fw);
1159 adev->gfx.rlc_fw = NULL;
1160 release_firmware(adev->gfx.mec_fw);
1161 adev->gfx.mec_fw = NULL;
1162 release_firmware(adev->gfx.mec2_fw);
1163 adev->gfx.mec2_fw = NULL;
1164
1165 kfree(adev->gfx.rlc.register_list_format);
1166 }
1167
gfx_v9_0_init_rlc_ext_microcode(struct amdgpu_device * adev)1168 static void gfx_v9_0_init_rlc_ext_microcode(struct amdgpu_device *adev)
1169 {
1170 const struct rlc_firmware_header_v2_1 *rlc_hdr;
1171
1172 rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data;
1173 adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver);
1174 adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver);
1175 adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes);
1176 adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes);
1177 adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver);
1178 adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver);
1179 adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes);
1180 adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes);
1181 adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver);
1182 adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver);
1183 adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes);
1184 adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes);
1185 adev->gfx.rlc.reg_list_format_direct_reg_list_length =
1186 le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length);
1187 }
1188
gfx_v9_0_check_fw_write_wait(struct amdgpu_device * adev)1189 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev)
1190 {
1191 adev->gfx.me_fw_write_wait = false;
1192 adev->gfx.mec_fw_write_wait = false;
1193
1194 if ((adev->asic_type != CHIP_ARCTURUS) &&
1195 ((adev->gfx.mec_fw_version < 0x000001a5) ||
1196 (adev->gfx.mec_feature_version < 46) ||
1197 (adev->gfx.pfp_fw_version < 0x000000b7) ||
1198 (adev->gfx.pfp_feature_version < 46)))
1199 DRM_WARN_ONCE("CP firmware version too old, please update!");
1200
1201 switch (adev->asic_type) {
1202 case CHIP_VEGA10:
1203 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1204 (adev->gfx.me_feature_version >= 42) &&
1205 (adev->gfx.pfp_fw_version >= 0x000000b1) &&
1206 (adev->gfx.pfp_feature_version >= 42))
1207 adev->gfx.me_fw_write_wait = true;
1208
1209 if ((adev->gfx.mec_fw_version >= 0x00000193) &&
1210 (adev->gfx.mec_feature_version >= 42))
1211 adev->gfx.mec_fw_write_wait = true;
1212 break;
1213 case CHIP_VEGA12:
1214 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1215 (adev->gfx.me_feature_version >= 44) &&
1216 (adev->gfx.pfp_fw_version >= 0x000000b2) &&
1217 (adev->gfx.pfp_feature_version >= 44))
1218 adev->gfx.me_fw_write_wait = true;
1219
1220 if ((adev->gfx.mec_fw_version >= 0x00000196) &&
1221 (adev->gfx.mec_feature_version >= 44))
1222 adev->gfx.mec_fw_write_wait = true;
1223 break;
1224 case CHIP_VEGA20:
1225 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1226 (adev->gfx.me_feature_version >= 44) &&
1227 (adev->gfx.pfp_fw_version >= 0x000000b2) &&
1228 (adev->gfx.pfp_feature_version >= 44))
1229 adev->gfx.me_fw_write_wait = true;
1230
1231 if ((adev->gfx.mec_fw_version >= 0x00000197) &&
1232 (adev->gfx.mec_feature_version >= 44))
1233 adev->gfx.mec_fw_write_wait = true;
1234 break;
1235 case CHIP_RAVEN:
1236 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1237 (adev->gfx.me_feature_version >= 42) &&
1238 (adev->gfx.pfp_fw_version >= 0x000000b1) &&
1239 (adev->gfx.pfp_feature_version >= 42))
1240 adev->gfx.me_fw_write_wait = true;
1241
1242 if ((adev->gfx.mec_fw_version >= 0x00000192) &&
1243 (adev->gfx.mec_feature_version >= 42))
1244 adev->gfx.mec_fw_write_wait = true;
1245 break;
1246 default:
1247 adev->gfx.me_fw_write_wait = true;
1248 adev->gfx.mec_fw_write_wait = true;
1249 break;
1250 }
1251 }
1252
1253 struct amdgpu_gfxoff_quirk {
1254 u16 chip_vendor;
1255 u16 chip_device;
1256 u16 subsys_vendor;
1257 u16 subsys_device;
1258 u8 revision;
1259 };
1260
1261 static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = {
1262 /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */
1263 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 },
1264 /* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */
1265 { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 },
1266 /* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */
1267 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 },
1268 { 0, 0, 0, 0, 0 },
1269 };
1270
gfx_v9_0_should_disable_gfxoff(struct pci_dev * pdev)1271 static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev)
1272 {
1273 const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list;
1274
1275 while (p && p->chip_device != 0) {
1276 if (pdev->vendor == p->chip_vendor &&
1277 pdev->device == p->chip_device &&
1278 pdev->subsystem_vendor == p->subsys_vendor &&
1279 pdev->subsystem_device == p->subsys_device &&
1280 pdev->revision == p->revision) {
1281 return true;
1282 }
1283 ++p;
1284 }
1285 return false;
1286 }
1287
is_raven_kicker(struct amdgpu_device * adev)1288 static bool is_raven_kicker(struct amdgpu_device *adev)
1289 {
1290 if (adev->pm.fw_version >= 0x41e2b)
1291 return true;
1292 else
1293 return false;
1294 }
1295
gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device * adev)1296 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev)
1297 {
1298 if (gfx_v9_0_should_disable_gfxoff(adev->pdev))
1299 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1300
1301 switch (adev->asic_type) {
1302 case CHIP_VEGA10:
1303 case CHIP_VEGA12:
1304 case CHIP_VEGA20:
1305 break;
1306 case CHIP_RAVEN:
1307 if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) ||
1308 (adev->apu_flags & AMD_APU_IS_PICASSO)) &&
1309 ((!is_raven_kicker(adev) &&
1310 adev->gfx.rlc_fw_version < 531) ||
1311 (adev->gfx.rlc_feature_version < 1) ||
1312 !adev->gfx.rlc.is_rlc_v2_1))
1313 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1314
1315 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1316 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1317 AMD_PG_SUPPORT_CP |
1318 AMD_PG_SUPPORT_RLC_SMU_HS;
1319 break;
1320 case CHIP_RENOIR:
1321 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1322 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1323 AMD_PG_SUPPORT_CP |
1324 AMD_PG_SUPPORT_RLC_SMU_HS;
1325 break;
1326 default:
1327 break;
1328 }
1329 }
1330
gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device * adev,const char * chip_name)1331 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev,
1332 const char *chip_name)
1333 {
1334 char fw_name[30];
1335 int err;
1336 struct amdgpu_firmware_info *info = NULL;
1337 const struct common_firmware_header *header = NULL;
1338 const struct gfx_firmware_header_v1_0 *cp_hdr;
1339
1340 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
1341 err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
1342 if (err)
1343 goto out;
1344 err = amdgpu_ucode_validate(adev->gfx.pfp_fw);
1345 if (err)
1346 goto out;
1347 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
1348 adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1349 adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1350
1351 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
1352 err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
1353 if (err)
1354 goto out;
1355 err = amdgpu_ucode_validate(adev->gfx.me_fw);
1356 if (err)
1357 goto out;
1358 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
1359 adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1360 adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1361
1362 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
1363 err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
1364 if (err)
1365 goto out;
1366 err = amdgpu_ucode_validate(adev->gfx.ce_fw);
1367 if (err)
1368 goto out;
1369 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
1370 adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1371 adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1372
1373 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1374 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP];
1375 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP;
1376 info->fw = adev->gfx.pfp_fw;
1377 header = (const struct common_firmware_header *)info->fw->data;
1378 adev->firmware.fw_size +=
1379 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1380
1381 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME];
1382 info->ucode_id = AMDGPU_UCODE_ID_CP_ME;
1383 info->fw = adev->gfx.me_fw;
1384 header = (const struct common_firmware_header *)info->fw->data;
1385 adev->firmware.fw_size +=
1386 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1387
1388 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE];
1389 info->ucode_id = AMDGPU_UCODE_ID_CP_CE;
1390 info->fw = adev->gfx.ce_fw;
1391 header = (const struct common_firmware_header *)info->fw->data;
1392 adev->firmware.fw_size +=
1393 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1394 }
1395
1396 out:
1397 if (err) {
1398 dev_err(adev->dev,
1399 "gfx9: Failed to load firmware \"%s\"\n",
1400 fw_name);
1401 release_firmware(adev->gfx.pfp_fw);
1402 adev->gfx.pfp_fw = NULL;
1403 release_firmware(adev->gfx.me_fw);
1404 adev->gfx.me_fw = NULL;
1405 release_firmware(adev->gfx.ce_fw);
1406 adev->gfx.ce_fw = NULL;
1407 }
1408 return err;
1409 }
1410
gfx_v9_0_init_rlc_microcode(struct amdgpu_device * adev,const char * chip_name)1411 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev,
1412 const char *chip_name)
1413 {
1414 char fw_name[30];
1415 int err;
1416 struct amdgpu_firmware_info *info = NULL;
1417 const struct common_firmware_header *header = NULL;
1418 const struct rlc_firmware_header_v2_0 *rlc_hdr;
1419 unsigned int *tmp = NULL;
1420 unsigned int i = 0;
1421 uint16_t version_major;
1422 uint16_t version_minor;
1423 uint32_t smu_version;
1424
1425 /*
1426 * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin
1427 * instead of picasso_rlc.bin.
1428 * Judgment method:
1429 * PCO AM4: revision >= 0xC8 && revision <= 0xCF
1430 * or revision >= 0xD8 && revision <= 0xDF
1431 * otherwise is PCO FP5
1432 */
1433 if (!strcmp(chip_name, "picasso") &&
1434 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) ||
1435 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF))))
1436 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc_am4.bin", chip_name);
1437 else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) &&
1438 (smu_version >= 0x41e2b))
1439 /**
1440 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly.
1441 */
1442 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_kicker_rlc.bin", chip_name);
1443 else
1444 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name);
1445 err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev);
1446 if (err)
1447 goto out;
1448 err = amdgpu_ucode_validate(adev->gfx.rlc_fw);
1449 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1450
1451 version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1452 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1453 if (version_major == 2 && version_minor == 1)
1454 adev->gfx.rlc.is_rlc_v2_1 = true;
1455
1456 adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
1457 adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
1458 adev->gfx.rlc.save_and_restore_offset =
1459 le32_to_cpu(rlc_hdr->save_and_restore_offset);
1460 adev->gfx.rlc.clear_state_descriptor_offset =
1461 le32_to_cpu(rlc_hdr->clear_state_descriptor_offset);
1462 adev->gfx.rlc.avail_scratch_ram_locations =
1463 le32_to_cpu(rlc_hdr->avail_scratch_ram_locations);
1464 adev->gfx.rlc.reg_restore_list_size =
1465 le32_to_cpu(rlc_hdr->reg_restore_list_size);
1466 adev->gfx.rlc.reg_list_format_start =
1467 le32_to_cpu(rlc_hdr->reg_list_format_start);
1468 adev->gfx.rlc.reg_list_format_separate_start =
1469 le32_to_cpu(rlc_hdr->reg_list_format_separate_start);
1470 adev->gfx.rlc.starting_offsets_start =
1471 le32_to_cpu(rlc_hdr->starting_offsets_start);
1472 adev->gfx.rlc.reg_list_format_size_bytes =
1473 le32_to_cpu(rlc_hdr->reg_list_format_size_bytes);
1474 adev->gfx.rlc.reg_list_size_bytes =
1475 le32_to_cpu(rlc_hdr->reg_list_size_bytes);
1476 adev->gfx.rlc.register_list_format =
1477 kmalloc(adev->gfx.rlc.reg_list_format_size_bytes +
1478 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL);
1479 if (!adev->gfx.rlc.register_list_format) {
1480 err = -ENOMEM;
1481 goto out;
1482 }
1483
1484 tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1485 le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));
1486 for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++)
1487 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]);
1488
1489 adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i;
1490
1491 tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1492 le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));
1493 for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++)
1494 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
1495
1496 if (adev->gfx.rlc.is_rlc_v2_1)
1497 gfx_v9_0_init_rlc_ext_microcode(adev);
1498
1499 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1500 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G];
1501 info->ucode_id = AMDGPU_UCODE_ID_RLC_G;
1502 info->fw = adev->gfx.rlc_fw;
1503 header = (const struct common_firmware_header *)info->fw->data;
1504 adev->firmware.fw_size +=
1505 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1506
1507 if (adev->gfx.rlc.is_rlc_v2_1 &&
1508 adev->gfx.rlc.save_restore_list_cntl_size_bytes &&
1509 adev->gfx.rlc.save_restore_list_gpm_size_bytes &&
1510 adev->gfx.rlc.save_restore_list_srm_size_bytes) {
1511 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL];
1512 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL;
1513 info->fw = adev->gfx.rlc_fw;
1514 adev->firmware.fw_size +=
1515 ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE);
1516
1517 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM];
1518 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM;
1519 info->fw = adev->gfx.rlc_fw;
1520 adev->firmware.fw_size +=
1521 ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE);
1522
1523 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM];
1524 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM;
1525 info->fw = adev->gfx.rlc_fw;
1526 adev->firmware.fw_size +=
1527 ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE);
1528 }
1529 }
1530
1531 out:
1532 if (err) {
1533 dev_err(adev->dev,
1534 "gfx9: Failed to load firmware \"%s\"\n",
1535 fw_name);
1536 release_firmware(adev->gfx.rlc_fw);
1537 adev->gfx.rlc_fw = NULL;
1538 }
1539 return err;
1540 }
1541
gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device * adev)1542 static bool gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device *adev)
1543 {
1544 if (adev->asic_type == CHIP_ALDEBARAN ||
1545 adev->asic_type == CHIP_ARCTURUS ||
1546 adev->asic_type == CHIP_RENOIR)
1547 return false;
1548
1549 return true;
1550 }
1551
gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device * adev,const char * chip_name)1552 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev,
1553 const char *chip_name)
1554 {
1555 char fw_name[30];
1556 int err;
1557 struct amdgpu_firmware_info *info = NULL;
1558 const struct common_firmware_header *header = NULL;
1559 const struct gfx_firmware_header_v1_0 *cp_hdr;
1560
1561 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
1562 err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
1563 if (err)
1564 goto out;
1565 err = amdgpu_ucode_validate(adev->gfx.mec_fw);
1566 if (err)
1567 goto out;
1568 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1569 adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1570 adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1571
1572
1573 if (gfx_v9_0_load_mec2_fw_bin_support(adev)) {
1574 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
1575 err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
1576 if (!err) {
1577 err = amdgpu_ucode_validate(adev->gfx.mec2_fw);
1578 if (err)
1579 goto out;
1580 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1581 adev->gfx.mec2_fw->data;
1582 adev->gfx.mec2_fw_version =
1583 le32_to_cpu(cp_hdr->header.ucode_version);
1584 adev->gfx.mec2_feature_version =
1585 le32_to_cpu(cp_hdr->ucode_feature_version);
1586 } else {
1587 err = 0;
1588 adev->gfx.mec2_fw = NULL;
1589 }
1590 } else {
1591 adev->gfx.mec2_fw_version = adev->gfx.mec_fw_version;
1592 adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version;
1593 }
1594
1595 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1596 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1];
1597 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1;
1598 info->fw = adev->gfx.mec_fw;
1599 header = (const struct common_firmware_header *)info->fw->data;
1600 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
1601 adev->firmware.fw_size +=
1602 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1603
1604 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT];
1605 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT;
1606 info->fw = adev->gfx.mec_fw;
1607 adev->firmware.fw_size +=
1608 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1609
1610 if (adev->gfx.mec2_fw) {
1611 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2];
1612 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
1613 info->fw = adev->gfx.mec2_fw;
1614 header = (const struct common_firmware_header *)info->fw->data;
1615 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
1616 adev->firmware.fw_size +=
1617 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1618
1619 /* TODO: Determine if MEC2 JT FW loading can be removed
1620 for all GFX V9 asic and above */
1621 if (gfx_v9_0_load_mec2_fw_bin_support(adev)) {
1622 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT];
1623 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT;
1624 info->fw = adev->gfx.mec2_fw;
1625 adev->firmware.fw_size +=
1626 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4,
1627 PAGE_SIZE);
1628 }
1629 }
1630 }
1631
1632 out:
1633 gfx_v9_0_check_if_need_gfxoff(adev);
1634 gfx_v9_0_check_fw_write_wait(adev);
1635 if (err) {
1636 dev_err(adev->dev,
1637 "gfx9: Failed to load firmware \"%s\"\n",
1638 fw_name);
1639 release_firmware(adev->gfx.mec_fw);
1640 adev->gfx.mec_fw = NULL;
1641 release_firmware(adev->gfx.mec2_fw);
1642 adev->gfx.mec2_fw = NULL;
1643 }
1644 return err;
1645 }
1646
gfx_v9_0_init_microcode(struct amdgpu_device * adev)1647 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
1648 {
1649 const char *chip_name;
1650 int r;
1651
1652 DRM_DEBUG("\n");
1653
1654 switch (adev->asic_type) {
1655 case CHIP_VEGA10:
1656 chip_name = "vega10";
1657 break;
1658 case CHIP_VEGA12:
1659 chip_name = "vega12";
1660 break;
1661 case CHIP_VEGA20:
1662 chip_name = "vega20";
1663 break;
1664 case CHIP_RAVEN:
1665 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1666 chip_name = "raven2";
1667 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
1668 chip_name = "picasso";
1669 else
1670 chip_name = "raven";
1671 break;
1672 case CHIP_ARCTURUS:
1673 chip_name = "arcturus";
1674 break;
1675 case CHIP_RENOIR:
1676 if (adev->apu_flags & AMD_APU_IS_RENOIR)
1677 chip_name = "renoir";
1678 else
1679 chip_name = "green_sardine";
1680 break;
1681 case CHIP_ALDEBARAN:
1682 chip_name = "aldebaran";
1683 break;
1684 default:
1685 BUG();
1686 }
1687
1688 /* No CPG in Arcturus */
1689 if (adev->gfx.num_gfx_rings) {
1690 r = gfx_v9_0_init_cp_gfx_microcode(adev, chip_name);
1691 if (r)
1692 return r;
1693 }
1694
1695 r = gfx_v9_0_init_rlc_microcode(adev, chip_name);
1696 if (r)
1697 return r;
1698
1699 r = gfx_v9_0_init_cp_compute_microcode(adev, chip_name);
1700 if (r)
1701 return r;
1702
1703 return r;
1704 }
1705
gfx_v9_0_get_csb_size(struct amdgpu_device * adev)1706 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev)
1707 {
1708 u32 count = 0;
1709 const struct cs_section_def *sect = NULL;
1710 const struct cs_extent_def *ext = NULL;
1711
1712 /* begin clear state */
1713 count += 2;
1714 /* context control state */
1715 count += 3;
1716
1717 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
1718 for (ext = sect->section; ext->extent != NULL; ++ext) {
1719 if (sect->id == SECT_CONTEXT)
1720 count += 2 + ext->reg_count;
1721 else
1722 return 0;
1723 }
1724 }
1725
1726 /* end clear state */
1727 count += 2;
1728 /* clear state */
1729 count += 2;
1730
1731 return count;
1732 }
1733
gfx_v9_0_get_csb_buffer(struct amdgpu_device * adev,volatile u32 * buffer)1734 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev,
1735 volatile u32 *buffer)
1736 {
1737 u32 count = 0, i;
1738 const struct cs_section_def *sect = NULL;
1739 const struct cs_extent_def *ext = NULL;
1740
1741 if (adev->gfx.rlc.cs_data == NULL)
1742 return;
1743 if (buffer == NULL)
1744 return;
1745
1746 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1747 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1748
1749 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
1750 buffer[count++] = cpu_to_le32(0x80000000);
1751 buffer[count++] = cpu_to_le32(0x80000000);
1752
1753 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
1754 for (ext = sect->section; ext->extent != NULL; ++ext) {
1755 if (sect->id == SECT_CONTEXT) {
1756 buffer[count++] =
1757 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
1758 buffer[count++] = cpu_to_le32(ext->reg_index -
1759 PACKET3_SET_CONTEXT_REG_START);
1760 for (i = 0; i < ext->reg_count; i++)
1761 buffer[count++] = cpu_to_le32(ext->extent[i]);
1762 } else {
1763 return;
1764 }
1765 }
1766 }
1767
1768 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1769 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
1770
1771 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
1772 buffer[count++] = cpu_to_le32(0);
1773 }
1774
gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device * adev)1775 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev)
1776 {
1777 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
1778 uint32_t pg_always_on_cu_num = 2;
1779 uint32_t always_on_cu_num;
1780 uint32_t i, j, k;
1781 uint32_t mask, cu_bitmap, counter;
1782
1783 if (adev->flags & AMD_IS_APU)
1784 always_on_cu_num = 4;
1785 else if (adev->asic_type == CHIP_VEGA12)
1786 always_on_cu_num = 8;
1787 else
1788 always_on_cu_num = 12;
1789
1790 mutex_lock(&adev->grbm_idx_mutex);
1791 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1792 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1793 mask = 1;
1794 cu_bitmap = 0;
1795 counter = 0;
1796 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
1797
1798 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
1799 if (cu_info->bitmap[i][j] & mask) {
1800 if (counter == pg_always_on_cu_num)
1801 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap);
1802 if (counter < always_on_cu_num)
1803 cu_bitmap |= mask;
1804 else
1805 break;
1806 counter++;
1807 }
1808 mask <<= 1;
1809 }
1810
1811 WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap);
1812 cu_info->ao_cu_bitmap[i][j] = cu_bitmap;
1813 }
1814 }
1815 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1816 mutex_unlock(&adev->grbm_idx_mutex);
1817 }
1818
gfx_v9_0_init_lbpw(struct amdgpu_device * adev)1819 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev)
1820 {
1821 uint32_t data;
1822
1823 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1824 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1825 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7);
1826 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1827 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16));
1828
1829 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1830 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1831
1832 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1833 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500);
1834
1835 mutex_lock(&adev->grbm_idx_mutex);
1836 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1837 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1838 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1839
1840 /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1841 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1842 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1843 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1844 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1845
1846 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1847 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1848 data &= 0x0000FFFF;
1849 data |= 0x00C00000;
1850 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1851
1852 /*
1853 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven),
1854 * programmed in gfx_v9_0_init_always_on_cu_mask()
1855 */
1856
1857 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1858 * but used for RLC_LB_CNTL configuration */
1859 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1860 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1861 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1862 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1863 mutex_unlock(&adev->grbm_idx_mutex);
1864
1865 gfx_v9_0_init_always_on_cu_mask(adev);
1866 }
1867
gfx_v9_4_init_lbpw(struct amdgpu_device * adev)1868 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev)
1869 {
1870 uint32_t data;
1871
1872 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1873 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1874 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8);
1875 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1876 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16));
1877
1878 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1879 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1880
1881 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1882 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800);
1883
1884 mutex_lock(&adev->grbm_idx_mutex);
1885 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1886 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1887 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1888
1889 /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1890 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1891 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1892 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1893 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1894
1895 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1896 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1897 data &= 0x0000FFFF;
1898 data |= 0x00C00000;
1899 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1900
1901 /*
1902 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON),
1903 * programmed in gfx_v9_0_init_always_on_cu_mask()
1904 */
1905
1906 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1907 * but used for RLC_LB_CNTL configuration */
1908 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1909 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1910 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1911 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1912 mutex_unlock(&adev->grbm_idx_mutex);
1913
1914 gfx_v9_0_init_always_on_cu_mask(adev);
1915 }
1916
gfx_v9_0_enable_lbpw(struct amdgpu_device * adev,bool enable)1917 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable)
1918 {
1919 WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0);
1920 }
1921
gfx_v9_0_cp_jump_table_num(struct amdgpu_device * adev)1922 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev)
1923 {
1924 if (gfx_v9_0_load_mec2_fw_bin_support(adev))
1925 return 5;
1926 else
1927 return 4;
1928 }
1929
gfx_v9_0_rlc_init(struct amdgpu_device * adev)1930 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev)
1931 {
1932 const struct cs_section_def *cs_data;
1933 int r;
1934
1935 adev->gfx.rlc.cs_data = gfx9_cs_data;
1936
1937 cs_data = adev->gfx.rlc.cs_data;
1938
1939 if (cs_data) {
1940 /* init clear state block */
1941 r = amdgpu_gfx_rlc_init_csb(adev);
1942 if (r)
1943 return r;
1944 }
1945
1946 if (adev->flags & AMD_IS_APU) {
1947 /* TODO: double check the cp_table_size for RV */
1948 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */
1949 r = amdgpu_gfx_rlc_init_cpt(adev);
1950 if (r)
1951 return r;
1952 }
1953
1954 switch (adev->asic_type) {
1955 case CHIP_RAVEN:
1956 gfx_v9_0_init_lbpw(adev);
1957 break;
1958 case CHIP_VEGA20:
1959 gfx_v9_4_init_lbpw(adev);
1960 break;
1961 default:
1962 break;
1963 }
1964
1965 /* init spm vmid with 0xf */
1966 if (adev->gfx.rlc.funcs->update_spm_vmid)
1967 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf);
1968
1969 return 0;
1970 }
1971
gfx_v9_0_mec_fini(struct amdgpu_device * adev)1972 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev)
1973 {
1974 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
1975 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
1976 }
1977
gfx_v9_0_mec_init(struct amdgpu_device * adev)1978 static int gfx_v9_0_mec_init(struct amdgpu_device *adev)
1979 {
1980 int r;
1981 u32 *hpd;
1982 const __le32 *fw_data;
1983 unsigned fw_size;
1984 u32 *fw;
1985 size_t mec_hpd_size;
1986
1987 const struct gfx_firmware_header_v1_0 *mec_hdr;
1988
1989 bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
1990
1991 /* take ownership of the relevant compute queues */
1992 amdgpu_gfx_compute_queue_acquire(adev);
1993 mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE;
1994 if (mec_hpd_size) {
1995 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
1996 AMDGPU_GEM_DOMAIN_VRAM,
1997 &adev->gfx.mec.hpd_eop_obj,
1998 &adev->gfx.mec.hpd_eop_gpu_addr,
1999 (void **)&hpd);
2000 if (r) {
2001 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
2002 gfx_v9_0_mec_fini(adev);
2003 return r;
2004 }
2005
2006 memset(hpd, 0, mec_hpd_size);
2007
2008 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
2009 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
2010 }
2011
2012 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
2013
2014 fw_data = (const __le32 *)
2015 (adev->gfx.mec_fw->data +
2016 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
2017 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
2018
2019 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
2020 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
2021 &adev->gfx.mec.mec_fw_obj,
2022 &adev->gfx.mec.mec_fw_gpu_addr,
2023 (void **)&fw);
2024 if (r) {
2025 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r);
2026 gfx_v9_0_mec_fini(adev);
2027 return r;
2028 }
2029
2030 memcpy(fw, fw_data, fw_size);
2031
2032 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
2033 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
2034
2035 return 0;
2036 }
2037
wave_read_ind(struct amdgpu_device * adev,uint32_t simd,uint32_t wave,uint32_t address)2038 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
2039 {
2040 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
2041 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
2042 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
2043 (address << SQ_IND_INDEX__INDEX__SHIFT) |
2044 (SQ_IND_INDEX__FORCE_READ_MASK));
2045 return RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
2046 }
2047
wave_read_regs(struct amdgpu_device * adev,uint32_t simd,uint32_t wave,uint32_t thread,uint32_t regno,uint32_t num,uint32_t * out)2048 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
2049 uint32_t wave, uint32_t thread,
2050 uint32_t regno, uint32_t num, uint32_t *out)
2051 {
2052 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
2053 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
2054 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
2055 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
2056 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
2057 (SQ_IND_INDEX__FORCE_READ_MASK) |
2058 (SQ_IND_INDEX__AUTO_INCR_MASK));
2059 while (num--)
2060 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
2061 }
2062
gfx_v9_0_read_wave_data(struct amdgpu_device * adev,uint32_t simd,uint32_t wave,uint32_t * dst,int * no_fields)2063 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
2064 {
2065 /* type 1 wave data */
2066 dst[(*no_fields)++] = 1;
2067 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
2068 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
2069 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
2070 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
2071 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
2072 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
2073 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
2074 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
2075 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
2076 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
2077 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
2078 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
2079 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
2080 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
2081 }
2082
gfx_v9_0_read_wave_sgprs(struct amdgpu_device * adev,uint32_t simd,uint32_t wave,uint32_t start,uint32_t size,uint32_t * dst)2083 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
2084 uint32_t wave, uint32_t start,
2085 uint32_t size, uint32_t *dst)
2086 {
2087 wave_read_regs(
2088 adev, simd, wave, 0,
2089 start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
2090 }
2091
gfx_v9_0_read_wave_vgprs(struct amdgpu_device * adev,uint32_t simd,uint32_t wave,uint32_t thread,uint32_t start,uint32_t size,uint32_t * dst)2092 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd,
2093 uint32_t wave, uint32_t thread,
2094 uint32_t start, uint32_t size,
2095 uint32_t *dst)
2096 {
2097 wave_read_regs(
2098 adev, simd, wave, thread,
2099 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
2100 }
2101
gfx_v9_0_select_me_pipe_q(struct amdgpu_device * adev,u32 me,u32 pipe,u32 q,u32 vm)2102 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev,
2103 u32 me, u32 pipe, u32 q, u32 vm)
2104 {
2105 soc15_grbm_select(adev, me, pipe, q, vm);
2106 }
2107
2108 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = {
2109 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,
2110 .select_se_sh = &gfx_v9_0_select_se_sh,
2111 .read_wave_data = &gfx_v9_0_read_wave_data,
2112 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
2113 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,
2114 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q,
2115 };
2116
2117 static const struct amdgpu_gfx_ras_funcs gfx_v9_0_ras_funcs = {
2118 .ras_late_init = amdgpu_gfx_ras_late_init,
2119 .ras_fini = amdgpu_gfx_ras_fini,
2120 .ras_error_inject = &gfx_v9_0_ras_error_inject,
2121 .query_ras_error_count = &gfx_v9_0_query_ras_error_count,
2122 .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count,
2123 };
2124
gfx_v9_0_gpu_early_init(struct amdgpu_device * adev)2125 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
2126 {
2127 u32 gb_addr_config;
2128 int err;
2129
2130 adev->gfx.funcs = &gfx_v9_0_gfx_funcs;
2131
2132 switch (adev->asic_type) {
2133 case CHIP_VEGA10:
2134 adev->gfx.config.max_hw_contexts = 8;
2135 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2136 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2137 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2138 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2139 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN;
2140 break;
2141 case CHIP_VEGA12:
2142 adev->gfx.config.max_hw_contexts = 8;
2143 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2144 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2145 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2146 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2147 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN;
2148 DRM_INFO("fix gfx.config for vega12\n");
2149 break;
2150 case CHIP_VEGA20:
2151 adev->gfx.ras_funcs = &gfx_v9_0_ras_funcs;
2152 adev->gfx.config.max_hw_contexts = 8;
2153 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2154 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2155 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2156 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2157 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2158 gb_addr_config &= ~0xf3e777ff;
2159 gb_addr_config |= 0x22014042;
2160 /* check vbios table if gpu info is not available */
2161 err = amdgpu_atomfirmware_get_gfx_info(adev);
2162 if (err)
2163 return err;
2164 break;
2165 case CHIP_RAVEN:
2166 adev->gfx.config.max_hw_contexts = 8;
2167 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2168 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2169 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2170 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2171 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
2172 gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN;
2173 else
2174 gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN;
2175 break;
2176 case CHIP_ARCTURUS:
2177 adev->gfx.ras_funcs = &gfx_v9_4_ras_funcs;
2178 adev->gfx.config.max_hw_contexts = 8;
2179 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2180 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2181 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2182 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2183 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2184 gb_addr_config &= ~0xf3e777ff;
2185 gb_addr_config |= 0x22014042;
2186 break;
2187 case CHIP_RENOIR:
2188 adev->gfx.config.max_hw_contexts = 8;
2189 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2190 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2191 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;
2192 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2193 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2194 gb_addr_config &= ~0xf3e777ff;
2195 gb_addr_config |= 0x22010042;
2196 break;
2197 case CHIP_ALDEBARAN:
2198 adev->gfx.ras_funcs = &gfx_v9_4_2_ras_funcs;
2199 adev->gfx.config.max_hw_contexts = 8;
2200 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2201 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2202 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2203 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2204 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2205 gb_addr_config &= ~0xf3e777ff;
2206 gb_addr_config |= 0x22014042;
2207 /* check vbios table if gpu info is not available */
2208 err = amdgpu_atomfirmware_get_gfx_info(adev);
2209 if (err)
2210 return err;
2211 break;
2212 default:
2213 BUG();
2214 break;
2215 }
2216
2217 adev->gfx.config.gb_addr_config = gb_addr_config;
2218
2219 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
2220 REG_GET_FIELD(
2221 adev->gfx.config.gb_addr_config,
2222 GB_ADDR_CONFIG,
2223 NUM_PIPES);
2224
2225 adev->gfx.config.max_tile_pipes =
2226 adev->gfx.config.gb_addr_config_fields.num_pipes;
2227
2228 adev->gfx.config.gb_addr_config_fields.num_banks = 1 <<
2229 REG_GET_FIELD(
2230 adev->gfx.config.gb_addr_config,
2231 GB_ADDR_CONFIG,
2232 NUM_BANKS);
2233 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
2234 REG_GET_FIELD(
2235 adev->gfx.config.gb_addr_config,
2236 GB_ADDR_CONFIG,
2237 MAX_COMPRESSED_FRAGS);
2238 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
2239 REG_GET_FIELD(
2240 adev->gfx.config.gb_addr_config,
2241 GB_ADDR_CONFIG,
2242 NUM_RB_PER_SE);
2243 adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
2244 REG_GET_FIELD(
2245 adev->gfx.config.gb_addr_config,
2246 GB_ADDR_CONFIG,
2247 NUM_SHADER_ENGINES);
2248 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
2249 REG_GET_FIELD(
2250 adev->gfx.config.gb_addr_config,
2251 GB_ADDR_CONFIG,
2252 PIPE_INTERLEAVE_SIZE));
2253
2254 return 0;
2255 }
2256
gfx_v9_0_compute_ring_init(struct amdgpu_device * adev,int ring_id,int mec,int pipe,int queue)2257 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
2258 int mec, int pipe, int queue)
2259 {
2260 unsigned irq_type;
2261 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
2262 unsigned int hw_prio;
2263
2264 ring = &adev->gfx.compute_ring[ring_id];
2265
2266 /* mec0 is me1 */
2267 ring->me = mec + 1;
2268 ring->pipe = pipe;
2269 ring->queue = queue;
2270
2271 ring->ring_obj = NULL;
2272 ring->use_doorbell = true;
2273 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
2274 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
2275 + (ring_id * GFX9_MEC_HPD_SIZE);
2276 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
2277
2278 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
2279 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
2280 + ring->pipe;
2281 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
2282 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
2283 /* type-2 packets are deprecated on MEC, use type-3 instead */
2284 return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
2285 hw_prio, NULL);
2286 }
2287
gfx_v9_0_sw_init(void * handle)2288 static int gfx_v9_0_sw_init(void *handle)
2289 {
2290 int i, j, k, r, ring_id;
2291 struct amdgpu_ring *ring;
2292 struct amdgpu_kiq *kiq;
2293 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2294
2295 switch (adev->asic_type) {
2296 case CHIP_VEGA10:
2297 case CHIP_VEGA12:
2298 case CHIP_VEGA20:
2299 case CHIP_RAVEN:
2300 case CHIP_ARCTURUS:
2301 case CHIP_RENOIR:
2302 case CHIP_ALDEBARAN:
2303 adev->gfx.mec.num_mec = 2;
2304 break;
2305 default:
2306 adev->gfx.mec.num_mec = 1;
2307 break;
2308 }
2309
2310 adev->gfx.mec.num_pipe_per_mec = 4;
2311 adev->gfx.mec.num_queue_per_pipe = 8;
2312
2313 /* EOP Event */
2314 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq);
2315 if (r)
2316 return r;
2317
2318 /* Privileged reg */
2319 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT,
2320 &adev->gfx.priv_reg_irq);
2321 if (r)
2322 return r;
2323
2324 /* Privileged inst */
2325 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT,
2326 &adev->gfx.priv_inst_irq);
2327 if (r)
2328 return r;
2329
2330 /* ECC error */
2331 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR,
2332 &adev->gfx.cp_ecc_error_irq);
2333 if (r)
2334 return r;
2335
2336 /* FUE error */
2337 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR,
2338 &adev->gfx.cp_ecc_error_irq);
2339 if (r)
2340 return r;
2341
2342 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
2343
2344 gfx_v9_0_scratch_init(adev);
2345
2346 r = gfx_v9_0_init_microcode(adev);
2347 if (r) {
2348 DRM_ERROR("Failed to load gfx firmware!\n");
2349 return r;
2350 }
2351
2352 r = adev->gfx.rlc.funcs->init(adev);
2353 if (r) {
2354 DRM_ERROR("Failed to init rlc BOs!\n");
2355 return r;
2356 }
2357
2358 r = gfx_v9_0_mec_init(adev);
2359 if (r) {
2360 DRM_ERROR("Failed to init MEC BOs!\n");
2361 return r;
2362 }
2363
2364 /* set up the gfx ring */
2365 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2366 ring = &adev->gfx.gfx_ring[i];
2367 ring->ring_obj = NULL;
2368 if (!i)
2369 sprintf(ring->name, "gfx");
2370 else
2371 sprintf(ring->name, "gfx_%d", i);
2372 ring->use_doorbell = true;
2373 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
2374 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
2375 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP,
2376 AMDGPU_RING_PRIO_DEFAULT, NULL);
2377 if (r)
2378 return r;
2379 }
2380
2381 /* set up the compute queues - allocate horizontally across pipes */
2382 ring_id = 0;
2383 for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
2384 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
2385 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
2386 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j))
2387 continue;
2388
2389 r = gfx_v9_0_compute_ring_init(adev,
2390 ring_id,
2391 i, k, j);
2392 if (r)
2393 return r;
2394
2395 ring_id++;
2396 }
2397 }
2398 }
2399
2400 r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE);
2401 if (r) {
2402 DRM_ERROR("Failed to init KIQ BOs!\n");
2403 return r;
2404 }
2405
2406 kiq = &adev->gfx.kiq;
2407 r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
2408 if (r)
2409 return r;
2410
2411 /* create MQD for all compute queues as wel as KIQ for SRIOV case */
2412 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation));
2413 if (r)
2414 return r;
2415
2416 adev->gfx.ce_ram_size = 0x8000;
2417
2418 r = gfx_v9_0_gpu_early_init(adev);
2419 if (r)
2420 return r;
2421
2422 return 0;
2423 }
2424
2425
gfx_v9_0_sw_fini(void * handle)2426 static int gfx_v9_0_sw_fini(void *handle)
2427 {
2428 int i;
2429 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2430
2431 if (adev->gfx.ras_funcs &&
2432 adev->gfx.ras_funcs->ras_fini)
2433 adev->gfx.ras_funcs->ras_fini(adev);
2434
2435 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2436 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
2437 for (i = 0; i < adev->gfx.num_compute_rings; i++)
2438 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
2439
2440 amdgpu_gfx_mqd_sw_fini(adev);
2441 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring);
2442 amdgpu_gfx_kiq_fini(adev);
2443
2444 gfx_v9_0_mec_fini(adev);
2445 amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj);
2446 if (adev->flags & AMD_IS_APU) {
2447 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
2448 &adev->gfx.rlc.cp_table_gpu_addr,
2449 (void **)&adev->gfx.rlc.cp_table_ptr);
2450 }
2451 gfx_v9_0_free_microcode(adev);
2452
2453 return 0;
2454 }
2455
2456
gfx_v9_0_tiling_mode_table_init(struct amdgpu_device * adev)2457 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev)
2458 {
2459 /* TODO */
2460 }
2461
gfx_v9_0_select_se_sh(struct amdgpu_device * adev,u32 se_num,u32 sh_num,u32 instance)2462 void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num,
2463 u32 instance)
2464 {
2465 u32 data;
2466
2467 if (instance == 0xffffffff)
2468 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
2469 else
2470 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
2471
2472 if (se_num == 0xffffffff)
2473 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
2474 else
2475 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
2476
2477 if (sh_num == 0xffffffff)
2478 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
2479 else
2480 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
2481
2482 WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data);
2483 }
2484
gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device * adev)2485 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev)
2486 {
2487 u32 data, mask;
2488
2489 data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE);
2490 data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE);
2491
2492 data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
2493 data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
2494
2495 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /
2496 adev->gfx.config.max_sh_per_se);
2497
2498 return (~data) & mask;
2499 }
2500
gfx_v9_0_setup_rb(struct amdgpu_device * adev)2501 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev)
2502 {
2503 int i, j;
2504 u32 data;
2505 u32 active_rbs = 0;
2506 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
2507 adev->gfx.config.max_sh_per_se;
2508
2509 mutex_lock(&adev->grbm_idx_mutex);
2510 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2511 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2512 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
2513 data = gfx_v9_0_get_rb_active_bitmap(adev);
2514 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
2515 rb_bitmap_width_per_sh);
2516 }
2517 }
2518 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2519 mutex_unlock(&adev->grbm_idx_mutex);
2520
2521 adev->gfx.config.backend_enable_mask = active_rbs;
2522 adev->gfx.config.num_rbs = hweight32(active_rbs);
2523 }
2524
2525 #define DEFAULT_SH_MEM_BASES (0x6000)
gfx_v9_0_init_compute_vmid(struct amdgpu_device * adev)2526 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)
2527 {
2528 int i;
2529 uint32_t sh_mem_config;
2530 uint32_t sh_mem_bases;
2531
2532 /*
2533 * Configure apertures:
2534 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB)
2535 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB)
2536 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB)
2537 */
2538 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
2539
2540 sh_mem_config = SH_MEM_ADDRESS_MODE_64 |
2541 SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
2542 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
2543
2544 mutex_lock(&adev->srbm_mutex);
2545 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2546 soc15_grbm_select(adev, 0, 0, 0, i);
2547 /* CP and shaders */
2548 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
2549 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
2550 }
2551 soc15_grbm_select(adev, 0, 0, 0, 0);
2552 mutex_unlock(&adev->srbm_mutex);
2553
2554 /* Initialize all compute VMIDs to have no GDS, GWS, or OA
2555 acccess. These should be enabled by FW for target VMIDs. */
2556 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2557 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0);
2558 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0);
2559 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0);
2560 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0);
2561 }
2562 }
2563
gfx_v9_0_init_gds_vmid(struct amdgpu_device * adev)2564 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev)
2565 {
2566 int vmid;
2567
2568 /*
2569 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
2570 * access. Compute VMIDs should be enabled by FW for target VMIDs,
2571 * the driver can enable them for graphics. VMID0 should maintain
2572 * access so that HWS firmware can save/restore entries.
2573 */
2574 for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) {
2575 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0);
2576 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0);
2577 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0);
2578 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0);
2579 }
2580 }
2581
gfx_v9_0_init_sq_config(struct amdgpu_device * adev)2582 static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev)
2583 {
2584 uint32_t tmp;
2585
2586 switch (adev->asic_type) {
2587 case CHIP_ARCTURUS:
2588 tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG);
2589 tmp = REG_SET_FIELD(tmp, SQ_CONFIG,
2590 DISABLE_BARRIER_WAITCNT, 1);
2591 WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp);
2592 break;
2593 default:
2594 break;
2595 }
2596 }
2597
gfx_v9_0_constants_init(struct amdgpu_device * adev)2598 static void gfx_v9_0_constants_init(struct amdgpu_device *adev)
2599 {
2600 u32 tmp;
2601 int i;
2602
2603 WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
2604
2605 gfx_v9_0_tiling_mode_table_init(adev);
2606
2607 gfx_v9_0_setup_rb(adev);
2608 gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info);
2609 adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2);
2610
2611 /* XXX SH_MEM regs */
2612 /* where to put LDS, scratch, GPUVM in FSA64 space */
2613 mutex_lock(&adev->srbm_mutex);
2614 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids; i++) {
2615 soc15_grbm_select(adev, 0, 0, 0, i);
2616 /* CP and shaders */
2617 if (i == 0) {
2618 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2619 SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2620 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2621 !!adev->gmc.noretry);
2622 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2623 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0);
2624 } else {
2625 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2626 SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2627 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2628 !!adev->gmc.noretry);
2629 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2630 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
2631 (adev->gmc.private_aperture_start >> 48));
2632 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
2633 (adev->gmc.shared_aperture_start >> 48));
2634 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp);
2635 }
2636 }
2637 soc15_grbm_select(adev, 0, 0, 0, 0);
2638
2639 mutex_unlock(&adev->srbm_mutex);
2640
2641 gfx_v9_0_init_compute_vmid(adev);
2642 gfx_v9_0_init_gds_vmid(adev);
2643 gfx_v9_0_init_sq_config(adev);
2644 }
2645
gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device * adev)2646 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
2647 {
2648 u32 i, j, k;
2649 u32 mask;
2650
2651 mutex_lock(&adev->grbm_idx_mutex);
2652 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2653 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2654 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
2655 for (k = 0; k < adev->usec_timeout; k++) {
2656 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0)
2657 break;
2658 udelay(1);
2659 }
2660 if (k == adev->usec_timeout) {
2661 gfx_v9_0_select_se_sh(adev, 0xffffffff,
2662 0xffffffff, 0xffffffff);
2663 mutex_unlock(&adev->grbm_idx_mutex);
2664 DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
2665 i, j);
2666 return;
2667 }
2668 }
2669 }
2670 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2671 mutex_unlock(&adev->grbm_idx_mutex);
2672
2673 mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
2674 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
2675 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
2676 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
2677 for (k = 0; k < adev->usec_timeout; k++) {
2678 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
2679 break;
2680 udelay(1);
2681 }
2682 }
2683
gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device * adev,bool enable)2684 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
2685 bool enable)
2686 {
2687 u32 tmp;
2688
2689 /* These interrupts should be enabled to drive DS clock */
2690
2691 tmp= RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0);
2692
2693 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
2694 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
2695 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
2696 if(adev->gfx.num_gfx_rings)
2697 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
2698
2699 WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp);
2700 }
2701
gfx_v9_0_init_csb(struct amdgpu_device * adev)2702 static void gfx_v9_0_init_csb(struct amdgpu_device *adev)
2703 {
2704 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
2705 /* csib */
2706 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI),
2707 adev->gfx.rlc.clear_state_gpu_addr >> 32);
2708 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO),
2709 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
2710 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH),
2711 adev->gfx.rlc.clear_state_size);
2712 }
2713
gfx_v9_1_parse_ind_reg_list(int * register_list_format,int indirect_offset,int list_size,int * unique_indirect_regs,int unique_indirect_reg_count,int * indirect_start_offsets,int * indirect_start_offsets_count,int max_start_offsets_count)2714 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format,
2715 int indirect_offset,
2716 int list_size,
2717 int *unique_indirect_regs,
2718 int unique_indirect_reg_count,
2719 int *indirect_start_offsets,
2720 int *indirect_start_offsets_count,
2721 int max_start_offsets_count)
2722 {
2723 int idx;
2724
2725 for (; indirect_offset < list_size; indirect_offset++) {
2726 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count);
2727 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset;
2728 *indirect_start_offsets_count = *indirect_start_offsets_count + 1;
2729
2730 while (register_list_format[indirect_offset] != 0xFFFFFFFF) {
2731 indirect_offset += 2;
2732
2733 /* look for the matching indice */
2734 for (idx = 0; idx < unique_indirect_reg_count; idx++) {
2735 if (unique_indirect_regs[idx] ==
2736 register_list_format[indirect_offset] ||
2737 !unique_indirect_regs[idx])
2738 break;
2739 }
2740
2741 BUG_ON(idx >= unique_indirect_reg_count);
2742
2743 if (!unique_indirect_regs[idx])
2744 unique_indirect_regs[idx] = register_list_format[indirect_offset];
2745
2746 indirect_offset++;
2747 }
2748 }
2749 }
2750
gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device * adev)2751 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev)
2752 {
2753 int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2754 int unique_indirect_reg_count = 0;
2755
2756 int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2757 int indirect_start_offsets_count = 0;
2758
2759 int list_size = 0;
2760 int i = 0, j = 0;
2761 u32 tmp = 0;
2762
2763 u32 *register_list_format =
2764 kmemdup(adev->gfx.rlc.register_list_format,
2765 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
2766 if (!register_list_format)
2767 return -ENOMEM;
2768
2769 /* setup unique_indirect_regs array and indirect_start_offsets array */
2770 unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs);
2771 gfx_v9_1_parse_ind_reg_list(register_list_format,
2772 adev->gfx.rlc.reg_list_format_direct_reg_list_length,
2773 adev->gfx.rlc.reg_list_format_size_bytes >> 2,
2774 unique_indirect_regs,
2775 unique_indirect_reg_count,
2776 indirect_start_offsets,
2777 &indirect_start_offsets_count,
2778 ARRAY_SIZE(indirect_start_offsets));
2779
2780 /* enable auto inc in case it is disabled */
2781 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));
2782 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
2783 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp);
2784
2785 /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */
2786 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR),
2787 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET);
2788 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)
2789 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA),
2790 adev->gfx.rlc.register_restore[i]);
2791
2792 /* load indirect register */
2793 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2794 adev->gfx.rlc.reg_list_format_start);
2795
2796 /* direct register portion */
2797 for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++)
2798 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2799 register_list_format[i]);
2800
2801 /* indirect register portion */
2802 while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) {
2803 if (register_list_format[i] == 0xFFFFFFFF) {
2804 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2805 continue;
2806 }
2807
2808 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2809 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2810
2811 for (j = 0; j < unique_indirect_reg_count; j++) {
2812 if (register_list_format[i] == unique_indirect_regs[j]) {
2813 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j);
2814 break;
2815 }
2816 }
2817
2818 BUG_ON(j >= unique_indirect_reg_count);
2819
2820 i++;
2821 }
2822
2823 /* set save/restore list size */
2824 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;
2825 list_size = list_size >> 1;
2826 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2827 adev->gfx.rlc.reg_restore_list_size);
2828 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size);
2829
2830 /* write the starting offsets to RLC scratch ram */
2831 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2832 adev->gfx.rlc.starting_offsets_start);
2833 for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
2834 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2835 indirect_start_offsets[i]);
2836
2837 /* load unique indirect regs*/
2838 for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) {
2839 if (unique_indirect_regs[i] != 0) {
2840 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0)
2841 + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i],
2842 unique_indirect_regs[i] & 0x3FFFF);
2843
2844 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0)
2845 + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i],
2846 unique_indirect_regs[i] >> 20);
2847 }
2848 }
2849
2850 kfree(register_list_format);
2851 return 0;
2852 }
2853
gfx_v9_0_enable_save_restore_machine(struct amdgpu_device * adev)2854 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev)
2855 {
2856 WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1);
2857 }
2858
pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device * adev,bool enable)2859 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev,
2860 bool enable)
2861 {
2862 uint32_t data = 0;
2863 uint32_t default_data = 0;
2864
2865 default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS));
2866 if (enable) {
2867 /* enable GFXIP control over CGPG */
2868 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2869 if(default_data != data)
2870 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2871
2872 /* update status */
2873 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK;
2874 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT);
2875 if(default_data != data)
2876 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2877 } else {
2878 /* restore GFXIP control over GCPG */
2879 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2880 if(default_data != data)
2881 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2882 }
2883 }
2884
gfx_v9_0_init_gfx_power_gating(struct amdgpu_device * adev)2885 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev)
2886 {
2887 uint32_t data = 0;
2888
2889 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2890 AMD_PG_SUPPORT_GFX_SMG |
2891 AMD_PG_SUPPORT_GFX_DMG)) {
2892 /* init IDLE_POLL_COUNT = 60 */
2893 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL));
2894 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK;
2895 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
2896 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data);
2897
2898 /* init RLC PG Delay */
2899 data = 0;
2900 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT);
2901 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT);
2902 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT);
2903 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT);
2904 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data);
2905
2906 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2));
2907 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK;
2908 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT);
2909 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data);
2910
2911 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3));
2912 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK;
2913 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT);
2914 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data);
2915
2916 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL));
2917 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK;
2918
2919 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */
2920 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT);
2921 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data);
2922 if (adev->asic_type != CHIP_RENOIR)
2923 pwr_10_0_gfxip_control_over_cgpg(adev, true);
2924 }
2925 }
2926
gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device * adev,bool enable)2927 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
2928 bool enable)
2929 {
2930 uint32_t data = 0;
2931 uint32_t default_data = 0;
2932
2933 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2934 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2935 SMU_CLK_SLOWDOWN_ON_PU_ENABLE,
2936 enable ? 1 : 0);
2937 if (default_data != data)
2938 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2939 }
2940
gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device * adev,bool enable)2941 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
2942 bool enable)
2943 {
2944 uint32_t data = 0;
2945 uint32_t default_data = 0;
2946
2947 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2948 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2949 SMU_CLK_SLOWDOWN_ON_PD_ENABLE,
2950 enable ? 1 : 0);
2951 if(default_data != data)
2952 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2953 }
2954
gfx_v9_0_enable_cp_power_gating(struct amdgpu_device * adev,bool enable)2955 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev,
2956 bool enable)
2957 {
2958 uint32_t data = 0;
2959 uint32_t default_data = 0;
2960
2961 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2962 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2963 CP_PG_DISABLE,
2964 enable ? 0 : 1);
2965 if(default_data != data)
2966 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2967 }
2968
gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device * adev,bool enable)2969 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
2970 bool enable)
2971 {
2972 uint32_t data, default_data;
2973
2974 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2975 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2976 GFX_POWER_GATING_ENABLE,
2977 enable ? 1 : 0);
2978 if(default_data != data)
2979 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2980 }
2981
gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device * adev,bool enable)2982 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev,
2983 bool enable)
2984 {
2985 uint32_t data, default_data;
2986
2987 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2988 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2989 GFX_PIPELINE_PG_ENABLE,
2990 enable ? 1 : 0);
2991 if(default_data != data)
2992 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2993
2994 if (!enable)
2995 /* read any GFX register to wake up GFX */
2996 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL));
2997 }
2998
gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device * adev,bool enable)2999 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
3000 bool enable)
3001 {
3002 uint32_t data, default_data;
3003
3004 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3005 data = REG_SET_FIELD(data, RLC_PG_CNTL,
3006 STATIC_PER_CU_PG_ENABLE,
3007 enable ? 1 : 0);
3008 if(default_data != data)
3009 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3010 }
3011
gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device * adev,bool enable)3012 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
3013 bool enable)
3014 {
3015 uint32_t data, default_data;
3016
3017 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3018 data = REG_SET_FIELD(data, RLC_PG_CNTL,
3019 DYN_PER_CU_PG_ENABLE,
3020 enable ? 1 : 0);
3021 if(default_data != data)
3022 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3023 }
3024
gfx_v9_0_init_pg(struct amdgpu_device * adev)3025 static void gfx_v9_0_init_pg(struct amdgpu_device *adev)
3026 {
3027 gfx_v9_0_init_csb(adev);
3028
3029 /*
3030 * Rlc save restore list is workable since v2_1.
3031 * And it's needed by gfxoff feature.
3032 */
3033 if (adev->gfx.rlc.is_rlc_v2_1) {
3034 if (adev->asic_type == CHIP_VEGA12 ||
3035 (adev->apu_flags & AMD_APU_IS_RAVEN2))
3036 gfx_v9_1_init_rlc_save_restore_list(adev);
3037 gfx_v9_0_enable_save_restore_machine(adev);
3038 }
3039
3040 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
3041 AMD_PG_SUPPORT_GFX_SMG |
3042 AMD_PG_SUPPORT_GFX_DMG |
3043 AMD_PG_SUPPORT_CP |
3044 AMD_PG_SUPPORT_GDS |
3045 AMD_PG_SUPPORT_RLC_SMU_HS)) {
3046 WREG32(mmRLC_JUMP_TABLE_RESTORE,
3047 adev->gfx.rlc.cp_table_gpu_addr >> 8);
3048 gfx_v9_0_init_gfx_power_gating(adev);
3049 }
3050 }
3051
gfx_v9_0_rlc_stop(struct amdgpu_device * adev)3052 static void gfx_v9_0_rlc_stop(struct amdgpu_device *adev)
3053 {
3054 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0);
3055 gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3056 gfx_v9_0_wait_for_rlc_serdes(adev);
3057 }
3058
gfx_v9_0_rlc_reset(struct amdgpu_device * adev)3059 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev)
3060 {
3061 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
3062 udelay(50);
3063 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
3064 udelay(50);
3065 }
3066
gfx_v9_0_rlc_start(struct amdgpu_device * adev)3067 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev)
3068 {
3069 #ifdef AMDGPU_RLC_DEBUG_RETRY
3070 u32 rlc_ucode_ver;
3071 #endif
3072
3073 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
3074 udelay(50);
3075
3076 /* carrizo do enable cp interrupt after cp inited */
3077 if (!(adev->flags & AMD_IS_APU)) {
3078 gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3079 udelay(50);
3080 }
3081
3082 #ifdef AMDGPU_RLC_DEBUG_RETRY
3083 /* RLC_GPM_GENERAL_6 : RLC Ucode version */
3084 rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6);
3085 if(rlc_ucode_ver == 0x108) {
3086 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n",
3087 rlc_ucode_ver, adev->gfx.rlc_fw_version);
3088 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles,
3089 * default is 0x9C4 to create a 100us interval */
3090 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4);
3091 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr
3092 * to disable the page fault retry interrupts, default is
3093 * 0x100 (256) */
3094 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100);
3095 }
3096 #endif
3097 }
3098
gfx_v9_0_rlc_load_microcode(struct amdgpu_device * adev)3099 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev)
3100 {
3101 const struct rlc_firmware_header_v2_0 *hdr;
3102 const __le32 *fw_data;
3103 unsigned i, fw_size;
3104
3105 if (!adev->gfx.rlc_fw)
3106 return -EINVAL;
3107
3108 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
3109 amdgpu_ucode_print_rlc_hdr(&hdr->header);
3110
3111 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
3112 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
3113 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
3114
3115 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR,
3116 RLCG_UCODE_LOADING_START_ADDRESS);
3117 for (i = 0; i < fw_size; i++)
3118 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
3119 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
3120
3121 return 0;
3122 }
3123
gfx_v9_0_rlc_resume(struct amdgpu_device * adev)3124 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)
3125 {
3126 int r;
3127
3128 if (amdgpu_sriov_vf(adev)) {
3129 gfx_v9_0_init_csb(adev);
3130 return 0;
3131 }
3132
3133 adev->gfx.rlc.funcs->stop(adev);
3134
3135 /* disable CG */
3136 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0);
3137
3138 gfx_v9_0_init_pg(adev);
3139
3140 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3141 /* legacy rlc firmware loading */
3142 r = gfx_v9_0_rlc_load_microcode(adev);
3143 if (r)
3144 return r;
3145 }
3146
3147 switch (adev->asic_type) {
3148 case CHIP_RAVEN:
3149 if (amdgpu_lbpw == 0)
3150 gfx_v9_0_enable_lbpw(adev, false);
3151 else
3152 gfx_v9_0_enable_lbpw(adev, true);
3153 break;
3154 case CHIP_VEGA20:
3155 if (amdgpu_lbpw > 0)
3156 gfx_v9_0_enable_lbpw(adev, true);
3157 else
3158 gfx_v9_0_enable_lbpw(adev, false);
3159 break;
3160 default:
3161 break;
3162 }
3163
3164 adev->gfx.rlc.funcs->start(adev);
3165
3166 return 0;
3167 }
3168
gfx_v9_0_cp_gfx_enable(struct amdgpu_device * adev,bool enable)3169 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
3170 {
3171 u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL);
3172
3173 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
3174 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
3175 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1);
3176 WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp);
3177 udelay(50);
3178 }
3179
gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device * adev)3180 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
3181 {
3182 const struct gfx_firmware_header_v1_0 *pfp_hdr;
3183 const struct gfx_firmware_header_v1_0 *ce_hdr;
3184 const struct gfx_firmware_header_v1_0 *me_hdr;
3185 const __le32 *fw_data;
3186 unsigned i, fw_size;
3187
3188 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
3189 return -EINVAL;
3190
3191 pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
3192 adev->gfx.pfp_fw->data;
3193 ce_hdr = (const struct gfx_firmware_header_v1_0 *)
3194 adev->gfx.ce_fw->data;
3195 me_hdr = (const struct gfx_firmware_header_v1_0 *)
3196 adev->gfx.me_fw->data;
3197
3198 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
3199 amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
3200 amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3201
3202 gfx_v9_0_cp_gfx_enable(adev, false);
3203
3204 /* PFP */
3205 fw_data = (const __le32 *)
3206 (adev->gfx.pfp_fw->data +
3207 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3208 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3209 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0);
3210 for (i = 0; i < fw_size; i++)
3211 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3212 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
3213
3214 /* CE */
3215 fw_data = (const __le32 *)
3216 (adev->gfx.ce_fw->data +
3217 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3218 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3219 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0);
3220 for (i = 0; i < fw_size; i++)
3221 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3222 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);
3223
3224 /* ME */
3225 fw_data = (const __le32 *)
3226 (adev->gfx.me_fw->data +
3227 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3228 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3229 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0);
3230 for (i = 0; i < fw_size; i++)
3231 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3232 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);
3233
3234 return 0;
3235 }
3236
gfx_v9_0_cp_gfx_start(struct amdgpu_device * adev)3237 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
3238 {
3239 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
3240 const struct cs_section_def *sect = NULL;
3241 const struct cs_extent_def *ext = NULL;
3242 int r, i, tmp;
3243
3244 /* init the CP */
3245 WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
3246 WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1);
3247
3248 gfx_v9_0_cp_gfx_enable(adev, true);
3249
3250 r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3);
3251 if (r) {
3252 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3253 return r;
3254 }
3255
3256 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3257 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3258
3259 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3260 amdgpu_ring_write(ring, 0x80000000);
3261 amdgpu_ring_write(ring, 0x80000000);
3262
3263 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
3264 for (ext = sect->section; ext->extent != NULL; ++ext) {
3265 if (sect->id == SECT_CONTEXT) {
3266 amdgpu_ring_write(ring,
3267 PACKET3(PACKET3_SET_CONTEXT_REG,
3268 ext->reg_count));
3269 amdgpu_ring_write(ring,
3270 ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
3271 for (i = 0; i < ext->reg_count; i++)
3272 amdgpu_ring_write(ring, ext->extent[i]);
3273 }
3274 }
3275 }
3276
3277 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3278 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3279
3280 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3281 amdgpu_ring_write(ring, 0);
3282
3283 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3284 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3285 amdgpu_ring_write(ring, 0x8000);
3286 amdgpu_ring_write(ring, 0x8000);
3287
3288 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1));
3289 tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE |
3290 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START));
3291 amdgpu_ring_write(ring, tmp);
3292 amdgpu_ring_write(ring, 0);
3293
3294 amdgpu_ring_commit(ring);
3295
3296 return 0;
3297 }
3298
gfx_v9_0_cp_gfx_resume(struct amdgpu_device * adev)3299 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev)
3300 {
3301 struct amdgpu_ring *ring;
3302 u32 tmp;
3303 u32 rb_bufsz;
3304 u64 rb_addr, rptr_addr, wptr_gpu_addr;
3305
3306 /* Set the write pointer delay */
3307 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0);
3308
3309 /* set the RB to use vmid 0 */
3310 WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0);
3311
3312 /* Set ring buffer size */
3313 ring = &adev->gfx.gfx_ring[0];
3314 rb_bufsz = order_base_2(ring->ring_size / 8);
3315 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
3316 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
3317 #ifdef __BIG_ENDIAN
3318 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
3319 #endif
3320 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3321
3322 /* Initialize the ring buffer's write pointers */
3323 ring->wptr = 0;
3324 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
3325 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3326
3327 /* set the wb address wether it's enabled or not */
3328 rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
3329 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
3330 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3331
3332 wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
3333 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));
3334 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));
3335
3336 mdelay(1);
3337 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3338
3339 rb_addr = ring->gpu_addr >> 8;
3340 WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr);
3341 WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
3342
3343 tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL);
3344 if (ring->use_doorbell) {
3345 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3346 DOORBELL_OFFSET, ring->doorbell_index);
3347 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3348 DOORBELL_EN, 1);
3349 } else {
3350 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);
3351 }
3352 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp);
3353
3354 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
3355 DOORBELL_RANGE_LOWER, ring->doorbell_index);
3356 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
3357
3358 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER,
3359 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
3360
3361
3362 /* start the ring */
3363 gfx_v9_0_cp_gfx_start(adev);
3364 ring->sched.ready = true;
3365
3366 return 0;
3367 }
3368
gfx_v9_0_cp_compute_enable(struct amdgpu_device * adev,bool enable)3369 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
3370 {
3371 if (enable) {
3372 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0);
3373 } else {
3374 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL,
3375 (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));
3376 adev->gfx.kiq.ring.sched.ready = false;
3377 }
3378 udelay(50);
3379 }
3380
gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device * adev)3381 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev)
3382 {
3383 const struct gfx_firmware_header_v1_0 *mec_hdr;
3384 const __le32 *fw_data;
3385 unsigned i;
3386 u32 tmp;
3387
3388 if (!adev->gfx.mec_fw)
3389 return -EINVAL;
3390
3391 gfx_v9_0_cp_compute_enable(adev, false);
3392
3393 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
3394 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3395
3396 fw_data = (const __le32 *)
3397 (adev->gfx.mec_fw->data +
3398 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
3399 tmp = 0;
3400 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
3401 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
3402 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp);
3403
3404 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO,
3405 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000);
3406 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI,
3407 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
3408
3409 /* MEC1 */
3410 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3411 mec_hdr->jt_offset);
3412 for (i = 0; i < mec_hdr->jt_size; i++)
3413 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA,
3414 le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
3415
3416 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3417 adev->gfx.mec_fw_version);
3418 /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */
3419
3420 return 0;
3421 }
3422
3423 /* KIQ functions */
gfx_v9_0_kiq_setting(struct amdgpu_ring * ring)3424 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring)
3425 {
3426 uint32_t tmp;
3427 struct amdgpu_device *adev = ring->adev;
3428
3429 /* tell RLC which is KIQ queue */
3430 tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS);
3431 tmp &= 0xffffff00;
3432 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
3433 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3434 tmp |= 0x80;
3435 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3436 }
3437
gfx_v9_0_mqd_set_priority(struct amdgpu_ring * ring,struct v9_mqd * mqd)3438 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd)
3439 {
3440 struct amdgpu_device *adev = ring->adev;
3441
3442 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
3443 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) {
3444 mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH;
3445 mqd->cp_hqd_queue_priority =
3446 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM;
3447 }
3448 }
3449 }
3450
gfx_v9_0_mqd_init(struct amdgpu_ring * ring)3451 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
3452 {
3453 struct amdgpu_device *adev = ring->adev;
3454 struct v9_mqd *mqd = ring->mqd_ptr;
3455 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
3456 uint32_t tmp;
3457
3458 mqd->header = 0xC0310800;
3459 mqd->compute_pipelinestat_enable = 0x00000001;
3460 mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
3461 mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
3462 mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
3463 mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
3464 mqd->compute_static_thread_mgmt_se4 = 0xffffffff;
3465 mqd->compute_static_thread_mgmt_se5 = 0xffffffff;
3466 mqd->compute_static_thread_mgmt_se6 = 0xffffffff;
3467 mqd->compute_static_thread_mgmt_se7 = 0xffffffff;
3468 mqd->compute_misc_reserved = 0x00000003;
3469
3470 mqd->dynamic_cu_mask_addr_lo =
3471 lower_32_bits(ring->mqd_gpu_addr
3472 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3473 mqd->dynamic_cu_mask_addr_hi =
3474 upper_32_bits(ring->mqd_gpu_addr
3475 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3476
3477 eop_base_addr = ring->eop_gpu_addr >> 8;
3478 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
3479 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
3480
3481 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3482 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL);
3483 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
3484 (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1));
3485
3486 mqd->cp_hqd_eop_control = tmp;
3487
3488 /* enable doorbell? */
3489 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3490
3491 if (ring->use_doorbell) {
3492 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3493 DOORBELL_OFFSET, ring->doorbell_index);
3494 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3495 DOORBELL_EN, 1);
3496 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3497 DOORBELL_SOURCE, 0);
3498 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3499 DOORBELL_HIT, 0);
3500 } else {
3501 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3502 DOORBELL_EN, 0);
3503 }
3504
3505 mqd->cp_hqd_pq_doorbell_control = tmp;
3506
3507 /* disable the queue if it's active */
3508 ring->wptr = 0;
3509 mqd->cp_hqd_dequeue_request = 0;
3510 mqd->cp_hqd_pq_rptr = 0;
3511 mqd->cp_hqd_pq_wptr_lo = 0;
3512 mqd->cp_hqd_pq_wptr_hi = 0;
3513
3514 /* set the pointer to the MQD */
3515 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
3516 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
3517
3518 /* set MQD vmid to 0 */
3519 tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL);
3520 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
3521 mqd->cp_mqd_control = tmp;
3522
3523 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3524 hqd_gpu_addr = ring->gpu_addr >> 8;
3525 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
3526 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3527
3528 /* set up the HQD, this is similar to CP_RB0_CNTL */
3529 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL);
3530 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
3531 (order_base_2(ring->ring_size / 4) - 1));
3532 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
3533 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
3534 #ifdef __BIG_ENDIAN
3535 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
3536 #endif
3537 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
3538 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
3539 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
3540 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
3541 mqd->cp_hqd_pq_control = tmp;
3542
3543 /* set the wb address whether it's enabled or not */
3544 wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
3545 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
3546 mqd->cp_hqd_pq_rptr_report_addr_hi =
3547 upper_32_bits(wb_gpu_addr) & 0xffff;
3548
3549 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3550 wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
3551 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3552 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3553
3554 tmp = 0;
3555 /* enable the doorbell if requested */
3556 if (ring->use_doorbell) {
3557 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3558 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3559 DOORBELL_OFFSET, ring->doorbell_index);
3560
3561 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3562 DOORBELL_EN, 1);
3563 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3564 DOORBELL_SOURCE, 0);
3565 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3566 DOORBELL_HIT, 0);
3567 }
3568
3569 mqd->cp_hqd_pq_doorbell_control = tmp;
3570
3571 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3572 ring->wptr = 0;
3573 mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR);
3574
3575 /* set the vmid for the queue */
3576 mqd->cp_hqd_vmid = 0;
3577
3578 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE);
3579 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
3580 mqd->cp_hqd_persistent_state = tmp;
3581
3582 /* set MIN_IB_AVAIL_SIZE */
3583 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL);
3584 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
3585 mqd->cp_hqd_ib_control = tmp;
3586
3587 /* set static priority for a queue/ring */
3588 gfx_v9_0_mqd_set_priority(ring, mqd);
3589 mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM);
3590
3591 /* map_queues packet doesn't need activate the queue,
3592 * so only kiq need set this field.
3593 */
3594 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
3595 mqd->cp_hqd_active = 1;
3596
3597 return 0;
3598 }
3599
gfx_v9_0_kiq_init_register(struct amdgpu_ring * ring)3600 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring)
3601 {
3602 struct amdgpu_device *adev = ring->adev;
3603 struct v9_mqd *mqd = ring->mqd_ptr;
3604 int j;
3605
3606 /* disable wptr polling */
3607 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3608
3609 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR,
3610 mqd->cp_hqd_eop_base_addr_lo);
3611 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI,
3612 mqd->cp_hqd_eop_base_addr_hi);
3613
3614 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3615 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL,
3616 mqd->cp_hqd_eop_control);
3617
3618 /* enable doorbell? */
3619 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3620 mqd->cp_hqd_pq_doorbell_control);
3621
3622 /* disable the queue if it's active */
3623 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3624 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3625 for (j = 0; j < adev->usec_timeout; j++) {
3626 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3627 break;
3628 udelay(1);
3629 }
3630 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3631 mqd->cp_hqd_dequeue_request);
3632 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR,
3633 mqd->cp_hqd_pq_rptr);
3634 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3635 mqd->cp_hqd_pq_wptr_lo);
3636 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3637 mqd->cp_hqd_pq_wptr_hi);
3638 }
3639
3640 /* set the pointer to the MQD */
3641 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR,
3642 mqd->cp_mqd_base_addr_lo);
3643 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI,
3644 mqd->cp_mqd_base_addr_hi);
3645
3646 /* set MQD vmid to 0 */
3647 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL,
3648 mqd->cp_mqd_control);
3649
3650 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3651 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE,
3652 mqd->cp_hqd_pq_base_lo);
3653 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI,
3654 mqd->cp_hqd_pq_base_hi);
3655
3656 /* set up the HQD, this is similar to CP_RB0_CNTL */
3657 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL,
3658 mqd->cp_hqd_pq_control);
3659
3660 /* set the wb address whether it's enabled or not */
3661 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,
3662 mqd->cp_hqd_pq_rptr_report_addr_lo);
3663 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3664 mqd->cp_hqd_pq_rptr_report_addr_hi);
3665
3666 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3667 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,
3668 mqd->cp_hqd_pq_wptr_poll_addr_lo);
3669 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
3670 mqd->cp_hqd_pq_wptr_poll_addr_hi);
3671
3672 /* enable the doorbell if requested */
3673 if (ring->use_doorbell) {
3674 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER,
3675 (adev->doorbell_index.kiq * 2) << 2);
3676 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3677 (adev->doorbell_index.userqueue_end * 2) << 2);
3678 }
3679
3680 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3681 mqd->cp_hqd_pq_doorbell_control);
3682
3683 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3684 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3685 mqd->cp_hqd_pq_wptr_lo);
3686 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3687 mqd->cp_hqd_pq_wptr_hi);
3688
3689 /* set the vmid for the queue */
3690 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);
3691
3692 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE,
3693 mqd->cp_hqd_persistent_state);
3694
3695 /* activate the queue */
3696 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE,
3697 mqd->cp_hqd_active);
3698
3699 if (ring->use_doorbell)
3700 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
3701
3702 return 0;
3703 }
3704
gfx_v9_0_kiq_fini_register(struct amdgpu_ring * ring)3705 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring)
3706 {
3707 struct amdgpu_device *adev = ring->adev;
3708 int j;
3709
3710 /* disable the queue if it's active */
3711 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3712
3713 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3714
3715 for (j = 0; j < adev->usec_timeout; j++) {
3716 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3717 break;
3718 udelay(1);
3719 }
3720
3721 if (j == AMDGPU_MAX_USEC_TIMEOUT) {
3722 DRM_DEBUG("KIQ dequeue request failed.\n");
3723
3724 /* Manual disable if dequeue request times out */
3725 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0);
3726 }
3727
3728 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3729 0);
3730 }
3731
3732 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0);
3733 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0);
3734 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0);
3735 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000);
3736 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
3737 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0);
3738 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0);
3739 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0);
3740
3741 return 0;
3742 }
3743
gfx_v9_0_kiq_init_queue(struct amdgpu_ring * ring)3744 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
3745 {
3746 struct amdgpu_device *adev = ring->adev;
3747 struct v9_mqd *mqd = ring->mqd_ptr;
3748 int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS;
3749 struct v9_mqd *tmp_mqd;
3750
3751 gfx_v9_0_kiq_setting(ring);
3752
3753 /* GPU could be in bad state during probe, driver trigger the reset
3754 * after load the SMU, in this case , the mqd is not be initialized.
3755 * driver need to re-init the mqd.
3756 * check mqd->cp_hqd_pq_control since this value should not be 0
3757 */
3758 tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx];
3759 if (amdgpu_in_reset(adev) && tmp_mqd->cp_hqd_pq_control){
3760 /* for GPU_RESET case , reset MQD to a clean status */
3761 if (adev->gfx.mec.mqd_backup[mqd_idx])
3762 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3763
3764 /* reset ring buffer */
3765 ring->wptr = 0;
3766 amdgpu_ring_clear_ring(ring);
3767
3768 mutex_lock(&adev->srbm_mutex);
3769 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3770 gfx_v9_0_kiq_init_register(ring);
3771 soc15_grbm_select(adev, 0, 0, 0, 0);
3772 mutex_unlock(&adev->srbm_mutex);
3773 } else {
3774 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3775 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3776 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3777 mutex_lock(&adev->srbm_mutex);
3778 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3779 gfx_v9_0_mqd_init(ring);
3780 gfx_v9_0_kiq_init_register(ring);
3781 soc15_grbm_select(adev, 0, 0, 0, 0);
3782 mutex_unlock(&adev->srbm_mutex);
3783
3784 if (adev->gfx.mec.mqd_backup[mqd_idx])
3785 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3786 }
3787
3788 return 0;
3789 }
3790
gfx_v9_0_kcq_init_queue(struct amdgpu_ring * ring)3791 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring)
3792 {
3793 struct amdgpu_device *adev = ring->adev;
3794 struct v9_mqd *mqd = ring->mqd_ptr;
3795 int mqd_idx = ring - &adev->gfx.compute_ring[0];
3796 struct v9_mqd *tmp_mqd;
3797
3798 /* Same as above kiq init, driver need to re-init the mqd if mqd->cp_hqd_pq_control
3799 * is not be initialized before
3800 */
3801 tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx];
3802
3803 if (!tmp_mqd->cp_hqd_pq_control ||
3804 (!amdgpu_in_reset(adev) && !adev->in_suspend)) {
3805 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3806 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3807 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3808 mutex_lock(&adev->srbm_mutex);
3809 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3810 gfx_v9_0_mqd_init(ring);
3811 soc15_grbm_select(adev, 0, 0, 0, 0);
3812 mutex_unlock(&adev->srbm_mutex);
3813
3814 if (adev->gfx.mec.mqd_backup[mqd_idx])
3815 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3816 } else if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
3817 /* reset MQD to a clean status */
3818 if (adev->gfx.mec.mqd_backup[mqd_idx])
3819 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3820
3821 /* reset ring buffer */
3822 ring->wptr = 0;
3823 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], 0);
3824 amdgpu_ring_clear_ring(ring);
3825 } else {
3826 amdgpu_ring_clear_ring(ring);
3827 }
3828
3829 return 0;
3830 }
3831
gfx_v9_0_kiq_resume(struct amdgpu_device * adev)3832 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev)
3833 {
3834 struct amdgpu_ring *ring;
3835 int r;
3836
3837 ring = &adev->gfx.kiq.ring;
3838
3839 r = amdgpu_bo_reserve(ring->mqd_obj, false);
3840 if (unlikely(r != 0))
3841 return r;
3842
3843 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3844 if (unlikely(r != 0))
3845 return r;
3846
3847 gfx_v9_0_kiq_init_queue(ring);
3848 amdgpu_bo_kunmap(ring->mqd_obj);
3849 ring->mqd_ptr = NULL;
3850 amdgpu_bo_unreserve(ring->mqd_obj);
3851 ring->sched.ready = true;
3852 return 0;
3853 }
3854
gfx_v9_0_kcq_resume(struct amdgpu_device * adev)3855 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev)
3856 {
3857 struct amdgpu_ring *ring = NULL;
3858 int r = 0, i;
3859
3860 gfx_v9_0_cp_compute_enable(adev, true);
3861
3862 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3863 ring = &adev->gfx.compute_ring[i];
3864
3865 r = amdgpu_bo_reserve(ring->mqd_obj, false);
3866 if (unlikely(r != 0))
3867 goto done;
3868 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3869 if (!r) {
3870 r = gfx_v9_0_kcq_init_queue(ring);
3871 amdgpu_bo_kunmap(ring->mqd_obj);
3872 ring->mqd_ptr = NULL;
3873 }
3874 amdgpu_bo_unreserve(ring->mqd_obj);
3875 if (r)
3876 goto done;
3877 }
3878
3879 r = amdgpu_gfx_enable_kcq(adev);
3880 done:
3881 return r;
3882 }
3883
gfx_v9_0_cp_resume(struct amdgpu_device * adev)3884 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev)
3885 {
3886 int r, i;
3887 struct amdgpu_ring *ring;
3888
3889 if (!(adev->flags & AMD_IS_APU))
3890 gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3891
3892 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3893 if (adev->gfx.num_gfx_rings) {
3894 /* legacy firmware loading */
3895 r = gfx_v9_0_cp_gfx_load_microcode(adev);
3896 if (r)
3897 return r;
3898 }
3899
3900 r = gfx_v9_0_cp_compute_load_microcode(adev);
3901 if (r)
3902 return r;
3903 }
3904
3905 r = gfx_v9_0_kiq_resume(adev);
3906 if (r)
3907 return r;
3908
3909 if (adev->gfx.num_gfx_rings) {
3910 r = gfx_v9_0_cp_gfx_resume(adev);
3911 if (r)
3912 return r;
3913 }
3914
3915 r = gfx_v9_0_kcq_resume(adev);
3916 if (r)
3917 return r;
3918
3919 if (adev->gfx.num_gfx_rings) {
3920 ring = &adev->gfx.gfx_ring[0];
3921 r = amdgpu_ring_test_helper(ring);
3922 if (r)
3923 return r;
3924 }
3925
3926 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3927 ring = &adev->gfx.compute_ring[i];
3928 amdgpu_ring_test_helper(ring);
3929 }
3930
3931 gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3932
3933 return 0;
3934 }
3935
gfx_v9_0_init_tcp_config(struct amdgpu_device * adev)3936 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev)
3937 {
3938 u32 tmp;
3939
3940 if (adev->asic_type != CHIP_ARCTURUS)
3941 return;
3942
3943 tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG);
3944 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH,
3945 adev->df.hash_status.hash_64k);
3946 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH,
3947 adev->df.hash_status.hash_2m);
3948 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH,
3949 adev->df.hash_status.hash_1g);
3950 WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp);
3951 }
3952
gfx_v9_0_cp_enable(struct amdgpu_device * adev,bool enable)3953 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable)
3954 {
3955 if (adev->gfx.num_gfx_rings)
3956 gfx_v9_0_cp_gfx_enable(adev, enable);
3957 gfx_v9_0_cp_compute_enable(adev, enable);
3958 }
3959
gfx_v9_0_hw_init(void * handle)3960 static int gfx_v9_0_hw_init(void *handle)
3961 {
3962 int r;
3963 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3964
3965 if (!amdgpu_sriov_vf(adev))
3966 gfx_v9_0_init_golden_registers(adev);
3967
3968 gfx_v9_0_constants_init(adev);
3969
3970 gfx_v9_0_init_tcp_config(adev);
3971
3972 r = adev->gfx.rlc.funcs->resume(adev);
3973 if (r)
3974 return r;
3975
3976 r = gfx_v9_0_cp_resume(adev);
3977 if (r)
3978 return r;
3979
3980 if (adev->asic_type == CHIP_ALDEBARAN)
3981 gfx_v9_4_2_set_power_brake_sequence(adev);
3982
3983 return r;
3984 }
3985
gfx_v9_0_hw_fini(void * handle)3986 static int gfx_v9_0_hw_fini(void *handle)
3987 {
3988 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3989
3990 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);
3991 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
3992 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
3993
3994 /* DF freeze and kcq disable will fail */
3995 if (!amdgpu_ras_intr_triggered())
3996 /* disable KCQ to avoid CPC touch memory not valid anymore */
3997 amdgpu_gfx_disable_kcq(adev);
3998
3999 if (amdgpu_sriov_vf(adev)) {
4000 gfx_v9_0_cp_gfx_enable(adev, false);
4001 /* must disable polling for SRIOV when hw finished, otherwise
4002 * CPC engine may still keep fetching WB address which is already
4003 * invalid after sw finished and trigger DMAR reading error in
4004 * hypervisor side.
4005 */
4006 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
4007 return 0;
4008 }
4009
4010 /* Use deinitialize sequence from CAIL when unbinding device from driver,
4011 * otherwise KIQ is hanging when binding back
4012 */
4013 if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
4014 mutex_lock(&adev->srbm_mutex);
4015 soc15_grbm_select(adev, adev->gfx.kiq.ring.me,
4016 adev->gfx.kiq.ring.pipe,
4017 adev->gfx.kiq.ring.queue, 0);
4018 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq.ring);
4019 soc15_grbm_select(adev, 0, 0, 0, 0);
4020 mutex_unlock(&adev->srbm_mutex);
4021 }
4022
4023 gfx_v9_0_cp_enable(adev, false);
4024
4025 /* Skip suspend with A+A reset */
4026 if (adev->gmc.xgmi.connected_to_cpu && amdgpu_in_reset(adev)) {
4027 dev_dbg(adev->dev, "Device in reset. Skipping RLC halt\n");
4028 return 0;
4029 }
4030
4031 adev->gfx.rlc.funcs->stop(adev);
4032 return 0;
4033 }
4034
gfx_v9_0_suspend(void * handle)4035 static int gfx_v9_0_suspend(void *handle)
4036 {
4037 return gfx_v9_0_hw_fini(handle);
4038 }
4039
gfx_v9_0_resume(void * handle)4040 static int gfx_v9_0_resume(void *handle)
4041 {
4042 return gfx_v9_0_hw_init(handle);
4043 }
4044
gfx_v9_0_is_idle(void * handle)4045 static bool gfx_v9_0_is_idle(void *handle)
4046 {
4047 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4048
4049 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS),
4050 GRBM_STATUS, GUI_ACTIVE))
4051 return false;
4052 else
4053 return true;
4054 }
4055
gfx_v9_0_wait_for_idle(void * handle)4056 static int gfx_v9_0_wait_for_idle(void *handle)
4057 {
4058 unsigned i;
4059 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4060
4061 for (i = 0; i < adev->usec_timeout; i++) {
4062 if (gfx_v9_0_is_idle(handle))
4063 return 0;
4064 udelay(1);
4065 }
4066 return -ETIMEDOUT;
4067 }
4068
gfx_v9_0_soft_reset(void * handle)4069 static int gfx_v9_0_soft_reset(void *handle)
4070 {
4071 u32 grbm_soft_reset = 0;
4072 u32 tmp;
4073 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4074
4075 /* GRBM_STATUS */
4076 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS);
4077 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
4078 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
4079 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
4080 GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
4081 GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
4082 GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) {
4083 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4084 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4085 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4086 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
4087 }
4088
4089 if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
4090 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4091 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4092 }
4093
4094 /* GRBM_STATUS2 */
4095 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);
4096 if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
4097 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4098 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
4099
4100
4101 if (grbm_soft_reset) {
4102 /* stop the rlc */
4103 adev->gfx.rlc.funcs->stop(adev);
4104
4105 if (adev->gfx.num_gfx_rings)
4106 /* Disable GFX parsing/prefetching */
4107 gfx_v9_0_cp_gfx_enable(adev, false);
4108
4109 /* Disable MEC parsing/prefetching */
4110 gfx_v9_0_cp_compute_enable(adev, false);
4111
4112 if (grbm_soft_reset) {
4113 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4114 tmp |= grbm_soft_reset;
4115 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4116 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4117 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4118
4119 udelay(50);
4120
4121 tmp &= ~grbm_soft_reset;
4122 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4123 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4124 }
4125
4126 /* Wait a little for things to settle down */
4127 udelay(50);
4128 }
4129 return 0;
4130 }
4131
gfx_v9_0_kiq_read_clock(struct amdgpu_device * adev)4132 static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev)
4133 {
4134 signed long r, cnt = 0;
4135 unsigned long flags;
4136 uint32_t seq, reg_val_offs = 0;
4137 uint64_t value = 0;
4138 struct amdgpu_kiq *kiq = &adev->gfx.kiq;
4139 struct amdgpu_ring *ring = &kiq->ring;
4140
4141 BUG_ON(!ring->funcs->emit_rreg);
4142
4143 spin_lock_irqsave(&kiq->ring_lock, flags);
4144 if (amdgpu_device_wb_get(adev, ®_val_offs)) {
4145 pr_err("critical bug! too many kiq readers\n");
4146 goto failed_unlock;
4147 }
4148 amdgpu_ring_alloc(ring, 32);
4149 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
4150 amdgpu_ring_write(ring, 9 | /* src: register*/
4151 (5 << 8) | /* dst: memory */
4152 (1 << 16) | /* count sel */
4153 (1 << 20)); /* write confirm */
4154 amdgpu_ring_write(ring, 0);
4155 amdgpu_ring_write(ring, 0);
4156 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
4157 reg_val_offs * 4));
4158 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
4159 reg_val_offs * 4));
4160 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
4161 if (r)
4162 goto failed_undo;
4163
4164 amdgpu_ring_commit(ring);
4165 spin_unlock_irqrestore(&kiq->ring_lock, flags);
4166
4167 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4168
4169 /* don't wait anymore for gpu reset case because this way may
4170 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
4171 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
4172 * never return if we keep waiting in virt_kiq_rreg, which cause
4173 * gpu_recover() hang there.
4174 *
4175 * also don't wait anymore for IRQ context
4176 * */
4177 if (r < 1 && (amdgpu_in_reset(adev)))
4178 goto failed_kiq_read;
4179
4180 might_sleep();
4181 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
4182 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
4183 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4184 }
4185
4186 if (cnt > MAX_KIQ_REG_TRY)
4187 goto failed_kiq_read;
4188
4189 mb();
4190 value = (uint64_t)adev->wb.wb[reg_val_offs] |
4191 (uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL;
4192 amdgpu_device_wb_free(adev, reg_val_offs);
4193 return value;
4194
4195 failed_undo:
4196 amdgpu_ring_undo(ring);
4197 failed_unlock:
4198 spin_unlock_irqrestore(&kiq->ring_lock, flags);
4199 failed_kiq_read:
4200 if (reg_val_offs)
4201 amdgpu_device_wb_free(adev, reg_val_offs);
4202 pr_err("failed to read gpu clock\n");
4203 return ~0;
4204 }
4205
gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device * adev)4206 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)
4207 {
4208 uint64_t clock;
4209
4210 amdgpu_gfx_off_ctrl(adev, false);
4211 mutex_lock(&adev->gfx.gpu_clock_mutex);
4212 if (adev->asic_type == CHIP_VEGA10 && amdgpu_sriov_runtime(adev)) {
4213 clock = gfx_v9_0_kiq_read_clock(adev);
4214 } else {
4215 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
4216 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |
4217 ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4218 }
4219 mutex_unlock(&adev->gfx.gpu_clock_mutex);
4220 amdgpu_gfx_off_ctrl(adev, true);
4221 return clock;
4222 }
4223
gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring * ring,uint32_t vmid,uint32_t gds_base,uint32_t gds_size,uint32_t gws_base,uint32_t gws_size,uint32_t oa_base,uint32_t oa_size)4224 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
4225 uint32_t vmid,
4226 uint32_t gds_base, uint32_t gds_size,
4227 uint32_t gws_base, uint32_t gws_size,
4228 uint32_t oa_base, uint32_t oa_size)
4229 {
4230 struct amdgpu_device *adev = ring->adev;
4231
4232 /* GDS Base */
4233 gfx_v9_0_write_data_to_reg(ring, 0, false,
4234 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid,
4235 gds_base);
4236
4237 /* GDS Size */
4238 gfx_v9_0_write_data_to_reg(ring, 0, false,
4239 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid,
4240 gds_size);
4241
4242 /* GWS */
4243 gfx_v9_0_write_data_to_reg(ring, 0, false,
4244 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid,
4245 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
4246
4247 /* OA */
4248 gfx_v9_0_write_data_to_reg(ring, 0, false,
4249 SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid,
4250 (1 << (oa_size + oa_base)) - (1 << oa_base));
4251 }
4252
4253 static const u32 vgpr_init_compute_shader[] =
4254 {
4255 0xb07c0000, 0xbe8000ff,
4256 0x000000f8, 0xbf110800,
4257 0x7e000280, 0x7e020280,
4258 0x7e040280, 0x7e060280,
4259 0x7e080280, 0x7e0a0280,
4260 0x7e0c0280, 0x7e0e0280,
4261 0x80808800, 0xbe803200,
4262 0xbf84fff5, 0xbf9c0000,
4263 0xd28c0001, 0x0001007f,
4264 0xd28d0001, 0x0002027e,
4265 0x10020288, 0xb8810904,
4266 0xb7814000, 0xd1196a01,
4267 0x00000301, 0xbe800087,
4268 0xbefc00c1, 0xd89c4000,
4269 0x00020201, 0xd89cc080,
4270 0x00040401, 0x320202ff,
4271 0x00000800, 0x80808100,
4272 0xbf84fff8, 0x7e020280,
4273 0xbf810000, 0x00000000,
4274 };
4275
4276 static const u32 sgpr_init_compute_shader[] =
4277 {
4278 0xb07c0000, 0xbe8000ff,
4279 0x0000005f, 0xbee50080,
4280 0xbe812c65, 0xbe822c65,
4281 0xbe832c65, 0xbe842c65,
4282 0xbe852c65, 0xb77c0005,
4283 0x80808500, 0xbf84fff8,
4284 0xbe800080, 0xbf810000,
4285 };
4286
4287 static const u32 vgpr_init_compute_shader_arcturus[] = {
4288 0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080,
4289 0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080,
4290 0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080,
4291 0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080,
4292 0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080,
4293 0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080,
4294 0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080,
4295 0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080,
4296 0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080,
4297 0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080,
4298 0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080,
4299 0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080,
4300 0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080,
4301 0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080,
4302 0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080,
4303 0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080,
4304 0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080,
4305 0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080,
4306 0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080,
4307 0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080,
4308 0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080,
4309 0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080,
4310 0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080,
4311 0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080,
4312 0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080,
4313 0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080,
4314 0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080,
4315 0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080,
4316 0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080,
4317 0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080,
4318 0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080,
4319 0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080,
4320 0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080,
4321 0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080,
4322 0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080,
4323 0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080,
4324 0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080,
4325 0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080,
4326 0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080,
4327 0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080,
4328 0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080,
4329 0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080,
4330 0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080,
4331 0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080,
4332 0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080,
4333 0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080,
4334 0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080,
4335 0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080,
4336 0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080,
4337 0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080,
4338 0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080,
4339 0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080,
4340 0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080,
4341 0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080,
4342 0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080,
4343 0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080,
4344 0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080,
4345 0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080,
4346 0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080,
4347 0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080,
4348 0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080,
4349 0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080,
4350 0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080,
4351 0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080,
4352 0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080,
4353 0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080,
4354 0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080,
4355 0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080,
4356 0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080,
4357 0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080,
4358 0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080,
4359 0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080,
4360 0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080,
4361 0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080,
4362 0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080,
4363 0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080,
4364 0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080,
4365 0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080,
4366 0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080,
4367 0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080,
4368 0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080,
4369 0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080,
4370 0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080,
4371 0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080,
4372 0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080,
4373 0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a,
4374 0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280,
4375 0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000,
4376 0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904,
4377 0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000,
4378 0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a,
4379 0xbf84fff8, 0xbf810000,
4380 };
4381
4382 /* When below register arrays changed, please update gpr_reg_size,
4383 and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds,
4384 to cover all gfx9 ASICs */
4385 static const struct soc15_reg_entry vgpr_init_regs[] = {
4386 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4387 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4388 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4389 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4390 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f },
4391 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */
4392 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4393 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4394 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4395 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4396 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4397 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4398 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4399 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4400 };
4401
4402 static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = {
4403 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4404 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4405 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4406 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4407 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf },
4408 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */
4409 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4410 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4411 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4412 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4413 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4414 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4415 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4416 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4417 };
4418
4419 static const struct soc15_reg_entry sgpr1_init_regs[] = {
4420 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4421 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4422 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4423 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4424 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4425 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4426 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff },
4427 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff },
4428 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff },
4429 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff },
4430 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff },
4431 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff },
4432 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff },
4433 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff },
4434 };
4435
4436 static const struct soc15_reg_entry sgpr2_init_regs[] = {
4437 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4438 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4439 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4440 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4441 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4442 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4443 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 },
4444 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 },
4445 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 },
4446 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 },
4447 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 },
4448 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 },
4449 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 },
4450 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 },
4451 };
4452
4453 static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = {
4454 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1},
4455 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1},
4456 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1},
4457 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1},
4458 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1},
4459 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1},
4460 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1},
4461 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1},
4462 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1},
4463 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1},
4464 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1},
4465 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1},
4466 { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1},
4467 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6},
4468 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16},
4469 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16},
4470 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16},
4471 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16},
4472 { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16},
4473 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16},
4474 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16},
4475 { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16},
4476 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6},
4477 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16},
4478 { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16},
4479 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1},
4480 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1},
4481 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32},
4482 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32},
4483 { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72},
4484 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16},
4485 { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2},
4486 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6},
4487 };
4488
gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device * adev)4489 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev)
4490 {
4491 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4492 int i, r;
4493
4494 /* only support when RAS is enabled */
4495 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4496 return 0;
4497
4498 r = amdgpu_ring_alloc(ring, 7);
4499 if (r) {
4500 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n",
4501 ring->name, r);
4502 return r;
4503 }
4504
4505 WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000);
4506 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size);
4507
4508 amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4509 amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC |
4510 PACKET3_DMA_DATA_DST_SEL(1) |
4511 PACKET3_DMA_DATA_SRC_SEL(2) |
4512 PACKET3_DMA_DATA_ENGINE(0)));
4513 amdgpu_ring_write(ring, 0);
4514 amdgpu_ring_write(ring, 0);
4515 amdgpu_ring_write(ring, 0);
4516 amdgpu_ring_write(ring, 0);
4517 amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT |
4518 adev->gds.gds_size);
4519
4520 amdgpu_ring_commit(ring);
4521
4522 for (i = 0; i < adev->usec_timeout; i++) {
4523 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring))
4524 break;
4525 udelay(1);
4526 }
4527
4528 if (i >= adev->usec_timeout)
4529 r = -ETIMEDOUT;
4530
4531 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000);
4532
4533 return r;
4534 }
4535
gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device * adev)4536 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
4537 {
4538 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4539 struct amdgpu_ib ib;
4540 struct dma_fence *f = NULL;
4541 int r, i;
4542 unsigned total_size, vgpr_offset, sgpr_offset;
4543 u64 gpu_addr;
4544
4545 int compute_dim_x = adev->gfx.config.max_shader_engines *
4546 adev->gfx.config.max_cu_per_sh *
4547 adev->gfx.config.max_sh_per_se;
4548 int sgpr_work_group_size = 5;
4549 int gpr_reg_size = adev->gfx.config.max_shader_engines + 6;
4550 int vgpr_init_shader_size;
4551 const u32 *vgpr_init_shader_ptr;
4552 const struct soc15_reg_entry *vgpr_init_regs_ptr;
4553
4554 /* only support when RAS is enabled */
4555 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4556 return 0;
4557
4558 /* bail if the compute ring is not ready */
4559 if (!ring->sched.ready)
4560 return 0;
4561
4562 if (adev->asic_type == CHIP_ARCTURUS ||
4563 adev->asic_type == CHIP_ALDEBARAN) {
4564 vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus;
4565 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus);
4566 vgpr_init_regs_ptr = vgpr_init_regs_arcturus;
4567 } else {
4568 vgpr_init_shader_ptr = vgpr_init_compute_shader;
4569 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader);
4570 vgpr_init_regs_ptr = vgpr_init_regs;
4571 }
4572
4573 total_size =
4574 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */
4575 total_size +=
4576 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */
4577 total_size +=
4578 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */
4579 total_size = ALIGN(total_size, 256);
4580 vgpr_offset = total_size;
4581 total_size += ALIGN(vgpr_init_shader_size, 256);
4582 sgpr_offset = total_size;
4583 total_size += sizeof(sgpr_init_compute_shader);
4584
4585 /* allocate an indirect buffer to put the commands in */
4586 memset(&ib, 0, sizeof(ib));
4587 r = amdgpu_ib_get(adev, NULL, total_size,
4588 AMDGPU_IB_POOL_DIRECT, &ib);
4589 if (r) {
4590 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
4591 return r;
4592 }
4593
4594 /* load the compute shaders */
4595 for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++)
4596 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i];
4597
4598 for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
4599 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
4600
4601 /* init the ib length to 0 */
4602 ib.length_dw = 0;
4603
4604 /* VGPR */
4605 /* write the register state for the compute dispatch */
4606 for (i = 0; i < gpr_reg_size; i++) {
4607 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4608 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i])
4609 - PACKET3_SET_SH_REG_START;
4610 ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value;
4611 }
4612 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4613 gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;
4614 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4615 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4616 - PACKET3_SET_SH_REG_START;
4617 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4618 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4619
4620 /* write dispatch packet */
4621 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4622 ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */
4623 ib.ptr[ib.length_dw++] = 1; /* y */
4624 ib.ptr[ib.length_dw++] = 1; /* z */
4625 ib.ptr[ib.length_dw++] =
4626 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4627
4628 /* write CS partial flush packet */
4629 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4630 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4631
4632 /* SGPR1 */
4633 /* write the register state for the compute dispatch */
4634 for (i = 0; i < gpr_reg_size; i++) {
4635 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4636 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i])
4637 - PACKET3_SET_SH_REG_START;
4638 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value;
4639 }
4640 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4641 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4642 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4643 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4644 - PACKET3_SET_SH_REG_START;
4645 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4646 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4647
4648 /* write dispatch packet */
4649 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4650 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4651 ib.ptr[ib.length_dw++] = 1; /* y */
4652 ib.ptr[ib.length_dw++] = 1; /* z */
4653 ib.ptr[ib.length_dw++] =
4654 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4655
4656 /* write CS partial flush packet */
4657 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4658 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4659
4660 /* SGPR2 */
4661 /* write the register state for the compute dispatch */
4662 for (i = 0; i < gpr_reg_size; i++) {
4663 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4664 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i])
4665 - PACKET3_SET_SH_REG_START;
4666 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value;
4667 }
4668 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4669 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4670 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4671 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4672 - PACKET3_SET_SH_REG_START;
4673 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4674 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4675
4676 /* write dispatch packet */
4677 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4678 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4679 ib.ptr[ib.length_dw++] = 1; /* y */
4680 ib.ptr[ib.length_dw++] = 1; /* z */
4681 ib.ptr[ib.length_dw++] =
4682 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4683
4684 /* write CS partial flush packet */
4685 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4686 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4687
4688 /* shedule the ib on the ring */
4689 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
4690 if (r) {
4691 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);
4692 goto fail;
4693 }
4694
4695 /* wait for the GPU to finish processing the IB */
4696 r = dma_fence_wait(f, false);
4697 if (r) {
4698 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
4699 goto fail;
4700 }
4701
4702 fail:
4703 amdgpu_ib_free(adev, &ib, NULL);
4704 dma_fence_put(f);
4705
4706 return r;
4707 }
4708
gfx_v9_0_early_init(void * handle)4709 static int gfx_v9_0_early_init(void *handle)
4710 {
4711 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4712
4713 if (adev->asic_type == CHIP_ARCTURUS ||
4714 adev->asic_type == CHIP_ALDEBARAN)
4715 adev->gfx.num_gfx_rings = 0;
4716 else
4717 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS;
4718 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
4719 AMDGPU_MAX_COMPUTE_RINGS);
4720 gfx_v9_0_set_kiq_pm4_funcs(adev);
4721 gfx_v9_0_set_ring_funcs(adev);
4722 gfx_v9_0_set_irq_funcs(adev);
4723 gfx_v9_0_set_gds_init(adev);
4724 gfx_v9_0_set_rlc_funcs(adev);
4725
4726 return 0;
4727 }
4728
gfx_v9_0_ecc_late_init(void * handle)4729 static int gfx_v9_0_ecc_late_init(void *handle)
4730 {
4731 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4732 int r;
4733
4734 /*
4735 * Temp workaround to fix the issue that CP firmware fails to
4736 * update read pointer when CPDMA is writing clearing operation
4737 * to GDS in suspend/resume sequence on several cards. So just
4738 * limit this operation in cold boot sequence.
4739 */
4740 if ((!adev->in_suspend) &&
4741 (adev->gds.gds_size)) {
4742 r = gfx_v9_0_do_edc_gds_workarounds(adev);
4743 if (r)
4744 return r;
4745 }
4746
4747 /* requires IBs so do in late init after IB pool is initialized */
4748 r = gfx_v9_0_do_edc_gpr_workarounds(adev);
4749 if (r)
4750 return r;
4751
4752 if (adev->gfx.ras_funcs &&
4753 adev->gfx.ras_funcs->ras_late_init) {
4754 r = adev->gfx.ras_funcs->ras_late_init(adev);
4755 if (r)
4756 return r;
4757 }
4758
4759 if (adev->gfx.ras_funcs &&
4760 adev->gfx.ras_funcs->enable_watchdog_timer)
4761 adev->gfx.ras_funcs->enable_watchdog_timer(adev);
4762
4763 return 0;
4764 }
4765
gfx_v9_0_late_init(void * handle)4766 static int gfx_v9_0_late_init(void *handle)
4767 {
4768 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4769 int r;
4770
4771 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
4772 if (r)
4773 return r;
4774
4775 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
4776 if (r)
4777 return r;
4778
4779 r = gfx_v9_0_ecc_late_init(handle);
4780 if (r)
4781 return r;
4782
4783 return 0;
4784 }
4785
gfx_v9_0_is_rlc_enabled(struct amdgpu_device * adev)4786 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev)
4787 {
4788 uint32_t rlc_setting;
4789
4790 /* if RLC is not enabled, do nothing */
4791 rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL);
4792 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
4793 return false;
4794
4795 return true;
4796 }
4797
gfx_v9_0_set_safe_mode(struct amdgpu_device * adev)4798 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev)
4799 {
4800 uint32_t data;
4801 unsigned i;
4802
4803 data = RLC_SAFE_MODE__CMD_MASK;
4804 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
4805 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4806
4807 /* wait for RLC_SAFE_MODE */
4808 for (i = 0; i < adev->usec_timeout; i++) {
4809 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
4810 break;
4811 udelay(1);
4812 }
4813 }
4814
gfx_v9_0_unset_safe_mode(struct amdgpu_device * adev)4815 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev)
4816 {
4817 uint32_t data;
4818
4819 data = RLC_SAFE_MODE__CMD_MASK;
4820 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4821 }
4822
gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device * adev,bool enable)4823 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev,
4824 bool enable)
4825 {
4826 amdgpu_gfx_rlc_enter_safe_mode(adev);
4827
4828 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {
4829 gfx_v9_0_enable_gfx_cg_power_gating(adev, true);
4830 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4831 gfx_v9_0_enable_gfx_pipeline_powergating(adev, true);
4832 } else {
4833 gfx_v9_0_enable_gfx_cg_power_gating(adev, false);
4834 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4835 gfx_v9_0_enable_gfx_pipeline_powergating(adev, false);
4836 }
4837
4838 amdgpu_gfx_rlc_exit_safe_mode(adev);
4839 }
4840
gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device * adev,bool enable)4841 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev,
4842 bool enable)
4843 {
4844 /* TODO: double check if we need to perform under safe mode */
4845 /* gfx_v9_0_enter_rlc_safe_mode(adev); */
4846
4847 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
4848 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true);
4849 else
4850 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false);
4851
4852 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
4853 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true);
4854 else
4855 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false);
4856
4857 /* gfx_v9_0_exit_rlc_safe_mode(adev); */
4858 }
4859
gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device * adev,bool enable)4860 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
4861 bool enable)
4862 {
4863 uint32_t data, def;
4864
4865 amdgpu_gfx_rlc_enter_safe_mode(adev);
4866
4867 /* It is disabled by HW by default */
4868 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
4869 /* 1 - RLC_CGTT_MGCG_OVERRIDE */
4870 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4871
4872 if (adev->asic_type != CHIP_VEGA12)
4873 data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4874
4875 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4876 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4877 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4878
4879 /* only for Vega10 & Raven1 */
4880 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK;
4881
4882 if (def != data)
4883 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4884
4885 /* MGLS is a global flag to control all MGLS in GFX */
4886 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
4887 /* 2 - RLC memory Light sleep */
4888 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
4889 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4890 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4891 if (def != data)
4892 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4893 }
4894 /* 3 - CP memory Light sleep */
4895 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
4896 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4897 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4898 if (def != data)
4899 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4900 }
4901 }
4902 } else {
4903 /* 1 - MGCG_OVERRIDE */
4904 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4905
4906 if (adev->asic_type != CHIP_VEGA12)
4907 data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4908
4909 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
4910 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4911 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4912 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4913
4914 if (def != data)
4915 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4916
4917 /* 2 - disable MGLS in RLC */
4918 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4919 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
4920 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4921 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4922 }
4923
4924 /* 3 - disable MGLS in CP */
4925 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4926 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
4927 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4928 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4929 }
4930 }
4931
4932 amdgpu_gfx_rlc_exit_safe_mode(adev);
4933 }
4934
gfx_v9_0_update_3d_clock_gating(struct amdgpu_device * adev,bool enable)4935 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev,
4936 bool enable)
4937 {
4938 uint32_t data, def;
4939
4940 if (!adev->gfx.num_gfx_rings)
4941 return;
4942
4943 amdgpu_gfx_rlc_enter_safe_mode(adev);
4944
4945 /* Enable 3D CGCG/CGLS */
4946 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)) {
4947 /* write cmd to clear cgcg/cgls ov */
4948 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4949 /* unset CGCG override */
4950 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
4951 /* update CGCG and CGLS override bits */
4952 if (def != data)
4953 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4954
4955 /* enable 3Dcgcg FSM(0x0000363f) */
4956 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
4957
4958 data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4959 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
4960 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
4961 data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
4962 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
4963 if (def != data)
4964 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
4965
4966 /* set IDLE_POLL_COUNT(0x00900100) */
4967 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
4968 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
4969 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
4970 if (def != data)
4971 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
4972 } else {
4973 /* Disable CGCG/CGLS */
4974 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
4975 /* disable cgcg, cgls should be disabled */
4976 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK |
4977 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK);
4978 /* disable cgcg and cgls in FSM */
4979 if (def != data)
4980 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
4981 }
4982
4983 amdgpu_gfx_rlc_exit_safe_mode(adev);
4984 }
4985
gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device * adev,bool enable)4986 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
4987 bool enable)
4988 {
4989 uint32_t def, data;
4990
4991 amdgpu_gfx_rlc_enter_safe_mode(adev);
4992
4993 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
4994 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4995 /* unset CGCG override */
4996 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
4997 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
4998 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
4999 else
5000 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
5001 /* update CGCG and CGLS override bits */
5002 if (def != data)
5003 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
5004
5005 /* enable cgcg FSM(0x0000363F) */
5006 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
5007
5008 if (adev->asic_type == CHIP_ARCTURUS)
5009 data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5010 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5011 else
5012 data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5013 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5014 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5015 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5016 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5017 if (def != data)
5018 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
5019
5020 /* set IDLE_POLL_COUNT(0x00900100) */
5021 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
5022 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
5023 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
5024 if (def != data)
5025 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
5026 } else {
5027 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
5028 /* reset CGCG/CGLS bits */
5029 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
5030 /* disable cgcg and cgls in FSM */
5031 if (def != data)
5032 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
5033 }
5034
5035 amdgpu_gfx_rlc_exit_safe_mode(adev);
5036 }
5037
gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device * adev,bool enable)5038 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev,
5039 bool enable)
5040 {
5041 if (enable) {
5042 /* CGCG/CGLS should be enabled after MGCG/MGLS
5043 * === MGCG + MGLS ===
5044 */
5045 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
5046 /* === CGCG /CGLS for GFX 3D Only === */
5047 gfx_v9_0_update_3d_clock_gating(adev, enable);
5048 /* === CGCG + CGLS === */
5049 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
5050 } else {
5051 /* CGCG/CGLS should be disabled before MGCG/MGLS
5052 * === CGCG + CGLS ===
5053 */
5054 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
5055 /* === CGCG /CGLS for GFX 3D Only === */
5056 gfx_v9_0_update_3d_clock_gating(adev, enable);
5057 /* === MGCG + MGLS === */
5058 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
5059 }
5060 return 0;
5061 }
5062
gfx_v9_0_update_spm_vmid(struct amdgpu_device * adev,unsigned vmid)5063 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid)
5064 {
5065 u32 reg, data;
5066
5067 reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL);
5068 if (amdgpu_sriov_is_pp_one_vf(adev))
5069 data = RREG32_NO_KIQ(reg);
5070 else
5071 data = RREG32(reg);
5072
5073 data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;
5074 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
5075
5076 if (amdgpu_sriov_is_pp_one_vf(adev))
5077 WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data);
5078 else
5079 WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data);
5080 }
5081
gfx_v9_0_check_rlcg_range(struct amdgpu_device * adev,uint32_t offset,struct soc15_reg_rlcg * entries,int arr_size)5082 static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev,
5083 uint32_t offset,
5084 struct soc15_reg_rlcg *entries, int arr_size)
5085 {
5086 int i;
5087 uint32_t reg;
5088
5089 if (!entries)
5090 return false;
5091
5092 for (i = 0; i < arr_size; i++) {
5093 const struct soc15_reg_rlcg *entry;
5094
5095 entry = &entries[i];
5096 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg;
5097 if (offset == reg)
5098 return true;
5099 }
5100
5101 return false;
5102 }
5103
gfx_v9_0_is_rlcg_access_range(struct amdgpu_device * adev,u32 offset)5104 static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset)
5105 {
5106 return gfx_v9_0_check_rlcg_range(adev, offset,
5107 (void *)rlcg_access_gc_9_0,
5108 ARRAY_SIZE(rlcg_access_gc_9_0));
5109 }
5110
5111 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = {
5112 .is_rlc_enabled = gfx_v9_0_is_rlc_enabled,
5113 .set_safe_mode = gfx_v9_0_set_safe_mode,
5114 .unset_safe_mode = gfx_v9_0_unset_safe_mode,
5115 .init = gfx_v9_0_rlc_init,
5116 .get_csb_size = gfx_v9_0_get_csb_size,
5117 .get_csb_buffer = gfx_v9_0_get_csb_buffer,
5118 .get_cp_table_num = gfx_v9_0_cp_jump_table_num,
5119 .resume = gfx_v9_0_rlc_resume,
5120 .stop = gfx_v9_0_rlc_stop,
5121 .reset = gfx_v9_0_rlc_reset,
5122 .start = gfx_v9_0_rlc_start,
5123 .update_spm_vmid = gfx_v9_0_update_spm_vmid,
5124 .rlcg_wreg = gfx_v9_0_rlcg_wreg,
5125 .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range,
5126 };
5127
gfx_v9_0_set_powergating_state(void * handle,enum amd_powergating_state state)5128 static int gfx_v9_0_set_powergating_state(void *handle,
5129 enum amd_powergating_state state)
5130 {
5131 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5132 bool enable = (state == AMD_PG_STATE_GATE);
5133
5134 switch (adev->asic_type) {
5135 case CHIP_RAVEN:
5136 case CHIP_RENOIR:
5137 if (!enable)
5138 amdgpu_gfx_off_ctrl(adev, false);
5139
5140 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
5141 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true);
5142 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true);
5143 } else {
5144 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false);
5145 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false);
5146 }
5147
5148 if (adev->pg_flags & AMD_PG_SUPPORT_CP)
5149 gfx_v9_0_enable_cp_power_gating(adev, true);
5150 else
5151 gfx_v9_0_enable_cp_power_gating(adev, false);
5152
5153 /* update gfx cgpg state */
5154 gfx_v9_0_update_gfx_cg_power_gating(adev, enable);
5155
5156 /* update mgcg state */
5157 gfx_v9_0_update_gfx_mg_power_gating(adev, enable);
5158
5159 if (enable)
5160 amdgpu_gfx_off_ctrl(adev, true);
5161 break;
5162 case CHIP_VEGA12:
5163 amdgpu_gfx_off_ctrl(adev, enable);
5164 break;
5165 default:
5166 break;
5167 }
5168
5169 return 0;
5170 }
5171
gfx_v9_0_set_clockgating_state(void * handle,enum amd_clockgating_state state)5172 static int gfx_v9_0_set_clockgating_state(void *handle,
5173 enum amd_clockgating_state state)
5174 {
5175 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5176
5177 if (amdgpu_sriov_vf(adev))
5178 return 0;
5179
5180 switch (adev->asic_type) {
5181 case CHIP_VEGA10:
5182 case CHIP_VEGA12:
5183 case CHIP_VEGA20:
5184 case CHIP_RAVEN:
5185 case CHIP_ARCTURUS:
5186 case CHIP_RENOIR:
5187 case CHIP_ALDEBARAN:
5188 gfx_v9_0_update_gfx_clock_gating(adev,
5189 state == AMD_CG_STATE_GATE);
5190 break;
5191 default:
5192 break;
5193 }
5194 return 0;
5195 }
5196
gfx_v9_0_get_clockgating_state(void * handle,u32 * flags)5197 static void gfx_v9_0_get_clockgating_state(void *handle, u32 *flags)
5198 {
5199 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5200 int data;
5201
5202 if (amdgpu_sriov_vf(adev))
5203 *flags = 0;
5204
5205 /* AMD_CG_SUPPORT_GFX_MGCG */
5206 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE));
5207 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
5208 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
5209
5210 /* AMD_CG_SUPPORT_GFX_CGCG */
5211 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL));
5212 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5213 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
5214
5215 /* AMD_CG_SUPPORT_GFX_CGLS */
5216 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5217 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
5218
5219 /* AMD_CG_SUPPORT_GFX_RLC_LS */
5220 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL));
5221 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
5222 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
5223
5224 /* AMD_CG_SUPPORT_GFX_CP_LS */
5225 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL));
5226 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
5227 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
5228
5229 if (adev->asic_type != CHIP_ARCTURUS) {
5230 /* AMD_CG_SUPPORT_GFX_3D_CGCG */
5231 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D));
5232 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
5233 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
5234
5235 /* AMD_CG_SUPPORT_GFX_3D_CGLS */
5236 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
5237 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
5238 }
5239 }
5240
gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring * ring)5241 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
5242 {
5243 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 is 32bit rptr*/
5244 }
5245
gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring * ring)5246 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
5247 {
5248 struct amdgpu_device *adev = ring->adev;
5249 u64 wptr;
5250
5251 /* XXX check if swapping is necessary on BE */
5252 if (ring->use_doorbell) {
5253 wptr = atomic64_read((atomic64_t *)&adev->wb.wb[ring->wptr_offs]);
5254 } else {
5255 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR);
5256 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32;
5257 }
5258
5259 return wptr;
5260 }
5261
gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring * ring)5262 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
5263 {
5264 struct amdgpu_device *adev = ring->adev;
5265
5266 if (ring->use_doorbell) {
5267 /* XXX check if swapping is necessary on BE */
5268 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr);
5269 WDOORBELL64(ring->doorbell_index, ring->wptr);
5270 } else {
5271 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
5272 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
5273 }
5274 }
5275
gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring * ring)5276 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
5277 {
5278 struct amdgpu_device *adev = ring->adev;
5279 u32 ref_and_mask, reg_mem_engine;
5280 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
5281
5282 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
5283 switch (ring->me) {
5284 case 1:
5285 ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
5286 break;
5287 case 2:
5288 ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
5289 break;
5290 default:
5291 return;
5292 }
5293 reg_mem_engine = 0;
5294 } else {
5295 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
5296 reg_mem_engine = 1; /* pfp */
5297 }
5298
5299 gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
5300 adev->nbio.funcs->get_hdp_flush_req_offset(adev),
5301 adev->nbio.funcs->get_hdp_flush_done_offset(adev),
5302 ref_and_mask, ref_and_mask, 0x20);
5303 }
5304
gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring * ring,struct amdgpu_job * job,struct amdgpu_ib * ib,uint32_t flags)5305 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
5306 struct amdgpu_job *job,
5307 struct amdgpu_ib *ib,
5308 uint32_t flags)
5309 {
5310 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5311 u32 header, control = 0;
5312
5313 if (ib->flags & AMDGPU_IB_FLAG_CE)
5314 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
5315 else
5316 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
5317
5318 control |= ib->length_dw | (vmid << 24);
5319
5320 if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
5321 control |= INDIRECT_BUFFER_PRE_ENB(1);
5322
5323 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
5324 gfx_v9_0_ring_emit_de_meta(ring);
5325 }
5326
5327 amdgpu_ring_write(ring, header);
5328 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5329 amdgpu_ring_write(ring,
5330 #ifdef __BIG_ENDIAN
5331 (2 << 0) |
5332 #endif
5333 lower_32_bits(ib->gpu_addr));
5334 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5335 amdgpu_ring_write(ring, control);
5336 }
5337
gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring * ring,struct amdgpu_job * job,struct amdgpu_ib * ib,uint32_t flags)5338 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
5339 struct amdgpu_job *job,
5340 struct amdgpu_ib *ib,
5341 uint32_t flags)
5342 {
5343 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5344 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
5345
5346 /* Currently, there is a high possibility to get wave ID mismatch
5347 * between ME and GDS, leading to a hw deadlock, because ME generates
5348 * different wave IDs than the GDS expects. This situation happens
5349 * randomly when at least 5 compute pipes use GDS ordered append.
5350 * The wave IDs generated by ME are also wrong after suspend/resume.
5351 * Those are probably bugs somewhere else in the kernel driver.
5352 *
5353 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
5354 * GDS to 0 for this ring (me/pipe).
5355 */
5356 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
5357 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
5358 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID);
5359 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
5360 }
5361
5362 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
5363 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5364 amdgpu_ring_write(ring,
5365 #ifdef __BIG_ENDIAN
5366 (2 << 0) |
5367 #endif
5368 lower_32_bits(ib->gpu_addr));
5369 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5370 amdgpu_ring_write(ring, control);
5371 }
5372
gfx_v9_0_ring_emit_fence(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned flags)5373 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
5374 u64 seq, unsigned flags)
5375 {
5376 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
5377 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
5378 bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY;
5379
5380 /* RELEASE_MEM - flush caches, send int */
5381 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
5382 amdgpu_ring_write(ring, ((writeback ? (EOP_TC_WB_ACTION_EN |
5383 EOP_TC_NC_ACTION_EN) :
5384 (EOP_TCL1_ACTION_EN |
5385 EOP_TC_ACTION_EN |
5386 EOP_TC_WB_ACTION_EN |
5387 EOP_TC_MD_ACTION_EN)) |
5388 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
5389 EVENT_INDEX(5)));
5390 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
5391
5392 /*
5393 * the address should be Qword aligned if 64bit write, Dword
5394 * aligned if only send 32bit data low (discard data high)
5395 */
5396 if (write64bit)
5397 BUG_ON(addr & 0x7);
5398 else
5399 BUG_ON(addr & 0x3);
5400 amdgpu_ring_write(ring, lower_32_bits(addr));
5401 amdgpu_ring_write(ring, upper_32_bits(addr));
5402 amdgpu_ring_write(ring, lower_32_bits(seq));
5403 amdgpu_ring_write(ring, upper_32_bits(seq));
5404 amdgpu_ring_write(ring, 0);
5405 }
5406
gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring * ring)5407 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
5408 {
5409 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5410 uint32_t seq = ring->fence_drv.sync_seq;
5411 uint64_t addr = ring->fence_drv.gpu_addr;
5412
5413 gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0,
5414 lower_32_bits(addr), upper_32_bits(addr),
5415 seq, 0xffffffff, 4);
5416 }
5417
gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring * ring,unsigned vmid,uint64_t pd_addr)5418 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
5419 unsigned vmid, uint64_t pd_addr)
5420 {
5421 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
5422
5423 /* compute doesn't have PFP */
5424 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
5425 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5426 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5427 amdgpu_ring_write(ring, 0x0);
5428 }
5429 }
5430
gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring * ring)5431 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
5432 {
5433 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 hardware is 32bit rptr */
5434 }
5435
gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring * ring)5436 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
5437 {
5438 u64 wptr;
5439
5440 /* XXX check if swapping is necessary on BE */
5441 if (ring->use_doorbell)
5442 wptr = atomic64_read((atomic64_t *)&ring->adev->wb.wb[ring->wptr_offs]);
5443 else
5444 BUG();
5445 return wptr;
5446 }
5447
gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring * ring)5448 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
5449 {
5450 struct amdgpu_device *adev = ring->adev;
5451
5452 /* XXX check if swapping is necessary on BE */
5453 if (ring->use_doorbell) {
5454 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr);
5455 WDOORBELL64(ring->doorbell_index, ring->wptr);
5456 } else{
5457 BUG(); /* only DOORBELL method supported on gfx9 now */
5458 }
5459 }
5460
gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned int flags)5461 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
5462 u64 seq, unsigned int flags)
5463 {
5464 struct amdgpu_device *adev = ring->adev;
5465
5466 /* we only allocate 32bit for each seq wb address */
5467 BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
5468
5469 /* write fence seq to the "addr" */
5470 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5471 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5472 WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
5473 amdgpu_ring_write(ring, lower_32_bits(addr));
5474 amdgpu_ring_write(ring, upper_32_bits(addr));
5475 amdgpu_ring_write(ring, lower_32_bits(seq));
5476
5477 if (flags & AMDGPU_FENCE_FLAG_INT) {
5478 /* set register to trigger INT */
5479 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5480 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5481 WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
5482 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS));
5483 amdgpu_ring_write(ring, 0);
5484 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
5485 }
5486 }
5487
gfx_v9_ring_emit_sb(struct amdgpu_ring * ring)5488 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)
5489 {
5490 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
5491 amdgpu_ring_write(ring, 0);
5492 }
5493
gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring * ring)5494 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
5495 {
5496 struct v9_ce_ib_state ce_payload = {0};
5497 uint64_t csa_addr;
5498 int cnt;
5499
5500 cnt = (sizeof(ce_payload) >> 2) + 4 - 2;
5501 csa_addr = amdgpu_csa_vaddr(ring->adev);
5502
5503 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5504 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
5505 WRITE_DATA_DST_SEL(8) |
5506 WR_CONFIRM) |
5507 WRITE_DATA_CACHE_POLICY(0));
5508 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload)));
5509 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload)));
5510 amdgpu_ring_write_multiple(ring, (void *)&ce_payload, sizeof(ce_payload) >> 2);
5511 }
5512
gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring * ring)5513 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring)
5514 {
5515 struct v9_de_ib_state de_payload = {0};
5516 uint64_t csa_addr, gds_addr;
5517 int cnt;
5518
5519 csa_addr = amdgpu_csa_vaddr(ring->adev);
5520 gds_addr = csa_addr + 4096;
5521 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
5522 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
5523
5524 cnt = (sizeof(de_payload) >> 2) + 4 - 2;
5525 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5526 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5527 WRITE_DATA_DST_SEL(8) |
5528 WR_CONFIRM) |
5529 WRITE_DATA_CACHE_POLICY(0));
5530 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload)));
5531 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload)));
5532 amdgpu_ring_write_multiple(ring, (void *)&de_payload, sizeof(de_payload) >> 2);
5533 }
5534
gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring * ring,bool start,bool secure)5535 static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,
5536 bool secure)
5537 {
5538 uint32_t v = secure ? FRAME_TMZ : 0;
5539
5540 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
5541 amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));
5542 }
5543
gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring * ring,uint32_t flags)5544 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
5545 {
5546 uint32_t dw2 = 0;
5547
5548 if (amdgpu_sriov_vf(ring->adev))
5549 gfx_v9_0_ring_emit_ce_meta(ring);
5550
5551 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
5552 if (flags & AMDGPU_HAVE_CTX_SWITCH) {
5553 /* set load_global_config & load_global_uconfig */
5554 dw2 |= 0x8001;
5555 /* set load_cs_sh_regs */
5556 dw2 |= 0x01000000;
5557 /* set load_per_context_state & load_gfx_sh_regs for GFX */
5558 dw2 |= 0x10002;
5559
5560 /* set load_ce_ram if preamble presented */
5561 if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
5562 dw2 |= 0x10000000;
5563 } else {
5564 /* still load_ce_ram if this is the first time preamble presented
5565 * although there is no context switch happens.
5566 */
5567 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
5568 dw2 |= 0x10000000;
5569 }
5570
5571 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5572 amdgpu_ring_write(ring, dw2);
5573 amdgpu_ring_write(ring, 0);
5574 }
5575
gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring * ring)5576 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring)
5577 {
5578 unsigned ret;
5579 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
5580 amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr));
5581 amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr));
5582 amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */
5583 ret = ring->wptr & ring->buf_mask;
5584 amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */
5585 return ret;
5586 }
5587
gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring * ring,unsigned offset)5588 static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset)
5589 {
5590 unsigned cur;
5591 BUG_ON(offset > ring->buf_mask);
5592 BUG_ON(ring->ring[offset] != 0x55aa55aa);
5593
5594 cur = (ring->wptr & ring->buf_mask) - 1;
5595 if (likely(cur > offset))
5596 ring->ring[offset] = cur - offset;
5597 else
5598 ring->ring[offset] = (ring->ring_size>>2) - offset + cur;
5599 }
5600
gfx_v9_0_ring_emit_rreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t reg_val_offs)5601 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
5602 uint32_t reg_val_offs)
5603 {
5604 struct amdgpu_device *adev = ring->adev;
5605
5606 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
5607 amdgpu_ring_write(ring, 0 | /* src: register*/
5608 (5 << 8) | /* dst: memory */
5609 (1 << 20)); /* write confirm */
5610 amdgpu_ring_write(ring, reg);
5611 amdgpu_ring_write(ring, 0);
5612 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
5613 reg_val_offs * 4));
5614 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
5615 reg_val_offs * 4));
5616 }
5617
gfx_v9_0_ring_emit_wreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t val)5618 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
5619 uint32_t val)
5620 {
5621 uint32_t cmd = 0;
5622
5623 switch (ring->funcs->type) {
5624 case AMDGPU_RING_TYPE_GFX:
5625 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
5626 break;
5627 case AMDGPU_RING_TYPE_KIQ:
5628 cmd = (1 << 16); /* no inc addr */
5629 break;
5630 default:
5631 cmd = WR_CONFIRM;
5632 break;
5633 }
5634 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5635 amdgpu_ring_write(ring, cmd);
5636 amdgpu_ring_write(ring, reg);
5637 amdgpu_ring_write(ring, 0);
5638 amdgpu_ring_write(ring, val);
5639 }
5640
gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring * ring,uint32_t reg,uint32_t val,uint32_t mask)5641 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
5642 uint32_t val, uint32_t mask)
5643 {
5644 gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
5645 }
5646
gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring * ring,uint32_t reg0,uint32_t reg1,uint32_t ref,uint32_t mask)5647 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
5648 uint32_t reg0, uint32_t reg1,
5649 uint32_t ref, uint32_t mask)
5650 {
5651 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5652 struct amdgpu_device *adev = ring->adev;
5653 bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ?
5654 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait;
5655
5656 if (fw_version_ok)
5657 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
5658 ref, mask, 0x20);
5659 else
5660 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1,
5661 ref, mask);
5662 }
5663
gfx_v9_0_ring_soft_recovery(struct amdgpu_ring * ring,unsigned vmid)5664 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
5665 {
5666 struct amdgpu_device *adev = ring->adev;
5667 uint32_t value = 0;
5668
5669 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
5670 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
5671 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
5672 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
5673 WREG32_SOC15(GC, 0, mmSQ_CMD, value);
5674 }
5675
gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device * adev,enum amdgpu_interrupt_state state)5676 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
5677 enum amdgpu_interrupt_state state)
5678 {
5679 switch (state) {
5680 case AMDGPU_IRQ_STATE_DISABLE:
5681 case AMDGPU_IRQ_STATE_ENABLE:
5682 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5683 TIME_STAMP_INT_ENABLE,
5684 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5685 break;
5686 default:
5687 break;
5688 }
5689 }
5690
gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device * adev,int me,int pipe,enum amdgpu_interrupt_state state)5691 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
5692 int me, int pipe,
5693 enum amdgpu_interrupt_state state)
5694 {
5695 u32 mec_int_cntl, mec_int_cntl_reg;
5696
5697 /*
5698 * amdgpu controls only the first MEC. That's why this function only
5699 * handles the setting of interrupts for this specific MEC. All other
5700 * pipes' interrupts are set by amdkfd.
5701 */
5702
5703 if (me == 1) {
5704 switch (pipe) {
5705 case 0:
5706 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
5707 break;
5708 case 1:
5709 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);
5710 break;
5711 case 2:
5712 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);
5713 break;
5714 case 3:
5715 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);
5716 break;
5717 default:
5718 DRM_DEBUG("invalid pipe %d\n", pipe);
5719 return;
5720 }
5721 } else {
5722 DRM_DEBUG("invalid me %d\n", me);
5723 return;
5724 }
5725
5726 switch (state) {
5727 case AMDGPU_IRQ_STATE_DISABLE:
5728 mec_int_cntl = RREG32(mec_int_cntl_reg);
5729 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5730 TIME_STAMP_INT_ENABLE, 0);
5731 WREG32(mec_int_cntl_reg, mec_int_cntl);
5732 break;
5733 case AMDGPU_IRQ_STATE_ENABLE:
5734 mec_int_cntl = RREG32(mec_int_cntl_reg);
5735 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5736 TIME_STAMP_INT_ENABLE, 1);
5737 WREG32(mec_int_cntl_reg, mec_int_cntl);
5738 break;
5739 default:
5740 break;
5741 }
5742 }
5743
gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)5744 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
5745 struct amdgpu_irq_src *source,
5746 unsigned type,
5747 enum amdgpu_interrupt_state state)
5748 {
5749 switch (state) {
5750 case AMDGPU_IRQ_STATE_DISABLE:
5751 case AMDGPU_IRQ_STATE_ENABLE:
5752 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5753 PRIV_REG_INT_ENABLE,
5754 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5755 break;
5756 default:
5757 break;
5758 }
5759
5760 return 0;
5761 }
5762
gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)5763 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
5764 struct amdgpu_irq_src *source,
5765 unsigned type,
5766 enum amdgpu_interrupt_state state)
5767 {
5768 switch (state) {
5769 case AMDGPU_IRQ_STATE_DISABLE:
5770 case AMDGPU_IRQ_STATE_ENABLE:
5771 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5772 PRIV_INSTR_INT_ENABLE,
5773 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5774 break;
5775 default:
5776 break;
5777 }
5778
5779 return 0;
5780 }
5781
5782 #define ENABLE_ECC_ON_ME_PIPE(me, pipe) \
5783 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
5784 CP_ECC_ERROR_INT_ENABLE, 1)
5785
5786 #define DISABLE_ECC_ON_ME_PIPE(me, pipe) \
5787 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
5788 CP_ECC_ERROR_INT_ENABLE, 0)
5789
gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)5790 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev,
5791 struct amdgpu_irq_src *source,
5792 unsigned type,
5793 enum amdgpu_interrupt_state state)
5794 {
5795 switch (state) {
5796 case AMDGPU_IRQ_STATE_DISABLE:
5797 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5798 CP_ECC_ERROR_INT_ENABLE, 0);
5799 DISABLE_ECC_ON_ME_PIPE(1, 0);
5800 DISABLE_ECC_ON_ME_PIPE(1, 1);
5801 DISABLE_ECC_ON_ME_PIPE(1, 2);
5802 DISABLE_ECC_ON_ME_PIPE(1, 3);
5803 break;
5804
5805 case AMDGPU_IRQ_STATE_ENABLE:
5806 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5807 CP_ECC_ERROR_INT_ENABLE, 1);
5808 ENABLE_ECC_ON_ME_PIPE(1, 0);
5809 ENABLE_ECC_ON_ME_PIPE(1, 1);
5810 ENABLE_ECC_ON_ME_PIPE(1, 2);
5811 ENABLE_ECC_ON_ME_PIPE(1, 3);
5812 break;
5813 default:
5814 break;
5815 }
5816
5817 return 0;
5818 }
5819
5820
gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device * adev,struct amdgpu_irq_src * src,unsigned type,enum amdgpu_interrupt_state state)5821 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev,
5822 struct amdgpu_irq_src *src,
5823 unsigned type,
5824 enum amdgpu_interrupt_state state)
5825 {
5826 switch (type) {
5827 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
5828 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state);
5829 break;
5830 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
5831 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
5832 break;
5833 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
5834 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
5835 break;
5836 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
5837 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
5838 break;
5839 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
5840 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
5841 break;
5842 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
5843 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
5844 break;
5845 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
5846 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
5847 break;
5848 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
5849 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
5850 break;
5851 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
5852 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
5853 break;
5854 default:
5855 break;
5856 }
5857 return 0;
5858 }
5859
gfx_v9_0_eop_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)5860 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev,
5861 struct amdgpu_irq_src *source,
5862 struct amdgpu_iv_entry *entry)
5863 {
5864 int i;
5865 u8 me_id, pipe_id, queue_id;
5866 struct amdgpu_ring *ring;
5867
5868 DRM_DEBUG("IH: CP EOP\n");
5869 me_id = (entry->ring_id & 0x0c) >> 2;
5870 pipe_id = (entry->ring_id & 0x03) >> 0;
5871 queue_id = (entry->ring_id & 0x70) >> 4;
5872
5873 switch (me_id) {
5874 case 0:
5875 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
5876 break;
5877 case 1:
5878 case 2:
5879 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5880 ring = &adev->gfx.compute_ring[i];
5881 /* Per-queue interrupt is supported for MEC starting from VI.
5882 * The interrupt can only be enabled/disabled per pipe instead of per queue.
5883 */
5884 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
5885 amdgpu_fence_process(ring);
5886 }
5887 break;
5888 }
5889 return 0;
5890 }
5891
gfx_v9_0_fault(struct amdgpu_device * adev,struct amdgpu_iv_entry * entry)5892 static void gfx_v9_0_fault(struct amdgpu_device *adev,
5893 struct amdgpu_iv_entry *entry)
5894 {
5895 u8 me_id, pipe_id, queue_id;
5896 struct amdgpu_ring *ring;
5897 int i;
5898
5899 me_id = (entry->ring_id & 0x0c) >> 2;
5900 pipe_id = (entry->ring_id & 0x03) >> 0;
5901 queue_id = (entry->ring_id & 0x70) >> 4;
5902
5903 switch (me_id) {
5904 case 0:
5905 drm_sched_fault(&adev->gfx.gfx_ring[0].sched);
5906 break;
5907 case 1:
5908 case 2:
5909 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5910 ring = &adev->gfx.compute_ring[i];
5911 if (ring->me == me_id && ring->pipe == pipe_id &&
5912 ring->queue == queue_id)
5913 drm_sched_fault(&ring->sched);
5914 }
5915 break;
5916 }
5917 }
5918
gfx_v9_0_priv_reg_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)5919 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev,
5920 struct amdgpu_irq_src *source,
5921 struct amdgpu_iv_entry *entry)
5922 {
5923 DRM_ERROR("Illegal register access in command stream\n");
5924 gfx_v9_0_fault(adev, entry);
5925 return 0;
5926 }
5927
gfx_v9_0_priv_inst_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)5928 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev,
5929 struct amdgpu_irq_src *source,
5930 struct amdgpu_iv_entry *entry)
5931 {
5932 DRM_ERROR("Illegal instruction in command stream\n");
5933 gfx_v9_0_fault(adev, entry);
5934 return 0;
5935 }
5936
5937
5938 static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = {
5939 { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT),
5940 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT),
5941 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT)
5942 },
5943 { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT),
5944 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT),
5945 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT)
5946 },
5947 { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
5948 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1),
5949 0, 0
5950 },
5951 { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
5952 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2),
5953 0, 0
5954 },
5955 { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT),
5956 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT),
5957 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT)
5958 },
5959 { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
5960 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT),
5961 0, 0
5962 },
5963 { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
5964 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT),
5965 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT)
5966 },
5967 { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT),
5968 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT),
5969 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT)
5970 },
5971 { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT),
5972 SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1),
5973 0, 0
5974 },
5975 { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT),
5976 SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1),
5977 0, 0
5978 },
5979 { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT),
5980 SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1),
5981 0, 0
5982 },
5983 { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
5984 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC),
5985 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED)
5986 },
5987 { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
5988 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED),
5989 0, 0
5990 },
5991 { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
5992 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC),
5993 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED)
5994 },
5995 { "GDS_OA_PHY_PHY_CMD_RAM_MEM",
5996 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
5997 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC),
5998 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED)
5999 },
6000 { "GDS_OA_PHY_PHY_DATA_RAM_MEM",
6001 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6002 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED),
6003 0, 0
6004 },
6005 { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM",
6006 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6007 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC),
6008 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED)
6009 },
6010 { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM",
6011 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6012 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC),
6013 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED)
6014 },
6015 { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM",
6016 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6017 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC),
6018 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED)
6019 },
6020 { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM",
6021 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6022 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC),
6023 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED)
6024 },
6025 { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT),
6026 SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT),
6027 0, 0
6028 },
6029 { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6030 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT),
6031 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT)
6032 },
6033 { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6034 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT),
6035 0, 0
6036 },
6037 { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6038 SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT),
6039 0, 0
6040 },
6041 { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6042 SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT),
6043 0, 0
6044 },
6045 { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6046 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT),
6047 0, 0
6048 },
6049 { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
6050 SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT),
6051 0, 0
6052 },
6053 { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
6054 SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT),
6055 0, 0
6056 },
6057 { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6058 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT),
6059 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT)
6060 },
6061 { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6062 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT),
6063 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT)
6064 },
6065 { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6066 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT),
6067 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT)
6068 },
6069 { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6070 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT),
6071 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT)
6072 },
6073 { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6074 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT),
6075 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT)
6076 },
6077 { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6078 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT),
6079 0, 0
6080 },
6081 { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6082 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT),
6083 0, 0
6084 },
6085 { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6086 SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT),
6087 0, 0
6088 },
6089 { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6090 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT),
6091 0, 0
6092 },
6093 { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6094 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT),
6095 0, 0
6096 },
6097 { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6098 SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT),
6099 0, 0
6100 },
6101 { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6102 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT),
6103 0, 0
6104 },
6105 { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6106 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT),
6107 0, 0
6108 },
6109 { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6110 SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT),
6111 0, 0
6112 },
6113 { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6114 SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT),
6115 0, 0
6116 },
6117 { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6118 SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT),
6119 0, 0
6120 },
6121 { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6122 SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT),
6123 0, 0
6124 },
6125 { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6126 SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT),
6127 0, 0
6128 },
6129 { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT),
6130 SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT),
6131 0, 0
6132 },
6133 { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6134 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT),
6135 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT)
6136 },
6137 { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6138 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT),
6139 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT)
6140 },
6141 { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6142 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT),
6143 0, 0
6144 },
6145 { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6146 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT),
6147 0, 0
6148 },
6149 { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6150 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT),
6151 0, 0
6152 },
6153 { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6154 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT),
6155 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT)
6156 },
6157 { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6158 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT),
6159 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT)
6160 },
6161 { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6162 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT),
6163 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT)
6164 },
6165 { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6166 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT),
6167 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT)
6168 },
6169 { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6170 SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT),
6171 0, 0
6172 },
6173 { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6174 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT),
6175 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT)
6176 },
6177 { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6178 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT),
6179 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT)
6180 },
6181 { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6182 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT),
6183 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT)
6184 },
6185 { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6186 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT),
6187 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT)
6188 },
6189 { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6190 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT),
6191 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT)
6192 },
6193 { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6194 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT),
6195 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT)
6196 },
6197 { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6198 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT),
6199 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT)
6200 },
6201 { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6202 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT),
6203 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT)
6204 },
6205 { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6206 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT),
6207 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT)
6208 },
6209 { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6210 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT),
6211 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT)
6212 },
6213 { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6214 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT),
6215 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT)
6216 },
6217 { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6218 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT),
6219 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT)
6220 },
6221 { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6222 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT),
6223 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT)
6224 },
6225 { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6226 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT),
6227 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT)
6228 },
6229 { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6230 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT),
6231 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT)
6232 },
6233 { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6234 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT),
6235 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT)
6236 },
6237 { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6238 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT),
6239 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT)
6240 },
6241 { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6242 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT),
6243 0, 0
6244 },
6245 { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6246 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT),
6247 0, 0
6248 },
6249 { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6250 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT),
6251 0, 0
6252 },
6253 { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6254 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT),
6255 0, 0
6256 },
6257 { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6258 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT),
6259 0, 0
6260 },
6261 { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6262 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT),
6263 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT)
6264 },
6265 { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6266 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT),
6267 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT)
6268 },
6269 { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6270 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT),
6271 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT)
6272 },
6273 { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6274 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT),
6275 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT)
6276 },
6277 { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6278 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT),
6279 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT)
6280 },
6281 { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6282 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT),
6283 0, 0
6284 },
6285 { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6286 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT),
6287 0, 0
6288 },
6289 { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6290 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT),
6291 0, 0
6292 },
6293 { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6294 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT),
6295 0, 0
6296 },
6297 { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6298 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT),
6299 0, 0
6300 },
6301 { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6302 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT),
6303 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT)
6304 },
6305 { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6306 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT),
6307 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT)
6308 },
6309 { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6310 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT),
6311 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT)
6312 },
6313 { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6314 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT),
6315 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT)
6316 },
6317 { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6318 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT),
6319 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT)
6320 },
6321 { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6322 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT),
6323 0, 0
6324 },
6325 { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6326 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT),
6327 0, 0
6328 },
6329 { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6330 SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT),
6331 0, 0
6332 },
6333 { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6334 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT),
6335 0, 0
6336 },
6337 { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6338 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT),
6339 0, 0
6340 },
6341 { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6342 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT),
6343 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT)
6344 },
6345 { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6346 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT),
6347 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT)
6348 },
6349 { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6350 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT),
6351 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT)
6352 },
6353 { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6354 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT),
6355 0, 0
6356 },
6357 { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6358 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT),
6359 0, 0
6360 },
6361 { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6362 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT),
6363 0, 0
6364 },
6365 { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6366 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT),
6367 0, 0
6368 },
6369 { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6370 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT),
6371 0, 0
6372 },
6373 { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6374 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT),
6375 0, 0
6376 }
6377 };
6378
gfx_v9_0_ras_error_inject(struct amdgpu_device * adev,void * inject_if)6379 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
6380 void *inject_if)
6381 {
6382 struct ras_inject_if *info = (struct ras_inject_if *)inject_if;
6383 int ret;
6384 struct ta_ras_trigger_error_input block_info = { 0 };
6385
6386 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6387 return -EINVAL;
6388
6389 if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks))
6390 return -EINVAL;
6391
6392 if (!ras_gfx_subblocks[info->head.sub_block_index].name)
6393 return -EPERM;
6394
6395 if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type &
6396 info->head.type)) {
6397 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n",
6398 ras_gfx_subblocks[info->head.sub_block_index].name,
6399 info->head.type);
6400 return -EPERM;
6401 }
6402
6403 if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type &
6404 info->head.type)) {
6405 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n",
6406 ras_gfx_subblocks[info->head.sub_block_index].name,
6407 info->head.type);
6408 return -EPERM;
6409 }
6410
6411 block_info.block_id = amdgpu_ras_block_to_ta(info->head.block);
6412 block_info.sub_block_index =
6413 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock;
6414 block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type);
6415 block_info.address = info->address;
6416 block_info.value = info->value;
6417
6418 mutex_lock(&adev->grbm_idx_mutex);
6419 ret = psp_ras_trigger_error(&adev->psp, &block_info);
6420 mutex_unlock(&adev->grbm_idx_mutex);
6421
6422 return ret;
6423 }
6424
6425 static const char *vml2_mems[] = {
6426 "UTC_VML2_BANK_CACHE_0_BIGK_MEM0",
6427 "UTC_VML2_BANK_CACHE_0_BIGK_MEM1",
6428 "UTC_VML2_BANK_CACHE_0_4K_MEM0",
6429 "UTC_VML2_BANK_CACHE_0_4K_MEM1",
6430 "UTC_VML2_BANK_CACHE_1_BIGK_MEM0",
6431 "UTC_VML2_BANK_CACHE_1_BIGK_MEM1",
6432 "UTC_VML2_BANK_CACHE_1_4K_MEM0",
6433 "UTC_VML2_BANK_CACHE_1_4K_MEM1",
6434 "UTC_VML2_BANK_CACHE_2_BIGK_MEM0",
6435 "UTC_VML2_BANK_CACHE_2_BIGK_MEM1",
6436 "UTC_VML2_BANK_CACHE_2_4K_MEM0",
6437 "UTC_VML2_BANK_CACHE_2_4K_MEM1",
6438 "UTC_VML2_BANK_CACHE_3_BIGK_MEM0",
6439 "UTC_VML2_BANK_CACHE_3_BIGK_MEM1",
6440 "UTC_VML2_BANK_CACHE_3_4K_MEM0",
6441 "UTC_VML2_BANK_CACHE_3_4K_MEM1",
6442 };
6443
6444 static const char *vml2_walker_mems[] = {
6445 "UTC_VML2_CACHE_PDE0_MEM0",
6446 "UTC_VML2_CACHE_PDE0_MEM1",
6447 "UTC_VML2_CACHE_PDE1_MEM0",
6448 "UTC_VML2_CACHE_PDE1_MEM1",
6449 "UTC_VML2_CACHE_PDE2_MEM0",
6450 "UTC_VML2_CACHE_PDE2_MEM1",
6451 "UTC_VML2_RDIF_LOG_FIFO",
6452 };
6453
6454 static const char *atc_l2_cache_2m_mems[] = {
6455 "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM",
6456 "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM",
6457 "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM",
6458 "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM",
6459 };
6460
6461 static const char *atc_l2_cache_4k_mems[] = {
6462 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0",
6463 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1",
6464 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2",
6465 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3",
6466 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4",
6467 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5",
6468 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6",
6469 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7",
6470 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0",
6471 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1",
6472 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2",
6473 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3",
6474 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4",
6475 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5",
6476 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6",
6477 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7",
6478 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0",
6479 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1",
6480 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2",
6481 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3",
6482 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4",
6483 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5",
6484 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6",
6485 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7",
6486 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0",
6487 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1",
6488 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2",
6489 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3",
6490 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4",
6491 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5",
6492 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6",
6493 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7",
6494 };
6495
gfx_v9_0_query_utc_edc_status(struct amdgpu_device * adev,struct ras_err_data * err_data)6496 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev,
6497 struct ras_err_data *err_data)
6498 {
6499 uint32_t i, data;
6500 uint32_t sec_count, ded_count;
6501
6502 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6503 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6504 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6505 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6506 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6507 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6508 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6509 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6510
6511 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6512 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6513 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6514
6515 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT);
6516 if (sec_count) {
6517 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6518 "SEC %d\n", i, vml2_mems[i], sec_count);
6519 err_data->ce_count += sec_count;
6520 }
6521
6522 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT);
6523 if (ded_count) {
6524 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6525 "DED %d\n", i, vml2_mems[i], ded_count);
6526 err_data->ue_count += ded_count;
6527 }
6528 }
6529
6530 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6531 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6532 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6533
6534 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6535 SEC_COUNT);
6536 if (sec_count) {
6537 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6538 "SEC %d\n", i, vml2_walker_mems[i], sec_count);
6539 err_data->ce_count += sec_count;
6540 }
6541
6542 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6543 DED_COUNT);
6544 if (ded_count) {
6545 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6546 "DED %d\n", i, vml2_walker_mems[i], ded_count);
6547 err_data->ue_count += ded_count;
6548 }
6549 }
6550
6551 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6552 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6553 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6554
6555 sec_count = (data & 0x00006000L) >> 0xd;
6556 if (sec_count) {
6557 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6558 "SEC %d\n", i, atc_l2_cache_2m_mems[i],
6559 sec_count);
6560 err_data->ce_count += sec_count;
6561 }
6562 }
6563
6564 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6565 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6566 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6567
6568 sec_count = (data & 0x00006000L) >> 0xd;
6569 if (sec_count) {
6570 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6571 "SEC %d\n", i, atc_l2_cache_4k_mems[i],
6572 sec_count);
6573 err_data->ce_count += sec_count;
6574 }
6575
6576 ded_count = (data & 0x00018000L) >> 0xf;
6577 if (ded_count) {
6578 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6579 "DED %d\n", i, atc_l2_cache_4k_mems[i],
6580 ded_count);
6581 err_data->ue_count += ded_count;
6582 }
6583 }
6584
6585 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6586 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6587 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6588 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6589
6590 return 0;
6591 }
6592
gfx_v9_0_ras_error_count(struct amdgpu_device * adev,const struct soc15_reg_entry * reg,uint32_t se_id,uint32_t inst_id,uint32_t value,uint32_t * sec_count,uint32_t * ded_count)6593 static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev,
6594 const struct soc15_reg_entry *reg,
6595 uint32_t se_id, uint32_t inst_id, uint32_t value,
6596 uint32_t *sec_count, uint32_t *ded_count)
6597 {
6598 uint32_t i;
6599 uint32_t sec_cnt, ded_cnt;
6600
6601 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) {
6602 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset ||
6603 gfx_v9_0_ras_fields[i].seg != reg->seg ||
6604 gfx_v9_0_ras_fields[i].inst != reg->inst)
6605 continue;
6606
6607 sec_cnt = (value &
6608 gfx_v9_0_ras_fields[i].sec_count_mask) >>
6609 gfx_v9_0_ras_fields[i].sec_count_shift;
6610 if (sec_cnt) {
6611 dev_info(adev->dev, "GFX SubBlock %s, "
6612 "Instance[%d][%d], SEC %d\n",
6613 gfx_v9_0_ras_fields[i].name,
6614 se_id, inst_id,
6615 sec_cnt);
6616 *sec_count += sec_cnt;
6617 }
6618
6619 ded_cnt = (value &
6620 gfx_v9_0_ras_fields[i].ded_count_mask) >>
6621 gfx_v9_0_ras_fields[i].ded_count_shift;
6622 if (ded_cnt) {
6623 dev_info(adev->dev, "GFX SubBlock %s, "
6624 "Instance[%d][%d], DED %d\n",
6625 gfx_v9_0_ras_fields[i].name,
6626 se_id, inst_id,
6627 ded_cnt);
6628 *ded_count += ded_cnt;
6629 }
6630 }
6631
6632 return 0;
6633 }
6634
gfx_v9_0_reset_ras_error_count(struct amdgpu_device * adev)6635 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev)
6636 {
6637 int i, j, k;
6638
6639 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6640 return;
6641
6642 /* read back registers to clear the counters */
6643 mutex_lock(&adev->grbm_idx_mutex);
6644 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
6645 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
6646 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
6647 gfx_v9_0_select_se_sh(adev, j, 0x0, k);
6648 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
6649 }
6650 }
6651 }
6652 WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000);
6653 mutex_unlock(&adev->grbm_idx_mutex);
6654
6655 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6656 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6657 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6658 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6659 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6660 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6661 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6662 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6663
6664 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6665 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6666 RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6667 }
6668
6669 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6670 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6671 RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6672 }
6673
6674 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6675 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6676 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6677 }
6678
6679 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6680 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6681 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6682 }
6683
6684 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6685 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6686 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6687 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6688 }
6689
gfx_v9_0_query_ras_error_count(struct amdgpu_device * adev,void * ras_error_status)6690 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
6691 void *ras_error_status)
6692 {
6693 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
6694 uint32_t sec_count = 0, ded_count = 0;
6695 uint32_t i, j, k;
6696 uint32_t reg_value;
6697
6698 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6699 return -EINVAL;
6700
6701 err_data->ue_count = 0;
6702 err_data->ce_count = 0;
6703
6704 mutex_lock(&adev->grbm_idx_mutex);
6705
6706 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
6707 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
6708 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
6709 gfx_v9_0_select_se_sh(adev, j, 0, k);
6710 reg_value =
6711 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
6712 if (reg_value)
6713 gfx_v9_0_ras_error_count(adev,
6714 &gfx_v9_0_edc_counter_regs[i],
6715 j, k, reg_value,
6716 &sec_count, &ded_count);
6717 }
6718 }
6719 }
6720
6721 err_data->ce_count += sec_count;
6722 err_data->ue_count += ded_count;
6723
6724 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
6725 mutex_unlock(&adev->grbm_idx_mutex);
6726
6727 gfx_v9_0_query_utc_edc_status(adev, err_data);
6728
6729 return 0;
6730 }
6731
gfx_v9_0_emit_mem_sync(struct amdgpu_ring * ring)6732 static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring)
6733 {
6734 const unsigned int cp_coher_cntl =
6735 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) |
6736 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) |
6737 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) |
6738 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) |
6739 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1);
6740
6741 /* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */
6742 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5));
6743 amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */
6744 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */
6745 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */
6746 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
6747 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */
6748 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
6749 }
6750
gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring * ring,uint32_t pipe,bool enable)6751 static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring,
6752 uint32_t pipe, bool enable)
6753 {
6754 struct amdgpu_device *adev = ring->adev;
6755 uint32_t val;
6756 uint32_t wcl_cs_reg;
6757
6758 /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */
6759 val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT;
6760
6761 switch (pipe) {
6762 case 0:
6763 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0);
6764 break;
6765 case 1:
6766 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1);
6767 break;
6768 case 2:
6769 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2);
6770 break;
6771 case 3:
6772 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3);
6773 break;
6774 default:
6775 DRM_DEBUG("invalid pipe %d\n", pipe);
6776 return;
6777 }
6778
6779 amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val);
6780
6781 }
gfx_v9_0_emit_wave_limit(struct amdgpu_ring * ring,bool enable)6782 static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable)
6783 {
6784 struct amdgpu_device *adev = ring->adev;
6785 uint32_t val;
6786 int i;
6787
6788
6789 /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit
6790 * number of gfx waves. Setting 5 bit will make sure gfx only gets
6791 * around 25% of gpu resources.
6792 */
6793 val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT;
6794 amdgpu_ring_emit_wreg(ring,
6795 SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX),
6796 val);
6797
6798 /* Restrict waves for normal/low priority compute queues as well
6799 * to get best QoS for high priority compute jobs.
6800 *
6801 * amdgpu controls only 1st ME(0-3 CS pipes).
6802 */
6803 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
6804 if (i != ring->pipe)
6805 gfx_v9_0_emit_wave_limit_cs(ring, i, enable);
6806
6807 }
6808 }
6809
6810 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = {
6811 .name = "gfx_v9_0",
6812 .early_init = gfx_v9_0_early_init,
6813 .late_init = gfx_v9_0_late_init,
6814 .sw_init = gfx_v9_0_sw_init,
6815 .sw_fini = gfx_v9_0_sw_fini,
6816 .hw_init = gfx_v9_0_hw_init,
6817 .hw_fini = gfx_v9_0_hw_fini,
6818 .suspend = gfx_v9_0_suspend,
6819 .resume = gfx_v9_0_resume,
6820 .is_idle = gfx_v9_0_is_idle,
6821 .wait_for_idle = gfx_v9_0_wait_for_idle,
6822 .soft_reset = gfx_v9_0_soft_reset,
6823 .set_clockgating_state = gfx_v9_0_set_clockgating_state,
6824 .set_powergating_state = gfx_v9_0_set_powergating_state,
6825 .get_clockgating_state = gfx_v9_0_get_clockgating_state,
6826 };
6827
6828 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = {
6829 .type = AMDGPU_RING_TYPE_GFX,
6830 .align_mask = 0xff,
6831 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6832 .support_64bit_ptrs = true,
6833 .vmhub = AMDGPU_GFXHUB_0,
6834 .get_rptr = gfx_v9_0_ring_get_rptr_gfx,
6835 .get_wptr = gfx_v9_0_ring_get_wptr_gfx,
6836 .set_wptr = gfx_v9_0_ring_set_wptr_gfx,
6837 .emit_frame_size = /* totally 242 maximum if 16 IBs */
6838 5 + /* COND_EXEC */
6839 7 + /* PIPELINE_SYNC */
6840 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6841 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6842 2 + /* VM_FLUSH */
6843 8 + /* FENCE for VM_FLUSH */
6844 20 + /* GDS switch */
6845 4 + /* double SWITCH_BUFFER,
6846 the first COND_EXEC jump to the place just
6847 prior to this double SWITCH_BUFFER */
6848 5 + /* COND_EXEC */
6849 7 + /* HDP_flush */
6850 4 + /* VGT_flush */
6851 14 + /* CE_META */
6852 31 + /* DE_META */
6853 3 + /* CNTX_CTRL */
6854 5 + /* HDP_INVL */
6855 8 + 8 + /* FENCE x2 */
6856 2 + /* SWITCH_BUFFER */
6857 7, /* gfx_v9_0_emit_mem_sync */
6858 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */
6859 .emit_ib = gfx_v9_0_ring_emit_ib_gfx,
6860 .emit_fence = gfx_v9_0_ring_emit_fence,
6861 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6862 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6863 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6864 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6865 .test_ring = gfx_v9_0_ring_test_ring,
6866 .test_ib = gfx_v9_0_ring_test_ib,
6867 .insert_nop = amdgpu_ring_insert_nop,
6868 .pad_ib = amdgpu_ring_generic_pad_ib,
6869 .emit_switch_buffer = gfx_v9_ring_emit_sb,
6870 .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
6871 .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
6872 .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec,
6873 .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl,
6874 .emit_wreg = gfx_v9_0_ring_emit_wreg,
6875 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6876 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6877 .soft_recovery = gfx_v9_0_ring_soft_recovery,
6878 .emit_mem_sync = gfx_v9_0_emit_mem_sync,
6879 };
6880
6881 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {
6882 .type = AMDGPU_RING_TYPE_COMPUTE,
6883 .align_mask = 0xff,
6884 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6885 .support_64bit_ptrs = true,
6886 .vmhub = AMDGPU_GFXHUB_0,
6887 .get_rptr = gfx_v9_0_ring_get_rptr_compute,
6888 .get_wptr = gfx_v9_0_ring_get_wptr_compute,
6889 .set_wptr = gfx_v9_0_ring_set_wptr_compute,
6890 .emit_frame_size =
6891 20 + /* gfx_v9_0_ring_emit_gds_switch */
6892 7 + /* gfx_v9_0_ring_emit_hdp_flush */
6893 5 + /* hdp invalidate */
6894 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
6895 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6896 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6897 2 + /* gfx_v9_0_ring_emit_vm_flush */
6898 8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */
6899 7 + /* gfx_v9_0_emit_mem_sync */
6900 5 + /* gfx_v9_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */
6901 15, /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */
6902 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
6903 .emit_ib = gfx_v9_0_ring_emit_ib_compute,
6904 .emit_fence = gfx_v9_0_ring_emit_fence,
6905 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6906 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6907 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6908 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6909 .test_ring = gfx_v9_0_ring_test_ring,
6910 .test_ib = gfx_v9_0_ring_test_ib,
6911 .insert_nop = amdgpu_ring_insert_nop,
6912 .pad_ib = amdgpu_ring_generic_pad_ib,
6913 .emit_wreg = gfx_v9_0_ring_emit_wreg,
6914 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6915 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6916 .emit_mem_sync = gfx_v9_0_emit_mem_sync,
6917 .emit_wave_limit = gfx_v9_0_emit_wave_limit,
6918 };
6919
6920 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {
6921 .type = AMDGPU_RING_TYPE_KIQ,
6922 .align_mask = 0xff,
6923 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6924 .support_64bit_ptrs = true,
6925 .vmhub = AMDGPU_GFXHUB_0,
6926 .get_rptr = gfx_v9_0_ring_get_rptr_compute,
6927 .get_wptr = gfx_v9_0_ring_get_wptr_compute,
6928 .set_wptr = gfx_v9_0_ring_set_wptr_compute,
6929 .emit_frame_size =
6930 20 + /* gfx_v9_0_ring_emit_gds_switch */
6931 7 + /* gfx_v9_0_ring_emit_hdp_flush */
6932 5 + /* hdp invalidate */
6933 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
6934 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6935 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6936 2 + /* gfx_v9_0_ring_emit_vm_flush */
6937 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */
6938 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
6939 .emit_fence = gfx_v9_0_ring_emit_fence_kiq,
6940 .test_ring = gfx_v9_0_ring_test_ring,
6941 .insert_nop = amdgpu_ring_insert_nop,
6942 .pad_ib = amdgpu_ring_generic_pad_ib,
6943 .emit_rreg = gfx_v9_0_ring_emit_rreg,
6944 .emit_wreg = gfx_v9_0_ring_emit_wreg,
6945 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6946 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6947 };
6948
gfx_v9_0_set_ring_funcs(struct amdgpu_device * adev)6949 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev)
6950 {
6951 int i;
6952
6953 adev->gfx.kiq.ring.funcs = &gfx_v9_0_ring_funcs_kiq;
6954
6955 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
6956 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx;
6957
6958 for (i = 0; i < adev->gfx.num_compute_rings; i++)
6959 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute;
6960 }
6961
6962 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = {
6963 .set = gfx_v9_0_set_eop_interrupt_state,
6964 .process = gfx_v9_0_eop_irq,
6965 };
6966
6967 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = {
6968 .set = gfx_v9_0_set_priv_reg_fault_state,
6969 .process = gfx_v9_0_priv_reg_irq,
6970 };
6971
6972 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = {
6973 .set = gfx_v9_0_set_priv_inst_fault_state,
6974 .process = gfx_v9_0_priv_inst_irq,
6975 };
6976
6977 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = {
6978 .set = gfx_v9_0_set_cp_ecc_error_state,
6979 .process = amdgpu_gfx_cp_ecc_error_irq,
6980 };
6981
6982
gfx_v9_0_set_irq_funcs(struct amdgpu_device * adev)6983 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev)
6984 {
6985 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
6986 adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs;
6987
6988 adev->gfx.priv_reg_irq.num_types = 1;
6989 adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs;
6990
6991 adev->gfx.priv_inst_irq.num_types = 1;
6992 adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs;
6993
6994 adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/
6995 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs;
6996 }
6997
gfx_v9_0_set_rlc_funcs(struct amdgpu_device * adev)6998 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev)
6999 {
7000 switch (adev->asic_type) {
7001 case CHIP_VEGA10:
7002 case CHIP_VEGA12:
7003 case CHIP_VEGA20:
7004 case CHIP_RAVEN:
7005 case CHIP_ARCTURUS:
7006 case CHIP_RENOIR:
7007 case CHIP_ALDEBARAN:
7008 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs;
7009 break;
7010 default:
7011 break;
7012 }
7013 }
7014
gfx_v9_0_set_gds_init(struct amdgpu_device * adev)7015 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev)
7016 {
7017 /* init asci gds info */
7018 switch (adev->asic_type) {
7019 case CHIP_VEGA10:
7020 case CHIP_VEGA12:
7021 case CHIP_VEGA20:
7022 adev->gds.gds_size = 0x10000;
7023 break;
7024 case CHIP_RAVEN:
7025 case CHIP_ARCTURUS:
7026 adev->gds.gds_size = 0x1000;
7027 break;
7028 case CHIP_ALDEBARAN:
7029 /* aldebaran removed all the GDS internal memory,
7030 * only support GWS opcode in kernel, like barrier
7031 * semaphore.etc */
7032 adev->gds.gds_size = 0;
7033 break;
7034 default:
7035 adev->gds.gds_size = 0x10000;
7036 break;
7037 }
7038
7039 switch (adev->asic_type) {
7040 case CHIP_VEGA10:
7041 case CHIP_VEGA20:
7042 adev->gds.gds_compute_max_wave_id = 0x7ff;
7043 break;
7044 case CHIP_VEGA12:
7045 adev->gds.gds_compute_max_wave_id = 0x27f;
7046 break;
7047 case CHIP_RAVEN:
7048 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
7049 adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */
7050 else
7051 adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */
7052 break;
7053 case CHIP_ARCTURUS:
7054 adev->gds.gds_compute_max_wave_id = 0xfff;
7055 break;
7056 case CHIP_ALDEBARAN:
7057 /* deprecated for Aldebaran, no usage at all */
7058 adev->gds.gds_compute_max_wave_id = 0;
7059 break;
7060 default:
7061 /* this really depends on the chip */
7062 adev->gds.gds_compute_max_wave_id = 0x7ff;
7063 break;
7064 }
7065
7066 adev->gds.gws_size = 64;
7067 adev->gds.oa_size = 16;
7068 }
7069
gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device * adev,u32 bitmap)7070 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
7071 u32 bitmap)
7072 {
7073 u32 data;
7074
7075 if (!bitmap)
7076 return;
7077
7078 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7079 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7080
7081 WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data);
7082 }
7083
gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device * adev)7084 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev)
7085 {
7086 u32 data, mask;
7087
7088 data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG);
7089 data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG);
7090
7091 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7092 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7093
7094 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
7095
7096 return (~data) & mask;
7097 }
7098
gfx_v9_0_get_cu_info(struct amdgpu_device * adev,struct amdgpu_cu_info * cu_info)7099 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
7100 struct amdgpu_cu_info *cu_info)
7101 {
7102 int i, j, k, counter, active_cu_number = 0;
7103 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
7104 unsigned disable_masks[4 * 4];
7105
7106 if (!adev || !cu_info)
7107 return -EINVAL;
7108
7109 /*
7110 * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs
7111 */
7112 if (adev->gfx.config.max_shader_engines *
7113 adev->gfx.config.max_sh_per_se > 16)
7114 return -EINVAL;
7115
7116 amdgpu_gfx_parse_disable_cu(disable_masks,
7117 adev->gfx.config.max_shader_engines,
7118 adev->gfx.config.max_sh_per_se);
7119
7120 mutex_lock(&adev->grbm_idx_mutex);
7121 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
7122 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
7123 mask = 1;
7124 ao_bitmap = 0;
7125 counter = 0;
7126 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
7127 gfx_v9_0_set_user_cu_inactive_bitmap(
7128 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]);
7129 bitmap = gfx_v9_0_get_cu_active_bitmap(adev);
7130
7131 /*
7132 * The bitmap(and ao_cu_bitmap) in cu_info structure is
7133 * 4x4 size array, and it's usually suitable for Vega
7134 * ASICs which has 4*2 SE/SH layout.
7135 * But for Arcturus, SE/SH layout is changed to 8*1.
7136 * To mostly reduce the impact, we make it compatible
7137 * with current bitmap array as below:
7138 * SE4,SH0 --> bitmap[0][1]
7139 * SE5,SH0 --> bitmap[1][1]
7140 * SE6,SH0 --> bitmap[2][1]
7141 * SE7,SH0 --> bitmap[3][1]
7142 */
7143 cu_info->bitmap[i % 4][j + i / 4] = bitmap;
7144
7145 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
7146 if (bitmap & mask) {
7147 if (counter < adev->gfx.config.max_cu_per_sh)
7148 ao_bitmap |= mask;
7149 counter ++;
7150 }
7151 mask <<= 1;
7152 }
7153 active_cu_number += counter;
7154 if (i < 2 && j < 2)
7155 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
7156 cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap;
7157 }
7158 }
7159 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
7160 mutex_unlock(&adev->grbm_idx_mutex);
7161
7162 cu_info->number = active_cu_number;
7163 cu_info->ao_cu_mask = ao_cu_mask;
7164 cu_info->simd_per_cu = NUM_SIMD_PER_CU;
7165
7166 return 0;
7167 }
7168
7169 const struct amdgpu_ip_block_version gfx_v9_0_ip_block =
7170 {
7171 .type = AMD_IP_BLOCK_TYPE_GFX,
7172 .major = 9,
7173 .minor = 0,
7174 .rev = 0,
7175 .funcs = &gfx_v9_0_ip_funcs,
7176 };
7177