1 /* 2 * Copyright 2016 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/delay.h> 25 #include <linux/kernel.h> 26 #include <linux/firmware.h> 27 #include <linux/module.h> 28 #include <linux/pci.h> 29 30 #include "amdgpu.h" 31 #include "amdgpu_gfx.h" 32 #include "soc15.h" 33 #include "soc15d.h" 34 #include "amdgpu_atomfirmware.h" 35 #include "amdgpu_pm.h" 36 37 #include "gc/gc_9_0_offset.h" 38 #include "gc/gc_9_0_sh_mask.h" 39 40 #include "vega10_enum.h" 41 42 #include "soc15_common.h" 43 #include "clearstate_gfx9.h" 44 #include "v9_structs.h" 45 46 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h" 47 48 #include "amdgpu_ras.h" 49 50 #include "amdgpu_ring_mux.h" 51 #include "gfx_v9_4.h" 52 #include "gfx_v9_0.h" 53 #include "gfx_v9_4_2.h" 54 55 #include "asic_reg/pwr/pwr_10_0_offset.h" 56 #include "asic_reg/pwr/pwr_10_0_sh_mask.h" 57 #include "asic_reg/gc/gc_9_0_default.h" 58 59 #define GFX9_NUM_GFX_RINGS 1 60 #define GFX9_NUM_SW_GFX_RINGS 2 61 #define GFX9_MEC_HPD_SIZE 4096 62 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L 63 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L 64 65 #define mmGCEA_PROBE_MAP 0x070c 66 #define mmGCEA_PROBE_MAP_BASE_IDX 0 67 68 MODULE_FIRMWARE("amdgpu/vega10_ce.bin"); 69 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin"); 70 MODULE_FIRMWARE("amdgpu/vega10_me.bin"); 71 MODULE_FIRMWARE("amdgpu/vega10_mec.bin"); 72 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin"); 73 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin"); 74 75 MODULE_FIRMWARE("amdgpu/vega12_ce.bin"); 76 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin"); 77 MODULE_FIRMWARE("amdgpu/vega12_me.bin"); 78 MODULE_FIRMWARE("amdgpu/vega12_mec.bin"); 79 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin"); 80 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin"); 81 82 MODULE_FIRMWARE("amdgpu/vega20_ce.bin"); 83 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin"); 84 MODULE_FIRMWARE("amdgpu/vega20_me.bin"); 85 MODULE_FIRMWARE("amdgpu/vega20_mec.bin"); 86 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin"); 87 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin"); 88 89 MODULE_FIRMWARE("amdgpu/raven_ce.bin"); 90 MODULE_FIRMWARE("amdgpu/raven_pfp.bin"); 91 MODULE_FIRMWARE("amdgpu/raven_me.bin"); 92 MODULE_FIRMWARE("amdgpu/raven_mec.bin"); 93 MODULE_FIRMWARE("amdgpu/raven_mec2.bin"); 94 MODULE_FIRMWARE("amdgpu/raven_rlc.bin"); 95 96 MODULE_FIRMWARE("amdgpu/picasso_ce.bin"); 97 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin"); 98 MODULE_FIRMWARE("amdgpu/picasso_me.bin"); 99 MODULE_FIRMWARE("amdgpu/picasso_mec.bin"); 100 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin"); 101 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin"); 102 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin"); 103 104 MODULE_FIRMWARE("amdgpu/raven2_ce.bin"); 105 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin"); 106 MODULE_FIRMWARE("amdgpu/raven2_me.bin"); 107 MODULE_FIRMWARE("amdgpu/raven2_mec.bin"); 108 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin"); 109 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin"); 110 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin"); 111 112 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin"); 113 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin"); 114 115 MODULE_FIRMWARE("amdgpu/renoir_ce.bin"); 116 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin"); 117 MODULE_FIRMWARE("amdgpu/renoir_me.bin"); 118 MODULE_FIRMWARE("amdgpu/renoir_mec.bin"); 119 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin"); 120 121 MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin"); 122 MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin"); 123 MODULE_FIRMWARE("amdgpu/green_sardine_me.bin"); 124 MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin"); 125 MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin"); 126 MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin"); 127 128 MODULE_FIRMWARE("amdgpu/aldebaran_mec.bin"); 129 MODULE_FIRMWARE("amdgpu/aldebaran_mec2.bin"); 130 MODULE_FIRMWARE("amdgpu/aldebaran_rlc.bin"); 131 MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec.bin"); 132 MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec2.bin"); 133 134 #define mmTCP_CHAN_STEER_0_ARCT 0x0b03 135 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX 0 136 #define mmTCP_CHAN_STEER_1_ARCT 0x0b04 137 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX 0 138 #define mmTCP_CHAN_STEER_2_ARCT 0x0b09 139 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX 0 140 #define mmTCP_CHAN_STEER_3_ARCT 0x0b0a 141 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX 0 142 #define mmTCP_CHAN_STEER_4_ARCT 0x0b0b 143 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX 0 144 #define mmTCP_CHAN_STEER_5_ARCT 0x0b0c 145 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX 0 146 147 #define mmGOLDEN_TSC_COUNT_UPPER_Renoir 0x0025 148 #define mmGOLDEN_TSC_COUNT_UPPER_Renoir_BASE_IDX 1 149 #define mmGOLDEN_TSC_COUNT_LOWER_Renoir 0x0026 150 #define mmGOLDEN_TSC_COUNT_LOWER_Renoir_BASE_IDX 1 151 152 enum ta_ras_gfx_subblock { 153 /*CPC*/ 154 TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0, 155 TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START, 156 TA_RAS_BLOCK__GFX_CPC_UCODE, 157 TA_RAS_BLOCK__GFX_DC_STATE_ME1, 158 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1, 159 TA_RAS_BLOCK__GFX_DC_RESTORE_ME1, 160 TA_RAS_BLOCK__GFX_DC_STATE_ME2, 161 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2, 162 TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 163 TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 164 /* CPF*/ 165 TA_RAS_BLOCK__GFX_CPF_INDEX_START, 166 TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START, 167 TA_RAS_BLOCK__GFX_CPF_ROQ_ME1, 168 TA_RAS_BLOCK__GFX_CPF_TAG, 169 TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG, 170 /* CPG*/ 171 TA_RAS_BLOCK__GFX_CPG_INDEX_START, 172 TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START, 173 TA_RAS_BLOCK__GFX_CPG_DMA_TAG, 174 TA_RAS_BLOCK__GFX_CPG_TAG, 175 TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG, 176 /* GDS*/ 177 TA_RAS_BLOCK__GFX_GDS_INDEX_START, 178 TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START, 179 TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE, 180 TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM, 181 TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM, 182 TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 183 TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 184 /* SPI*/ 185 TA_RAS_BLOCK__GFX_SPI_SR_MEM, 186 /* SQ*/ 187 TA_RAS_BLOCK__GFX_SQ_INDEX_START, 188 TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START, 189 TA_RAS_BLOCK__GFX_SQ_LDS_D, 190 TA_RAS_BLOCK__GFX_SQ_LDS_I, 191 TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/ 192 TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR, 193 /* SQC (3 ranges)*/ 194 TA_RAS_BLOCK__GFX_SQC_INDEX_START, 195 /* SQC range 0*/ 196 TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START, 197 TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO = 198 TA_RAS_BLOCK__GFX_SQC_INDEX0_START, 199 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 200 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO, 201 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 202 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO, 203 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 204 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 205 TA_RAS_BLOCK__GFX_SQC_INDEX0_END = 206 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 207 /* SQC range 1*/ 208 TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 209 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM = 210 TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 211 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 212 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO, 213 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM, 214 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM, 215 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO, 216 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO, 217 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 218 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 219 TA_RAS_BLOCK__GFX_SQC_INDEX1_END = 220 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 221 /* SQC range 2*/ 222 TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 223 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM = 224 TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 225 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 226 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO, 227 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM, 228 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM, 229 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO, 230 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO, 231 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 232 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 233 TA_RAS_BLOCK__GFX_SQC_INDEX2_END = 234 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 235 TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END, 236 /* TA*/ 237 TA_RAS_BLOCK__GFX_TA_INDEX_START, 238 TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START, 239 TA_RAS_BLOCK__GFX_TA_FS_AFIFO, 240 TA_RAS_BLOCK__GFX_TA_FL_LFIFO, 241 TA_RAS_BLOCK__GFX_TA_FX_LFIFO, 242 TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 243 TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 244 /* TCA*/ 245 TA_RAS_BLOCK__GFX_TCA_INDEX_START, 246 TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START, 247 TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 248 TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 249 /* TCC (5 sub-ranges)*/ 250 TA_RAS_BLOCK__GFX_TCC_INDEX_START, 251 /* TCC range 0*/ 252 TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START, 253 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START, 254 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1, 255 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0, 256 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1, 257 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0, 258 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1, 259 TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG, 260 TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 261 TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 262 /* TCC range 1*/ 263 TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 264 TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 265 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 266 TA_RAS_BLOCK__GFX_TCC_INDEX1_END = 267 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 268 /* TCC range 2*/ 269 TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 270 TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 271 TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL, 272 TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO, 273 TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN, 274 TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ, 275 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO, 276 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM, 277 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 278 TA_RAS_BLOCK__GFX_TCC_INDEX2_END = 279 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 280 /* TCC range 3*/ 281 TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 282 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 283 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 284 TA_RAS_BLOCK__GFX_TCC_INDEX3_END = 285 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 286 /* TCC range 4*/ 287 TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 288 TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN = 289 TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 290 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 291 TA_RAS_BLOCK__GFX_TCC_INDEX4_END = 292 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 293 TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END, 294 /* TCI*/ 295 TA_RAS_BLOCK__GFX_TCI_WRITE_RAM, 296 /* TCP*/ 297 TA_RAS_BLOCK__GFX_TCP_INDEX_START, 298 TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START, 299 TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM, 300 TA_RAS_BLOCK__GFX_TCP_CMD_FIFO, 301 TA_RAS_BLOCK__GFX_TCP_VM_FIFO, 302 TA_RAS_BLOCK__GFX_TCP_DB_RAM, 303 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0, 304 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 305 TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 306 /* TD*/ 307 TA_RAS_BLOCK__GFX_TD_INDEX_START, 308 TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START, 309 TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI, 310 TA_RAS_BLOCK__GFX_TD_CS_FIFO, 311 TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO, 312 /* EA (3 sub-ranges)*/ 313 TA_RAS_BLOCK__GFX_EA_INDEX_START, 314 /* EA range 0*/ 315 TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START, 316 TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START, 317 TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM, 318 TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM, 319 TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM, 320 TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM, 321 TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM, 322 TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM, 323 TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 324 TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 325 /* EA range 1*/ 326 TA_RAS_BLOCK__GFX_EA_INDEX1_START, 327 TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START, 328 TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM, 329 TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM, 330 TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM, 331 TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM, 332 TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM, 333 TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 334 TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 335 /* EA range 2*/ 336 TA_RAS_BLOCK__GFX_EA_INDEX2_START, 337 TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START, 338 TA_RAS_BLOCK__GFX_EA_MAM_D1MEM, 339 TA_RAS_BLOCK__GFX_EA_MAM_D2MEM, 340 TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 341 TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 342 TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END, 343 /* UTC VM L2 bank*/ 344 TA_RAS_BLOCK__UTC_VML2_BANK_CACHE, 345 /* UTC VM walker*/ 346 TA_RAS_BLOCK__UTC_VML2_WALKER, 347 /* UTC ATC L2 2MB cache*/ 348 TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK, 349 /* UTC ATC L2 4KB cache*/ 350 TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK, 351 TA_RAS_BLOCK__GFX_MAX 352 }; 353 354 struct ras_gfx_subblock { 355 unsigned char *name; 356 int ta_subblock; 357 int hw_supported_error_type; 358 int sw_supported_error_type; 359 }; 360 361 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h) \ 362 [AMDGPU_RAS_BLOCK__##subblock] = { \ 363 #subblock, \ 364 TA_RAS_BLOCK__##subblock, \ 365 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)), \ 366 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)), \ 367 } 368 369 static const struct ras_gfx_subblock ras_gfx_subblocks[] = { 370 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1), 371 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1), 372 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 373 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 374 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 375 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 376 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 377 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 378 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 379 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 380 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1), 381 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0), 382 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1), 383 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1), 384 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 385 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0), 386 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0, 387 0), 388 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0, 389 0), 390 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 391 AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0), 392 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0), 393 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1), 394 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0), 395 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0), 396 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1), 397 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 398 0, 0), 399 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 400 0), 401 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 402 0, 0), 403 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0, 404 0), 405 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 406 0, 0), 407 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 408 0), 409 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 410 1), 411 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 412 0, 0, 0), 413 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 414 0), 415 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 416 0), 417 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 418 0), 419 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 420 0), 421 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 422 0), 423 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 424 0, 0), 425 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 426 0), 427 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 428 0), 429 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 430 0, 0, 0), 431 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 432 0), 433 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 434 0), 435 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 436 0), 437 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 438 0), 439 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 440 0), 441 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 442 0, 0), 443 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 444 0), 445 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1), 446 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 447 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 448 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 449 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 450 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0), 451 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 452 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1), 453 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0, 454 1), 455 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0, 456 1), 457 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0, 458 1), 459 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0, 460 0), 461 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0, 462 0), 463 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 464 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 465 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0), 466 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0), 467 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0), 468 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0), 469 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 470 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0), 471 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0), 472 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 473 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0), 474 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0, 475 0), 476 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 477 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0, 478 0), 479 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0, 480 0, 0), 481 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0, 482 0), 483 AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 484 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1), 485 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0), 486 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 487 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 488 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 489 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0), 490 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0), 491 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1), 492 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0), 493 AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 494 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1), 495 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 496 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 497 AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 498 AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 499 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 500 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 501 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 502 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 503 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 504 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 505 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 506 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0), 507 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 508 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 509 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0), 510 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0), 511 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0), 512 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0), 513 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0), 514 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0), 515 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0), 516 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0), 517 }; 518 519 static const struct soc15_reg_golden golden_settings_gc_9_0[] = 520 { 521 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 522 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000), 523 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 524 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 525 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 526 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 527 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 528 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 529 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 530 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87), 531 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f), 532 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 533 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 534 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 535 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 536 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 537 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff), 538 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 539 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 540 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 541 }; 542 543 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] = 544 { 545 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107), 546 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 547 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 548 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 549 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 550 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 551 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042), 552 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 553 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000), 554 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 555 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 556 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 557 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 558 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 559 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 560 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107), 561 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800), 562 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080) 563 }; 564 565 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] = 566 { 567 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080), 568 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 569 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 570 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042), 571 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042), 572 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400), 573 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000), 574 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000), 575 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107), 576 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000), 577 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000) 578 }; 579 580 static const struct soc15_reg_golden golden_settings_gc_9_1[] = 581 { 582 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 583 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 584 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 585 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 586 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 587 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 588 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 589 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 590 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 591 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 592 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 593 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 594 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 595 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 596 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 597 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 598 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 599 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 600 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 601 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff), 602 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080), 603 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 604 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 605 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 606 }; 607 608 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] = 609 { 610 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 611 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042), 612 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042), 613 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000), 614 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000), 615 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 616 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800) 617 }; 618 619 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] = 620 { 621 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000), 622 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 623 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 624 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080), 625 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080), 626 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080), 627 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041), 628 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041), 629 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 630 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 631 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080), 632 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080), 633 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080), 634 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080), 635 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080), 636 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 637 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010), 638 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 639 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 640 }; 641 642 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] = 643 { 644 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 645 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 646 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 647 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042), 648 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042), 649 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 650 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 651 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 652 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 653 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 654 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 655 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc), 656 }; 657 658 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] = 659 { 660 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff), 661 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000), 662 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382) 663 }; 664 665 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] = 666 { 667 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 668 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 669 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 670 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 671 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 672 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 673 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 674 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 675 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87), 676 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f), 677 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 678 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 679 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 680 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 681 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 682 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff) 683 }; 684 685 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] = 686 { 687 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080), 688 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 689 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 690 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041), 691 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041), 692 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 693 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107), 694 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 695 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410), 696 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 697 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 698 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 699 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 700 }; 701 702 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] = 703 { 704 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 705 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000), 706 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e), 707 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca), 708 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098), 709 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3), 710 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1), 711 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135), 712 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000), 713 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00), 714 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000) 715 }; 716 717 static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = { 718 {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)}, 719 {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)}, 720 }; 721 722 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] = 723 { 724 mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 725 mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 726 mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 727 mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 728 mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 729 mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 730 mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 731 mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 732 }; 733 734 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] = 735 { 736 mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0, 737 mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0, 738 mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0, 739 mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0, 740 mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0, 741 mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0, 742 mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0, 743 mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0, 744 }; 745 746 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042 747 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041 748 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042 749 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041 750 751 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev); 752 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev); 753 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev); 754 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev); 755 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 756 struct amdgpu_cu_info *cu_info); 757 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev); 758 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume, bool usegds); 759 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring); 760 static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 761 void *ras_error_status); 762 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 763 void *inject_if, uint32_t instance_mask); 764 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev); 765 static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev, 766 unsigned int vmid); 767 768 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring, 769 uint64_t queue_mask) 770 { 771 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6)); 772 amdgpu_ring_write(kiq_ring, 773 PACKET3_SET_RESOURCES_VMID_MASK(0) | 774 /* vmid_mask:0* queue_type:0 (KIQ) */ 775 PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); 776 amdgpu_ring_write(kiq_ring, 777 lower_32_bits(queue_mask)); /* queue mask lo */ 778 amdgpu_ring_write(kiq_ring, 779 upper_32_bits(queue_mask)); /* queue mask hi */ 780 amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */ 781 amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */ 782 amdgpu_ring_write(kiq_ring, 0); /* oac mask */ 783 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */ 784 } 785 786 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring, 787 struct amdgpu_ring *ring) 788 { 789 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); 790 uint64_t wptr_addr = ring->wptr_gpu_addr; 791 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 792 793 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); 794 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/ 795 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 796 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */ 797 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */ 798 PACKET3_MAP_QUEUES_QUEUE(ring->queue) | 799 PACKET3_MAP_QUEUES_PIPE(ring->pipe) | 800 PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) | 801 /*queue_type: normal compute queue */ 802 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | 803 /* alloc format: all_on_one_pipe */ 804 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | 805 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) | 806 /* num_queues: must be 1 */ 807 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); 808 amdgpu_ring_write(kiq_ring, 809 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index)); 810 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr)); 811 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr)); 812 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr)); 813 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr)); 814 } 815 816 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring, 817 struct amdgpu_ring *ring, 818 enum amdgpu_unmap_queues_action action, 819 u64 gpu_addr, u64 seq) 820 { 821 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 822 823 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4)); 824 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 825 PACKET3_UNMAP_QUEUES_ACTION(action) | 826 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) | 827 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) | 828 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1)); 829 amdgpu_ring_write(kiq_ring, 830 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index)); 831 832 if (action == PREEMPT_QUEUES_NO_UNMAP) { 833 amdgpu_ring_write(kiq_ring, lower_32_bits(ring->wptr & ring->buf_mask)); 834 amdgpu_ring_write(kiq_ring, 0); 835 amdgpu_ring_write(kiq_ring, 0); 836 837 } else { 838 amdgpu_ring_write(kiq_ring, 0); 839 amdgpu_ring_write(kiq_ring, 0); 840 amdgpu_ring_write(kiq_ring, 0); 841 } 842 } 843 844 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring, 845 struct amdgpu_ring *ring, 846 u64 addr, 847 u64 seq) 848 { 849 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 850 851 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5)); 852 amdgpu_ring_write(kiq_ring, 853 PACKET3_QUERY_STATUS_CONTEXT_ID(0) | 854 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) | 855 PACKET3_QUERY_STATUS_COMMAND(2)); 856 /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 857 amdgpu_ring_write(kiq_ring, 858 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) | 859 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel)); 860 amdgpu_ring_write(kiq_ring, lower_32_bits(addr)); 861 amdgpu_ring_write(kiq_ring, upper_32_bits(addr)); 862 amdgpu_ring_write(kiq_ring, lower_32_bits(seq)); 863 amdgpu_ring_write(kiq_ring, upper_32_bits(seq)); 864 } 865 866 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring, 867 uint16_t pasid, uint32_t flush_type, 868 bool all_hub) 869 { 870 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0)); 871 amdgpu_ring_write(kiq_ring, 872 PACKET3_INVALIDATE_TLBS_DST_SEL(1) | 873 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) | 874 PACKET3_INVALIDATE_TLBS_PASID(pasid) | 875 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type)); 876 } 877 878 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = { 879 .kiq_set_resources = gfx_v9_0_kiq_set_resources, 880 .kiq_map_queues = gfx_v9_0_kiq_map_queues, 881 .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues, 882 .kiq_query_status = gfx_v9_0_kiq_query_status, 883 .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs, 884 .set_resources_size = 8, 885 .map_queues_size = 7, 886 .unmap_queues_size = 6, 887 .query_status_size = 7, 888 .invalidate_tlbs_size = 2, 889 }; 890 891 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev) 892 { 893 adev->gfx.kiq[0].pmf = &gfx_v9_0_kiq_pm4_funcs; 894 } 895 896 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev) 897 { 898 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 899 case IP_VERSION(9, 0, 1): 900 soc15_program_register_sequence(adev, 901 golden_settings_gc_9_0, 902 ARRAY_SIZE(golden_settings_gc_9_0)); 903 soc15_program_register_sequence(adev, 904 golden_settings_gc_9_0_vg10, 905 ARRAY_SIZE(golden_settings_gc_9_0_vg10)); 906 break; 907 case IP_VERSION(9, 2, 1): 908 soc15_program_register_sequence(adev, 909 golden_settings_gc_9_2_1, 910 ARRAY_SIZE(golden_settings_gc_9_2_1)); 911 soc15_program_register_sequence(adev, 912 golden_settings_gc_9_2_1_vg12, 913 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12)); 914 break; 915 case IP_VERSION(9, 4, 0): 916 soc15_program_register_sequence(adev, 917 golden_settings_gc_9_0, 918 ARRAY_SIZE(golden_settings_gc_9_0)); 919 soc15_program_register_sequence(adev, 920 golden_settings_gc_9_0_vg20, 921 ARRAY_SIZE(golden_settings_gc_9_0_vg20)); 922 break; 923 case IP_VERSION(9, 4, 1): 924 soc15_program_register_sequence(adev, 925 golden_settings_gc_9_4_1_arct, 926 ARRAY_SIZE(golden_settings_gc_9_4_1_arct)); 927 break; 928 case IP_VERSION(9, 2, 2): 929 case IP_VERSION(9, 1, 0): 930 soc15_program_register_sequence(adev, golden_settings_gc_9_1, 931 ARRAY_SIZE(golden_settings_gc_9_1)); 932 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 933 soc15_program_register_sequence(adev, 934 golden_settings_gc_9_1_rv2, 935 ARRAY_SIZE(golden_settings_gc_9_1_rv2)); 936 else 937 soc15_program_register_sequence(adev, 938 golden_settings_gc_9_1_rv1, 939 ARRAY_SIZE(golden_settings_gc_9_1_rv1)); 940 break; 941 case IP_VERSION(9, 3, 0): 942 soc15_program_register_sequence(adev, 943 golden_settings_gc_9_1_rn, 944 ARRAY_SIZE(golden_settings_gc_9_1_rn)); 945 return; /* for renoir, don't need common goldensetting */ 946 case IP_VERSION(9, 4, 2): 947 gfx_v9_4_2_init_golden_registers(adev, 948 adev->smuio.funcs->get_die_id(adev)); 949 break; 950 default: 951 break; 952 } 953 954 if ((amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) && 955 (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2))) 956 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common, 957 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common)); 958 } 959 960 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel, 961 bool wc, uint32_t reg, uint32_t val) 962 { 963 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 964 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) | 965 WRITE_DATA_DST_SEL(0) | 966 (wc ? WR_CONFIRM : 0)); 967 amdgpu_ring_write(ring, reg); 968 amdgpu_ring_write(ring, 0); 969 amdgpu_ring_write(ring, val); 970 } 971 972 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, 973 int mem_space, int opt, uint32_t addr0, 974 uint32_t addr1, uint32_t ref, uint32_t mask, 975 uint32_t inv) 976 { 977 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 978 amdgpu_ring_write(ring, 979 /* memory (1) or register (0) */ 980 (WAIT_REG_MEM_MEM_SPACE(mem_space) | 981 WAIT_REG_MEM_OPERATION(opt) | /* wait */ 982 WAIT_REG_MEM_FUNCTION(3) | /* equal */ 983 WAIT_REG_MEM_ENGINE(eng_sel))); 984 985 if (mem_space) 986 BUG_ON(addr0 & 0x3); /* Dword align */ 987 amdgpu_ring_write(ring, addr0); 988 amdgpu_ring_write(ring, addr1); 989 amdgpu_ring_write(ring, ref); 990 amdgpu_ring_write(ring, mask); 991 amdgpu_ring_write(ring, inv); /* poll interval */ 992 } 993 994 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring) 995 { 996 struct amdgpu_device *adev = ring->adev; 997 uint32_t scratch = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0); 998 uint32_t tmp = 0; 999 unsigned i; 1000 int r; 1001 1002 WREG32(scratch, 0xCAFEDEAD); 1003 r = amdgpu_ring_alloc(ring, 3); 1004 if (r) 1005 return r; 1006 1007 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 1008 amdgpu_ring_write(ring, scratch - PACKET3_SET_UCONFIG_REG_START); 1009 amdgpu_ring_write(ring, 0xDEADBEEF); 1010 amdgpu_ring_commit(ring); 1011 1012 for (i = 0; i < adev->usec_timeout; i++) { 1013 tmp = RREG32(scratch); 1014 if (tmp == 0xDEADBEEF) 1015 break; 1016 udelay(1); 1017 } 1018 1019 if (i >= adev->usec_timeout) 1020 r = -ETIMEDOUT; 1021 return r; 1022 } 1023 1024 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) 1025 { 1026 struct amdgpu_device *adev = ring->adev; 1027 struct amdgpu_ib ib; 1028 struct dma_fence *f = NULL; 1029 1030 unsigned index; 1031 uint64_t gpu_addr; 1032 uint32_t tmp; 1033 long r; 1034 1035 r = amdgpu_device_wb_get(adev, &index); 1036 if (r) 1037 return r; 1038 1039 gpu_addr = adev->wb.gpu_addr + (index * 4); 1040 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD); 1041 memset(&ib, 0, sizeof(ib)); 1042 1043 r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib); 1044 if (r) 1045 goto err1; 1046 1047 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3); 1048 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM; 1049 ib.ptr[2] = lower_32_bits(gpu_addr); 1050 ib.ptr[3] = upper_32_bits(gpu_addr); 1051 ib.ptr[4] = 0xDEADBEEF; 1052 ib.length_dw = 5; 1053 1054 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 1055 if (r) 1056 goto err2; 1057 1058 r = dma_fence_wait_timeout(f, false, timeout); 1059 if (r == 0) { 1060 r = -ETIMEDOUT; 1061 goto err2; 1062 } else if (r < 0) { 1063 goto err2; 1064 } 1065 1066 tmp = adev->wb.wb[index]; 1067 if (tmp == 0xDEADBEEF) 1068 r = 0; 1069 else 1070 r = -EINVAL; 1071 1072 err2: 1073 amdgpu_ib_free(adev, &ib, NULL); 1074 dma_fence_put(f); 1075 err1: 1076 amdgpu_device_wb_free(adev, index); 1077 return r; 1078 } 1079 1080 1081 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev) 1082 { 1083 amdgpu_ucode_release(&adev->gfx.pfp_fw); 1084 amdgpu_ucode_release(&adev->gfx.me_fw); 1085 amdgpu_ucode_release(&adev->gfx.ce_fw); 1086 amdgpu_ucode_release(&adev->gfx.rlc_fw); 1087 amdgpu_ucode_release(&adev->gfx.mec_fw); 1088 amdgpu_ucode_release(&adev->gfx.mec2_fw); 1089 1090 kfree(adev->gfx.rlc.register_list_format); 1091 } 1092 1093 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev) 1094 { 1095 adev->gfx.me_fw_write_wait = false; 1096 adev->gfx.mec_fw_write_wait = false; 1097 1098 if ((amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) && 1099 ((adev->gfx.mec_fw_version < 0x000001a5) || 1100 (adev->gfx.mec_feature_version < 46) || 1101 (adev->gfx.pfp_fw_version < 0x000000b7) || 1102 (adev->gfx.pfp_feature_version < 46))) 1103 DRM_WARN_ONCE("CP firmware version too old, please update!"); 1104 1105 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1106 case IP_VERSION(9, 0, 1): 1107 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1108 (adev->gfx.me_feature_version >= 42) && 1109 (adev->gfx.pfp_fw_version >= 0x000000b1) && 1110 (adev->gfx.pfp_feature_version >= 42)) 1111 adev->gfx.me_fw_write_wait = true; 1112 1113 if ((adev->gfx.mec_fw_version >= 0x00000193) && 1114 (adev->gfx.mec_feature_version >= 42)) 1115 adev->gfx.mec_fw_write_wait = true; 1116 break; 1117 case IP_VERSION(9, 2, 1): 1118 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1119 (adev->gfx.me_feature_version >= 44) && 1120 (adev->gfx.pfp_fw_version >= 0x000000b2) && 1121 (adev->gfx.pfp_feature_version >= 44)) 1122 adev->gfx.me_fw_write_wait = true; 1123 1124 if ((adev->gfx.mec_fw_version >= 0x00000196) && 1125 (adev->gfx.mec_feature_version >= 44)) 1126 adev->gfx.mec_fw_write_wait = true; 1127 break; 1128 case IP_VERSION(9, 4, 0): 1129 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1130 (adev->gfx.me_feature_version >= 44) && 1131 (adev->gfx.pfp_fw_version >= 0x000000b2) && 1132 (adev->gfx.pfp_feature_version >= 44)) 1133 adev->gfx.me_fw_write_wait = true; 1134 1135 if ((adev->gfx.mec_fw_version >= 0x00000197) && 1136 (adev->gfx.mec_feature_version >= 44)) 1137 adev->gfx.mec_fw_write_wait = true; 1138 break; 1139 case IP_VERSION(9, 1, 0): 1140 case IP_VERSION(9, 2, 2): 1141 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1142 (adev->gfx.me_feature_version >= 42) && 1143 (adev->gfx.pfp_fw_version >= 0x000000b1) && 1144 (adev->gfx.pfp_feature_version >= 42)) 1145 adev->gfx.me_fw_write_wait = true; 1146 1147 if ((adev->gfx.mec_fw_version >= 0x00000192) && 1148 (adev->gfx.mec_feature_version >= 42)) 1149 adev->gfx.mec_fw_write_wait = true; 1150 break; 1151 default: 1152 adev->gfx.me_fw_write_wait = true; 1153 adev->gfx.mec_fw_write_wait = true; 1154 break; 1155 } 1156 } 1157 1158 struct amdgpu_gfxoff_quirk { 1159 u16 chip_vendor; 1160 u16 chip_device; 1161 u16 subsys_vendor; 1162 u16 subsys_device; 1163 u8 revision; 1164 }; 1165 1166 static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = { 1167 /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */ 1168 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 }, 1169 /* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */ 1170 { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 }, 1171 /* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */ 1172 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 }, 1173 /* Apple MacBook Pro (15-inch, 2019) Radeon Pro Vega 20 4 GB */ 1174 { 0x1002, 0x69af, 0x106b, 0x019a, 0xc0 }, 1175 { 0, 0, 0, 0, 0 }, 1176 }; 1177 1178 static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev) 1179 { 1180 const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list; 1181 1182 while (p && p->chip_device != 0) { 1183 if (pdev->vendor == p->chip_vendor && 1184 pdev->device == p->chip_device && 1185 pdev->subsystem_vendor == p->subsys_vendor && 1186 pdev->subsystem_device == p->subsys_device && 1187 pdev->revision == p->revision) { 1188 return true; 1189 } 1190 ++p; 1191 } 1192 return false; 1193 } 1194 1195 static bool is_raven_kicker(struct amdgpu_device *adev) 1196 { 1197 if (adev->pm.fw_version >= 0x41e2b) 1198 return true; 1199 else 1200 return false; 1201 } 1202 1203 static bool check_if_enlarge_doorbell_range(struct amdgpu_device *adev) 1204 { 1205 if ((amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 3, 0)) && 1206 (adev->gfx.me_fw_version >= 0x000000a5) && 1207 (adev->gfx.me_feature_version >= 52)) 1208 return true; 1209 else 1210 return false; 1211 } 1212 1213 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev) 1214 { 1215 if (gfx_v9_0_should_disable_gfxoff(adev->pdev)) 1216 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1217 1218 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1219 case IP_VERSION(9, 0, 1): 1220 case IP_VERSION(9, 2, 1): 1221 case IP_VERSION(9, 4, 0): 1222 break; 1223 case IP_VERSION(9, 2, 2): 1224 case IP_VERSION(9, 1, 0): 1225 if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) || 1226 (adev->apu_flags & AMD_APU_IS_PICASSO)) && 1227 ((!is_raven_kicker(adev) && 1228 adev->gfx.rlc_fw_version < 531) || 1229 (adev->gfx.rlc_feature_version < 1) || 1230 !adev->gfx.rlc.is_rlc_v2_1)) 1231 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1232 1233 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1234 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1235 AMD_PG_SUPPORT_CP | 1236 AMD_PG_SUPPORT_RLC_SMU_HS; 1237 break; 1238 case IP_VERSION(9, 3, 0): 1239 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1240 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1241 AMD_PG_SUPPORT_CP | 1242 AMD_PG_SUPPORT_RLC_SMU_HS; 1243 break; 1244 default: 1245 break; 1246 } 1247 } 1248 1249 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev, 1250 char *chip_name) 1251 { 1252 char fw_name[30]; 1253 int err; 1254 1255 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name); 1256 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, fw_name); 1257 if (err) 1258 goto out; 1259 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP); 1260 1261 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name); 1262 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, fw_name); 1263 if (err) 1264 goto out; 1265 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME); 1266 1267 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name); 1268 err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw, fw_name); 1269 if (err) 1270 goto out; 1271 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_CE); 1272 1273 out: 1274 if (err) { 1275 amdgpu_ucode_release(&adev->gfx.pfp_fw); 1276 amdgpu_ucode_release(&adev->gfx.me_fw); 1277 amdgpu_ucode_release(&adev->gfx.ce_fw); 1278 } 1279 return err; 1280 } 1281 1282 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev, 1283 char *chip_name) 1284 { 1285 char fw_name[30]; 1286 int err; 1287 const struct rlc_firmware_header_v2_0 *rlc_hdr; 1288 uint16_t version_major; 1289 uint16_t version_minor; 1290 uint32_t smu_version; 1291 1292 /* 1293 * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin 1294 * instead of picasso_rlc.bin. 1295 * Judgment method: 1296 * PCO AM4: revision >= 0xC8 && revision <= 0xCF 1297 * or revision >= 0xD8 && revision <= 0xDF 1298 * otherwise is PCO FP5 1299 */ 1300 if (!strcmp(chip_name, "picasso") && 1301 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) || 1302 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF)))) 1303 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc_am4.bin", chip_name); 1304 else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) && 1305 (smu_version >= 0x41e2b)) 1306 /** 1307 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly. 1308 */ 1309 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_kicker_rlc.bin", chip_name); 1310 else 1311 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name); 1312 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, fw_name); 1313 if (err) 1314 goto out; 1315 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 1316 1317 version_major = le16_to_cpu(rlc_hdr->header.header_version_major); 1318 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); 1319 err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor); 1320 out: 1321 if (err) 1322 amdgpu_ucode_release(&adev->gfx.rlc_fw); 1323 1324 return err; 1325 } 1326 1327 static bool gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device *adev) 1328 { 1329 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) || 1330 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) || 1331 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 3, 0)) 1332 return false; 1333 1334 return true; 1335 } 1336 1337 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev, 1338 char *chip_name) 1339 { 1340 char fw_name[30]; 1341 int err; 1342 1343 if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN)) 1344 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sjt_mec.bin", chip_name); 1345 else 1346 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name); 1347 1348 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, fw_name); 1349 if (err) 1350 goto out; 1351 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1); 1352 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT); 1353 1354 if (gfx_v9_0_load_mec2_fw_bin_support(adev)) { 1355 if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN)) 1356 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sjt_mec2.bin", chip_name); 1357 else 1358 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name); 1359 1360 /* ignore failures to load */ 1361 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw, fw_name); 1362 if (!err) { 1363 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2); 1364 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2_JT); 1365 } else { 1366 err = 0; 1367 amdgpu_ucode_release(&adev->gfx.mec2_fw); 1368 } 1369 } else { 1370 adev->gfx.mec2_fw_version = adev->gfx.mec_fw_version; 1371 adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version; 1372 } 1373 1374 gfx_v9_0_check_if_need_gfxoff(adev); 1375 gfx_v9_0_check_fw_write_wait(adev); 1376 1377 out: 1378 if (err) 1379 amdgpu_ucode_release(&adev->gfx.mec_fw); 1380 return err; 1381 } 1382 1383 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev) 1384 { 1385 char ucode_prefix[30]; 1386 int r; 1387 1388 DRM_DEBUG("\n"); 1389 amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix)); 1390 1391 /* No CPG in Arcturus */ 1392 if (adev->gfx.num_gfx_rings) { 1393 r = gfx_v9_0_init_cp_gfx_microcode(adev, ucode_prefix); 1394 if (r) 1395 return r; 1396 } 1397 1398 r = gfx_v9_0_init_rlc_microcode(adev, ucode_prefix); 1399 if (r) 1400 return r; 1401 1402 r = gfx_v9_0_init_cp_compute_microcode(adev, ucode_prefix); 1403 if (r) 1404 return r; 1405 1406 return r; 1407 } 1408 1409 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev) 1410 { 1411 u32 count = 0; 1412 const struct cs_section_def *sect = NULL; 1413 const struct cs_extent_def *ext = NULL; 1414 1415 /* begin clear state */ 1416 count += 2; 1417 /* context control state */ 1418 count += 3; 1419 1420 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 1421 for (ext = sect->section; ext->extent != NULL; ++ext) { 1422 if (sect->id == SECT_CONTEXT) 1423 count += 2 + ext->reg_count; 1424 else 1425 return 0; 1426 } 1427 } 1428 1429 /* end clear state */ 1430 count += 2; 1431 /* clear state */ 1432 count += 2; 1433 1434 return count; 1435 } 1436 1437 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev, 1438 volatile u32 *buffer) 1439 { 1440 u32 count = 0, i; 1441 const struct cs_section_def *sect = NULL; 1442 const struct cs_extent_def *ext = NULL; 1443 1444 if (adev->gfx.rlc.cs_data == NULL) 1445 return; 1446 if (buffer == NULL) 1447 return; 1448 1449 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1450 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 1451 1452 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 1453 buffer[count++] = cpu_to_le32(0x80000000); 1454 buffer[count++] = cpu_to_le32(0x80000000); 1455 1456 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) { 1457 for (ext = sect->section; ext->extent != NULL; ++ext) { 1458 if (sect->id == SECT_CONTEXT) { 1459 buffer[count++] = 1460 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); 1461 buffer[count++] = cpu_to_le32(ext->reg_index - 1462 PACKET3_SET_CONTEXT_REG_START); 1463 for (i = 0; i < ext->reg_count; i++) 1464 buffer[count++] = cpu_to_le32(ext->extent[i]); 1465 } else { 1466 return; 1467 } 1468 } 1469 } 1470 1471 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1472 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE); 1473 1474 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0)); 1475 buffer[count++] = cpu_to_le32(0); 1476 } 1477 1478 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev) 1479 { 1480 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info; 1481 uint32_t pg_always_on_cu_num = 2; 1482 uint32_t always_on_cu_num; 1483 uint32_t i, j, k; 1484 uint32_t mask, cu_bitmap, counter; 1485 1486 if (adev->flags & AMD_IS_APU) 1487 always_on_cu_num = 4; 1488 else if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 2, 1)) 1489 always_on_cu_num = 8; 1490 else 1491 always_on_cu_num = 12; 1492 1493 mutex_lock(&adev->grbm_idx_mutex); 1494 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 1495 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 1496 mask = 1; 1497 cu_bitmap = 0; 1498 counter = 0; 1499 amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0); 1500 1501 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 1502 if (cu_info->bitmap[0][i][j] & mask) { 1503 if (counter == pg_always_on_cu_num) 1504 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap); 1505 if (counter < always_on_cu_num) 1506 cu_bitmap |= mask; 1507 else 1508 break; 1509 counter++; 1510 } 1511 mask <<= 1; 1512 } 1513 1514 WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap); 1515 cu_info->ao_cu_bitmap[i][j] = cu_bitmap; 1516 } 1517 } 1518 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 1519 mutex_unlock(&adev->grbm_idx_mutex); 1520 } 1521 1522 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev) 1523 { 1524 uint32_t data; 1525 1526 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1527 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1528 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7); 1529 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1530 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16)); 1531 1532 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1533 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1534 1535 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1536 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500); 1537 1538 mutex_lock(&adev->grbm_idx_mutex); 1539 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1540 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 1541 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1542 1543 /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1544 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1545 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1546 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1547 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1548 1549 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1550 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1551 data &= 0x0000FFFF; 1552 data |= 0x00C00000; 1553 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1554 1555 /* 1556 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven), 1557 * programmed in gfx_v9_0_init_always_on_cu_mask() 1558 */ 1559 1560 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1561 * but used for RLC_LB_CNTL configuration */ 1562 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1563 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1564 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1565 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1566 mutex_unlock(&adev->grbm_idx_mutex); 1567 1568 gfx_v9_0_init_always_on_cu_mask(adev); 1569 } 1570 1571 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev) 1572 { 1573 uint32_t data; 1574 1575 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1576 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1577 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8); 1578 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1579 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16)); 1580 1581 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1582 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1583 1584 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1585 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800); 1586 1587 mutex_lock(&adev->grbm_idx_mutex); 1588 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1589 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 1590 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1591 1592 /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1593 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1594 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1595 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1596 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1597 1598 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1599 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1600 data &= 0x0000FFFF; 1601 data |= 0x00C00000; 1602 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1603 1604 /* 1605 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON), 1606 * programmed in gfx_v9_0_init_always_on_cu_mask() 1607 */ 1608 1609 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1610 * but used for RLC_LB_CNTL configuration */ 1611 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1612 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1613 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1614 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1615 mutex_unlock(&adev->grbm_idx_mutex); 1616 1617 gfx_v9_0_init_always_on_cu_mask(adev); 1618 } 1619 1620 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable) 1621 { 1622 WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0); 1623 } 1624 1625 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev) 1626 { 1627 if (gfx_v9_0_load_mec2_fw_bin_support(adev)) 1628 return 5; 1629 else 1630 return 4; 1631 } 1632 1633 static void gfx_v9_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev) 1634 { 1635 struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl; 1636 1637 reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0]; 1638 reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0); 1639 reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG1); 1640 reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG2); 1641 reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG3); 1642 reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_CNTL); 1643 reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX); 1644 reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, mmRLC_SPARE_INT); 1645 adev->gfx.rlc.rlcg_reg_access_supported = true; 1646 } 1647 1648 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev) 1649 { 1650 const struct cs_section_def *cs_data; 1651 int r; 1652 1653 adev->gfx.rlc.cs_data = gfx9_cs_data; 1654 1655 cs_data = adev->gfx.rlc.cs_data; 1656 1657 if (cs_data) { 1658 /* init clear state block */ 1659 r = amdgpu_gfx_rlc_init_csb(adev); 1660 if (r) 1661 return r; 1662 } 1663 1664 if (adev->flags & AMD_IS_APU) { 1665 /* TODO: double check the cp_table_size for RV */ 1666 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */ 1667 r = amdgpu_gfx_rlc_init_cpt(adev); 1668 if (r) 1669 return r; 1670 } 1671 1672 return 0; 1673 } 1674 1675 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev) 1676 { 1677 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); 1678 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL); 1679 } 1680 1681 static int gfx_v9_0_mec_init(struct amdgpu_device *adev) 1682 { 1683 int r; 1684 u32 *hpd; 1685 const __le32 *fw_data; 1686 unsigned fw_size; 1687 u32 *fw; 1688 size_t mec_hpd_size; 1689 1690 const struct gfx_firmware_header_v1_0 *mec_hdr; 1691 1692 bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 1693 1694 /* take ownership of the relevant compute queues */ 1695 amdgpu_gfx_compute_queue_acquire(adev); 1696 mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE; 1697 if (mec_hpd_size) { 1698 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, 1699 AMDGPU_GEM_DOMAIN_VRAM | 1700 AMDGPU_GEM_DOMAIN_GTT, 1701 &adev->gfx.mec.hpd_eop_obj, 1702 &adev->gfx.mec.hpd_eop_gpu_addr, 1703 (void **)&hpd); 1704 if (r) { 1705 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); 1706 gfx_v9_0_mec_fini(adev); 1707 return r; 1708 } 1709 1710 memset(hpd, 0, mec_hpd_size); 1711 1712 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); 1713 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); 1714 } 1715 1716 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1717 1718 fw_data = (const __le32 *) 1719 (adev->gfx.mec_fw->data + 1720 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 1721 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes); 1722 1723 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes, 1724 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 1725 &adev->gfx.mec.mec_fw_obj, 1726 &adev->gfx.mec.mec_fw_gpu_addr, 1727 (void **)&fw); 1728 if (r) { 1729 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r); 1730 gfx_v9_0_mec_fini(adev); 1731 return r; 1732 } 1733 1734 memcpy(fw, fw_data, fw_size); 1735 1736 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); 1737 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); 1738 1739 return 0; 1740 } 1741 1742 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address) 1743 { 1744 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX, 1745 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 1746 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 1747 (address << SQ_IND_INDEX__INDEX__SHIFT) | 1748 (SQ_IND_INDEX__FORCE_READ_MASK)); 1749 return RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 1750 } 1751 1752 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd, 1753 uint32_t wave, uint32_t thread, 1754 uint32_t regno, uint32_t num, uint32_t *out) 1755 { 1756 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX, 1757 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 1758 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 1759 (regno << SQ_IND_INDEX__INDEX__SHIFT) | 1760 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) | 1761 (SQ_IND_INDEX__FORCE_READ_MASK) | 1762 (SQ_IND_INDEX__AUTO_INCR_MASK)); 1763 while (num--) 1764 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 1765 } 1766 1767 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields) 1768 { 1769 /* type 1 wave data */ 1770 dst[(*no_fields)++] = 1; 1771 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS); 1772 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO); 1773 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI); 1774 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO); 1775 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI); 1776 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID); 1777 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0); 1778 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1); 1779 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC); 1780 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC); 1781 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS); 1782 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS); 1783 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0); 1784 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0); 1785 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE); 1786 } 1787 1788 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, 1789 uint32_t wave, uint32_t start, 1790 uint32_t size, uint32_t *dst) 1791 { 1792 wave_read_regs( 1793 adev, simd, wave, 0, 1794 start + SQIND_WAVE_SGPRS_OFFSET, size, dst); 1795 } 1796 1797 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, 1798 uint32_t wave, uint32_t thread, 1799 uint32_t start, uint32_t size, 1800 uint32_t *dst) 1801 { 1802 wave_read_regs( 1803 adev, simd, wave, thread, 1804 start + SQIND_WAVE_VGPRS_OFFSET, size, dst); 1805 } 1806 1807 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev, 1808 u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id) 1809 { 1810 soc15_grbm_select(adev, me, pipe, q, vm, 0); 1811 } 1812 1813 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = { 1814 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter, 1815 .select_se_sh = &gfx_v9_0_select_se_sh, 1816 .read_wave_data = &gfx_v9_0_read_wave_data, 1817 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs, 1818 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs, 1819 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q, 1820 }; 1821 1822 const struct amdgpu_ras_block_hw_ops gfx_v9_0_ras_ops = { 1823 .ras_error_inject = &gfx_v9_0_ras_error_inject, 1824 .query_ras_error_count = &gfx_v9_0_query_ras_error_count, 1825 .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count, 1826 }; 1827 1828 static struct amdgpu_gfx_ras gfx_v9_0_ras = { 1829 .ras_block = { 1830 .hw_ops = &gfx_v9_0_ras_ops, 1831 }, 1832 }; 1833 1834 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev) 1835 { 1836 u32 gb_addr_config; 1837 int err; 1838 1839 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1840 case IP_VERSION(9, 0, 1): 1841 adev->gfx.config.max_hw_contexts = 8; 1842 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1843 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1844 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1845 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 1846 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN; 1847 break; 1848 case IP_VERSION(9, 2, 1): 1849 adev->gfx.config.max_hw_contexts = 8; 1850 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1851 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1852 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1853 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 1854 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN; 1855 DRM_INFO("fix gfx.config for vega12\n"); 1856 break; 1857 case IP_VERSION(9, 4, 0): 1858 adev->gfx.ras = &gfx_v9_0_ras; 1859 adev->gfx.config.max_hw_contexts = 8; 1860 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1861 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1862 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1863 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 1864 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 1865 gb_addr_config &= ~0xf3e777ff; 1866 gb_addr_config |= 0x22014042; 1867 /* check vbios table if gpu info is not available */ 1868 err = amdgpu_atomfirmware_get_gfx_info(adev); 1869 if (err) 1870 return err; 1871 break; 1872 case IP_VERSION(9, 2, 2): 1873 case IP_VERSION(9, 1, 0): 1874 adev->gfx.config.max_hw_contexts = 8; 1875 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1876 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1877 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1878 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 1879 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 1880 gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN; 1881 else 1882 gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN; 1883 break; 1884 case IP_VERSION(9, 4, 1): 1885 adev->gfx.ras = &gfx_v9_4_ras; 1886 adev->gfx.config.max_hw_contexts = 8; 1887 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1888 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1889 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1890 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 1891 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 1892 gb_addr_config &= ~0xf3e777ff; 1893 gb_addr_config |= 0x22014042; 1894 break; 1895 case IP_VERSION(9, 3, 0): 1896 adev->gfx.config.max_hw_contexts = 8; 1897 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1898 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1899 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80; 1900 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 1901 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 1902 gb_addr_config &= ~0xf3e777ff; 1903 gb_addr_config |= 0x22010042; 1904 break; 1905 case IP_VERSION(9, 4, 2): 1906 adev->gfx.ras = &gfx_v9_4_2_ras; 1907 adev->gfx.config.max_hw_contexts = 8; 1908 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1909 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1910 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1911 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 1912 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 1913 gb_addr_config &= ~0xf3e777ff; 1914 gb_addr_config |= 0x22014042; 1915 /* check vbios table if gpu info is not available */ 1916 err = amdgpu_atomfirmware_get_gfx_info(adev); 1917 if (err) 1918 return err; 1919 break; 1920 default: 1921 BUG(); 1922 break; 1923 } 1924 1925 adev->gfx.config.gb_addr_config = gb_addr_config; 1926 1927 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 << 1928 REG_GET_FIELD( 1929 adev->gfx.config.gb_addr_config, 1930 GB_ADDR_CONFIG, 1931 NUM_PIPES); 1932 1933 adev->gfx.config.max_tile_pipes = 1934 adev->gfx.config.gb_addr_config_fields.num_pipes; 1935 1936 adev->gfx.config.gb_addr_config_fields.num_banks = 1 << 1937 REG_GET_FIELD( 1938 adev->gfx.config.gb_addr_config, 1939 GB_ADDR_CONFIG, 1940 NUM_BANKS); 1941 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 << 1942 REG_GET_FIELD( 1943 adev->gfx.config.gb_addr_config, 1944 GB_ADDR_CONFIG, 1945 MAX_COMPRESSED_FRAGS); 1946 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 << 1947 REG_GET_FIELD( 1948 adev->gfx.config.gb_addr_config, 1949 GB_ADDR_CONFIG, 1950 NUM_RB_PER_SE); 1951 adev->gfx.config.gb_addr_config_fields.num_se = 1 << 1952 REG_GET_FIELD( 1953 adev->gfx.config.gb_addr_config, 1954 GB_ADDR_CONFIG, 1955 NUM_SHADER_ENGINES); 1956 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 + 1957 REG_GET_FIELD( 1958 adev->gfx.config.gb_addr_config, 1959 GB_ADDR_CONFIG, 1960 PIPE_INTERLEAVE_SIZE)); 1961 1962 return 0; 1963 } 1964 1965 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, 1966 int mec, int pipe, int queue) 1967 { 1968 unsigned irq_type; 1969 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id]; 1970 unsigned int hw_prio; 1971 1972 ring = &adev->gfx.compute_ring[ring_id]; 1973 1974 /* mec0 is me1 */ 1975 ring->me = mec + 1; 1976 ring->pipe = pipe; 1977 ring->queue = queue; 1978 1979 ring->ring_obj = NULL; 1980 ring->use_doorbell = true; 1981 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1; 1982 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr 1983 + (ring_id * GFX9_MEC_HPD_SIZE); 1984 ring->vm_hub = AMDGPU_GFXHUB(0); 1985 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue); 1986 1987 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 1988 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) 1989 + ring->pipe; 1990 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ? 1991 AMDGPU_RING_PRIO_2 : AMDGPU_RING_PRIO_DEFAULT; 1992 /* type-2 packets are deprecated on MEC, use type-3 instead */ 1993 return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type, 1994 hw_prio, NULL); 1995 } 1996 1997 static int gfx_v9_0_sw_init(void *handle) 1998 { 1999 int i, j, k, r, ring_id; 2000 struct amdgpu_ring *ring; 2001 struct amdgpu_kiq *kiq; 2002 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2003 unsigned int hw_prio; 2004 2005 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 2006 case IP_VERSION(9, 0, 1): 2007 case IP_VERSION(9, 2, 1): 2008 case IP_VERSION(9, 4, 0): 2009 case IP_VERSION(9, 2, 2): 2010 case IP_VERSION(9, 1, 0): 2011 case IP_VERSION(9, 4, 1): 2012 case IP_VERSION(9, 3, 0): 2013 case IP_VERSION(9, 4, 2): 2014 adev->gfx.mec.num_mec = 2; 2015 break; 2016 default: 2017 adev->gfx.mec.num_mec = 1; 2018 break; 2019 } 2020 2021 adev->gfx.mec.num_pipe_per_mec = 4; 2022 adev->gfx.mec.num_queue_per_pipe = 8; 2023 2024 /* EOP Event */ 2025 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq); 2026 if (r) 2027 return r; 2028 2029 /* Privileged reg */ 2030 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT, 2031 &adev->gfx.priv_reg_irq); 2032 if (r) 2033 return r; 2034 2035 /* Privileged inst */ 2036 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT, 2037 &adev->gfx.priv_inst_irq); 2038 if (r) 2039 return r; 2040 2041 /* ECC error */ 2042 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR, 2043 &adev->gfx.cp_ecc_error_irq); 2044 if (r) 2045 return r; 2046 2047 /* FUE error */ 2048 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR, 2049 &adev->gfx.cp_ecc_error_irq); 2050 if (r) 2051 return r; 2052 2053 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; 2054 2055 if (adev->gfx.rlc.funcs) { 2056 if (adev->gfx.rlc.funcs->init) { 2057 r = adev->gfx.rlc.funcs->init(adev); 2058 if (r) { 2059 dev_err(adev->dev, "Failed to init rlc BOs!\n"); 2060 return r; 2061 } 2062 } 2063 } 2064 2065 r = gfx_v9_0_mec_init(adev); 2066 if (r) { 2067 DRM_ERROR("Failed to init MEC BOs!\n"); 2068 return r; 2069 } 2070 2071 /* set up the gfx ring */ 2072 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 2073 ring = &adev->gfx.gfx_ring[i]; 2074 ring->ring_obj = NULL; 2075 if (!i) 2076 sprintf(ring->name, "gfx"); 2077 else 2078 sprintf(ring->name, "gfx_%d", i); 2079 ring->use_doorbell = true; 2080 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1; 2081 2082 /* disable scheduler on the real ring */ 2083 ring->no_scheduler = true; 2084 ring->vm_hub = AMDGPU_GFXHUB(0); 2085 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, 2086 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, 2087 AMDGPU_RING_PRIO_DEFAULT, NULL); 2088 if (r) 2089 return r; 2090 } 2091 2092 /* set up the software rings */ 2093 if (adev->gfx.num_gfx_rings) { 2094 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) { 2095 ring = &adev->gfx.sw_gfx_ring[i]; 2096 ring->ring_obj = NULL; 2097 sprintf(ring->name, amdgpu_sw_ring_name(i)); 2098 ring->use_doorbell = true; 2099 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1; 2100 ring->is_sw_ring = true; 2101 hw_prio = amdgpu_sw_ring_priority(i); 2102 ring->vm_hub = AMDGPU_GFXHUB(0); 2103 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, 2104 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, hw_prio, 2105 NULL); 2106 if (r) 2107 return r; 2108 ring->wptr = 0; 2109 } 2110 2111 /* init the muxer and add software rings */ 2112 r = amdgpu_ring_mux_init(&adev->gfx.muxer, &adev->gfx.gfx_ring[0], 2113 GFX9_NUM_SW_GFX_RINGS); 2114 if (r) { 2115 DRM_ERROR("amdgpu_ring_mux_init failed(%d)\n", r); 2116 return r; 2117 } 2118 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) { 2119 r = amdgpu_ring_mux_add_sw_ring(&adev->gfx.muxer, 2120 &adev->gfx.sw_gfx_ring[i]); 2121 if (r) { 2122 DRM_ERROR("amdgpu_ring_mux_add_sw_ring failed(%d)\n", r); 2123 return r; 2124 } 2125 } 2126 } 2127 2128 /* set up the compute queues - allocate horizontally across pipes */ 2129 ring_id = 0; 2130 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 2131 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 2132 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 2133 if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i, 2134 k, j)) 2135 continue; 2136 2137 r = gfx_v9_0_compute_ring_init(adev, 2138 ring_id, 2139 i, k, j); 2140 if (r) 2141 return r; 2142 2143 ring_id++; 2144 } 2145 } 2146 } 2147 2148 r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE, 0); 2149 if (r) { 2150 DRM_ERROR("Failed to init KIQ BOs!\n"); 2151 return r; 2152 } 2153 2154 kiq = &adev->gfx.kiq[0]; 2155 r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq, 0); 2156 if (r) 2157 return r; 2158 2159 /* create MQD for all compute queues as wel as KIQ for SRIOV case */ 2160 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation), 0); 2161 if (r) 2162 return r; 2163 2164 adev->gfx.ce_ram_size = 0x8000; 2165 2166 r = gfx_v9_0_gpu_early_init(adev); 2167 if (r) 2168 return r; 2169 2170 if (amdgpu_gfx_ras_sw_init(adev)) { 2171 dev_err(adev->dev, "Failed to initialize gfx ras block!\n"); 2172 return -EINVAL; 2173 } 2174 2175 return 0; 2176 } 2177 2178 2179 static int gfx_v9_0_sw_fini(void *handle) 2180 { 2181 int i; 2182 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2183 2184 if (adev->gfx.num_gfx_rings) { 2185 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) 2186 amdgpu_ring_fini(&adev->gfx.sw_gfx_ring[i]); 2187 amdgpu_ring_mux_fini(&adev->gfx.muxer); 2188 } 2189 2190 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 2191 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); 2192 for (i = 0; i < adev->gfx.num_compute_rings; i++) 2193 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); 2194 2195 amdgpu_gfx_mqd_sw_fini(adev, 0); 2196 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring); 2197 amdgpu_gfx_kiq_fini(adev, 0); 2198 2199 gfx_v9_0_mec_fini(adev); 2200 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj, 2201 &adev->gfx.rlc.clear_state_gpu_addr, 2202 (void **)&adev->gfx.rlc.cs_ptr); 2203 if (adev->flags & AMD_IS_APU) { 2204 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, 2205 &adev->gfx.rlc.cp_table_gpu_addr, 2206 (void **)&adev->gfx.rlc.cp_table_ptr); 2207 } 2208 gfx_v9_0_free_microcode(adev); 2209 2210 return 0; 2211 } 2212 2213 2214 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev) 2215 { 2216 /* TODO */ 2217 } 2218 2219 void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, 2220 u32 instance, int xcc_id) 2221 { 2222 u32 data; 2223 2224 if (instance == 0xffffffff) 2225 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1); 2226 else 2227 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance); 2228 2229 if (se_num == 0xffffffff) 2230 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1); 2231 else 2232 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num); 2233 2234 if (sh_num == 0xffffffff) 2235 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1); 2236 else 2237 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num); 2238 2239 WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data); 2240 } 2241 2242 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev) 2243 { 2244 u32 data, mask; 2245 2246 data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE); 2247 data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE); 2248 2249 data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK; 2250 data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT; 2251 2252 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se / 2253 adev->gfx.config.max_sh_per_se); 2254 2255 return (~data) & mask; 2256 } 2257 2258 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev) 2259 { 2260 int i, j; 2261 u32 data; 2262 u32 active_rbs = 0; 2263 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se / 2264 adev->gfx.config.max_sh_per_se; 2265 2266 mutex_lock(&adev->grbm_idx_mutex); 2267 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2268 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2269 amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0); 2270 data = gfx_v9_0_get_rb_active_bitmap(adev); 2271 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) * 2272 rb_bitmap_width_per_sh); 2273 } 2274 } 2275 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 2276 mutex_unlock(&adev->grbm_idx_mutex); 2277 2278 adev->gfx.config.backend_enable_mask = active_rbs; 2279 adev->gfx.config.num_rbs = hweight32(active_rbs); 2280 } 2281 2282 static void gfx_v9_0_debug_trap_config_init(struct amdgpu_device *adev, 2283 uint32_t first_vmid, 2284 uint32_t last_vmid) 2285 { 2286 uint32_t data; 2287 uint32_t trap_config_vmid_mask = 0; 2288 int i; 2289 2290 /* Calculate trap config vmid mask */ 2291 for (i = first_vmid; i < last_vmid; i++) 2292 trap_config_vmid_mask |= (1 << i); 2293 2294 data = REG_SET_FIELD(0, SPI_GDBG_TRAP_CONFIG, 2295 VMID_SEL, trap_config_vmid_mask); 2296 data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG, 2297 TRAP_EN, 1); 2298 WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_CONFIG), data); 2299 WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), 0); 2300 2301 WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA0), 0); 2302 WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA1), 0); 2303 } 2304 2305 #define DEFAULT_SH_MEM_BASES (0x6000) 2306 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev) 2307 { 2308 int i; 2309 uint32_t sh_mem_config; 2310 uint32_t sh_mem_bases; 2311 2312 /* 2313 * Configure apertures: 2314 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) 2315 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) 2316 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) 2317 */ 2318 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16); 2319 2320 sh_mem_config = SH_MEM_ADDRESS_MODE_64 | 2321 SH_MEM_ALIGNMENT_MODE_UNALIGNED << 2322 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT; 2323 2324 mutex_lock(&adev->srbm_mutex); 2325 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2326 soc15_grbm_select(adev, 0, 0, 0, i, 0); 2327 /* CP and shaders */ 2328 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config); 2329 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases); 2330 } 2331 soc15_grbm_select(adev, 0, 0, 0, 0, 0); 2332 mutex_unlock(&adev->srbm_mutex); 2333 2334 /* Initialize all compute VMIDs to have no GDS, GWS, or OA 2335 access. These should be enabled by FW for target VMIDs. */ 2336 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2337 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0); 2338 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0); 2339 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0); 2340 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0); 2341 } 2342 } 2343 2344 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev) 2345 { 2346 int vmid; 2347 2348 /* 2349 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA 2350 * access. Compute VMIDs should be enabled by FW for target VMIDs, 2351 * the driver can enable them for graphics. VMID0 should maintain 2352 * access so that HWS firmware can save/restore entries. 2353 */ 2354 for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) { 2355 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0); 2356 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0); 2357 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0); 2358 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0); 2359 } 2360 } 2361 2362 static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev) 2363 { 2364 uint32_t tmp; 2365 2366 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 2367 case IP_VERSION(9, 4, 1): 2368 tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG); 2369 tmp = REG_SET_FIELD(tmp, SQ_CONFIG, DISABLE_BARRIER_WAITCNT, 2370 !READ_ONCE(adev->barrier_has_auto_waitcnt)); 2371 WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp); 2372 break; 2373 default: 2374 break; 2375 } 2376 } 2377 2378 static void gfx_v9_0_constants_init(struct amdgpu_device *adev) 2379 { 2380 u32 tmp; 2381 int i; 2382 2383 WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff); 2384 2385 gfx_v9_0_tiling_mode_table_init(adev); 2386 2387 if (adev->gfx.num_gfx_rings) 2388 gfx_v9_0_setup_rb(adev); 2389 gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info); 2390 adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2); 2391 2392 /* XXX SH_MEM regs */ 2393 /* where to put LDS, scratch, GPUVM in FSA64 space */ 2394 mutex_lock(&adev->srbm_mutex); 2395 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) { 2396 soc15_grbm_select(adev, 0, 0, 0, i, 0); 2397 /* CP and shaders */ 2398 if (i == 0) { 2399 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2400 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2401 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2402 !!adev->gmc.noretry); 2403 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2404 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0); 2405 } else { 2406 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2407 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2408 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2409 !!adev->gmc.noretry); 2410 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2411 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE, 2412 (adev->gmc.private_aperture_start >> 48)); 2413 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE, 2414 (adev->gmc.shared_aperture_start >> 48)); 2415 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp); 2416 } 2417 } 2418 soc15_grbm_select(adev, 0, 0, 0, 0, 0); 2419 2420 mutex_unlock(&adev->srbm_mutex); 2421 2422 gfx_v9_0_init_compute_vmid(adev); 2423 gfx_v9_0_init_gds_vmid(adev); 2424 gfx_v9_0_init_sq_config(adev); 2425 } 2426 2427 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev) 2428 { 2429 u32 i, j, k; 2430 u32 mask; 2431 2432 mutex_lock(&adev->grbm_idx_mutex); 2433 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2434 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2435 amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0); 2436 for (k = 0; k < adev->usec_timeout; k++) { 2437 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0) 2438 break; 2439 udelay(1); 2440 } 2441 if (k == adev->usec_timeout) { 2442 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 2443 0xffffffff, 0xffffffff, 0); 2444 mutex_unlock(&adev->grbm_idx_mutex); 2445 DRM_INFO("Timeout wait for RLC serdes %u,%u\n", 2446 i, j); 2447 return; 2448 } 2449 } 2450 } 2451 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 2452 mutex_unlock(&adev->grbm_idx_mutex); 2453 2454 mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK | 2455 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK | 2456 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK | 2457 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK; 2458 for (k = 0; k < adev->usec_timeout; k++) { 2459 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0) 2460 break; 2461 udelay(1); 2462 } 2463 } 2464 2465 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev, 2466 bool enable) 2467 { 2468 u32 tmp; 2469 2470 /* These interrupts should be enabled to drive DS clock */ 2471 2472 tmp= RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0); 2473 2474 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0); 2475 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0); 2476 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0); 2477 if(adev->gfx.num_gfx_rings) 2478 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0); 2479 2480 WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp); 2481 } 2482 2483 static void gfx_v9_0_init_csb(struct amdgpu_device *adev) 2484 { 2485 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); 2486 /* csib */ 2487 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI), 2488 adev->gfx.rlc.clear_state_gpu_addr >> 32); 2489 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO), 2490 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); 2491 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH), 2492 adev->gfx.rlc.clear_state_size); 2493 } 2494 2495 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format, 2496 int indirect_offset, 2497 int list_size, 2498 int *unique_indirect_regs, 2499 int unique_indirect_reg_count, 2500 int *indirect_start_offsets, 2501 int *indirect_start_offsets_count, 2502 int max_start_offsets_count) 2503 { 2504 int idx; 2505 2506 for (; indirect_offset < list_size; indirect_offset++) { 2507 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count); 2508 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset; 2509 *indirect_start_offsets_count = *indirect_start_offsets_count + 1; 2510 2511 while (register_list_format[indirect_offset] != 0xFFFFFFFF) { 2512 indirect_offset += 2; 2513 2514 /* look for the matching indice */ 2515 for (idx = 0; idx < unique_indirect_reg_count; idx++) { 2516 if (unique_indirect_regs[idx] == 2517 register_list_format[indirect_offset] || 2518 !unique_indirect_regs[idx]) 2519 break; 2520 } 2521 2522 BUG_ON(idx >= unique_indirect_reg_count); 2523 2524 if (!unique_indirect_regs[idx]) 2525 unique_indirect_regs[idx] = register_list_format[indirect_offset]; 2526 2527 indirect_offset++; 2528 } 2529 } 2530 } 2531 2532 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev) 2533 { 2534 int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2535 int unique_indirect_reg_count = 0; 2536 2537 int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2538 int indirect_start_offsets_count = 0; 2539 2540 int list_size = 0; 2541 int i = 0, j = 0; 2542 u32 tmp = 0; 2543 2544 u32 *register_list_format = 2545 kmemdup(adev->gfx.rlc.register_list_format, 2546 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL); 2547 if (!register_list_format) 2548 return -ENOMEM; 2549 2550 /* setup unique_indirect_regs array and indirect_start_offsets array */ 2551 unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs); 2552 gfx_v9_1_parse_ind_reg_list(register_list_format, 2553 adev->gfx.rlc.reg_list_format_direct_reg_list_length, 2554 adev->gfx.rlc.reg_list_format_size_bytes >> 2, 2555 unique_indirect_regs, 2556 unique_indirect_reg_count, 2557 indirect_start_offsets, 2558 &indirect_start_offsets_count, 2559 ARRAY_SIZE(indirect_start_offsets)); 2560 2561 /* enable auto inc in case it is disabled */ 2562 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL)); 2563 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK; 2564 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp); 2565 2566 /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */ 2567 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR), 2568 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET); 2569 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++) 2570 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA), 2571 adev->gfx.rlc.register_restore[i]); 2572 2573 /* load indirect register */ 2574 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2575 adev->gfx.rlc.reg_list_format_start); 2576 2577 /* direct register portion */ 2578 for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++) 2579 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2580 register_list_format[i]); 2581 2582 /* indirect register portion */ 2583 while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) { 2584 if (register_list_format[i] == 0xFFFFFFFF) { 2585 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2586 continue; 2587 } 2588 2589 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2590 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2591 2592 for (j = 0; j < unique_indirect_reg_count; j++) { 2593 if (register_list_format[i] == unique_indirect_regs[j]) { 2594 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j); 2595 break; 2596 } 2597 } 2598 2599 BUG_ON(j >= unique_indirect_reg_count); 2600 2601 i++; 2602 } 2603 2604 /* set save/restore list size */ 2605 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2; 2606 list_size = list_size >> 1; 2607 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2608 adev->gfx.rlc.reg_restore_list_size); 2609 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size); 2610 2611 /* write the starting offsets to RLC scratch ram */ 2612 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2613 adev->gfx.rlc.starting_offsets_start); 2614 for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++) 2615 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2616 indirect_start_offsets[i]); 2617 2618 /* load unique indirect regs*/ 2619 for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) { 2620 if (unique_indirect_regs[i] != 0) { 2621 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0) 2622 + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i], 2623 unique_indirect_regs[i] & 0x3FFFF); 2624 2625 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0) 2626 + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i], 2627 unique_indirect_regs[i] >> 20); 2628 } 2629 } 2630 2631 kfree(register_list_format); 2632 return 0; 2633 } 2634 2635 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev) 2636 { 2637 WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1); 2638 } 2639 2640 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev, 2641 bool enable) 2642 { 2643 uint32_t data = 0; 2644 uint32_t default_data = 0; 2645 2646 default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS)); 2647 if (enable) { 2648 /* enable GFXIP control over CGPG */ 2649 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2650 if(default_data != data) 2651 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2652 2653 /* update status */ 2654 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK; 2655 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT); 2656 if(default_data != data) 2657 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2658 } else { 2659 /* restore GFXIP control over GCPG */ 2660 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2661 if(default_data != data) 2662 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2663 } 2664 } 2665 2666 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev) 2667 { 2668 uint32_t data = 0; 2669 2670 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 2671 AMD_PG_SUPPORT_GFX_SMG | 2672 AMD_PG_SUPPORT_GFX_DMG)) { 2673 /* init IDLE_POLL_COUNT = 60 */ 2674 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL)); 2675 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK; 2676 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 2677 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data); 2678 2679 /* init RLC PG Delay */ 2680 data = 0; 2681 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT); 2682 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT); 2683 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT); 2684 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT); 2685 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data); 2686 2687 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2)); 2688 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK; 2689 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT); 2690 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data); 2691 2692 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3)); 2693 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK; 2694 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT); 2695 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data); 2696 2697 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL)); 2698 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK; 2699 2700 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */ 2701 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT); 2702 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data); 2703 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 3, 0)) 2704 pwr_10_0_gfxip_control_over_cgpg(adev, true); 2705 } 2706 } 2707 2708 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev, 2709 bool enable) 2710 { 2711 uint32_t data = 0; 2712 uint32_t default_data = 0; 2713 2714 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2715 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2716 SMU_CLK_SLOWDOWN_ON_PU_ENABLE, 2717 enable ? 1 : 0); 2718 if (default_data != data) 2719 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2720 } 2721 2722 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev, 2723 bool enable) 2724 { 2725 uint32_t data = 0; 2726 uint32_t default_data = 0; 2727 2728 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2729 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2730 SMU_CLK_SLOWDOWN_ON_PD_ENABLE, 2731 enable ? 1 : 0); 2732 if(default_data != data) 2733 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2734 } 2735 2736 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev, 2737 bool enable) 2738 { 2739 uint32_t data = 0; 2740 uint32_t default_data = 0; 2741 2742 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2743 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2744 CP_PG_DISABLE, 2745 enable ? 0 : 1); 2746 if(default_data != data) 2747 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2748 } 2749 2750 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev, 2751 bool enable) 2752 { 2753 uint32_t data, default_data; 2754 2755 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2756 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2757 GFX_POWER_GATING_ENABLE, 2758 enable ? 1 : 0); 2759 if(default_data != data) 2760 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2761 } 2762 2763 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev, 2764 bool enable) 2765 { 2766 uint32_t data, default_data; 2767 2768 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2769 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2770 GFX_PIPELINE_PG_ENABLE, 2771 enable ? 1 : 0); 2772 if(default_data != data) 2773 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2774 2775 if (!enable) 2776 /* read any GFX register to wake up GFX */ 2777 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL)); 2778 } 2779 2780 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev, 2781 bool enable) 2782 { 2783 uint32_t data, default_data; 2784 2785 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2786 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2787 STATIC_PER_CU_PG_ENABLE, 2788 enable ? 1 : 0); 2789 if(default_data != data) 2790 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2791 } 2792 2793 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev, 2794 bool enable) 2795 { 2796 uint32_t data, default_data; 2797 2798 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2799 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2800 DYN_PER_CU_PG_ENABLE, 2801 enable ? 1 : 0); 2802 if(default_data != data) 2803 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2804 } 2805 2806 static void gfx_v9_0_init_pg(struct amdgpu_device *adev) 2807 { 2808 gfx_v9_0_init_csb(adev); 2809 2810 /* 2811 * Rlc save restore list is workable since v2_1. 2812 * And it's needed by gfxoff feature. 2813 */ 2814 if (adev->gfx.rlc.is_rlc_v2_1) { 2815 if (amdgpu_ip_version(adev, GC_HWIP, 0) == 2816 IP_VERSION(9, 2, 1) || 2817 (adev->apu_flags & AMD_APU_IS_RAVEN2)) 2818 gfx_v9_1_init_rlc_save_restore_list(adev); 2819 gfx_v9_0_enable_save_restore_machine(adev); 2820 } 2821 2822 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 2823 AMD_PG_SUPPORT_GFX_SMG | 2824 AMD_PG_SUPPORT_GFX_DMG | 2825 AMD_PG_SUPPORT_CP | 2826 AMD_PG_SUPPORT_GDS | 2827 AMD_PG_SUPPORT_RLC_SMU_HS)) { 2828 WREG32_SOC15(GC, 0, mmRLC_JUMP_TABLE_RESTORE, 2829 adev->gfx.rlc.cp_table_gpu_addr >> 8); 2830 gfx_v9_0_init_gfx_power_gating(adev); 2831 } 2832 } 2833 2834 static void gfx_v9_0_rlc_stop(struct amdgpu_device *adev) 2835 { 2836 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0); 2837 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 2838 gfx_v9_0_wait_for_rlc_serdes(adev); 2839 } 2840 2841 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev) 2842 { 2843 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 2844 udelay(50); 2845 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0); 2846 udelay(50); 2847 } 2848 2849 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev) 2850 { 2851 #ifdef AMDGPU_RLC_DEBUG_RETRY 2852 u32 rlc_ucode_ver; 2853 #endif 2854 2855 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1); 2856 udelay(50); 2857 2858 /* carrizo do enable cp interrupt after cp inited */ 2859 if (!(adev->flags & AMD_IS_APU)) { 2860 gfx_v9_0_enable_gui_idle_interrupt(adev, true); 2861 udelay(50); 2862 } 2863 2864 #ifdef AMDGPU_RLC_DEBUG_RETRY 2865 /* RLC_GPM_GENERAL_6 : RLC Ucode version */ 2866 rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6); 2867 if(rlc_ucode_ver == 0x108) { 2868 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n", 2869 rlc_ucode_ver, adev->gfx.rlc_fw_version); 2870 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles, 2871 * default is 0x9C4 to create a 100us interval */ 2872 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4); 2873 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr 2874 * to disable the page fault retry interrupts, default is 2875 * 0x100 (256) */ 2876 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100); 2877 } 2878 #endif 2879 } 2880 2881 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev) 2882 { 2883 const struct rlc_firmware_header_v2_0 *hdr; 2884 const __le32 *fw_data; 2885 unsigned i, fw_size; 2886 2887 if (!adev->gfx.rlc_fw) 2888 return -EINVAL; 2889 2890 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 2891 amdgpu_ucode_print_rlc_hdr(&hdr->header); 2892 2893 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 2894 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 2895 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 2896 2897 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, 2898 RLCG_UCODE_LOADING_START_ADDRESS); 2899 for (i = 0; i < fw_size; i++) 2900 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++)); 2901 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version); 2902 2903 return 0; 2904 } 2905 2906 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev) 2907 { 2908 int r; 2909 2910 if (amdgpu_sriov_vf(adev)) { 2911 gfx_v9_0_init_csb(adev); 2912 return 0; 2913 } 2914 2915 adev->gfx.rlc.funcs->stop(adev); 2916 2917 /* disable CG */ 2918 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0); 2919 2920 gfx_v9_0_init_pg(adev); 2921 2922 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 2923 /* legacy rlc firmware loading */ 2924 r = gfx_v9_0_rlc_load_microcode(adev); 2925 if (r) 2926 return r; 2927 } 2928 2929 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 2930 case IP_VERSION(9, 2, 2): 2931 case IP_VERSION(9, 1, 0): 2932 gfx_v9_0_init_lbpw(adev); 2933 if (amdgpu_lbpw == 0) 2934 gfx_v9_0_enable_lbpw(adev, false); 2935 else 2936 gfx_v9_0_enable_lbpw(adev, true); 2937 break; 2938 case IP_VERSION(9, 4, 0): 2939 gfx_v9_4_init_lbpw(adev); 2940 if (amdgpu_lbpw > 0) 2941 gfx_v9_0_enable_lbpw(adev, true); 2942 else 2943 gfx_v9_0_enable_lbpw(adev, false); 2944 break; 2945 default: 2946 break; 2947 } 2948 2949 gfx_v9_0_update_spm_vmid_internal(adev, 0xf); 2950 2951 adev->gfx.rlc.funcs->start(adev); 2952 2953 return 0; 2954 } 2955 2956 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) 2957 { 2958 u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL); 2959 2960 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1); 2961 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1); 2962 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1); 2963 WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp); 2964 udelay(50); 2965 } 2966 2967 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev) 2968 { 2969 const struct gfx_firmware_header_v1_0 *pfp_hdr; 2970 const struct gfx_firmware_header_v1_0 *ce_hdr; 2971 const struct gfx_firmware_header_v1_0 *me_hdr; 2972 const __le32 *fw_data; 2973 unsigned i, fw_size; 2974 2975 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw) 2976 return -EINVAL; 2977 2978 pfp_hdr = (const struct gfx_firmware_header_v1_0 *) 2979 adev->gfx.pfp_fw->data; 2980 ce_hdr = (const struct gfx_firmware_header_v1_0 *) 2981 adev->gfx.ce_fw->data; 2982 me_hdr = (const struct gfx_firmware_header_v1_0 *) 2983 adev->gfx.me_fw->data; 2984 2985 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header); 2986 amdgpu_ucode_print_gfx_hdr(&ce_hdr->header); 2987 amdgpu_ucode_print_gfx_hdr(&me_hdr->header); 2988 2989 gfx_v9_0_cp_gfx_enable(adev, false); 2990 2991 /* PFP */ 2992 fw_data = (const __le32 *) 2993 (adev->gfx.pfp_fw->data + 2994 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); 2995 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4; 2996 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0); 2997 for (i = 0; i < fw_size; i++) 2998 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++)); 2999 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version); 3000 3001 /* CE */ 3002 fw_data = (const __le32 *) 3003 (adev->gfx.ce_fw->data + 3004 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); 3005 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4; 3006 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0); 3007 for (i = 0; i < fw_size; i++) 3008 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++)); 3009 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version); 3010 3011 /* ME */ 3012 fw_data = (const __le32 *) 3013 (adev->gfx.me_fw->data + 3014 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); 3015 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4; 3016 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0); 3017 for (i = 0; i < fw_size; i++) 3018 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++)); 3019 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version); 3020 3021 return 0; 3022 } 3023 3024 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev) 3025 { 3026 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0]; 3027 const struct cs_section_def *sect = NULL; 3028 const struct cs_extent_def *ext = NULL; 3029 int r, i, tmp; 3030 3031 /* init the CP */ 3032 WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1); 3033 WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1); 3034 3035 gfx_v9_0_cp_gfx_enable(adev, true); 3036 3037 r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3); 3038 if (r) { 3039 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r); 3040 return r; 3041 } 3042 3043 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3044 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 3045 3046 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 3047 amdgpu_ring_write(ring, 0x80000000); 3048 amdgpu_ring_write(ring, 0x80000000); 3049 3050 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 3051 for (ext = sect->section; ext->extent != NULL; ++ext) { 3052 if (sect->id == SECT_CONTEXT) { 3053 amdgpu_ring_write(ring, 3054 PACKET3(PACKET3_SET_CONTEXT_REG, 3055 ext->reg_count)); 3056 amdgpu_ring_write(ring, 3057 ext->reg_index - PACKET3_SET_CONTEXT_REG_START); 3058 for (i = 0; i < ext->reg_count; i++) 3059 amdgpu_ring_write(ring, ext->extent[i]); 3060 } 3061 } 3062 } 3063 3064 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3065 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 3066 3067 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3068 amdgpu_ring_write(ring, 0); 3069 3070 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 3071 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 3072 amdgpu_ring_write(ring, 0x8000); 3073 amdgpu_ring_write(ring, 0x8000); 3074 3075 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1)); 3076 tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE | 3077 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START)); 3078 amdgpu_ring_write(ring, tmp); 3079 amdgpu_ring_write(ring, 0); 3080 3081 amdgpu_ring_commit(ring); 3082 3083 return 0; 3084 } 3085 3086 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev) 3087 { 3088 struct amdgpu_ring *ring; 3089 u32 tmp; 3090 u32 rb_bufsz; 3091 u64 rb_addr, rptr_addr, wptr_gpu_addr; 3092 3093 /* Set the write pointer delay */ 3094 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0); 3095 3096 /* set the RB to use vmid 0 */ 3097 WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0); 3098 3099 /* Set ring buffer size */ 3100 ring = &adev->gfx.gfx_ring[0]; 3101 rb_bufsz = order_base_2(ring->ring_size / 8); 3102 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz); 3103 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2); 3104 #ifdef __BIG_ENDIAN 3105 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1); 3106 #endif 3107 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3108 3109 /* Initialize the ring buffer's write pointers */ 3110 ring->wptr = 0; 3111 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 3112 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 3113 3114 /* set the wb address wether it's enabled or not */ 3115 rptr_addr = ring->rptr_gpu_addr; 3116 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr)); 3117 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK); 3118 3119 wptr_gpu_addr = ring->wptr_gpu_addr; 3120 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr)); 3121 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr)); 3122 3123 mdelay(1); 3124 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3125 3126 rb_addr = ring->gpu_addr >> 8; 3127 WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr); 3128 WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr)); 3129 3130 tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL); 3131 if (ring->use_doorbell) { 3132 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3133 DOORBELL_OFFSET, ring->doorbell_index); 3134 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3135 DOORBELL_EN, 1); 3136 } else { 3137 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0); 3138 } 3139 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp); 3140 3141 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER, 3142 DOORBELL_RANGE_LOWER, ring->doorbell_index); 3143 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp); 3144 3145 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER, 3146 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK); 3147 3148 3149 /* start the ring */ 3150 gfx_v9_0_cp_gfx_start(adev); 3151 3152 return 0; 3153 } 3154 3155 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable) 3156 { 3157 if (enable) { 3158 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0); 3159 } else { 3160 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 3161 (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK)); 3162 adev->gfx.kiq[0].ring.sched.ready = false; 3163 } 3164 udelay(50); 3165 } 3166 3167 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev) 3168 { 3169 const struct gfx_firmware_header_v1_0 *mec_hdr; 3170 const __le32 *fw_data; 3171 unsigned i; 3172 u32 tmp; 3173 3174 if (!adev->gfx.mec_fw) 3175 return -EINVAL; 3176 3177 gfx_v9_0_cp_compute_enable(adev, false); 3178 3179 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 3180 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header); 3181 3182 fw_data = (const __le32 *) 3183 (adev->gfx.mec_fw->data + 3184 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 3185 tmp = 0; 3186 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); 3187 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 3188 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp); 3189 3190 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO, 3191 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000); 3192 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI, 3193 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr)); 3194 3195 /* MEC1 */ 3196 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3197 mec_hdr->jt_offset); 3198 for (i = 0; i < mec_hdr->jt_size; i++) 3199 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA, 3200 le32_to_cpup(fw_data + mec_hdr->jt_offset + i)); 3201 3202 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3203 adev->gfx.mec_fw_version); 3204 /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */ 3205 3206 return 0; 3207 } 3208 3209 /* KIQ functions */ 3210 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring) 3211 { 3212 uint32_t tmp; 3213 struct amdgpu_device *adev = ring->adev; 3214 3215 /* tell RLC which is KIQ queue */ 3216 tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS); 3217 tmp &= 0xffffff00; 3218 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); 3219 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp); 3220 tmp |= 0x80; 3221 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp); 3222 } 3223 3224 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd) 3225 { 3226 struct amdgpu_device *adev = ring->adev; 3227 3228 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 3229 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) { 3230 mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH; 3231 mqd->cp_hqd_queue_priority = 3232 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM; 3233 } 3234 } 3235 } 3236 3237 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring) 3238 { 3239 struct amdgpu_device *adev = ring->adev; 3240 struct v9_mqd *mqd = ring->mqd_ptr; 3241 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr; 3242 uint32_t tmp; 3243 3244 mqd->header = 0xC0310800; 3245 mqd->compute_pipelinestat_enable = 0x00000001; 3246 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; 3247 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; 3248 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; 3249 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; 3250 mqd->compute_static_thread_mgmt_se4 = 0xffffffff; 3251 mqd->compute_static_thread_mgmt_se5 = 0xffffffff; 3252 mqd->compute_static_thread_mgmt_se6 = 0xffffffff; 3253 mqd->compute_static_thread_mgmt_se7 = 0xffffffff; 3254 mqd->compute_misc_reserved = 0x00000003; 3255 3256 mqd->dynamic_cu_mask_addr_lo = 3257 lower_32_bits(ring->mqd_gpu_addr 3258 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3259 mqd->dynamic_cu_mask_addr_hi = 3260 upper_32_bits(ring->mqd_gpu_addr 3261 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3262 3263 eop_base_addr = ring->eop_gpu_addr >> 8; 3264 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr; 3265 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); 3266 3267 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3268 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL); 3269 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE, 3270 (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1)); 3271 3272 mqd->cp_hqd_eop_control = tmp; 3273 3274 /* enable doorbell? */ 3275 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL); 3276 3277 if (ring->use_doorbell) { 3278 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3279 DOORBELL_OFFSET, ring->doorbell_index); 3280 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3281 DOORBELL_EN, 1); 3282 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3283 DOORBELL_SOURCE, 0); 3284 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3285 DOORBELL_HIT, 0); 3286 } else { 3287 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3288 DOORBELL_EN, 0); 3289 } 3290 3291 mqd->cp_hqd_pq_doorbell_control = tmp; 3292 3293 /* disable the queue if it's active */ 3294 ring->wptr = 0; 3295 mqd->cp_hqd_dequeue_request = 0; 3296 mqd->cp_hqd_pq_rptr = 0; 3297 mqd->cp_hqd_pq_wptr_lo = 0; 3298 mqd->cp_hqd_pq_wptr_hi = 0; 3299 3300 /* set the pointer to the MQD */ 3301 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc; 3302 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr); 3303 3304 /* set MQD vmid to 0 */ 3305 tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL); 3306 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); 3307 mqd->cp_mqd_control = tmp; 3308 3309 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3310 hqd_gpu_addr = ring->gpu_addr >> 8; 3311 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; 3312 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 3313 3314 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3315 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL); 3316 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE, 3317 (order_base_2(ring->ring_size / 4) - 1)); 3318 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE, 3319 (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1)); 3320 #ifdef __BIG_ENDIAN 3321 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1); 3322 #endif 3323 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0); 3324 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0); 3325 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1); 3326 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1); 3327 mqd->cp_hqd_pq_control = tmp; 3328 3329 /* set the wb address whether it's enabled or not */ 3330 wb_gpu_addr = ring->rptr_gpu_addr; 3331 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; 3332 mqd->cp_hqd_pq_rptr_report_addr_hi = 3333 upper_32_bits(wb_gpu_addr) & 0xffff; 3334 3335 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3336 wb_gpu_addr = ring->wptr_gpu_addr; 3337 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 3338 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 3339 3340 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3341 ring->wptr = 0; 3342 mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR); 3343 3344 /* set the vmid for the queue */ 3345 mqd->cp_hqd_vmid = 0; 3346 3347 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE); 3348 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53); 3349 mqd->cp_hqd_persistent_state = tmp; 3350 3351 /* set MIN_IB_AVAIL_SIZE */ 3352 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL); 3353 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3); 3354 mqd->cp_hqd_ib_control = tmp; 3355 3356 /* set static priority for a queue/ring */ 3357 gfx_v9_0_mqd_set_priority(ring, mqd); 3358 mqd->cp_hqd_quantum = RREG32_SOC15(GC, 0, mmCP_HQD_QUANTUM); 3359 3360 /* map_queues packet doesn't need activate the queue, 3361 * so only kiq need set this field. 3362 */ 3363 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 3364 mqd->cp_hqd_active = 1; 3365 3366 return 0; 3367 } 3368 3369 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring) 3370 { 3371 struct amdgpu_device *adev = ring->adev; 3372 struct v9_mqd *mqd = ring->mqd_ptr; 3373 int j; 3374 3375 /* disable wptr polling */ 3376 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 3377 3378 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR, 3379 mqd->cp_hqd_eop_base_addr_lo); 3380 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI, 3381 mqd->cp_hqd_eop_base_addr_hi); 3382 3383 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3384 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL, 3385 mqd->cp_hqd_eop_control); 3386 3387 /* enable doorbell? */ 3388 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3389 mqd->cp_hqd_pq_doorbell_control); 3390 3391 /* disable the queue if it's active */ 3392 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3393 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3394 for (j = 0; j < adev->usec_timeout; j++) { 3395 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3396 break; 3397 udelay(1); 3398 } 3399 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3400 mqd->cp_hqd_dequeue_request); 3401 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 3402 mqd->cp_hqd_pq_rptr); 3403 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3404 mqd->cp_hqd_pq_wptr_lo); 3405 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3406 mqd->cp_hqd_pq_wptr_hi); 3407 } 3408 3409 /* set the pointer to the MQD */ 3410 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR, 3411 mqd->cp_mqd_base_addr_lo); 3412 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI, 3413 mqd->cp_mqd_base_addr_hi); 3414 3415 /* set MQD vmid to 0 */ 3416 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL, 3417 mqd->cp_mqd_control); 3418 3419 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3420 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE, 3421 mqd->cp_hqd_pq_base_lo); 3422 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI, 3423 mqd->cp_hqd_pq_base_hi); 3424 3425 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3426 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL, 3427 mqd->cp_hqd_pq_control); 3428 3429 /* set the wb address whether it's enabled or not */ 3430 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR, 3431 mqd->cp_hqd_pq_rptr_report_addr_lo); 3432 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI, 3433 mqd->cp_hqd_pq_rptr_report_addr_hi); 3434 3435 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3436 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR, 3437 mqd->cp_hqd_pq_wptr_poll_addr_lo); 3438 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI, 3439 mqd->cp_hqd_pq_wptr_poll_addr_hi); 3440 3441 /* enable the doorbell if requested */ 3442 if (ring->use_doorbell) { 3443 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER, 3444 (adev->doorbell_index.kiq * 2) << 2); 3445 /* If GC has entered CGPG, ringing doorbell > first page 3446 * doesn't wakeup GC. Enlarge CP_MEC_DOORBELL_RANGE_UPPER to 3447 * workaround this issue. And this change has to align with firmware 3448 * update. 3449 */ 3450 if (check_if_enlarge_doorbell_range(adev)) 3451 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER, 3452 (adev->doorbell.size - 4)); 3453 else 3454 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER, 3455 (adev->doorbell_index.userqueue_end * 2) << 2); 3456 } 3457 3458 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3459 mqd->cp_hqd_pq_doorbell_control); 3460 3461 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3462 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3463 mqd->cp_hqd_pq_wptr_lo); 3464 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3465 mqd->cp_hqd_pq_wptr_hi); 3466 3467 /* set the vmid for the queue */ 3468 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid); 3469 3470 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 3471 mqd->cp_hqd_persistent_state); 3472 3473 /* activate the queue */ 3474 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 3475 mqd->cp_hqd_active); 3476 3477 if (ring->use_doorbell) 3478 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1); 3479 3480 return 0; 3481 } 3482 3483 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring) 3484 { 3485 struct amdgpu_device *adev = ring->adev; 3486 int j; 3487 3488 /* disable the queue if it's active */ 3489 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3490 3491 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3492 3493 for (j = 0; j < adev->usec_timeout; j++) { 3494 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3495 break; 3496 udelay(1); 3497 } 3498 3499 if (j == AMDGPU_MAX_USEC_TIMEOUT) { 3500 DRM_DEBUG("KIQ dequeue request failed.\n"); 3501 3502 /* Manual disable if dequeue request times out */ 3503 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0); 3504 } 3505 3506 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3507 0); 3508 } 3509 3510 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0); 3511 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0); 3512 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0); 3513 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000); 3514 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0); 3515 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0); 3516 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0); 3517 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0); 3518 3519 return 0; 3520 } 3521 3522 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring) 3523 { 3524 struct amdgpu_device *adev = ring->adev; 3525 struct v9_mqd *mqd = ring->mqd_ptr; 3526 struct v9_mqd *tmp_mqd; 3527 3528 gfx_v9_0_kiq_setting(ring); 3529 3530 /* GPU could be in bad state during probe, driver trigger the reset 3531 * after load the SMU, in this case , the mqd is not be initialized. 3532 * driver need to re-init the mqd. 3533 * check mqd->cp_hqd_pq_control since this value should not be 0 3534 */ 3535 tmp_mqd = (struct v9_mqd *)adev->gfx.kiq[0].mqd_backup; 3536 if (amdgpu_in_reset(adev) && tmp_mqd->cp_hqd_pq_control){ 3537 /* for GPU_RESET case , reset MQD to a clean status */ 3538 if (adev->gfx.kiq[0].mqd_backup) 3539 memcpy(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(struct v9_mqd_allocation)); 3540 3541 /* reset ring buffer */ 3542 ring->wptr = 0; 3543 amdgpu_ring_clear_ring(ring); 3544 3545 mutex_lock(&adev->srbm_mutex); 3546 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0); 3547 gfx_v9_0_kiq_init_register(ring); 3548 soc15_grbm_select(adev, 0, 0, 0, 0, 0); 3549 mutex_unlock(&adev->srbm_mutex); 3550 } else { 3551 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3552 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3553 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3554 if (amdgpu_sriov_vf(adev) && adev->in_suspend) 3555 amdgpu_ring_clear_ring(ring); 3556 mutex_lock(&adev->srbm_mutex); 3557 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0); 3558 gfx_v9_0_mqd_init(ring); 3559 gfx_v9_0_kiq_init_register(ring); 3560 soc15_grbm_select(adev, 0, 0, 0, 0, 0); 3561 mutex_unlock(&adev->srbm_mutex); 3562 3563 if (adev->gfx.kiq[0].mqd_backup) 3564 memcpy(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(struct v9_mqd_allocation)); 3565 } 3566 3567 return 0; 3568 } 3569 3570 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring) 3571 { 3572 struct amdgpu_device *adev = ring->adev; 3573 struct v9_mqd *mqd = ring->mqd_ptr; 3574 int mqd_idx = ring - &adev->gfx.compute_ring[0]; 3575 struct v9_mqd *tmp_mqd; 3576 3577 /* Same as above kiq init, driver need to re-init the mqd if mqd->cp_hqd_pq_control 3578 * is not be initialized before 3579 */ 3580 tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx]; 3581 3582 if (!tmp_mqd->cp_hqd_pq_control || 3583 (!amdgpu_in_reset(adev) && !adev->in_suspend)) { 3584 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3585 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3586 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3587 mutex_lock(&adev->srbm_mutex); 3588 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0); 3589 gfx_v9_0_mqd_init(ring); 3590 soc15_grbm_select(adev, 0, 0, 0, 0, 0); 3591 mutex_unlock(&adev->srbm_mutex); 3592 3593 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3594 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); 3595 } else { 3596 /* restore MQD to a clean status */ 3597 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3598 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 3599 /* reset ring buffer */ 3600 ring->wptr = 0; 3601 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0); 3602 amdgpu_ring_clear_ring(ring); 3603 } 3604 3605 return 0; 3606 } 3607 3608 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev) 3609 { 3610 struct amdgpu_ring *ring; 3611 int r; 3612 3613 ring = &adev->gfx.kiq[0].ring; 3614 3615 r = amdgpu_bo_reserve(ring->mqd_obj, false); 3616 if (unlikely(r != 0)) 3617 return r; 3618 3619 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 3620 if (unlikely(r != 0)) { 3621 amdgpu_bo_unreserve(ring->mqd_obj); 3622 return r; 3623 } 3624 3625 gfx_v9_0_kiq_init_queue(ring); 3626 amdgpu_bo_kunmap(ring->mqd_obj); 3627 ring->mqd_ptr = NULL; 3628 amdgpu_bo_unreserve(ring->mqd_obj); 3629 return 0; 3630 } 3631 3632 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev) 3633 { 3634 struct amdgpu_ring *ring = NULL; 3635 int r = 0, i; 3636 3637 gfx_v9_0_cp_compute_enable(adev, true); 3638 3639 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3640 ring = &adev->gfx.compute_ring[i]; 3641 3642 r = amdgpu_bo_reserve(ring->mqd_obj, false); 3643 if (unlikely(r != 0)) 3644 goto done; 3645 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 3646 if (!r) { 3647 r = gfx_v9_0_kcq_init_queue(ring); 3648 amdgpu_bo_kunmap(ring->mqd_obj); 3649 ring->mqd_ptr = NULL; 3650 } 3651 amdgpu_bo_unreserve(ring->mqd_obj); 3652 if (r) 3653 goto done; 3654 } 3655 3656 r = amdgpu_gfx_enable_kcq(adev, 0); 3657 done: 3658 return r; 3659 } 3660 3661 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev) 3662 { 3663 int r, i; 3664 struct amdgpu_ring *ring; 3665 3666 if (!(adev->flags & AMD_IS_APU)) 3667 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 3668 3669 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 3670 if (adev->gfx.num_gfx_rings) { 3671 /* legacy firmware loading */ 3672 r = gfx_v9_0_cp_gfx_load_microcode(adev); 3673 if (r) 3674 return r; 3675 } 3676 3677 r = gfx_v9_0_cp_compute_load_microcode(adev); 3678 if (r) 3679 return r; 3680 } 3681 3682 r = gfx_v9_0_kiq_resume(adev); 3683 if (r) 3684 return r; 3685 3686 if (adev->gfx.num_gfx_rings) { 3687 r = gfx_v9_0_cp_gfx_resume(adev); 3688 if (r) 3689 return r; 3690 } 3691 3692 r = gfx_v9_0_kcq_resume(adev); 3693 if (r) 3694 return r; 3695 3696 if (adev->gfx.num_gfx_rings) { 3697 ring = &adev->gfx.gfx_ring[0]; 3698 r = amdgpu_ring_test_helper(ring); 3699 if (r) 3700 return r; 3701 } 3702 3703 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3704 ring = &adev->gfx.compute_ring[i]; 3705 amdgpu_ring_test_helper(ring); 3706 } 3707 3708 gfx_v9_0_enable_gui_idle_interrupt(adev, true); 3709 3710 return 0; 3711 } 3712 3713 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev) 3714 { 3715 u32 tmp; 3716 3717 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1) && 3718 amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2)) 3719 return; 3720 3721 tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG); 3722 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH, 3723 adev->df.hash_status.hash_64k); 3724 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH, 3725 adev->df.hash_status.hash_2m); 3726 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH, 3727 adev->df.hash_status.hash_1g); 3728 WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp); 3729 } 3730 3731 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable) 3732 { 3733 if (adev->gfx.num_gfx_rings) 3734 gfx_v9_0_cp_gfx_enable(adev, enable); 3735 gfx_v9_0_cp_compute_enable(adev, enable); 3736 } 3737 3738 static int gfx_v9_0_hw_init(void *handle) 3739 { 3740 int r; 3741 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3742 3743 if (!amdgpu_sriov_vf(adev)) 3744 gfx_v9_0_init_golden_registers(adev); 3745 3746 gfx_v9_0_constants_init(adev); 3747 3748 gfx_v9_0_init_tcp_config(adev); 3749 3750 r = adev->gfx.rlc.funcs->resume(adev); 3751 if (r) 3752 return r; 3753 3754 r = gfx_v9_0_cp_resume(adev); 3755 if (r) 3756 return r; 3757 3758 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2)) 3759 gfx_v9_4_2_set_power_brake_sequence(adev); 3760 3761 return r; 3762 } 3763 3764 static int gfx_v9_0_hw_fini(void *handle) 3765 { 3766 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3767 3768 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 3769 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); 3770 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 3771 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 3772 3773 /* DF freeze and kcq disable will fail */ 3774 if (!amdgpu_ras_intr_triggered()) 3775 /* disable KCQ to avoid CPC touch memory not valid anymore */ 3776 amdgpu_gfx_disable_kcq(adev, 0); 3777 3778 if (amdgpu_sriov_vf(adev)) { 3779 gfx_v9_0_cp_gfx_enable(adev, false); 3780 /* must disable polling for SRIOV when hw finished, otherwise 3781 * CPC engine may still keep fetching WB address which is already 3782 * invalid after sw finished and trigger DMAR reading error in 3783 * hypervisor side. 3784 */ 3785 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 3786 return 0; 3787 } 3788 3789 /* Use deinitialize sequence from CAIL when unbinding device from driver, 3790 * otherwise KIQ is hanging when binding back 3791 */ 3792 if (!amdgpu_in_reset(adev) && !adev->in_suspend) { 3793 mutex_lock(&adev->srbm_mutex); 3794 soc15_grbm_select(adev, adev->gfx.kiq[0].ring.me, 3795 adev->gfx.kiq[0].ring.pipe, 3796 adev->gfx.kiq[0].ring.queue, 0, 0); 3797 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq[0].ring); 3798 soc15_grbm_select(adev, 0, 0, 0, 0, 0); 3799 mutex_unlock(&adev->srbm_mutex); 3800 } 3801 3802 gfx_v9_0_cp_enable(adev, false); 3803 3804 /* Skip stopping RLC with A+A reset or when RLC controls GFX clock */ 3805 if ((adev->gmc.xgmi.connected_to_cpu && amdgpu_in_reset(adev)) || 3806 (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(9, 4, 2))) { 3807 dev_dbg(adev->dev, "Skipping RLC halt\n"); 3808 return 0; 3809 } 3810 3811 adev->gfx.rlc.funcs->stop(adev); 3812 return 0; 3813 } 3814 3815 static int gfx_v9_0_suspend(void *handle) 3816 { 3817 return gfx_v9_0_hw_fini(handle); 3818 } 3819 3820 static int gfx_v9_0_resume(void *handle) 3821 { 3822 return gfx_v9_0_hw_init(handle); 3823 } 3824 3825 static bool gfx_v9_0_is_idle(void *handle) 3826 { 3827 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3828 3829 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS), 3830 GRBM_STATUS, GUI_ACTIVE)) 3831 return false; 3832 else 3833 return true; 3834 } 3835 3836 static int gfx_v9_0_wait_for_idle(void *handle) 3837 { 3838 unsigned i; 3839 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3840 3841 for (i = 0; i < adev->usec_timeout; i++) { 3842 if (gfx_v9_0_is_idle(handle)) 3843 return 0; 3844 udelay(1); 3845 } 3846 return -ETIMEDOUT; 3847 } 3848 3849 static int gfx_v9_0_soft_reset(void *handle) 3850 { 3851 u32 grbm_soft_reset = 0; 3852 u32 tmp; 3853 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3854 3855 /* GRBM_STATUS */ 3856 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS); 3857 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK | 3858 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK | 3859 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK | 3860 GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK | 3861 GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK | 3862 GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) { 3863 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 3864 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 3865 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 3866 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1); 3867 } 3868 3869 if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) { 3870 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 3871 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 3872 } 3873 3874 /* GRBM_STATUS2 */ 3875 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2); 3876 if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY)) 3877 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 3878 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 3879 3880 3881 if (grbm_soft_reset) { 3882 /* stop the rlc */ 3883 adev->gfx.rlc.funcs->stop(adev); 3884 3885 if (adev->gfx.num_gfx_rings) 3886 /* Disable GFX parsing/prefetching */ 3887 gfx_v9_0_cp_gfx_enable(adev, false); 3888 3889 /* Disable MEC parsing/prefetching */ 3890 gfx_v9_0_cp_compute_enable(adev, false); 3891 3892 if (grbm_soft_reset) { 3893 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 3894 tmp |= grbm_soft_reset; 3895 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 3896 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 3897 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 3898 3899 udelay(50); 3900 3901 tmp &= ~grbm_soft_reset; 3902 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 3903 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 3904 } 3905 3906 /* Wait a little for things to settle down */ 3907 udelay(50); 3908 } 3909 return 0; 3910 } 3911 3912 static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev) 3913 { 3914 signed long r, cnt = 0; 3915 unsigned long flags; 3916 uint32_t seq, reg_val_offs = 0; 3917 uint64_t value = 0; 3918 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0]; 3919 struct amdgpu_ring *ring = &kiq->ring; 3920 3921 BUG_ON(!ring->funcs->emit_rreg); 3922 3923 spin_lock_irqsave(&kiq->ring_lock, flags); 3924 if (amdgpu_device_wb_get(adev, ®_val_offs)) { 3925 pr_err("critical bug! too many kiq readers\n"); 3926 goto failed_unlock; 3927 } 3928 amdgpu_ring_alloc(ring, 32); 3929 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 3930 amdgpu_ring_write(ring, 9 | /* src: register*/ 3931 (5 << 8) | /* dst: memory */ 3932 (1 << 16) | /* count sel */ 3933 (1 << 20)); /* write confirm */ 3934 amdgpu_ring_write(ring, 0); 3935 amdgpu_ring_write(ring, 0); 3936 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 3937 reg_val_offs * 4)); 3938 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 3939 reg_val_offs * 4)); 3940 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 3941 if (r) 3942 goto failed_undo; 3943 3944 amdgpu_ring_commit(ring); 3945 spin_unlock_irqrestore(&kiq->ring_lock, flags); 3946 3947 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 3948 3949 /* don't wait anymore for gpu reset case because this way may 3950 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg 3951 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will 3952 * never return if we keep waiting in virt_kiq_rreg, which cause 3953 * gpu_recover() hang there. 3954 * 3955 * also don't wait anymore for IRQ context 3956 * */ 3957 if (r < 1 && (amdgpu_in_reset(adev))) 3958 goto failed_kiq_read; 3959 3960 might_sleep(); 3961 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 3962 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 3963 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 3964 } 3965 3966 if (cnt > MAX_KIQ_REG_TRY) 3967 goto failed_kiq_read; 3968 3969 mb(); 3970 value = (uint64_t)adev->wb.wb[reg_val_offs] | 3971 (uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL; 3972 amdgpu_device_wb_free(adev, reg_val_offs); 3973 return value; 3974 3975 failed_undo: 3976 amdgpu_ring_undo(ring); 3977 failed_unlock: 3978 spin_unlock_irqrestore(&kiq->ring_lock, flags); 3979 failed_kiq_read: 3980 if (reg_val_offs) 3981 amdgpu_device_wb_free(adev, reg_val_offs); 3982 pr_err("failed to read gpu clock\n"); 3983 return ~0; 3984 } 3985 3986 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev) 3987 { 3988 uint64_t clock, clock_lo, clock_hi, hi_check; 3989 3990 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 3991 case IP_VERSION(9, 3, 0): 3992 preempt_disable(); 3993 clock_hi = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir); 3994 clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir); 3995 hi_check = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir); 3996 /* The SMUIO TSC clock frequency is 100MHz, which sets 32-bit carry over 3997 * roughly every 42 seconds. 3998 */ 3999 if (hi_check != clock_hi) { 4000 clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir); 4001 clock_hi = hi_check; 4002 } 4003 preempt_enable(); 4004 clock = clock_lo | (clock_hi << 32ULL); 4005 break; 4006 default: 4007 amdgpu_gfx_off_ctrl(adev, false); 4008 mutex_lock(&adev->gfx.gpu_clock_mutex); 4009 if (amdgpu_ip_version(adev, GC_HWIP, 0) == 4010 IP_VERSION(9, 0, 1) && 4011 amdgpu_sriov_runtime(adev)) { 4012 clock = gfx_v9_0_kiq_read_clock(adev); 4013 } else { 4014 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1); 4015 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) | 4016 ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 4017 } 4018 mutex_unlock(&adev->gfx.gpu_clock_mutex); 4019 amdgpu_gfx_off_ctrl(adev, true); 4020 break; 4021 } 4022 return clock; 4023 } 4024 4025 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring, 4026 uint32_t vmid, 4027 uint32_t gds_base, uint32_t gds_size, 4028 uint32_t gws_base, uint32_t gws_size, 4029 uint32_t oa_base, uint32_t oa_size) 4030 { 4031 struct amdgpu_device *adev = ring->adev; 4032 4033 /* GDS Base */ 4034 gfx_v9_0_write_data_to_reg(ring, 0, false, 4035 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid, 4036 gds_base); 4037 4038 /* GDS Size */ 4039 gfx_v9_0_write_data_to_reg(ring, 0, false, 4040 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid, 4041 gds_size); 4042 4043 /* GWS */ 4044 gfx_v9_0_write_data_to_reg(ring, 0, false, 4045 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid, 4046 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base); 4047 4048 /* OA */ 4049 gfx_v9_0_write_data_to_reg(ring, 0, false, 4050 SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid, 4051 (1 << (oa_size + oa_base)) - (1 << oa_base)); 4052 } 4053 4054 static const u32 vgpr_init_compute_shader[] = 4055 { 4056 0xb07c0000, 0xbe8000ff, 4057 0x000000f8, 0xbf110800, 4058 0x7e000280, 0x7e020280, 4059 0x7e040280, 0x7e060280, 4060 0x7e080280, 0x7e0a0280, 4061 0x7e0c0280, 0x7e0e0280, 4062 0x80808800, 0xbe803200, 4063 0xbf84fff5, 0xbf9c0000, 4064 0xd28c0001, 0x0001007f, 4065 0xd28d0001, 0x0002027e, 4066 0x10020288, 0xb8810904, 4067 0xb7814000, 0xd1196a01, 4068 0x00000301, 0xbe800087, 4069 0xbefc00c1, 0xd89c4000, 4070 0x00020201, 0xd89cc080, 4071 0x00040401, 0x320202ff, 4072 0x00000800, 0x80808100, 4073 0xbf84fff8, 0x7e020280, 4074 0xbf810000, 0x00000000, 4075 }; 4076 4077 static const u32 sgpr_init_compute_shader[] = 4078 { 4079 0xb07c0000, 0xbe8000ff, 4080 0x0000005f, 0xbee50080, 4081 0xbe812c65, 0xbe822c65, 4082 0xbe832c65, 0xbe842c65, 4083 0xbe852c65, 0xb77c0005, 4084 0x80808500, 0xbf84fff8, 4085 0xbe800080, 0xbf810000, 4086 }; 4087 4088 static const u32 vgpr_init_compute_shader_arcturus[] = { 4089 0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080, 4090 0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080, 4091 0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080, 4092 0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080, 4093 0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080, 4094 0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080, 4095 0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080, 4096 0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080, 4097 0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080, 4098 0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080, 4099 0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080, 4100 0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080, 4101 0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080, 4102 0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080, 4103 0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080, 4104 0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080, 4105 0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080, 4106 0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080, 4107 0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080, 4108 0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080, 4109 0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080, 4110 0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080, 4111 0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080, 4112 0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080, 4113 0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080, 4114 0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080, 4115 0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080, 4116 0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080, 4117 0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080, 4118 0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080, 4119 0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080, 4120 0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080, 4121 0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080, 4122 0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080, 4123 0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080, 4124 0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080, 4125 0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080, 4126 0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080, 4127 0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080, 4128 0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080, 4129 0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080, 4130 0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080, 4131 0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080, 4132 0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080, 4133 0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080, 4134 0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080, 4135 0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080, 4136 0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080, 4137 0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080, 4138 0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080, 4139 0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080, 4140 0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080, 4141 0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080, 4142 0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080, 4143 0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080, 4144 0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080, 4145 0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080, 4146 0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080, 4147 0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080, 4148 0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080, 4149 0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080, 4150 0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080, 4151 0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080, 4152 0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080, 4153 0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080, 4154 0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080, 4155 0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080, 4156 0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080, 4157 0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080, 4158 0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080, 4159 0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080, 4160 0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080, 4161 0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080, 4162 0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080, 4163 0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080, 4164 0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080, 4165 0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080, 4166 0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080, 4167 0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080, 4168 0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080, 4169 0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080, 4170 0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080, 4171 0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080, 4172 0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080, 4173 0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080, 4174 0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a, 4175 0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280, 4176 0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000, 4177 0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904, 4178 0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000, 4179 0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a, 4180 0xbf84fff8, 0xbf810000, 4181 }; 4182 4183 /* When below register arrays changed, please update gpr_reg_size, 4184 and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds, 4185 to cover all gfx9 ASICs */ 4186 static const struct soc15_reg_entry vgpr_init_regs[] = { 4187 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4188 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4189 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 }, 4190 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4191 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f }, 4192 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 4193 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 4194 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 4195 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 4196 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 4197 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, 4198 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, 4199 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, 4200 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, 4201 }; 4202 4203 static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = { 4204 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4205 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4206 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 }, 4207 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4208 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf }, 4209 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 4210 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 4211 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 4212 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 4213 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 4214 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, 4215 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, 4216 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, 4217 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, 4218 }; 4219 4220 static const struct soc15_reg_entry sgpr1_init_regs[] = { 4221 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4222 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4223 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 }, 4224 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4225 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */ 4226 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4227 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff }, 4228 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff }, 4229 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff }, 4230 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff }, 4231 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff }, 4232 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff }, 4233 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff }, 4234 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff }, 4235 }; 4236 4237 static const struct soc15_reg_entry sgpr2_init_regs[] = { 4238 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4239 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4240 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 }, 4241 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4242 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */ 4243 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4244 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 }, 4245 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 }, 4246 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 }, 4247 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 }, 4248 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 }, 4249 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 }, 4250 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 }, 4251 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 }, 4252 }; 4253 4254 static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = { 4255 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1}, 4256 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1}, 4257 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1}, 4258 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1}, 4259 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1}, 4260 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1}, 4261 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1}, 4262 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1}, 4263 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1}, 4264 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1}, 4265 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1}, 4266 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1}, 4267 { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1}, 4268 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6}, 4269 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16}, 4270 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16}, 4271 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16}, 4272 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16}, 4273 { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16}, 4274 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16}, 4275 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16}, 4276 { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16}, 4277 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6}, 4278 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16}, 4279 { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16}, 4280 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1}, 4281 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1}, 4282 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32}, 4283 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32}, 4284 { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72}, 4285 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16}, 4286 { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2}, 4287 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6}, 4288 }; 4289 4290 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev) 4291 { 4292 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4293 int i, r; 4294 4295 /* only support when RAS is enabled */ 4296 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4297 return 0; 4298 4299 r = amdgpu_ring_alloc(ring, 7); 4300 if (r) { 4301 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n", 4302 ring->name, r); 4303 return r; 4304 } 4305 4306 WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000); 4307 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size); 4308 4309 amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5)); 4310 amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC | 4311 PACKET3_DMA_DATA_DST_SEL(1) | 4312 PACKET3_DMA_DATA_SRC_SEL(2) | 4313 PACKET3_DMA_DATA_ENGINE(0))); 4314 amdgpu_ring_write(ring, 0); 4315 amdgpu_ring_write(ring, 0); 4316 amdgpu_ring_write(ring, 0); 4317 amdgpu_ring_write(ring, 0); 4318 amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT | 4319 adev->gds.gds_size); 4320 4321 amdgpu_ring_commit(ring); 4322 4323 for (i = 0; i < adev->usec_timeout; i++) { 4324 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring)) 4325 break; 4326 udelay(1); 4327 } 4328 4329 if (i >= adev->usec_timeout) 4330 r = -ETIMEDOUT; 4331 4332 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000); 4333 4334 return r; 4335 } 4336 4337 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev) 4338 { 4339 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4340 struct amdgpu_ib ib; 4341 struct dma_fence *f = NULL; 4342 int r, i; 4343 unsigned total_size, vgpr_offset, sgpr_offset; 4344 u64 gpu_addr; 4345 4346 int compute_dim_x = adev->gfx.config.max_shader_engines * 4347 adev->gfx.config.max_cu_per_sh * 4348 adev->gfx.config.max_sh_per_se; 4349 int sgpr_work_group_size = 5; 4350 int gpr_reg_size = adev->gfx.config.max_shader_engines + 6; 4351 int vgpr_init_shader_size; 4352 const u32 *vgpr_init_shader_ptr; 4353 const struct soc15_reg_entry *vgpr_init_regs_ptr; 4354 4355 /* only support when RAS is enabled */ 4356 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4357 return 0; 4358 4359 /* bail if the compute ring is not ready */ 4360 if (!ring->sched.ready) 4361 return 0; 4362 4363 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1)) { 4364 vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus; 4365 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus); 4366 vgpr_init_regs_ptr = vgpr_init_regs_arcturus; 4367 } else { 4368 vgpr_init_shader_ptr = vgpr_init_compute_shader; 4369 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader); 4370 vgpr_init_regs_ptr = vgpr_init_regs; 4371 } 4372 4373 total_size = 4374 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */ 4375 total_size += 4376 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */ 4377 total_size += 4378 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */ 4379 total_size = ALIGN(total_size, 256); 4380 vgpr_offset = total_size; 4381 total_size += ALIGN(vgpr_init_shader_size, 256); 4382 sgpr_offset = total_size; 4383 total_size += sizeof(sgpr_init_compute_shader); 4384 4385 /* allocate an indirect buffer to put the commands in */ 4386 memset(&ib, 0, sizeof(ib)); 4387 r = amdgpu_ib_get(adev, NULL, total_size, 4388 AMDGPU_IB_POOL_DIRECT, &ib); 4389 if (r) { 4390 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r); 4391 return r; 4392 } 4393 4394 /* load the compute shaders */ 4395 for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++) 4396 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i]; 4397 4398 for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++) 4399 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i]; 4400 4401 /* init the ib length to 0 */ 4402 ib.length_dw = 0; 4403 4404 /* VGPR */ 4405 /* write the register state for the compute dispatch */ 4406 for (i = 0; i < gpr_reg_size; i++) { 4407 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4408 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i]) 4409 - PACKET3_SET_SH_REG_START; 4410 ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value; 4411 } 4412 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4413 gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8; 4414 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4415 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4416 - PACKET3_SET_SH_REG_START; 4417 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4418 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4419 4420 /* write dispatch packet */ 4421 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4422 ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */ 4423 ib.ptr[ib.length_dw++] = 1; /* y */ 4424 ib.ptr[ib.length_dw++] = 1; /* z */ 4425 ib.ptr[ib.length_dw++] = 4426 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4427 4428 /* write CS partial flush packet */ 4429 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4430 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4431 4432 /* SGPR1 */ 4433 /* write the register state for the compute dispatch */ 4434 for (i = 0; i < gpr_reg_size; i++) { 4435 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4436 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i]) 4437 - PACKET3_SET_SH_REG_START; 4438 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value; 4439 } 4440 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4441 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4442 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4443 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4444 - PACKET3_SET_SH_REG_START; 4445 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4446 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4447 4448 /* write dispatch packet */ 4449 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4450 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ 4451 ib.ptr[ib.length_dw++] = 1; /* y */ 4452 ib.ptr[ib.length_dw++] = 1; /* z */ 4453 ib.ptr[ib.length_dw++] = 4454 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4455 4456 /* write CS partial flush packet */ 4457 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4458 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4459 4460 /* SGPR2 */ 4461 /* write the register state for the compute dispatch */ 4462 for (i = 0; i < gpr_reg_size; i++) { 4463 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4464 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i]) 4465 - PACKET3_SET_SH_REG_START; 4466 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value; 4467 } 4468 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4469 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4470 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4471 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4472 - PACKET3_SET_SH_REG_START; 4473 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4474 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4475 4476 /* write dispatch packet */ 4477 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4478 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ 4479 ib.ptr[ib.length_dw++] = 1; /* y */ 4480 ib.ptr[ib.length_dw++] = 1; /* z */ 4481 ib.ptr[ib.length_dw++] = 4482 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4483 4484 /* write CS partial flush packet */ 4485 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4486 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4487 4488 /* shedule the ib on the ring */ 4489 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 4490 if (r) { 4491 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r); 4492 goto fail; 4493 } 4494 4495 /* wait for the GPU to finish processing the IB */ 4496 r = dma_fence_wait(f, false); 4497 if (r) { 4498 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); 4499 goto fail; 4500 } 4501 4502 fail: 4503 amdgpu_ib_free(adev, &ib, NULL); 4504 dma_fence_put(f); 4505 4506 return r; 4507 } 4508 4509 static int gfx_v9_0_early_init(void *handle) 4510 { 4511 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4512 4513 adev->gfx.funcs = &gfx_v9_0_gfx_funcs; 4514 4515 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) || 4516 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2)) 4517 adev->gfx.num_gfx_rings = 0; 4518 else 4519 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS; 4520 adev->gfx.xcc_mask = 1; 4521 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev), 4522 AMDGPU_MAX_COMPUTE_RINGS); 4523 gfx_v9_0_set_kiq_pm4_funcs(adev); 4524 gfx_v9_0_set_ring_funcs(adev); 4525 gfx_v9_0_set_irq_funcs(adev); 4526 gfx_v9_0_set_gds_init(adev); 4527 gfx_v9_0_set_rlc_funcs(adev); 4528 4529 /* init rlcg reg access ctrl */ 4530 gfx_v9_0_init_rlcg_reg_access_ctrl(adev); 4531 4532 return gfx_v9_0_init_microcode(adev); 4533 } 4534 4535 static int gfx_v9_0_ecc_late_init(void *handle) 4536 { 4537 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4538 int r; 4539 4540 /* 4541 * Temp workaround to fix the issue that CP firmware fails to 4542 * update read pointer when CPDMA is writing clearing operation 4543 * to GDS in suspend/resume sequence on several cards. So just 4544 * limit this operation in cold boot sequence. 4545 */ 4546 if ((!adev->in_suspend) && 4547 (adev->gds.gds_size)) { 4548 r = gfx_v9_0_do_edc_gds_workarounds(adev); 4549 if (r) 4550 return r; 4551 } 4552 4553 /* requires IBs so do in late init after IB pool is initialized */ 4554 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2)) 4555 r = gfx_v9_4_2_do_edc_gpr_workarounds(adev); 4556 else 4557 r = gfx_v9_0_do_edc_gpr_workarounds(adev); 4558 4559 if (r) 4560 return r; 4561 4562 if (adev->gfx.ras && 4563 adev->gfx.ras->enable_watchdog_timer) 4564 adev->gfx.ras->enable_watchdog_timer(adev); 4565 4566 return 0; 4567 } 4568 4569 static int gfx_v9_0_late_init(void *handle) 4570 { 4571 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4572 int r; 4573 4574 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); 4575 if (r) 4576 return r; 4577 4578 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); 4579 if (r) 4580 return r; 4581 4582 r = gfx_v9_0_ecc_late_init(handle); 4583 if (r) 4584 return r; 4585 4586 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2)) 4587 gfx_v9_4_2_debug_trap_config_init(adev, 4588 adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID); 4589 else 4590 gfx_v9_0_debug_trap_config_init(adev, 4591 adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID); 4592 4593 return 0; 4594 } 4595 4596 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev) 4597 { 4598 uint32_t rlc_setting; 4599 4600 /* if RLC is not enabled, do nothing */ 4601 rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL); 4602 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK)) 4603 return false; 4604 4605 return true; 4606 } 4607 4608 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id) 4609 { 4610 uint32_t data; 4611 unsigned i; 4612 4613 data = RLC_SAFE_MODE__CMD_MASK; 4614 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT); 4615 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4616 4617 /* wait for RLC_SAFE_MODE */ 4618 for (i = 0; i < adev->usec_timeout; i++) { 4619 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD)) 4620 break; 4621 udelay(1); 4622 } 4623 } 4624 4625 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id) 4626 { 4627 uint32_t data; 4628 4629 data = RLC_SAFE_MODE__CMD_MASK; 4630 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4631 } 4632 4633 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev, 4634 bool enable) 4635 { 4636 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 4637 4638 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) { 4639 gfx_v9_0_enable_gfx_cg_power_gating(adev, true); 4640 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4641 gfx_v9_0_enable_gfx_pipeline_powergating(adev, true); 4642 } else { 4643 gfx_v9_0_enable_gfx_cg_power_gating(adev, false); 4644 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4645 gfx_v9_0_enable_gfx_pipeline_powergating(adev, false); 4646 } 4647 4648 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 4649 } 4650 4651 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev, 4652 bool enable) 4653 { 4654 /* TODO: double check if we need to perform under safe mode */ 4655 /* gfx_v9_0_enter_rlc_safe_mode(adev); */ 4656 4657 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable) 4658 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true); 4659 else 4660 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false); 4661 4662 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable) 4663 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true); 4664 else 4665 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false); 4666 4667 /* gfx_v9_0_exit_rlc_safe_mode(adev); */ 4668 } 4669 4670 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, 4671 bool enable) 4672 { 4673 uint32_t data, def; 4674 4675 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 4676 4677 /* It is disabled by HW by default */ 4678 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) { 4679 /* 1 - RLC_CGTT_MGCG_OVERRIDE */ 4680 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4681 4682 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 2, 1)) 4683 data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 4684 4685 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 4686 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 4687 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 4688 4689 /* only for Vega10 & Raven1 */ 4690 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK; 4691 4692 if (def != data) 4693 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4694 4695 /* MGLS is a global flag to control all MGLS in GFX */ 4696 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { 4697 /* 2 - RLC memory Light sleep */ 4698 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) { 4699 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4700 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 4701 if (def != data) 4702 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 4703 } 4704 /* 3 - CP memory Light sleep */ 4705 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) { 4706 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 4707 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 4708 if (def != data) 4709 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 4710 } 4711 } 4712 } else { 4713 /* 1 - MGCG_OVERRIDE */ 4714 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4715 4716 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 2, 1)) 4717 data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 4718 4719 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK | 4720 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 4721 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 4722 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 4723 4724 if (def != data) 4725 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4726 4727 /* 2 - disable MGLS in RLC */ 4728 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4729 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) { 4730 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 4731 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 4732 } 4733 4734 /* 3 - disable MGLS in CP */ 4735 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 4736 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) { 4737 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 4738 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 4739 } 4740 } 4741 4742 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 4743 } 4744 4745 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev, 4746 bool enable) 4747 { 4748 uint32_t data, def; 4749 4750 if (!adev->gfx.num_gfx_rings) 4751 return; 4752 4753 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 4754 4755 /* Enable 3D CGCG/CGLS */ 4756 if (enable) { 4757 /* write cmd to clear cgcg/cgls ov */ 4758 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4759 /* unset CGCG override */ 4760 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK; 4761 /* update CGCG and CGLS override bits */ 4762 if (def != data) 4763 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4764 4765 /* enable 3Dcgcg FSM(0x0000363f) */ 4766 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 4767 4768 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) 4769 data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4770 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK; 4771 else 4772 data = 0x0 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT; 4773 4774 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) 4775 data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 4776 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK; 4777 if (def != data) 4778 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 4779 4780 /* set IDLE_POLL_COUNT(0x00900100) */ 4781 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 4782 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 4783 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 4784 if (def != data) 4785 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 4786 } else { 4787 /* Disable CGCG/CGLS */ 4788 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 4789 /* disable cgcg, cgls should be disabled */ 4790 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK | 4791 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK); 4792 /* disable cgcg and cgls in FSM */ 4793 if (def != data) 4794 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 4795 } 4796 4797 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 4798 } 4799 4800 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev, 4801 bool enable) 4802 { 4803 uint32_t def, data; 4804 4805 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 4806 4807 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) { 4808 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4809 /* unset CGCG override */ 4810 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK; 4811 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 4812 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 4813 else 4814 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 4815 /* update CGCG and CGLS override bits */ 4816 if (def != data) 4817 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4818 4819 /* enable cgcg FSM(0x0000363F) */ 4820 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 4821 4822 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1)) 4823 data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4824 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 4825 else 4826 data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4827 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 4828 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 4829 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 4830 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 4831 if (def != data) 4832 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 4833 4834 /* set IDLE_POLL_COUNT(0x00900100) */ 4835 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 4836 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 4837 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 4838 if (def != data) 4839 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 4840 } else { 4841 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 4842 /* reset CGCG/CGLS bits */ 4843 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK); 4844 /* disable cgcg and cgls in FSM */ 4845 if (def != data) 4846 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 4847 } 4848 4849 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 4850 } 4851 4852 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev, 4853 bool enable) 4854 { 4855 if (enable) { 4856 /* CGCG/CGLS should be enabled after MGCG/MGLS 4857 * === MGCG + MGLS === 4858 */ 4859 gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 4860 /* === CGCG /CGLS for GFX 3D Only === */ 4861 gfx_v9_0_update_3d_clock_gating(adev, enable); 4862 /* === CGCG + CGLS === */ 4863 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 4864 } else { 4865 /* CGCG/CGLS should be disabled before MGCG/MGLS 4866 * === CGCG + CGLS === 4867 */ 4868 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 4869 /* === CGCG /CGLS for GFX 3D Only === */ 4870 gfx_v9_0_update_3d_clock_gating(adev, enable); 4871 /* === MGCG + MGLS === */ 4872 gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 4873 } 4874 return 0; 4875 } 4876 4877 static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev, 4878 unsigned int vmid) 4879 { 4880 u32 reg, data; 4881 4882 reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL); 4883 if (amdgpu_sriov_is_pp_one_vf(adev)) 4884 data = RREG32_NO_KIQ(reg); 4885 else 4886 data = RREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL); 4887 4888 data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK; 4889 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT; 4890 4891 if (amdgpu_sriov_is_pp_one_vf(adev)) 4892 WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data); 4893 else 4894 WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data); 4895 } 4896 4897 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, unsigned int vmid) 4898 { 4899 amdgpu_gfx_off_ctrl(adev, false); 4900 4901 gfx_v9_0_update_spm_vmid_internal(adev, vmid); 4902 4903 amdgpu_gfx_off_ctrl(adev, true); 4904 } 4905 4906 static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev, 4907 uint32_t offset, 4908 struct soc15_reg_rlcg *entries, int arr_size) 4909 { 4910 int i; 4911 uint32_t reg; 4912 4913 if (!entries) 4914 return false; 4915 4916 for (i = 0; i < arr_size; i++) { 4917 const struct soc15_reg_rlcg *entry; 4918 4919 entry = &entries[i]; 4920 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg; 4921 if (offset == reg) 4922 return true; 4923 } 4924 4925 return false; 4926 } 4927 4928 static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset) 4929 { 4930 return gfx_v9_0_check_rlcg_range(adev, offset, 4931 (void *)rlcg_access_gc_9_0, 4932 ARRAY_SIZE(rlcg_access_gc_9_0)); 4933 } 4934 4935 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = { 4936 .is_rlc_enabled = gfx_v9_0_is_rlc_enabled, 4937 .set_safe_mode = gfx_v9_0_set_safe_mode, 4938 .unset_safe_mode = gfx_v9_0_unset_safe_mode, 4939 .init = gfx_v9_0_rlc_init, 4940 .get_csb_size = gfx_v9_0_get_csb_size, 4941 .get_csb_buffer = gfx_v9_0_get_csb_buffer, 4942 .get_cp_table_num = gfx_v9_0_cp_jump_table_num, 4943 .resume = gfx_v9_0_rlc_resume, 4944 .stop = gfx_v9_0_rlc_stop, 4945 .reset = gfx_v9_0_rlc_reset, 4946 .start = gfx_v9_0_rlc_start, 4947 .update_spm_vmid = gfx_v9_0_update_spm_vmid, 4948 .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range, 4949 }; 4950 4951 static int gfx_v9_0_set_powergating_state(void *handle, 4952 enum amd_powergating_state state) 4953 { 4954 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4955 bool enable = (state == AMD_PG_STATE_GATE); 4956 4957 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 4958 case IP_VERSION(9, 2, 2): 4959 case IP_VERSION(9, 1, 0): 4960 case IP_VERSION(9, 3, 0): 4961 if (!enable) 4962 amdgpu_gfx_off_ctrl(adev, false); 4963 4964 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) { 4965 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true); 4966 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true); 4967 } else { 4968 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false); 4969 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false); 4970 } 4971 4972 if (adev->pg_flags & AMD_PG_SUPPORT_CP) 4973 gfx_v9_0_enable_cp_power_gating(adev, true); 4974 else 4975 gfx_v9_0_enable_cp_power_gating(adev, false); 4976 4977 /* update gfx cgpg state */ 4978 gfx_v9_0_update_gfx_cg_power_gating(adev, enable); 4979 4980 /* update mgcg state */ 4981 gfx_v9_0_update_gfx_mg_power_gating(adev, enable); 4982 4983 if (enable) 4984 amdgpu_gfx_off_ctrl(adev, true); 4985 break; 4986 case IP_VERSION(9, 2, 1): 4987 amdgpu_gfx_off_ctrl(adev, enable); 4988 break; 4989 default: 4990 break; 4991 } 4992 4993 return 0; 4994 } 4995 4996 static int gfx_v9_0_set_clockgating_state(void *handle, 4997 enum amd_clockgating_state state) 4998 { 4999 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5000 5001 if (amdgpu_sriov_vf(adev)) 5002 return 0; 5003 5004 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 5005 case IP_VERSION(9, 0, 1): 5006 case IP_VERSION(9, 2, 1): 5007 case IP_VERSION(9, 4, 0): 5008 case IP_VERSION(9, 2, 2): 5009 case IP_VERSION(9, 1, 0): 5010 case IP_VERSION(9, 4, 1): 5011 case IP_VERSION(9, 3, 0): 5012 case IP_VERSION(9, 4, 2): 5013 gfx_v9_0_update_gfx_clock_gating(adev, 5014 state == AMD_CG_STATE_GATE); 5015 break; 5016 default: 5017 break; 5018 } 5019 return 0; 5020 } 5021 5022 static void gfx_v9_0_get_clockgating_state(void *handle, u64 *flags) 5023 { 5024 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5025 int data; 5026 5027 if (amdgpu_sriov_vf(adev)) 5028 *flags = 0; 5029 5030 /* AMD_CG_SUPPORT_GFX_MGCG */ 5031 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE)); 5032 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK)) 5033 *flags |= AMD_CG_SUPPORT_GFX_MGCG; 5034 5035 /* AMD_CG_SUPPORT_GFX_CGCG */ 5036 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL)); 5037 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK) 5038 *flags |= AMD_CG_SUPPORT_GFX_CGCG; 5039 5040 /* AMD_CG_SUPPORT_GFX_CGLS */ 5041 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK) 5042 *flags |= AMD_CG_SUPPORT_GFX_CGLS; 5043 5044 /* AMD_CG_SUPPORT_GFX_RLC_LS */ 5045 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL)); 5046 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) 5047 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS; 5048 5049 /* AMD_CG_SUPPORT_GFX_CP_LS */ 5050 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL)); 5051 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) 5052 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS; 5053 5054 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) { 5055 /* AMD_CG_SUPPORT_GFX_3D_CGCG */ 5056 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D)); 5057 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK) 5058 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG; 5059 5060 /* AMD_CG_SUPPORT_GFX_3D_CGLS */ 5061 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK) 5062 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS; 5063 } 5064 } 5065 5066 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring) 5067 { 5068 return *ring->rptr_cpu_addr; /* gfx9 is 32bit rptr*/ 5069 } 5070 5071 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring) 5072 { 5073 struct amdgpu_device *adev = ring->adev; 5074 u64 wptr; 5075 5076 /* XXX check if swapping is necessary on BE */ 5077 if (ring->use_doorbell) { 5078 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr); 5079 } else { 5080 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR); 5081 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32; 5082 } 5083 5084 return wptr; 5085 } 5086 5087 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring) 5088 { 5089 struct amdgpu_device *adev = ring->adev; 5090 5091 if (ring->use_doorbell) { 5092 /* XXX check if swapping is necessary on BE */ 5093 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, ring->wptr); 5094 WDOORBELL64(ring->doorbell_index, ring->wptr); 5095 } else { 5096 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 5097 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 5098 } 5099 } 5100 5101 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) 5102 { 5103 struct amdgpu_device *adev = ring->adev; 5104 u32 ref_and_mask, reg_mem_engine; 5105 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg; 5106 5107 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 5108 switch (ring->me) { 5109 case 1: 5110 ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe; 5111 break; 5112 case 2: 5113 ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe; 5114 break; 5115 default: 5116 return; 5117 } 5118 reg_mem_engine = 0; 5119 } else { 5120 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0; 5121 reg_mem_engine = 1; /* pfp */ 5122 } 5123 5124 gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1, 5125 adev->nbio.funcs->get_hdp_flush_req_offset(adev), 5126 adev->nbio.funcs->get_hdp_flush_done_offset(adev), 5127 ref_and_mask, ref_and_mask, 0x20); 5128 } 5129 5130 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, 5131 struct amdgpu_job *job, 5132 struct amdgpu_ib *ib, 5133 uint32_t flags) 5134 { 5135 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5136 u32 header, control = 0; 5137 5138 if (ib->flags & AMDGPU_IB_FLAG_CE) 5139 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 5140 else 5141 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 5142 5143 control |= ib->length_dw | (vmid << 24); 5144 5145 if (ib->flags & AMDGPU_IB_FLAG_PREEMPT) { 5146 control |= INDIRECT_BUFFER_PRE_ENB(1); 5147 5148 if (flags & AMDGPU_IB_PREEMPTED) 5149 control |= INDIRECT_BUFFER_PRE_RESUME(1); 5150 5151 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid) 5152 gfx_v9_0_ring_emit_de_meta(ring, 5153 (!amdgpu_sriov_vf(ring->adev) && 5154 flags & AMDGPU_IB_PREEMPTED) ? 5155 true : false, 5156 job->gds_size > 0 && job->gds_base != 0); 5157 } 5158 5159 amdgpu_ring_write(ring, header); 5160 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 5161 amdgpu_ring_write(ring, 5162 #ifdef __BIG_ENDIAN 5163 (2 << 0) | 5164 #endif 5165 lower_32_bits(ib->gpu_addr)); 5166 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5167 amdgpu_ring_ib_on_emit_cntl(ring); 5168 amdgpu_ring_write(ring, control); 5169 } 5170 5171 static void gfx_v9_0_ring_patch_cntl(struct amdgpu_ring *ring, 5172 unsigned offset) 5173 { 5174 u32 control = ring->ring[offset]; 5175 5176 control |= INDIRECT_BUFFER_PRE_RESUME(1); 5177 ring->ring[offset] = control; 5178 } 5179 5180 static void gfx_v9_0_ring_patch_ce_meta(struct amdgpu_ring *ring, 5181 unsigned offset) 5182 { 5183 struct amdgpu_device *adev = ring->adev; 5184 void *ce_payload_cpu_addr; 5185 uint64_t payload_offset, payload_size; 5186 5187 payload_size = sizeof(struct v9_ce_ib_state); 5188 5189 if (ring->is_mes_queue) { 5190 payload_offset = offsetof(struct amdgpu_mes_ctx_meta_data, 5191 gfx[0].gfx_meta_data) + 5192 offsetof(struct v9_gfx_meta_data, ce_payload); 5193 ce_payload_cpu_addr = 5194 amdgpu_mes_ctx_get_offs_cpu_addr(ring, payload_offset); 5195 } else { 5196 payload_offset = offsetof(struct v9_gfx_meta_data, ce_payload); 5197 ce_payload_cpu_addr = adev->virt.csa_cpu_addr + payload_offset; 5198 } 5199 5200 if (offset + (payload_size >> 2) <= ring->buf_mask + 1) { 5201 memcpy((void *)&ring->ring[offset], ce_payload_cpu_addr, payload_size); 5202 } else { 5203 memcpy((void *)&ring->ring[offset], ce_payload_cpu_addr, 5204 (ring->buf_mask + 1 - offset) << 2); 5205 payload_size -= (ring->buf_mask + 1 - offset) << 2; 5206 memcpy((void *)&ring->ring[0], 5207 ce_payload_cpu_addr + ((ring->buf_mask + 1 - offset) << 2), 5208 payload_size); 5209 } 5210 } 5211 5212 static void gfx_v9_0_ring_patch_de_meta(struct amdgpu_ring *ring, 5213 unsigned offset) 5214 { 5215 struct amdgpu_device *adev = ring->adev; 5216 void *de_payload_cpu_addr; 5217 uint64_t payload_offset, payload_size; 5218 5219 payload_size = sizeof(struct v9_de_ib_state); 5220 5221 if (ring->is_mes_queue) { 5222 payload_offset = offsetof(struct amdgpu_mes_ctx_meta_data, 5223 gfx[0].gfx_meta_data) + 5224 offsetof(struct v9_gfx_meta_data, de_payload); 5225 de_payload_cpu_addr = 5226 amdgpu_mes_ctx_get_offs_cpu_addr(ring, payload_offset); 5227 } else { 5228 payload_offset = offsetof(struct v9_gfx_meta_data, de_payload); 5229 de_payload_cpu_addr = adev->virt.csa_cpu_addr + payload_offset; 5230 } 5231 5232 ((struct v9_de_ib_state *)de_payload_cpu_addr)->ib_completion_status = 5233 IB_COMPLETION_STATUS_PREEMPTED; 5234 5235 if (offset + (payload_size >> 2) <= ring->buf_mask + 1) { 5236 memcpy((void *)&ring->ring[offset], de_payload_cpu_addr, payload_size); 5237 } else { 5238 memcpy((void *)&ring->ring[offset], de_payload_cpu_addr, 5239 (ring->buf_mask + 1 - offset) << 2); 5240 payload_size -= (ring->buf_mask + 1 - offset) << 2; 5241 memcpy((void *)&ring->ring[0], 5242 de_payload_cpu_addr + ((ring->buf_mask + 1 - offset) << 2), 5243 payload_size); 5244 } 5245 } 5246 5247 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring, 5248 struct amdgpu_job *job, 5249 struct amdgpu_ib *ib, 5250 uint32_t flags) 5251 { 5252 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5253 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); 5254 5255 /* Currently, there is a high possibility to get wave ID mismatch 5256 * between ME and GDS, leading to a hw deadlock, because ME generates 5257 * different wave IDs than the GDS expects. This situation happens 5258 * randomly when at least 5 compute pipes use GDS ordered append. 5259 * The wave IDs generated by ME are also wrong after suspend/resume. 5260 * Those are probably bugs somewhere else in the kernel driver. 5261 * 5262 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and 5263 * GDS to 0 for this ring (me/pipe). 5264 */ 5265 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { 5266 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 5267 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID); 5268 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); 5269 } 5270 5271 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 5272 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 5273 amdgpu_ring_write(ring, 5274 #ifdef __BIG_ENDIAN 5275 (2 << 0) | 5276 #endif 5277 lower_32_bits(ib->gpu_addr)); 5278 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5279 amdgpu_ring_write(ring, control); 5280 } 5281 5282 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, 5283 u64 seq, unsigned flags) 5284 { 5285 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 5286 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 5287 bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY; 5288 bool exec = flags & AMDGPU_FENCE_FLAG_EXEC; 5289 uint32_t dw2 = 0; 5290 5291 /* RELEASE_MEM - flush caches, send int */ 5292 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6)); 5293 5294 if (writeback) { 5295 dw2 = EOP_TC_NC_ACTION_EN; 5296 } else { 5297 dw2 = EOP_TCL1_ACTION_EN | EOP_TC_ACTION_EN | 5298 EOP_TC_MD_ACTION_EN; 5299 } 5300 dw2 |= EOP_TC_WB_ACTION_EN | EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 5301 EVENT_INDEX(5); 5302 if (exec) 5303 dw2 |= EOP_EXEC; 5304 5305 amdgpu_ring_write(ring, dw2); 5306 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0)); 5307 5308 /* 5309 * the address should be Qword aligned if 64bit write, Dword 5310 * aligned if only send 32bit data low (discard data high) 5311 */ 5312 if (write64bit) 5313 BUG_ON(addr & 0x7); 5314 else 5315 BUG_ON(addr & 0x3); 5316 amdgpu_ring_write(ring, lower_32_bits(addr)); 5317 amdgpu_ring_write(ring, upper_32_bits(addr)); 5318 amdgpu_ring_write(ring, lower_32_bits(seq)); 5319 amdgpu_ring_write(ring, upper_32_bits(seq)); 5320 amdgpu_ring_write(ring, 0); 5321 } 5322 5323 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) 5324 { 5325 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5326 uint32_t seq = ring->fence_drv.sync_seq; 5327 uint64_t addr = ring->fence_drv.gpu_addr; 5328 5329 gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0, 5330 lower_32_bits(addr), upper_32_bits(addr), 5331 seq, 0xffffffff, 4); 5332 } 5333 5334 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 5335 unsigned vmid, uint64_t pd_addr) 5336 { 5337 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 5338 5339 /* compute doesn't have PFP */ 5340 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) { 5341 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 5342 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 5343 amdgpu_ring_write(ring, 0x0); 5344 } 5345 } 5346 5347 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring) 5348 { 5349 return *ring->rptr_cpu_addr; /* gfx9 hardware is 32bit rptr */ 5350 } 5351 5352 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring) 5353 { 5354 u64 wptr; 5355 5356 /* XXX check if swapping is necessary on BE */ 5357 if (ring->use_doorbell) 5358 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr); 5359 else 5360 BUG(); 5361 return wptr; 5362 } 5363 5364 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring) 5365 { 5366 struct amdgpu_device *adev = ring->adev; 5367 5368 /* XXX check if swapping is necessary on BE */ 5369 if (ring->use_doorbell) { 5370 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, ring->wptr); 5371 WDOORBELL64(ring->doorbell_index, ring->wptr); 5372 } else{ 5373 BUG(); /* only DOORBELL method supported on gfx9 now */ 5374 } 5375 } 5376 5377 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, 5378 u64 seq, unsigned int flags) 5379 { 5380 struct amdgpu_device *adev = ring->adev; 5381 5382 /* we only allocate 32bit for each seq wb address */ 5383 BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); 5384 5385 /* write fence seq to the "addr" */ 5386 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5387 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5388 WRITE_DATA_DST_SEL(5) | WR_CONFIRM)); 5389 amdgpu_ring_write(ring, lower_32_bits(addr)); 5390 amdgpu_ring_write(ring, upper_32_bits(addr)); 5391 amdgpu_ring_write(ring, lower_32_bits(seq)); 5392 5393 if (flags & AMDGPU_FENCE_FLAG_INT) { 5394 /* set register to trigger INT */ 5395 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5396 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5397 WRITE_DATA_DST_SEL(0) | WR_CONFIRM)); 5398 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS)); 5399 amdgpu_ring_write(ring, 0); 5400 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */ 5401 } 5402 } 5403 5404 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring) 5405 { 5406 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 5407 amdgpu_ring_write(ring, 0); 5408 } 5409 5410 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring, bool resume) 5411 { 5412 struct amdgpu_device *adev = ring->adev; 5413 struct v9_ce_ib_state ce_payload = {0}; 5414 uint64_t offset, ce_payload_gpu_addr; 5415 void *ce_payload_cpu_addr; 5416 int cnt; 5417 5418 cnt = (sizeof(ce_payload) >> 2) + 4 - 2; 5419 5420 if (ring->is_mes_queue) { 5421 offset = offsetof(struct amdgpu_mes_ctx_meta_data, 5422 gfx[0].gfx_meta_data) + 5423 offsetof(struct v9_gfx_meta_data, ce_payload); 5424 ce_payload_gpu_addr = 5425 amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset); 5426 ce_payload_cpu_addr = 5427 amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset); 5428 } else { 5429 offset = offsetof(struct v9_gfx_meta_data, ce_payload); 5430 ce_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset; 5431 ce_payload_cpu_addr = adev->virt.csa_cpu_addr + offset; 5432 } 5433 5434 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5435 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) | 5436 WRITE_DATA_DST_SEL(8) | 5437 WR_CONFIRM) | 5438 WRITE_DATA_CACHE_POLICY(0)); 5439 amdgpu_ring_write(ring, lower_32_bits(ce_payload_gpu_addr)); 5440 amdgpu_ring_write(ring, upper_32_bits(ce_payload_gpu_addr)); 5441 5442 amdgpu_ring_ib_on_emit_ce(ring); 5443 5444 if (resume) 5445 amdgpu_ring_write_multiple(ring, ce_payload_cpu_addr, 5446 sizeof(ce_payload) >> 2); 5447 else 5448 amdgpu_ring_write_multiple(ring, (void *)&ce_payload, 5449 sizeof(ce_payload) >> 2); 5450 } 5451 5452 static int gfx_v9_0_ring_preempt_ib(struct amdgpu_ring *ring) 5453 { 5454 int i, r = 0; 5455 struct amdgpu_device *adev = ring->adev; 5456 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0]; 5457 struct amdgpu_ring *kiq_ring = &kiq->ring; 5458 unsigned long flags; 5459 5460 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues) 5461 return -EINVAL; 5462 5463 spin_lock_irqsave(&kiq->ring_lock, flags); 5464 5465 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) { 5466 spin_unlock_irqrestore(&kiq->ring_lock, flags); 5467 return -ENOMEM; 5468 } 5469 5470 /* assert preemption condition */ 5471 amdgpu_ring_set_preempt_cond_exec(ring, false); 5472 5473 ring->trail_seq += 1; 5474 amdgpu_ring_alloc(ring, 13); 5475 gfx_v9_0_ring_emit_fence(ring, ring->trail_fence_gpu_addr, 5476 ring->trail_seq, AMDGPU_FENCE_FLAG_EXEC | AMDGPU_FENCE_FLAG_INT); 5477 5478 /* assert IB preemption, emit the trailing fence */ 5479 kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP, 5480 ring->trail_fence_gpu_addr, 5481 ring->trail_seq); 5482 5483 amdgpu_ring_commit(kiq_ring); 5484 spin_unlock_irqrestore(&kiq->ring_lock, flags); 5485 5486 /* poll the trailing fence */ 5487 for (i = 0; i < adev->usec_timeout; i++) { 5488 if (ring->trail_seq == 5489 le32_to_cpu(*ring->trail_fence_cpu_addr)) 5490 break; 5491 udelay(1); 5492 } 5493 5494 if (i >= adev->usec_timeout) { 5495 r = -EINVAL; 5496 DRM_WARN("ring %d timeout to preempt ib\n", ring->idx); 5497 } 5498 5499 /*reset the CP_VMID_PREEMPT after trailing fence*/ 5500 amdgpu_ring_emit_wreg(ring, 5501 SOC15_REG_OFFSET(GC, 0, mmCP_VMID_PREEMPT), 5502 0x0); 5503 amdgpu_ring_commit(ring); 5504 5505 /* deassert preemption condition */ 5506 amdgpu_ring_set_preempt_cond_exec(ring, true); 5507 return r; 5508 } 5509 5510 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume, bool usegds) 5511 { 5512 struct amdgpu_device *adev = ring->adev; 5513 struct v9_de_ib_state de_payload = {0}; 5514 uint64_t offset, gds_addr, de_payload_gpu_addr; 5515 void *de_payload_cpu_addr; 5516 int cnt; 5517 5518 if (ring->is_mes_queue) { 5519 offset = offsetof(struct amdgpu_mes_ctx_meta_data, 5520 gfx[0].gfx_meta_data) + 5521 offsetof(struct v9_gfx_meta_data, de_payload); 5522 de_payload_gpu_addr = 5523 amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset); 5524 de_payload_cpu_addr = 5525 amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset); 5526 5527 offset = offsetof(struct amdgpu_mes_ctx_meta_data, 5528 gfx[0].gds_backup) + 5529 offsetof(struct v9_gfx_meta_data, de_payload); 5530 gds_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset); 5531 } else { 5532 offset = offsetof(struct v9_gfx_meta_data, de_payload); 5533 de_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset; 5534 de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset; 5535 5536 gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) + 5537 AMDGPU_CSA_SIZE - adev->gds.gds_size, 5538 PAGE_SIZE); 5539 } 5540 5541 if (usegds) { 5542 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr); 5543 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr); 5544 } 5545 5546 cnt = (sizeof(de_payload) >> 2) + 4 - 2; 5547 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5548 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | 5549 WRITE_DATA_DST_SEL(8) | 5550 WR_CONFIRM) | 5551 WRITE_DATA_CACHE_POLICY(0)); 5552 amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr)); 5553 amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr)); 5554 5555 amdgpu_ring_ib_on_emit_de(ring); 5556 if (resume) 5557 amdgpu_ring_write_multiple(ring, de_payload_cpu_addr, 5558 sizeof(de_payload) >> 2); 5559 else 5560 amdgpu_ring_write_multiple(ring, (void *)&de_payload, 5561 sizeof(de_payload) >> 2); 5562 } 5563 5564 static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, 5565 bool secure) 5566 { 5567 uint32_t v = secure ? FRAME_TMZ : 0; 5568 5569 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0)); 5570 amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1)); 5571 } 5572 5573 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags) 5574 { 5575 uint32_t dw2 = 0; 5576 5577 gfx_v9_0_ring_emit_ce_meta(ring, 5578 (!amdgpu_sriov_vf(ring->adev) && 5579 flags & AMDGPU_IB_PREEMPTED) ? true : false); 5580 5581 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */ 5582 if (flags & AMDGPU_HAVE_CTX_SWITCH) { 5583 /* set load_global_config & load_global_uconfig */ 5584 dw2 |= 0x8001; 5585 /* set load_cs_sh_regs */ 5586 dw2 |= 0x01000000; 5587 /* set load_per_context_state & load_gfx_sh_regs for GFX */ 5588 dw2 |= 0x10002; 5589 5590 /* set load_ce_ram if preamble presented */ 5591 if (AMDGPU_PREAMBLE_IB_PRESENT & flags) 5592 dw2 |= 0x10000000; 5593 } else { 5594 /* still load_ce_ram if this is the first time preamble presented 5595 * although there is no context switch happens. 5596 */ 5597 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags) 5598 dw2 |= 0x10000000; 5599 } 5600 5601 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 5602 amdgpu_ring_write(ring, dw2); 5603 amdgpu_ring_write(ring, 0); 5604 } 5605 5606 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring) 5607 { 5608 unsigned ret; 5609 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3)); 5610 amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr)); 5611 amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr)); 5612 amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */ 5613 ret = ring->wptr & ring->buf_mask; 5614 amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */ 5615 return ret; 5616 } 5617 5618 static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset) 5619 { 5620 unsigned cur; 5621 BUG_ON(offset > ring->buf_mask); 5622 BUG_ON(ring->ring[offset] != 0x55aa55aa); 5623 5624 cur = (ring->wptr - 1) & ring->buf_mask; 5625 if (likely(cur > offset)) 5626 ring->ring[offset] = cur - offset; 5627 else 5628 ring->ring[offset] = (ring->ring_size>>2) - offset + cur; 5629 } 5630 5631 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg, 5632 uint32_t reg_val_offs) 5633 { 5634 struct amdgpu_device *adev = ring->adev; 5635 5636 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 5637 amdgpu_ring_write(ring, 0 | /* src: register*/ 5638 (5 << 8) | /* dst: memory */ 5639 (1 << 20)); /* write confirm */ 5640 amdgpu_ring_write(ring, reg); 5641 amdgpu_ring_write(ring, 0); 5642 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 5643 reg_val_offs * 4)); 5644 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 5645 reg_val_offs * 4)); 5646 } 5647 5648 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 5649 uint32_t val) 5650 { 5651 uint32_t cmd = 0; 5652 5653 switch (ring->funcs->type) { 5654 case AMDGPU_RING_TYPE_GFX: 5655 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM; 5656 break; 5657 case AMDGPU_RING_TYPE_KIQ: 5658 cmd = (1 << 16); /* no inc addr */ 5659 break; 5660 default: 5661 cmd = WR_CONFIRM; 5662 break; 5663 } 5664 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5665 amdgpu_ring_write(ring, cmd); 5666 amdgpu_ring_write(ring, reg); 5667 amdgpu_ring_write(ring, 0); 5668 amdgpu_ring_write(ring, val); 5669 } 5670 5671 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, 5672 uint32_t val, uint32_t mask) 5673 { 5674 gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20); 5675 } 5676 5677 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring, 5678 uint32_t reg0, uint32_t reg1, 5679 uint32_t ref, uint32_t mask) 5680 { 5681 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5682 struct amdgpu_device *adev = ring->adev; 5683 bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ? 5684 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait; 5685 5686 if (fw_version_ok) 5687 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1, 5688 ref, mask, 0x20); 5689 else 5690 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1, 5691 ref, mask); 5692 } 5693 5694 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid) 5695 { 5696 struct amdgpu_device *adev = ring->adev; 5697 uint32_t value = 0; 5698 5699 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03); 5700 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01); 5701 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1); 5702 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid); 5703 WREG32_SOC15(GC, 0, mmSQ_CMD, value); 5704 } 5705 5706 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev, 5707 enum amdgpu_interrupt_state state) 5708 { 5709 switch (state) { 5710 case AMDGPU_IRQ_STATE_DISABLE: 5711 case AMDGPU_IRQ_STATE_ENABLE: 5712 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5713 TIME_STAMP_INT_ENABLE, 5714 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5715 break; 5716 default: 5717 break; 5718 } 5719 } 5720 5721 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev, 5722 int me, int pipe, 5723 enum amdgpu_interrupt_state state) 5724 { 5725 u32 mec_int_cntl, mec_int_cntl_reg; 5726 5727 /* 5728 * amdgpu controls only the first MEC. That's why this function only 5729 * handles the setting of interrupts for this specific MEC. All other 5730 * pipes' interrupts are set by amdkfd. 5731 */ 5732 5733 if (me == 1) { 5734 switch (pipe) { 5735 case 0: 5736 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL); 5737 break; 5738 case 1: 5739 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL); 5740 break; 5741 case 2: 5742 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL); 5743 break; 5744 case 3: 5745 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL); 5746 break; 5747 default: 5748 DRM_DEBUG("invalid pipe %d\n", pipe); 5749 return; 5750 } 5751 } else { 5752 DRM_DEBUG("invalid me %d\n", me); 5753 return; 5754 } 5755 5756 switch (state) { 5757 case AMDGPU_IRQ_STATE_DISABLE: 5758 mec_int_cntl = RREG32_SOC15_IP(GC,mec_int_cntl_reg); 5759 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 5760 TIME_STAMP_INT_ENABLE, 0); 5761 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl); 5762 break; 5763 case AMDGPU_IRQ_STATE_ENABLE: 5764 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg); 5765 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 5766 TIME_STAMP_INT_ENABLE, 1); 5767 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl); 5768 break; 5769 default: 5770 break; 5771 } 5772 } 5773 5774 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev, 5775 struct amdgpu_irq_src *source, 5776 unsigned type, 5777 enum amdgpu_interrupt_state state) 5778 { 5779 switch (state) { 5780 case AMDGPU_IRQ_STATE_DISABLE: 5781 case AMDGPU_IRQ_STATE_ENABLE: 5782 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5783 PRIV_REG_INT_ENABLE, 5784 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5785 break; 5786 default: 5787 break; 5788 } 5789 5790 return 0; 5791 } 5792 5793 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev, 5794 struct amdgpu_irq_src *source, 5795 unsigned type, 5796 enum amdgpu_interrupt_state state) 5797 { 5798 switch (state) { 5799 case AMDGPU_IRQ_STATE_DISABLE: 5800 case AMDGPU_IRQ_STATE_ENABLE: 5801 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5802 PRIV_INSTR_INT_ENABLE, 5803 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5804 break; 5805 default: 5806 break; 5807 } 5808 5809 return 0; 5810 } 5811 5812 #define ENABLE_ECC_ON_ME_PIPE(me, pipe) \ 5813 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 5814 CP_ECC_ERROR_INT_ENABLE, 1) 5815 5816 #define DISABLE_ECC_ON_ME_PIPE(me, pipe) \ 5817 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 5818 CP_ECC_ERROR_INT_ENABLE, 0) 5819 5820 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev, 5821 struct amdgpu_irq_src *source, 5822 unsigned type, 5823 enum amdgpu_interrupt_state state) 5824 { 5825 switch (state) { 5826 case AMDGPU_IRQ_STATE_DISABLE: 5827 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5828 CP_ECC_ERROR_INT_ENABLE, 0); 5829 DISABLE_ECC_ON_ME_PIPE(1, 0); 5830 DISABLE_ECC_ON_ME_PIPE(1, 1); 5831 DISABLE_ECC_ON_ME_PIPE(1, 2); 5832 DISABLE_ECC_ON_ME_PIPE(1, 3); 5833 break; 5834 5835 case AMDGPU_IRQ_STATE_ENABLE: 5836 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5837 CP_ECC_ERROR_INT_ENABLE, 1); 5838 ENABLE_ECC_ON_ME_PIPE(1, 0); 5839 ENABLE_ECC_ON_ME_PIPE(1, 1); 5840 ENABLE_ECC_ON_ME_PIPE(1, 2); 5841 ENABLE_ECC_ON_ME_PIPE(1, 3); 5842 break; 5843 default: 5844 break; 5845 } 5846 5847 return 0; 5848 } 5849 5850 5851 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev, 5852 struct amdgpu_irq_src *src, 5853 unsigned type, 5854 enum amdgpu_interrupt_state state) 5855 { 5856 switch (type) { 5857 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP: 5858 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state); 5859 break; 5860 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP: 5861 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state); 5862 break; 5863 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP: 5864 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state); 5865 break; 5866 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP: 5867 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state); 5868 break; 5869 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP: 5870 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state); 5871 break; 5872 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP: 5873 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state); 5874 break; 5875 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP: 5876 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state); 5877 break; 5878 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP: 5879 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state); 5880 break; 5881 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP: 5882 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state); 5883 break; 5884 default: 5885 break; 5886 } 5887 return 0; 5888 } 5889 5890 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev, 5891 struct amdgpu_irq_src *source, 5892 struct amdgpu_iv_entry *entry) 5893 { 5894 int i; 5895 u8 me_id, pipe_id, queue_id; 5896 struct amdgpu_ring *ring; 5897 5898 DRM_DEBUG("IH: CP EOP\n"); 5899 me_id = (entry->ring_id & 0x0c) >> 2; 5900 pipe_id = (entry->ring_id & 0x03) >> 0; 5901 queue_id = (entry->ring_id & 0x70) >> 4; 5902 5903 switch (me_id) { 5904 case 0: 5905 if (adev->gfx.num_gfx_rings && 5906 !amdgpu_mcbp_handle_trailing_fence_irq(&adev->gfx.muxer)) { 5907 /* Fence signals are handled on the software rings*/ 5908 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) 5909 amdgpu_fence_process(&adev->gfx.sw_gfx_ring[i]); 5910 } 5911 break; 5912 case 1: 5913 case 2: 5914 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5915 ring = &adev->gfx.compute_ring[i]; 5916 /* Per-queue interrupt is supported for MEC starting from VI. 5917 * The interrupt can only be enabled/disabled per pipe instead of per queue. 5918 */ 5919 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id)) 5920 amdgpu_fence_process(ring); 5921 } 5922 break; 5923 } 5924 return 0; 5925 } 5926 5927 static void gfx_v9_0_fault(struct amdgpu_device *adev, 5928 struct amdgpu_iv_entry *entry) 5929 { 5930 u8 me_id, pipe_id, queue_id; 5931 struct amdgpu_ring *ring; 5932 int i; 5933 5934 me_id = (entry->ring_id & 0x0c) >> 2; 5935 pipe_id = (entry->ring_id & 0x03) >> 0; 5936 queue_id = (entry->ring_id & 0x70) >> 4; 5937 5938 switch (me_id) { 5939 case 0: 5940 drm_sched_fault(&adev->gfx.gfx_ring[0].sched); 5941 break; 5942 case 1: 5943 case 2: 5944 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5945 ring = &adev->gfx.compute_ring[i]; 5946 if (ring->me == me_id && ring->pipe == pipe_id && 5947 ring->queue == queue_id) 5948 drm_sched_fault(&ring->sched); 5949 } 5950 break; 5951 } 5952 } 5953 5954 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev, 5955 struct amdgpu_irq_src *source, 5956 struct amdgpu_iv_entry *entry) 5957 { 5958 DRM_ERROR("Illegal register access in command stream\n"); 5959 gfx_v9_0_fault(adev, entry); 5960 return 0; 5961 } 5962 5963 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev, 5964 struct amdgpu_irq_src *source, 5965 struct amdgpu_iv_entry *entry) 5966 { 5967 DRM_ERROR("Illegal instruction in command stream\n"); 5968 gfx_v9_0_fault(adev, entry); 5969 return 0; 5970 } 5971 5972 5973 static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = { 5974 { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 5975 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT), 5976 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT) 5977 }, 5978 { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 5979 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT), 5980 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT) 5981 }, 5982 { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 5983 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1), 5984 0, 0 5985 }, 5986 { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 5987 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2), 5988 0, 0 5989 }, 5990 { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 5991 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT), 5992 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT) 5993 }, 5994 { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 5995 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT), 5996 0, 0 5997 }, 5998 { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 5999 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT), 6000 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT) 6001 }, 6002 { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 6003 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT), 6004 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT) 6005 }, 6006 { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 6007 SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1), 6008 0, 0 6009 }, 6010 { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 6011 SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1), 6012 0, 0 6013 }, 6014 { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 6015 SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1), 6016 0, 0 6017 }, 6018 { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 6019 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC), 6020 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED) 6021 }, 6022 { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 6023 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED), 6024 0, 0 6025 }, 6026 { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 6027 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC), 6028 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED) 6029 }, 6030 { "GDS_OA_PHY_PHY_CMD_RAM_MEM", 6031 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 6032 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC), 6033 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED) 6034 }, 6035 { "GDS_OA_PHY_PHY_DATA_RAM_MEM", 6036 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 6037 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED), 6038 0, 0 6039 }, 6040 { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM", 6041 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6042 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC), 6043 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED) 6044 }, 6045 { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM", 6046 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6047 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC), 6048 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED) 6049 }, 6050 { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM", 6051 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6052 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC), 6053 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED) 6054 }, 6055 { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM", 6056 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6057 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC), 6058 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED) 6059 }, 6060 { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 6061 SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT), 6062 0, 0 6063 }, 6064 { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6065 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT), 6066 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT) 6067 }, 6068 { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6069 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT), 6070 0, 0 6071 }, 6072 { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6073 SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT), 6074 0, 0 6075 }, 6076 { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6077 SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT), 6078 0, 0 6079 }, 6080 { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6081 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT), 6082 0, 0 6083 }, 6084 { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 6085 SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT), 6086 0, 0 6087 }, 6088 { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 6089 SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT), 6090 0, 0 6091 }, 6092 { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6093 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT), 6094 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT) 6095 }, 6096 { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6097 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT), 6098 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT) 6099 }, 6100 { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6101 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT), 6102 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT) 6103 }, 6104 { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6105 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT), 6106 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT) 6107 }, 6108 { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6109 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT), 6110 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT) 6111 }, 6112 { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6113 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT), 6114 0, 0 6115 }, 6116 { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6117 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT), 6118 0, 0 6119 }, 6120 { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6121 SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT), 6122 0, 0 6123 }, 6124 { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6125 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT), 6126 0, 0 6127 }, 6128 { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6129 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT), 6130 0, 0 6131 }, 6132 { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6133 SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT), 6134 0, 0 6135 }, 6136 { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6137 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT), 6138 0, 0 6139 }, 6140 { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6141 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT), 6142 0, 0 6143 }, 6144 { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6145 SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT), 6146 0, 0 6147 }, 6148 { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6149 SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT), 6150 0, 0 6151 }, 6152 { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6153 SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT), 6154 0, 0 6155 }, 6156 { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6157 SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT), 6158 0, 0 6159 }, 6160 { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6161 SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT), 6162 0, 0 6163 }, 6164 { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 6165 SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT), 6166 0, 0 6167 }, 6168 { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6169 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT), 6170 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT) 6171 }, 6172 { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6173 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT), 6174 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT) 6175 }, 6176 { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6177 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT), 6178 0, 0 6179 }, 6180 { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6181 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT), 6182 0, 0 6183 }, 6184 { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6185 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT), 6186 0, 0 6187 }, 6188 { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6189 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT), 6190 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT) 6191 }, 6192 { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6193 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT), 6194 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT) 6195 }, 6196 { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6197 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT), 6198 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT) 6199 }, 6200 { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6201 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT), 6202 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT) 6203 }, 6204 { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6205 SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT), 6206 0, 0 6207 }, 6208 { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6209 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT), 6210 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT) 6211 }, 6212 { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6213 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT), 6214 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT) 6215 }, 6216 { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6217 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT), 6218 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT) 6219 }, 6220 { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6221 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT), 6222 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT) 6223 }, 6224 { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6225 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT), 6226 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT) 6227 }, 6228 { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6229 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT), 6230 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT) 6231 }, 6232 { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6233 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT), 6234 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT) 6235 }, 6236 { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6237 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT), 6238 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT) 6239 }, 6240 { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6241 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT), 6242 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT) 6243 }, 6244 { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6245 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT), 6246 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT) 6247 }, 6248 { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6249 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT), 6250 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT) 6251 }, 6252 { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6253 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT), 6254 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT) 6255 }, 6256 { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6257 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT), 6258 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT) 6259 }, 6260 { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6261 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT), 6262 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT) 6263 }, 6264 { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6265 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT), 6266 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT) 6267 }, 6268 { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6269 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT), 6270 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT) 6271 }, 6272 { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6273 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT), 6274 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT) 6275 }, 6276 { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6277 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT), 6278 0, 0 6279 }, 6280 { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6281 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT), 6282 0, 0 6283 }, 6284 { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6285 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT), 6286 0, 0 6287 }, 6288 { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6289 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT), 6290 0, 0 6291 }, 6292 { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6293 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT), 6294 0, 0 6295 }, 6296 { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6297 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT), 6298 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT) 6299 }, 6300 { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6301 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT), 6302 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT) 6303 }, 6304 { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6305 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT), 6306 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT) 6307 }, 6308 { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6309 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT), 6310 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT) 6311 }, 6312 { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6313 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT), 6314 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT) 6315 }, 6316 { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6317 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT), 6318 0, 0 6319 }, 6320 { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6321 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT), 6322 0, 0 6323 }, 6324 { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6325 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT), 6326 0, 0 6327 }, 6328 { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6329 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT), 6330 0, 0 6331 }, 6332 { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6333 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT), 6334 0, 0 6335 }, 6336 { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6337 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT), 6338 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT) 6339 }, 6340 { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6341 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT), 6342 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT) 6343 }, 6344 { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6345 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT), 6346 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT) 6347 }, 6348 { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6349 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT), 6350 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT) 6351 }, 6352 { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6353 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT), 6354 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT) 6355 }, 6356 { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6357 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT), 6358 0, 0 6359 }, 6360 { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6361 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT), 6362 0, 0 6363 }, 6364 { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6365 SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT), 6366 0, 0 6367 }, 6368 { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6369 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT), 6370 0, 0 6371 }, 6372 { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6373 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT), 6374 0, 0 6375 }, 6376 { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6377 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT), 6378 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT) 6379 }, 6380 { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6381 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT), 6382 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT) 6383 }, 6384 { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6385 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT), 6386 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT) 6387 }, 6388 { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6389 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT), 6390 0, 0 6391 }, 6392 { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6393 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT), 6394 0, 0 6395 }, 6396 { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6397 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT), 6398 0, 0 6399 }, 6400 { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6401 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT), 6402 0, 0 6403 }, 6404 { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6405 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT), 6406 0, 0 6407 }, 6408 { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6409 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT), 6410 0, 0 6411 } 6412 }; 6413 6414 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 6415 void *inject_if, uint32_t instance_mask) 6416 { 6417 struct ras_inject_if *info = (struct ras_inject_if *)inject_if; 6418 int ret; 6419 struct ta_ras_trigger_error_input block_info = { 0 }; 6420 6421 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6422 return -EINVAL; 6423 6424 if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks)) 6425 return -EINVAL; 6426 6427 if (!ras_gfx_subblocks[info->head.sub_block_index].name) 6428 return -EPERM; 6429 6430 if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type & 6431 info->head.type)) { 6432 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n", 6433 ras_gfx_subblocks[info->head.sub_block_index].name, 6434 info->head.type); 6435 return -EPERM; 6436 } 6437 6438 if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type & 6439 info->head.type)) { 6440 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n", 6441 ras_gfx_subblocks[info->head.sub_block_index].name, 6442 info->head.type); 6443 return -EPERM; 6444 } 6445 6446 block_info.block_id = amdgpu_ras_block_to_ta(info->head.block); 6447 block_info.sub_block_index = 6448 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock; 6449 block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type); 6450 block_info.address = info->address; 6451 block_info.value = info->value; 6452 6453 mutex_lock(&adev->grbm_idx_mutex); 6454 ret = psp_ras_trigger_error(&adev->psp, &block_info, instance_mask); 6455 mutex_unlock(&adev->grbm_idx_mutex); 6456 6457 return ret; 6458 } 6459 6460 static const char * const vml2_mems[] = { 6461 "UTC_VML2_BANK_CACHE_0_BIGK_MEM0", 6462 "UTC_VML2_BANK_CACHE_0_BIGK_MEM1", 6463 "UTC_VML2_BANK_CACHE_0_4K_MEM0", 6464 "UTC_VML2_BANK_CACHE_0_4K_MEM1", 6465 "UTC_VML2_BANK_CACHE_1_BIGK_MEM0", 6466 "UTC_VML2_BANK_CACHE_1_BIGK_MEM1", 6467 "UTC_VML2_BANK_CACHE_1_4K_MEM0", 6468 "UTC_VML2_BANK_CACHE_1_4K_MEM1", 6469 "UTC_VML2_BANK_CACHE_2_BIGK_MEM0", 6470 "UTC_VML2_BANK_CACHE_2_BIGK_MEM1", 6471 "UTC_VML2_BANK_CACHE_2_4K_MEM0", 6472 "UTC_VML2_BANK_CACHE_2_4K_MEM1", 6473 "UTC_VML2_BANK_CACHE_3_BIGK_MEM0", 6474 "UTC_VML2_BANK_CACHE_3_BIGK_MEM1", 6475 "UTC_VML2_BANK_CACHE_3_4K_MEM0", 6476 "UTC_VML2_BANK_CACHE_3_4K_MEM1", 6477 }; 6478 6479 static const char * const vml2_walker_mems[] = { 6480 "UTC_VML2_CACHE_PDE0_MEM0", 6481 "UTC_VML2_CACHE_PDE0_MEM1", 6482 "UTC_VML2_CACHE_PDE1_MEM0", 6483 "UTC_VML2_CACHE_PDE1_MEM1", 6484 "UTC_VML2_CACHE_PDE2_MEM0", 6485 "UTC_VML2_CACHE_PDE2_MEM1", 6486 "UTC_VML2_RDIF_LOG_FIFO", 6487 }; 6488 6489 static const char * const atc_l2_cache_2m_mems[] = { 6490 "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM", 6491 "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM", 6492 "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM", 6493 "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM", 6494 }; 6495 6496 static const char *atc_l2_cache_4k_mems[] = { 6497 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0", 6498 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1", 6499 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2", 6500 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3", 6501 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4", 6502 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5", 6503 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6", 6504 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7", 6505 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0", 6506 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1", 6507 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2", 6508 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3", 6509 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4", 6510 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5", 6511 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6", 6512 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7", 6513 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0", 6514 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1", 6515 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2", 6516 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3", 6517 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4", 6518 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5", 6519 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6", 6520 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7", 6521 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0", 6522 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1", 6523 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2", 6524 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3", 6525 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4", 6526 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5", 6527 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6", 6528 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7", 6529 }; 6530 6531 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev, 6532 struct ras_err_data *err_data) 6533 { 6534 uint32_t i, data; 6535 uint32_t sec_count, ded_count; 6536 6537 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6538 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 6539 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6540 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 6541 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6542 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 6543 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6544 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 6545 6546 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { 6547 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 6548 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 6549 6550 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT); 6551 if (sec_count) { 6552 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6553 "SEC %d\n", i, vml2_mems[i], sec_count); 6554 err_data->ce_count += sec_count; 6555 } 6556 6557 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT); 6558 if (ded_count) { 6559 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6560 "DED %d\n", i, vml2_mems[i], ded_count); 6561 err_data->ue_count += ded_count; 6562 } 6563 } 6564 6565 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { 6566 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 6567 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 6568 6569 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6570 SEC_COUNT); 6571 if (sec_count) { 6572 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6573 "SEC %d\n", i, vml2_walker_mems[i], sec_count); 6574 err_data->ce_count += sec_count; 6575 } 6576 6577 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6578 DED_COUNT); 6579 if (ded_count) { 6580 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6581 "DED %d\n", i, vml2_walker_mems[i], ded_count); 6582 err_data->ue_count += ded_count; 6583 } 6584 } 6585 6586 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { 6587 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 6588 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 6589 6590 sec_count = (data & 0x00006000L) >> 0xd; 6591 if (sec_count) { 6592 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6593 "SEC %d\n", i, atc_l2_cache_2m_mems[i], 6594 sec_count); 6595 err_data->ce_count += sec_count; 6596 } 6597 } 6598 6599 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { 6600 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 6601 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 6602 6603 sec_count = (data & 0x00006000L) >> 0xd; 6604 if (sec_count) { 6605 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6606 "SEC %d\n", i, atc_l2_cache_4k_mems[i], 6607 sec_count); 6608 err_data->ce_count += sec_count; 6609 } 6610 6611 ded_count = (data & 0x00018000L) >> 0xf; 6612 if (ded_count) { 6613 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6614 "DED %d\n", i, atc_l2_cache_4k_mems[i], 6615 ded_count); 6616 err_data->ue_count += ded_count; 6617 } 6618 } 6619 6620 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6621 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6622 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6623 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6624 6625 return 0; 6626 } 6627 6628 static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev, 6629 const struct soc15_reg_entry *reg, 6630 uint32_t se_id, uint32_t inst_id, uint32_t value, 6631 uint32_t *sec_count, uint32_t *ded_count) 6632 { 6633 uint32_t i; 6634 uint32_t sec_cnt, ded_cnt; 6635 6636 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) { 6637 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset || 6638 gfx_v9_0_ras_fields[i].seg != reg->seg || 6639 gfx_v9_0_ras_fields[i].inst != reg->inst) 6640 continue; 6641 6642 sec_cnt = (value & 6643 gfx_v9_0_ras_fields[i].sec_count_mask) >> 6644 gfx_v9_0_ras_fields[i].sec_count_shift; 6645 if (sec_cnt) { 6646 dev_info(adev->dev, "GFX SubBlock %s, " 6647 "Instance[%d][%d], SEC %d\n", 6648 gfx_v9_0_ras_fields[i].name, 6649 se_id, inst_id, 6650 sec_cnt); 6651 *sec_count += sec_cnt; 6652 } 6653 6654 ded_cnt = (value & 6655 gfx_v9_0_ras_fields[i].ded_count_mask) >> 6656 gfx_v9_0_ras_fields[i].ded_count_shift; 6657 if (ded_cnt) { 6658 dev_info(adev->dev, "GFX SubBlock %s, " 6659 "Instance[%d][%d], DED %d\n", 6660 gfx_v9_0_ras_fields[i].name, 6661 se_id, inst_id, 6662 ded_cnt); 6663 *ded_count += ded_cnt; 6664 } 6665 } 6666 6667 return 0; 6668 } 6669 6670 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev) 6671 { 6672 int i, j, k; 6673 6674 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6675 return; 6676 6677 /* read back registers to clear the counters */ 6678 mutex_lock(&adev->grbm_idx_mutex); 6679 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { 6680 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { 6681 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { 6682 amdgpu_gfx_select_se_sh(adev, j, 0x0, k, 0); 6683 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); 6684 } 6685 } 6686 } 6687 WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000); 6688 mutex_unlock(&adev->grbm_idx_mutex); 6689 6690 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6691 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 6692 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6693 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 6694 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6695 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 6696 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6697 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 6698 6699 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { 6700 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 6701 RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 6702 } 6703 6704 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { 6705 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 6706 RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 6707 } 6708 6709 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { 6710 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 6711 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 6712 } 6713 6714 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { 6715 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 6716 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 6717 } 6718 6719 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6720 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6721 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6722 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6723 } 6724 6725 static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 6726 void *ras_error_status) 6727 { 6728 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; 6729 uint32_t sec_count = 0, ded_count = 0; 6730 uint32_t i, j, k; 6731 uint32_t reg_value; 6732 6733 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6734 return; 6735 6736 err_data->ue_count = 0; 6737 err_data->ce_count = 0; 6738 6739 mutex_lock(&adev->grbm_idx_mutex); 6740 6741 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { 6742 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { 6743 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { 6744 amdgpu_gfx_select_se_sh(adev, j, 0, k, 0); 6745 reg_value = 6746 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); 6747 if (reg_value) 6748 gfx_v9_0_ras_error_count(adev, 6749 &gfx_v9_0_edc_counter_regs[i], 6750 j, k, reg_value, 6751 &sec_count, &ded_count); 6752 } 6753 } 6754 } 6755 6756 err_data->ce_count += sec_count; 6757 err_data->ue_count += ded_count; 6758 6759 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 6760 mutex_unlock(&adev->grbm_idx_mutex); 6761 6762 gfx_v9_0_query_utc_edc_status(adev, err_data); 6763 } 6764 6765 static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring) 6766 { 6767 const unsigned int cp_coher_cntl = 6768 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) | 6769 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) | 6770 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) | 6771 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) | 6772 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1); 6773 6774 /* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */ 6775 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5)); 6776 amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */ 6777 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */ 6778 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */ 6779 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ 6780 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */ 6781 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */ 6782 } 6783 6784 static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring, 6785 uint32_t pipe, bool enable) 6786 { 6787 struct amdgpu_device *adev = ring->adev; 6788 uint32_t val; 6789 uint32_t wcl_cs_reg; 6790 6791 /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */ 6792 val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT; 6793 6794 switch (pipe) { 6795 case 0: 6796 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0); 6797 break; 6798 case 1: 6799 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1); 6800 break; 6801 case 2: 6802 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2); 6803 break; 6804 case 3: 6805 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3); 6806 break; 6807 default: 6808 DRM_DEBUG("invalid pipe %d\n", pipe); 6809 return; 6810 } 6811 6812 amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val); 6813 6814 } 6815 static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable) 6816 { 6817 struct amdgpu_device *adev = ring->adev; 6818 uint32_t val; 6819 int i; 6820 6821 6822 /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit 6823 * number of gfx waves. Setting 5 bit will make sure gfx only gets 6824 * around 25% of gpu resources. 6825 */ 6826 val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT; 6827 amdgpu_ring_emit_wreg(ring, 6828 SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX), 6829 val); 6830 6831 /* Restrict waves for normal/low priority compute queues as well 6832 * to get best QoS for high priority compute jobs. 6833 * 6834 * amdgpu controls only 1st ME(0-3 CS pipes). 6835 */ 6836 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { 6837 if (i != ring->pipe) 6838 gfx_v9_0_emit_wave_limit_cs(ring, i, enable); 6839 6840 } 6841 } 6842 6843 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = { 6844 .name = "gfx_v9_0", 6845 .early_init = gfx_v9_0_early_init, 6846 .late_init = gfx_v9_0_late_init, 6847 .sw_init = gfx_v9_0_sw_init, 6848 .sw_fini = gfx_v9_0_sw_fini, 6849 .hw_init = gfx_v9_0_hw_init, 6850 .hw_fini = gfx_v9_0_hw_fini, 6851 .suspend = gfx_v9_0_suspend, 6852 .resume = gfx_v9_0_resume, 6853 .is_idle = gfx_v9_0_is_idle, 6854 .wait_for_idle = gfx_v9_0_wait_for_idle, 6855 .soft_reset = gfx_v9_0_soft_reset, 6856 .set_clockgating_state = gfx_v9_0_set_clockgating_state, 6857 .set_powergating_state = gfx_v9_0_set_powergating_state, 6858 .get_clockgating_state = gfx_v9_0_get_clockgating_state, 6859 }; 6860 6861 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = { 6862 .type = AMDGPU_RING_TYPE_GFX, 6863 .align_mask = 0xff, 6864 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6865 .support_64bit_ptrs = true, 6866 .secure_submission_supported = true, 6867 .get_rptr = gfx_v9_0_ring_get_rptr_gfx, 6868 .get_wptr = gfx_v9_0_ring_get_wptr_gfx, 6869 .set_wptr = gfx_v9_0_ring_set_wptr_gfx, 6870 .emit_frame_size = /* totally 242 maximum if 16 IBs */ 6871 5 + /* COND_EXEC */ 6872 7 + /* PIPELINE_SYNC */ 6873 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6874 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6875 2 + /* VM_FLUSH */ 6876 8 + /* FENCE for VM_FLUSH */ 6877 20 + /* GDS switch */ 6878 4 + /* double SWITCH_BUFFER, 6879 the first COND_EXEC jump to the place just 6880 prior to this double SWITCH_BUFFER */ 6881 5 + /* COND_EXEC */ 6882 7 + /* HDP_flush */ 6883 4 + /* VGT_flush */ 6884 14 + /* CE_META */ 6885 31 + /* DE_META */ 6886 3 + /* CNTX_CTRL */ 6887 5 + /* HDP_INVL */ 6888 8 + 8 + /* FENCE x2 */ 6889 2 + /* SWITCH_BUFFER */ 6890 7, /* gfx_v9_0_emit_mem_sync */ 6891 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */ 6892 .emit_ib = gfx_v9_0_ring_emit_ib_gfx, 6893 .emit_fence = gfx_v9_0_ring_emit_fence, 6894 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 6895 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 6896 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 6897 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 6898 .test_ring = gfx_v9_0_ring_test_ring, 6899 .insert_nop = amdgpu_ring_insert_nop, 6900 .pad_ib = amdgpu_ring_generic_pad_ib, 6901 .emit_switch_buffer = gfx_v9_ring_emit_sb, 6902 .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl, 6903 .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec, 6904 .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec, 6905 .preempt_ib = gfx_v9_0_ring_preempt_ib, 6906 .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl, 6907 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6908 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6909 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6910 .soft_recovery = gfx_v9_0_ring_soft_recovery, 6911 .emit_mem_sync = gfx_v9_0_emit_mem_sync, 6912 }; 6913 6914 static const struct amdgpu_ring_funcs gfx_v9_0_sw_ring_funcs_gfx = { 6915 .type = AMDGPU_RING_TYPE_GFX, 6916 .align_mask = 0xff, 6917 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6918 .support_64bit_ptrs = true, 6919 .secure_submission_supported = true, 6920 .get_rptr = amdgpu_sw_ring_get_rptr_gfx, 6921 .get_wptr = amdgpu_sw_ring_get_wptr_gfx, 6922 .set_wptr = amdgpu_sw_ring_set_wptr_gfx, 6923 .emit_frame_size = /* totally 242 maximum if 16 IBs */ 6924 5 + /* COND_EXEC */ 6925 7 + /* PIPELINE_SYNC */ 6926 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6927 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6928 2 + /* VM_FLUSH */ 6929 8 + /* FENCE for VM_FLUSH */ 6930 20 + /* GDS switch */ 6931 4 + /* double SWITCH_BUFFER, 6932 * the first COND_EXEC jump to the place just 6933 * prior to this double SWITCH_BUFFER 6934 */ 6935 5 + /* COND_EXEC */ 6936 7 + /* HDP_flush */ 6937 4 + /* VGT_flush */ 6938 14 + /* CE_META */ 6939 31 + /* DE_META */ 6940 3 + /* CNTX_CTRL */ 6941 5 + /* HDP_INVL */ 6942 8 + 8 + /* FENCE x2 */ 6943 2 + /* SWITCH_BUFFER */ 6944 7, /* gfx_v9_0_emit_mem_sync */ 6945 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */ 6946 .emit_ib = gfx_v9_0_ring_emit_ib_gfx, 6947 .emit_fence = gfx_v9_0_ring_emit_fence, 6948 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 6949 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 6950 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 6951 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 6952 .test_ring = gfx_v9_0_ring_test_ring, 6953 .test_ib = gfx_v9_0_ring_test_ib, 6954 .insert_nop = amdgpu_sw_ring_insert_nop, 6955 .pad_ib = amdgpu_ring_generic_pad_ib, 6956 .emit_switch_buffer = gfx_v9_ring_emit_sb, 6957 .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl, 6958 .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec, 6959 .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec, 6960 .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl, 6961 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6962 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6963 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6964 .soft_recovery = gfx_v9_0_ring_soft_recovery, 6965 .emit_mem_sync = gfx_v9_0_emit_mem_sync, 6966 .patch_cntl = gfx_v9_0_ring_patch_cntl, 6967 .patch_de = gfx_v9_0_ring_patch_de_meta, 6968 .patch_ce = gfx_v9_0_ring_patch_ce_meta, 6969 }; 6970 6971 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = { 6972 .type = AMDGPU_RING_TYPE_COMPUTE, 6973 .align_mask = 0xff, 6974 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6975 .support_64bit_ptrs = true, 6976 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 6977 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 6978 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 6979 .emit_frame_size = 6980 20 + /* gfx_v9_0_ring_emit_gds_switch */ 6981 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 6982 5 + /* hdp invalidate */ 6983 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 6984 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6985 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6986 2 + /* gfx_v9_0_ring_emit_vm_flush */ 6987 8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */ 6988 7 + /* gfx_v9_0_emit_mem_sync */ 6989 5 + /* gfx_v9_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */ 6990 15, /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */ 6991 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 6992 .emit_ib = gfx_v9_0_ring_emit_ib_compute, 6993 .emit_fence = gfx_v9_0_ring_emit_fence, 6994 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 6995 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 6996 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 6997 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 6998 .test_ring = gfx_v9_0_ring_test_ring, 6999 .test_ib = gfx_v9_0_ring_test_ib, 7000 .insert_nop = amdgpu_ring_insert_nop, 7001 .pad_ib = amdgpu_ring_generic_pad_ib, 7002 .emit_wreg = gfx_v9_0_ring_emit_wreg, 7003 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 7004 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 7005 .emit_mem_sync = gfx_v9_0_emit_mem_sync, 7006 .emit_wave_limit = gfx_v9_0_emit_wave_limit, 7007 }; 7008 7009 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = { 7010 .type = AMDGPU_RING_TYPE_KIQ, 7011 .align_mask = 0xff, 7012 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 7013 .support_64bit_ptrs = true, 7014 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 7015 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 7016 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 7017 .emit_frame_size = 7018 20 + /* gfx_v9_0_ring_emit_gds_switch */ 7019 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 7020 5 + /* hdp invalidate */ 7021 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 7022 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 7023 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 7024 2 + /* gfx_v9_0_ring_emit_vm_flush */ 7025 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */ 7026 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 7027 .emit_fence = gfx_v9_0_ring_emit_fence_kiq, 7028 .test_ring = gfx_v9_0_ring_test_ring, 7029 .insert_nop = amdgpu_ring_insert_nop, 7030 .pad_ib = amdgpu_ring_generic_pad_ib, 7031 .emit_rreg = gfx_v9_0_ring_emit_rreg, 7032 .emit_wreg = gfx_v9_0_ring_emit_wreg, 7033 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 7034 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 7035 }; 7036 7037 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev) 7038 { 7039 int i; 7040 7041 adev->gfx.kiq[0].ring.funcs = &gfx_v9_0_ring_funcs_kiq; 7042 7043 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 7044 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx; 7045 7046 if (adev->gfx.num_gfx_rings) { 7047 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) 7048 adev->gfx.sw_gfx_ring[i].funcs = &gfx_v9_0_sw_ring_funcs_gfx; 7049 } 7050 7051 for (i = 0; i < adev->gfx.num_compute_rings; i++) 7052 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute; 7053 } 7054 7055 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = { 7056 .set = gfx_v9_0_set_eop_interrupt_state, 7057 .process = gfx_v9_0_eop_irq, 7058 }; 7059 7060 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = { 7061 .set = gfx_v9_0_set_priv_reg_fault_state, 7062 .process = gfx_v9_0_priv_reg_irq, 7063 }; 7064 7065 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = { 7066 .set = gfx_v9_0_set_priv_inst_fault_state, 7067 .process = gfx_v9_0_priv_inst_irq, 7068 }; 7069 7070 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = { 7071 .set = gfx_v9_0_set_cp_ecc_error_state, 7072 .process = amdgpu_gfx_cp_ecc_error_irq, 7073 }; 7074 7075 7076 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev) 7077 { 7078 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; 7079 adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs; 7080 7081 adev->gfx.priv_reg_irq.num_types = 1; 7082 adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs; 7083 7084 adev->gfx.priv_inst_irq.num_types = 1; 7085 adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs; 7086 7087 adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/ 7088 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs; 7089 } 7090 7091 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev) 7092 { 7093 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 7094 case IP_VERSION(9, 0, 1): 7095 case IP_VERSION(9, 2, 1): 7096 case IP_VERSION(9, 4, 0): 7097 case IP_VERSION(9, 2, 2): 7098 case IP_VERSION(9, 1, 0): 7099 case IP_VERSION(9, 4, 1): 7100 case IP_VERSION(9, 3, 0): 7101 case IP_VERSION(9, 4, 2): 7102 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs; 7103 break; 7104 default: 7105 break; 7106 } 7107 } 7108 7109 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev) 7110 { 7111 /* init asci gds info */ 7112 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 7113 case IP_VERSION(9, 0, 1): 7114 case IP_VERSION(9, 2, 1): 7115 case IP_VERSION(9, 4, 0): 7116 adev->gds.gds_size = 0x10000; 7117 break; 7118 case IP_VERSION(9, 2, 2): 7119 case IP_VERSION(9, 1, 0): 7120 case IP_VERSION(9, 4, 1): 7121 adev->gds.gds_size = 0x1000; 7122 break; 7123 case IP_VERSION(9, 4, 2): 7124 /* aldebaran removed all the GDS internal memory, 7125 * only support GWS opcode in kernel, like barrier 7126 * semaphore.etc */ 7127 adev->gds.gds_size = 0; 7128 break; 7129 default: 7130 adev->gds.gds_size = 0x10000; 7131 break; 7132 } 7133 7134 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 7135 case IP_VERSION(9, 0, 1): 7136 case IP_VERSION(9, 4, 0): 7137 adev->gds.gds_compute_max_wave_id = 0x7ff; 7138 break; 7139 case IP_VERSION(9, 2, 1): 7140 adev->gds.gds_compute_max_wave_id = 0x27f; 7141 break; 7142 case IP_VERSION(9, 2, 2): 7143 case IP_VERSION(9, 1, 0): 7144 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 7145 adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */ 7146 else 7147 adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */ 7148 break; 7149 case IP_VERSION(9, 4, 1): 7150 adev->gds.gds_compute_max_wave_id = 0xfff; 7151 break; 7152 case IP_VERSION(9, 4, 2): 7153 /* deprecated for Aldebaran, no usage at all */ 7154 adev->gds.gds_compute_max_wave_id = 0; 7155 break; 7156 default: 7157 /* this really depends on the chip */ 7158 adev->gds.gds_compute_max_wave_id = 0x7ff; 7159 break; 7160 } 7161 7162 adev->gds.gws_size = 64; 7163 adev->gds.oa_size = 16; 7164 } 7165 7166 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev, 7167 u32 bitmap) 7168 { 7169 u32 data; 7170 7171 if (!bitmap) 7172 return; 7173 7174 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 7175 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 7176 7177 WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data); 7178 } 7179 7180 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev) 7181 { 7182 u32 data, mask; 7183 7184 data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG); 7185 data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG); 7186 7187 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 7188 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 7189 7190 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh); 7191 7192 return (~data) & mask; 7193 } 7194 7195 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 7196 struct amdgpu_cu_info *cu_info) 7197 { 7198 int i, j, k, counter, active_cu_number = 0; 7199 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0; 7200 unsigned disable_masks[4 * 4]; 7201 7202 if (!adev || !cu_info) 7203 return -EINVAL; 7204 7205 /* 7206 * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs 7207 */ 7208 if (adev->gfx.config.max_shader_engines * 7209 adev->gfx.config.max_sh_per_se > 16) 7210 return -EINVAL; 7211 7212 amdgpu_gfx_parse_disable_cu(disable_masks, 7213 adev->gfx.config.max_shader_engines, 7214 adev->gfx.config.max_sh_per_se); 7215 7216 mutex_lock(&adev->grbm_idx_mutex); 7217 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 7218 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 7219 mask = 1; 7220 ao_bitmap = 0; 7221 counter = 0; 7222 amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0); 7223 gfx_v9_0_set_user_cu_inactive_bitmap( 7224 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]); 7225 bitmap = gfx_v9_0_get_cu_active_bitmap(adev); 7226 7227 /* 7228 * The bitmap(and ao_cu_bitmap) in cu_info structure is 7229 * 4x4 size array, and it's usually suitable for Vega 7230 * ASICs which has 4*2 SE/SH layout. 7231 * But for Arcturus, SE/SH layout is changed to 8*1. 7232 * To mostly reduce the impact, we make it compatible 7233 * with current bitmap array as below: 7234 * SE4,SH0 --> bitmap[0][1] 7235 * SE5,SH0 --> bitmap[1][1] 7236 * SE6,SH0 --> bitmap[2][1] 7237 * SE7,SH0 --> bitmap[3][1] 7238 */ 7239 cu_info->bitmap[0][i % 4][j + i / 4] = bitmap; 7240 7241 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 7242 if (bitmap & mask) { 7243 if (counter < adev->gfx.config.max_cu_per_sh) 7244 ao_bitmap |= mask; 7245 counter ++; 7246 } 7247 mask <<= 1; 7248 } 7249 active_cu_number += counter; 7250 if (i < 2 && j < 2) 7251 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8)); 7252 cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap; 7253 } 7254 } 7255 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 7256 mutex_unlock(&adev->grbm_idx_mutex); 7257 7258 cu_info->number = active_cu_number; 7259 cu_info->ao_cu_mask = ao_cu_mask; 7260 cu_info->simd_per_cu = NUM_SIMD_PER_CU; 7261 7262 return 0; 7263 } 7264 7265 const struct amdgpu_ip_block_version gfx_v9_0_ip_block = 7266 { 7267 .type = AMD_IP_BLOCK_TYPE_GFX, 7268 .major = 9, 7269 .minor = 0, 7270 .rev = 0, 7271 .funcs = &gfx_v9_0_ip_funcs, 7272 }; 7273