1fb4d8502Sjsg /* 2fb4d8502Sjsg * Copyright 2016 Advanced Micro Devices, Inc. 3fb4d8502Sjsg * 4fb4d8502Sjsg * Permission is hereby granted, free of charge, to any person obtaining a 5fb4d8502Sjsg * copy of this software and associated documentation files (the "Software"), 6fb4d8502Sjsg * to deal in the Software without restriction, including without limitation 7fb4d8502Sjsg * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8fb4d8502Sjsg * and/or sell copies of the Software, and to permit persons to whom the 9fb4d8502Sjsg * Software is furnished to do so, subject to the following conditions: 10fb4d8502Sjsg * 11fb4d8502Sjsg * The above copyright notice and this permission notice shall be included in 12fb4d8502Sjsg * all copies or substantial portions of the Software. 13fb4d8502Sjsg * 14fb4d8502Sjsg * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15fb4d8502Sjsg * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16fb4d8502Sjsg * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17fb4d8502Sjsg * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18fb4d8502Sjsg * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19fb4d8502Sjsg * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20fb4d8502Sjsg * OTHER DEALINGS IN THE SOFTWARE. 21fb4d8502Sjsg * 22fb4d8502Sjsg */ 23c349dbc7Sjsg 24c349dbc7Sjsg #include <linux/delay.h> 25fb4d8502Sjsg #include <linux/kernel.h> 26fb4d8502Sjsg #include <linux/firmware.h> 27c349dbc7Sjsg #include <linux/module.h> 28c349dbc7Sjsg #include <linux/pci.h> 29c349dbc7Sjsg 30fb4d8502Sjsg #include "amdgpu.h" 31fb4d8502Sjsg #include "amdgpu_gfx.h" 32fb4d8502Sjsg #include "soc15.h" 33fb4d8502Sjsg #include "soc15d.h" 34fb4d8502Sjsg #include "amdgpu_atomfirmware.h" 35c349dbc7Sjsg #include "amdgpu_pm.h" 36fb4d8502Sjsg 37fb4d8502Sjsg #include "gc/gc_9_0_offset.h" 38fb4d8502Sjsg #include "gc/gc_9_0_sh_mask.h" 39c349dbc7Sjsg 40fb4d8502Sjsg #include "vega10_enum.h" 41fb4d8502Sjsg #include "hdp/hdp_4_0_offset.h" 42fb4d8502Sjsg 43fb4d8502Sjsg #include "soc15_common.h" 44fb4d8502Sjsg #include "clearstate_gfx9.h" 45fb4d8502Sjsg #include "v9_structs.h" 46fb4d8502Sjsg 47fb4d8502Sjsg #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h" 48fb4d8502Sjsg 49c349dbc7Sjsg #include "amdgpu_ras.h" 50c349dbc7Sjsg 51c349dbc7Sjsg #include "gfx_v9_4.h" 52ad8b1aafSjsg #include "gfx_v9_0.h" 53ad8b1aafSjsg 54ad8b1aafSjsg #include "asic_reg/pwr/pwr_10_0_offset.h" 55ad8b1aafSjsg #include "asic_reg/pwr/pwr_10_0_sh_mask.h" 56c349dbc7Sjsg 57fb4d8502Sjsg #define GFX9_NUM_GFX_RINGS 1 58c349dbc7Sjsg #define GFX9_MEC_HPD_SIZE 4096 59fb4d8502Sjsg #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L 60fb4d8502Sjsg #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L 61fb4d8502Sjsg 62c349dbc7Sjsg #define mmGCEA_PROBE_MAP 0x070c 63c349dbc7Sjsg #define mmGCEA_PROBE_MAP_BASE_IDX 0 64c349dbc7Sjsg 65fb4d8502Sjsg MODULE_FIRMWARE("amdgpu/vega10_ce.bin"); 66fb4d8502Sjsg MODULE_FIRMWARE("amdgpu/vega10_pfp.bin"); 67fb4d8502Sjsg MODULE_FIRMWARE("amdgpu/vega10_me.bin"); 68fb4d8502Sjsg MODULE_FIRMWARE("amdgpu/vega10_mec.bin"); 69fb4d8502Sjsg MODULE_FIRMWARE("amdgpu/vega10_mec2.bin"); 70fb4d8502Sjsg MODULE_FIRMWARE("amdgpu/vega10_rlc.bin"); 71fb4d8502Sjsg 72fb4d8502Sjsg MODULE_FIRMWARE("amdgpu/vega12_ce.bin"); 73fb4d8502Sjsg MODULE_FIRMWARE("amdgpu/vega12_pfp.bin"); 74fb4d8502Sjsg MODULE_FIRMWARE("amdgpu/vega12_me.bin"); 75fb4d8502Sjsg MODULE_FIRMWARE("amdgpu/vega12_mec.bin"); 76fb4d8502Sjsg MODULE_FIRMWARE("amdgpu/vega12_mec2.bin"); 77fb4d8502Sjsg MODULE_FIRMWARE("amdgpu/vega12_rlc.bin"); 78fb4d8502Sjsg 79fb4d8502Sjsg MODULE_FIRMWARE("amdgpu/vega20_ce.bin"); 80fb4d8502Sjsg MODULE_FIRMWARE("amdgpu/vega20_pfp.bin"); 81fb4d8502Sjsg MODULE_FIRMWARE("amdgpu/vega20_me.bin"); 82fb4d8502Sjsg MODULE_FIRMWARE("amdgpu/vega20_mec.bin"); 83fb4d8502Sjsg MODULE_FIRMWARE("amdgpu/vega20_mec2.bin"); 84fb4d8502Sjsg MODULE_FIRMWARE("amdgpu/vega20_rlc.bin"); 85fb4d8502Sjsg 86fb4d8502Sjsg MODULE_FIRMWARE("amdgpu/raven_ce.bin"); 87fb4d8502Sjsg MODULE_FIRMWARE("amdgpu/raven_pfp.bin"); 88fb4d8502Sjsg MODULE_FIRMWARE("amdgpu/raven_me.bin"); 89fb4d8502Sjsg MODULE_FIRMWARE("amdgpu/raven_mec.bin"); 90fb4d8502Sjsg MODULE_FIRMWARE("amdgpu/raven_mec2.bin"); 91fb4d8502Sjsg MODULE_FIRMWARE("amdgpu/raven_rlc.bin"); 92fb4d8502Sjsg 9302fbaceeSjsg MODULE_FIRMWARE("amdgpu/picasso_ce.bin"); 9402fbaceeSjsg MODULE_FIRMWARE("amdgpu/picasso_pfp.bin"); 9502fbaceeSjsg MODULE_FIRMWARE("amdgpu/picasso_me.bin"); 9602fbaceeSjsg MODULE_FIRMWARE("amdgpu/picasso_mec.bin"); 9702fbaceeSjsg MODULE_FIRMWARE("amdgpu/picasso_mec2.bin"); 9802fbaceeSjsg MODULE_FIRMWARE("amdgpu/picasso_rlc.bin"); 99c349dbc7Sjsg MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin"); 100c349dbc7Sjsg 101c349dbc7Sjsg MODULE_FIRMWARE("amdgpu/raven2_ce.bin"); 102c349dbc7Sjsg MODULE_FIRMWARE("amdgpu/raven2_pfp.bin"); 103c349dbc7Sjsg MODULE_FIRMWARE("amdgpu/raven2_me.bin"); 104c349dbc7Sjsg MODULE_FIRMWARE("amdgpu/raven2_mec.bin"); 105c349dbc7Sjsg MODULE_FIRMWARE("amdgpu/raven2_mec2.bin"); 106c349dbc7Sjsg MODULE_FIRMWARE("amdgpu/raven2_rlc.bin"); 107c349dbc7Sjsg MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin"); 108c349dbc7Sjsg 109c349dbc7Sjsg MODULE_FIRMWARE("amdgpu/arcturus_mec.bin"); 110c349dbc7Sjsg MODULE_FIRMWARE("amdgpu/arcturus_mec2.bin"); 111c349dbc7Sjsg MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin"); 112c349dbc7Sjsg 113c349dbc7Sjsg MODULE_FIRMWARE("amdgpu/renoir_ce.bin"); 114c349dbc7Sjsg MODULE_FIRMWARE("amdgpu/renoir_pfp.bin"); 115c349dbc7Sjsg MODULE_FIRMWARE("amdgpu/renoir_me.bin"); 116c349dbc7Sjsg MODULE_FIRMWARE("amdgpu/renoir_mec.bin"); 117c349dbc7Sjsg MODULE_FIRMWARE("amdgpu/renoir_mec2.bin"); 118c349dbc7Sjsg MODULE_FIRMWARE("amdgpu/renoir_rlc.bin"); 119c349dbc7Sjsg 120ad8b1aafSjsg MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin"); 121ad8b1aafSjsg MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin"); 122ad8b1aafSjsg MODULE_FIRMWARE("amdgpu/green_sardine_me.bin"); 123ad8b1aafSjsg MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin"); 124ad8b1aafSjsg MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin"); 125ad8b1aafSjsg MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin"); 126ad8b1aafSjsg 127c349dbc7Sjsg #define mmTCP_CHAN_STEER_0_ARCT 0x0b03 128c349dbc7Sjsg #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX 0 129c349dbc7Sjsg #define mmTCP_CHAN_STEER_1_ARCT 0x0b04 130c349dbc7Sjsg #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX 0 131c349dbc7Sjsg #define mmTCP_CHAN_STEER_2_ARCT 0x0b09 132c349dbc7Sjsg #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX 0 133c349dbc7Sjsg #define mmTCP_CHAN_STEER_3_ARCT 0x0b0a 134c349dbc7Sjsg #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX 0 135c349dbc7Sjsg #define mmTCP_CHAN_STEER_4_ARCT 0x0b0b 136c349dbc7Sjsg #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX 0 137c349dbc7Sjsg #define mmTCP_CHAN_STEER_5_ARCT 0x0b0c 138c349dbc7Sjsg #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX 0 139c349dbc7Sjsg 140c349dbc7Sjsg enum ta_ras_gfx_subblock { 141c349dbc7Sjsg /*CPC*/ 142c349dbc7Sjsg TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0, 143c349dbc7Sjsg TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START, 144c349dbc7Sjsg TA_RAS_BLOCK__GFX_CPC_UCODE, 145c349dbc7Sjsg TA_RAS_BLOCK__GFX_DC_STATE_ME1, 146c349dbc7Sjsg TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1, 147c349dbc7Sjsg TA_RAS_BLOCK__GFX_DC_RESTORE_ME1, 148c349dbc7Sjsg TA_RAS_BLOCK__GFX_DC_STATE_ME2, 149c349dbc7Sjsg TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2, 150c349dbc7Sjsg TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 151c349dbc7Sjsg TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 152c349dbc7Sjsg /* CPF*/ 153c349dbc7Sjsg TA_RAS_BLOCK__GFX_CPF_INDEX_START, 154c349dbc7Sjsg TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START, 155c349dbc7Sjsg TA_RAS_BLOCK__GFX_CPF_ROQ_ME1, 156c349dbc7Sjsg TA_RAS_BLOCK__GFX_CPF_TAG, 157c349dbc7Sjsg TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG, 158c349dbc7Sjsg /* CPG*/ 159c349dbc7Sjsg TA_RAS_BLOCK__GFX_CPG_INDEX_START, 160c349dbc7Sjsg TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START, 161c349dbc7Sjsg TA_RAS_BLOCK__GFX_CPG_DMA_TAG, 162c349dbc7Sjsg TA_RAS_BLOCK__GFX_CPG_TAG, 163c349dbc7Sjsg TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG, 164c349dbc7Sjsg /* GDS*/ 165c349dbc7Sjsg TA_RAS_BLOCK__GFX_GDS_INDEX_START, 166c349dbc7Sjsg TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START, 167c349dbc7Sjsg TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE, 168c349dbc7Sjsg TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM, 169c349dbc7Sjsg TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM, 170c349dbc7Sjsg TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 171c349dbc7Sjsg TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 172c349dbc7Sjsg /* SPI*/ 173c349dbc7Sjsg TA_RAS_BLOCK__GFX_SPI_SR_MEM, 174c349dbc7Sjsg /* SQ*/ 175c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQ_INDEX_START, 176c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START, 177c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQ_LDS_D, 178c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQ_LDS_I, 179c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/ 180c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR, 181c349dbc7Sjsg /* SQC (3 ranges)*/ 182c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_INDEX_START, 183c349dbc7Sjsg /* SQC range 0*/ 184c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START, 185c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO = 186c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_INDEX0_START, 187c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 188c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO, 189c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 190c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO, 191c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 192c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 193c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_INDEX0_END = 194c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 195c349dbc7Sjsg /* SQC range 1*/ 196c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 197c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM = 198c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 199c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 200c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO, 201c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM, 202c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM, 203c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO, 204c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO, 205c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 206c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 207c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_INDEX1_END = 208c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 209c349dbc7Sjsg /* SQC range 2*/ 210c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 211c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM = 212c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 213c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 214c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO, 215c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM, 216c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM, 217c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO, 218c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO, 219c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 220c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 221c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_INDEX2_END = 222c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 223c349dbc7Sjsg TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END, 224c349dbc7Sjsg /* TA*/ 225c349dbc7Sjsg TA_RAS_BLOCK__GFX_TA_INDEX_START, 226c349dbc7Sjsg TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START, 227c349dbc7Sjsg TA_RAS_BLOCK__GFX_TA_FS_AFIFO, 228c349dbc7Sjsg TA_RAS_BLOCK__GFX_TA_FL_LFIFO, 229c349dbc7Sjsg TA_RAS_BLOCK__GFX_TA_FX_LFIFO, 230c349dbc7Sjsg TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 231c349dbc7Sjsg TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 232c349dbc7Sjsg /* TCA*/ 233c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCA_INDEX_START, 234c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START, 235c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 236c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 237c349dbc7Sjsg /* TCC (5 sub-ranges)*/ 238c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_INDEX_START, 239c349dbc7Sjsg /* TCC range 0*/ 240c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START, 241c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START, 242c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1, 243c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0, 244c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1, 245c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0, 246c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1, 247c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG, 248c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 249c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 250c349dbc7Sjsg /* TCC range 1*/ 251c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 252c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 253c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 254c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_INDEX1_END = 255c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 256c349dbc7Sjsg /* TCC range 2*/ 257c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 258c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 259c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL, 260c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO, 261c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN, 262c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ, 263c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_SRC_FIFO, 264c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM, 265c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 266c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_INDEX2_END = 267c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 268c349dbc7Sjsg /* TCC range 3*/ 269c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 270c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 271c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 272c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_INDEX3_END = 273c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 274c349dbc7Sjsg /* TCC range 4*/ 275c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 276c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN = 277c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 278c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 279c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_INDEX4_END = 280c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 281c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END, 282c349dbc7Sjsg /* TCI*/ 283c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCI_WRITE_RAM, 284c349dbc7Sjsg /* TCP*/ 285c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCP_INDEX_START, 286c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START, 287c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM, 288c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCP_CMD_FIFO, 289c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCP_VM_FIFO, 290c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCP_DB_RAM, 291c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0, 292c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 293c349dbc7Sjsg TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 294c349dbc7Sjsg /* TD*/ 295c349dbc7Sjsg TA_RAS_BLOCK__GFX_TD_INDEX_START, 296c349dbc7Sjsg TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START, 297c349dbc7Sjsg TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI, 298c349dbc7Sjsg TA_RAS_BLOCK__GFX_TD_CS_FIFO, 299c349dbc7Sjsg TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO, 300c349dbc7Sjsg /* EA (3 sub-ranges)*/ 301c349dbc7Sjsg TA_RAS_BLOCK__GFX_EA_INDEX_START, 302c349dbc7Sjsg /* EA range 0*/ 303c349dbc7Sjsg TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START, 304c349dbc7Sjsg TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START, 305c349dbc7Sjsg TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM, 306c349dbc7Sjsg TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM, 307c349dbc7Sjsg TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM, 308c349dbc7Sjsg TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM, 309c349dbc7Sjsg TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM, 310c349dbc7Sjsg TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM, 311c349dbc7Sjsg TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 312c349dbc7Sjsg TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 313c349dbc7Sjsg /* EA range 1*/ 314c349dbc7Sjsg TA_RAS_BLOCK__GFX_EA_INDEX1_START, 315c349dbc7Sjsg TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START, 316c349dbc7Sjsg TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM, 317c349dbc7Sjsg TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM, 318c349dbc7Sjsg TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM, 319c349dbc7Sjsg TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM, 320c349dbc7Sjsg TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM, 321c349dbc7Sjsg TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 322c349dbc7Sjsg TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 323c349dbc7Sjsg /* EA range 2*/ 324c349dbc7Sjsg TA_RAS_BLOCK__GFX_EA_INDEX2_START, 325c349dbc7Sjsg TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START, 326c349dbc7Sjsg TA_RAS_BLOCK__GFX_EA_MAM_D1MEM, 327c349dbc7Sjsg TA_RAS_BLOCK__GFX_EA_MAM_D2MEM, 328c349dbc7Sjsg TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 329c349dbc7Sjsg TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 330c349dbc7Sjsg TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END, 331c349dbc7Sjsg /* UTC VM L2 bank*/ 332c349dbc7Sjsg TA_RAS_BLOCK__UTC_VML2_BANK_CACHE, 333c349dbc7Sjsg /* UTC VM walker*/ 334c349dbc7Sjsg TA_RAS_BLOCK__UTC_VML2_WALKER, 335c349dbc7Sjsg /* UTC ATC L2 2MB cache*/ 336c349dbc7Sjsg TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK, 337c349dbc7Sjsg /* UTC ATC L2 4KB cache*/ 338c349dbc7Sjsg TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK, 339c349dbc7Sjsg TA_RAS_BLOCK__GFX_MAX 340c349dbc7Sjsg }; 341c349dbc7Sjsg 342c349dbc7Sjsg struct ras_gfx_subblock { 343c349dbc7Sjsg unsigned char *name; 344c349dbc7Sjsg int ta_subblock; 345c349dbc7Sjsg int hw_supported_error_type; 346c349dbc7Sjsg int sw_supported_error_type; 347c349dbc7Sjsg }; 348c349dbc7Sjsg 349c349dbc7Sjsg #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h) \ 350c349dbc7Sjsg [AMDGPU_RAS_BLOCK__##subblock] = { \ 351c349dbc7Sjsg #subblock, \ 352c349dbc7Sjsg TA_RAS_BLOCK__##subblock, \ 353c349dbc7Sjsg ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)), \ 354c349dbc7Sjsg (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)), \ 355c349dbc7Sjsg } 356c349dbc7Sjsg 357c349dbc7Sjsg static const struct ras_gfx_subblock ras_gfx_subblocks[] = { 358c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1), 359c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1), 360c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 361c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 362c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 363c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 364c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 365c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 366c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 367c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 368c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1), 369c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0), 370c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1), 371c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1), 372c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 373c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0), 374c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0, 375c349dbc7Sjsg 0), 376c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0, 377c349dbc7Sjsg 0), 378c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 379c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0), 380c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0), 381c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1), 382c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0), 383c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0), 384c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1), 385c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 386c349dbc7Sjsg 0, 0), 387c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 388c349dbc7Sjsg 0), 389c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 390c349dbc7Sjsg 0, 0), 391c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0, 392c349dbc7Sjsg 0), 393c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 394c349dbc7Sjsg 0, 0), 395c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 396c349dbc7Sjsg 0), 397c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 398c349dbc7Sjsg 1), 399c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 400c349dbc7Sjsg 0, 0, 0), 401c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 402c349dbc7Sjsg 0), 403c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 404c349dbc7Sjsg 0), 405c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 406c349dbc7Sjsg 0), 407c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 408c349dbc7Sjsg 0), 409c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 410c349dbc7Sjsg 0), 411c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 412c349dbc7Sjsg 0, 0), 413c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 414c349dbc7Sjsg 0), 415c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 416c349dbc7Sjsg 0), 417c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 418c349dbc7Sjsg 0, 0, 0), 419c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 420c349dbc7Sjsg 0), 421c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 422c349dbc7Sjsg 0), 423c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 424c349dbc7Sjsg 0), 425c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 426c349dbc7Sjsg 0), 427c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 428c349dbc7Sjsg 0), 429c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 430c349dbc7Sjsg 0, 0), 431c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 432c349dbc7Sjsg 0), 433c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1), 434c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 435c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 436c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 437c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 438c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0), 439c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 440c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1), 441c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0, 442c349dbc7Sjsg 1), 443c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0, 444c349dbc7Sjsg 1), 445c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0, 446c349dbc7Sjsg 1), 447c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0, 448c349dbc7Sjsg 0), 449c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0, 450c349dbc7Sjsg 0), 451c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 452c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 453c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0), 454c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0), 455c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0), 456c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0), 457c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 458c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0), 459c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0), 460c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 461c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0), 462c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0, 463c349dbc7Sjsg 0), 464c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 465c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0, 466c349dbc7Sjsg 0), 467c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0, 468c349dbc7Sjsg 0, 0), 469c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0, 470c349dbc7Sjsg 0), 471c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 472c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1), 473c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0), 474c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 475c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 476c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 477c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0), 478c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0), 479c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1), 480c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0), 481c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 482c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1), 483c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 484c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 485c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 486c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 487c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 488c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 489c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 490c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 491c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 492c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 493c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 494c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0), 495c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 496c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 497c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0), 498c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0), 499c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0), 500c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0), 501c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0), 502c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0), 503c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0), 504c349dbc7Sjsg AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0), 505c349dbc7Sjsg }; 50602fbaceeSjsg 507fb4d8502Sjsg static const struct soc15_reg_golden golden_settings_gc_9_0[] = 508fb4d8502Sjsg { 5097a066a52Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 510bbba1d3bSjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000), 511fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 512fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 513fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 514fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 515fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 516fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 517fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 518ad8b1aafSjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87), 519ad8b1aafSjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f), 520fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 521fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 522fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 523fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 524fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 525c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff), 526c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 527c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 528c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 529fb4d8502Sjsg }; 530fb4d8502Sjsg 531fb4d8502Sjsg static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] = 532fb4d8502Sjsg { 533fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107), 534fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 535fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 536fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 537fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 538fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 539fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042), 540fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 541fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000), 542fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 543fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 544fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 545fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 546fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 547fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 548fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107), 549fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800), 550fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080) 551fb4d8502Sjsg }; 552fb4d8502Sjsg 553fb4d8502Sjsg static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] = 554fb4d8502Sjsg { 555fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080), 556fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 557fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 558fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042), 559fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042), 560fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400), 561fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000), 562fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000), 563fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107), 564fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000), 565fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000) 566fb4d8502Sjsg }; 567fb4d8502Sjsg 568fb4d8502Sjsg static const struct soc15_reg_golden golden_settings_gc_9_1[] = 569fb4d8502Sjsg { 570fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 571fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 572fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 573fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 574fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 575fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 576fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 577fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 578fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 579fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 580fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 581fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 582fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 583fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 584fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 585fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 586fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 587fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 588fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 589fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff), 590c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080), 591c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 592c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 593c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 594fb4d8502Sjsg }; 595fb4d8502Sjsg 596fb4d8502Sjsg static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] = 597fb4d8502Sjsg { 598fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 599fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042), 600fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042), 601fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000), 602fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000), 603fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 604fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800) 605fb4d8502Sjsg }; 606fb4d8502Sjsg 607c349dbc7Sjsg static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] = 608c349dbc7Sjsg { 609c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000), 610c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 611c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 612c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080), 613c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080), 614c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080), 615c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041), 616c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041), 617c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 618c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 619c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080), 620c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080), 621c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080), 622c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080), 623c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080), 624c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 625c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010), 626c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 627c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 628c349dbc7Sjsg }; 629c349dbc7Sjsg 630c349dbc7Sjsg static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] = 631c349dbc7Sjsg { 632c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 633c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 634c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 635c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042), 636c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042), 637c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 638c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 639c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 640c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 641c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 642c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 643c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc), 644c349dbc7Sjsg }; 645c349dbc7Sjsg 646fb4d8502Sjsg static const struct soc15_reg_golden golden_settings_gc_9_x_common[] = 647fb4d8502Sjsg { 648c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff), 649fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000), 650fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382) 651fb4d8502Sjsg }; 652fb4d8502Sjsg 653fb4d8502Sjsg static const struct soc15_reg_golden golden_settings_gc_9_2_1[] = 654fb4d8502Sjsg { 655fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 656fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 657fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 658fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 659fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 660fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 661fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 662fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 663fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87), 664fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f), 665fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 666fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 667fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 668fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 669fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 670fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff) 671fb4d8502Sjsg }; 672fb4d8502Sjsg 673fb4d8502Sjsg static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] = 674fb4d8502Sjsg { 675fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080), 676fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 677fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 678fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041), 679fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041), 680fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 681fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107), 682fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 683fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410), 684c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 685c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 686c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 687c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 688c349dbc7Sjsg }; 689c349dbc7Sjsg 690c349dbc7Sjsg static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] = 691c349dbc7Sjsg { 692c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 693c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000), 694c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e), 695c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca), 696c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098), 697c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3), 698c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1), 699c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135), 700c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000), 701c349dbc7Sjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00), 702ad8b1aafSjsg SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000) 703c349dbc7Sjsg }; 704c349dbc7Sjsg 705c349dbc7Sjsg static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = { 706c349dbc7Sjsg {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)}, 707c349dbc7Sjsg {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)}, 708fb4d8502Sjsg }; 709fb4d8502Sjsg 710fb4d8502Sjsg static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] = 711fb4d8502Sjsg { 712fb4d8502Sjsg mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 713fb4d8502Sjsg mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 714fb4d8502Sjsg mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 715fb4d8502Sjsg mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 716fb4d8502Sjsg mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 717fb4d8502Sjsg mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 718fb4d8502Sjsg mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 719fb4d8502Sjsg mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 720fb4d8502Sjsg }; 721fb4d8502Sjsg 722fb4d8502Sjsg static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] = 723fb4d8502Sjsg { 724fb4d8502Sjsg mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0, 725fb4d8502Sjsg mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0, 726fb4d8502Sjsg mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0, 727fb4d8502Sjsg mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0, 728fb4d8502Sjsg mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0, 729fb4d8502Sjsg mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0, 730fb4d8502Sjsg mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0, 731fb4d8502Sjsg mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0, 732fb4d8502Sjsg }; 733fb4d8502Sjsg 734ad8b1aafSjsg static void gfx_v9_0_rlcg_wreg(struct amdgpu_device *adev, u32 offset, u32 v) 735c349dbc7Sjsg { 736c349dbc7Sjsg static void *scratch_reg0; 737c349dbc7Sjsg static void *scratch_reg1; 738c349dbc7Sjsg static void *scratch_reg2; 739c349dbc7Sjsg static void *scratch_reg3; 740c349dbc7Sjsg static void *spare_int; 741c349dbc7Sjsg static uint32_t grbm_cntl; 742c349dbc7Sjsg static uint32_t grbm_idx; 743c349dbc7Sjsg 744c349dbc7Sjsg scratch_reg0 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] + mmSCRATCH_REG0)*4; 745c349dbc7Sjsg scratch_reg1 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG1)*4; 746c349dbc7Sjsg scratch_reg2 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG2)*4; 747c349dbc7Sjsg scratch_reg3 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG3)*4; 748c349dbc7Sjsg spare_int = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] + mmRLC_SPARE_INT)*4; 749c349dbc7Sjsg 750c349dbc7Sjsg grbm_cntl = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] + mmGRBM_GFX_CNTL; 751c349dbc7Sjsg grbm_idx = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] + mmGRBM_GFX_INDEX; 752c349dbc7Sjsg 753c349dbc7Sjsg if (amdgpu_sriov_runtime(adev)) { 754c349dbc7Sjsg pr_err("shouldn't call rlcg write register during runtime\n"); 755c349dbc7Sjsg return; 756c349dbc7Sjsg } 757c349dbc7Sjsg 758c349dbc7Sjsg if (offset == grbm_cntl || offset == grbm_idx) { 759c349dbc7Sjsg if (offset == grbm_cntl) 760e54dbfe7Sjsg writel(v, scratch_reg2); 761c349dbc7Sjsg else if (offset == grbm_idx) 762e54dbfe7Sjsg writel(v, scratch_reg3); 763c349dbc7Sjsg 764e54dbfe7Sjsg writel(v, ((void __iomem *)adev->rmmio) + (offset * 4)); 765c349dbc7Sjsg } else { 766c349dbc7Sjsg uint32_t i = 0; 767c349dbc7Sjsg uint32_t retries = 50000; 768c349dbc7Sjsg 769e54dbfe7Sjsg writel(v, scratch_reg0); 770e54dbfe7Sjsg writel(offset | 0x80000000, scratch_reg1); 771e54dbfe7Sjsg writel(1, spare_int); 772c349dbc7Sjsg for (i = 0; i < retries; i++) { 773c349dbc7Sjsg u32 tmp; 774c349dbc7Sjsg 775e54dbfe7Sjsg tmp = readl(scratch_reg1); 776c349dbc7Sjsg if (!(tmp & 0x80000000)) 777c349dbc7Sjsg break; 778c349dbc7Sjsg 779c349dbc7Sjsg udelay(10); 780c349dbc7Sjsg } 781c349dbc7Sjsg if (i >= retries) 782c349dbc7Sjsg pr_err("timeout: rlcg program reg:0x%05x failed !\n", offset); 783c349dbc7Sjsg } 784c349dbc7Sjsg 785c349dbc7Sjsg } 786c349dbc7Sjsg 787fb4d8502Sjsg #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042 788fb4d8502Sjsg #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041 789fb4d8502Sjsg #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042 790c349dbc7Sjsg #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041 791fb4d8502Sjsg 792fb4d8502Sjsg static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev); 793fb4d8502Sjsg static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev); 794fb4d8502Sjsg static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev); 795fb4d8502Sjsg static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev); 796fb4d8502Sjsg static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 797fb4d8502Sjsg struct amdgpu_cu_info *cu_info); 798fb4d8502Sjsg static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev); 799fb4d8502Sjsg static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring); 800c349dbc7Sjsg static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring); 801c349dbc7Sjsg static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 802c349dbc7Sjsg void *ras_error_status); 803c349dbc7Sjsg static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 804c349dbc7Sjsg void *inject_if); 805c349dbc7Sjsg static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev); 806c349dbc7Sjsg 807c349dbc7Sjsg static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring, 808c349dbc7Sjsg uint64_t queue_mask) 809c349dbc7Sjsg { 810c349dbc7Sjsg amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6)); 811c349dbc7Sjsg amdgpu_ring_write(kiq_ring, 812c349dbc7Sjsg PACKET3_SET_RESOURCES_VMID_MASK(0) | 813c349dbc7Sjsg /* vmid_mask:0* queue_type:0 (KIQ) */ 814c349dbc7Sjsg PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); 815c349dbc7Sjsg amdgpu_ring_write(kiq_ring, 816c349dbc7Sjsg lower_32_bits(queue_mask)); /* queue mask lo */ 817c349dbc7Sjsg amdgpu_ring_write(kiq_ring, 818c349dbc7Sjsg upper_32_bits(queue_mask)); /* queue mask hi */ 819c349dbc7Sjsg amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */ 820c349dbc7Sjsg amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */ 821c349dbc7Sjsg amdgpu_ring_write(kiq_ring, 0); /* oac mask */ 822c349dbc7Sjsg amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */ 823c349dbc7Sjsg } 824c349dbc7Sjsg 825c349dbc7Sjsg static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring, 826c349dbc7Sjsg struct amdgpu_ring *ring) 827c349dbc7Sjsg { 828c349dbc7Sjsg struct amdgpu_device *adev = kiq_ring->adev; 829c349dbc7Sjsg uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); 830c349dbc7Sjsg uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 831c349dbc7Sjsg uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 832c349dbc7Sjsg 833c349dbc7Sjsg amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); 834c349dbc7Sjsg /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/ 835c349dbc7Sjsg amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 836c349dbc7Sjsg PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */ 837c349dbc7Sjsg PACKET3_MAP_QUEUES_VMID(0) | /* VMID */ 838c349dbc7Sjsg PACKET3_MAP_QUEUES_QUEUE(ring->queue) | 839c349dbc7Sjsg PACKET3_MAP_QUEUES_PIPE(ring->pipe) | 840c349dbc7Sjsg PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) | 841c349dbc7Sjsg /*queue_type: normal compute queue */ 842c349dbc7Sjsg PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | 843c349dbc7Sjsg /* alloc format: all_on_one_pipe */ 844c349dbc7Sjsg PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | 845c349dbc7Sjsg PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) | 846c349dbc7Sjsg /* num_queues: must be 1 */ 847c349dbc7Sjsg PACKET3_MAP_QUEUES_NUM_QUEUES(1)); 848c349dbc7Sjsg amdgpu_ring_write(kiq_ring, 849c349dbc7Sjsg PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index)); 850c349dbc7Sjsg amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr)); 851c349dbc7Sjsg amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr)); 852c349dbc7Sjsg amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr)); 853c349dbc7Sjsg amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr)); 854c349dbc7Sjsg } 855c349dbc7Sjsg 856c349dbc7Sjsg static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring, 857c349dbc7Sjsg struct amdgpu_ring *ring, 858c349dbc7Sjsg enum amdgpu_unmap_queues_action action, 859c349dbc7Sjsg u64 gpu_addr, u64 seq) 860c349dbc7Sjsg { 861c349dbc7Sjsg uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 862c349dbc7Sjsg 863c349dbc7Sjsg amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4)); 864c349dbc7Sjsg amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 865c349dbc7Sjsg PACKET3_UNMAP_QUEUES_ACTION(action) | 866c349dbc7Sjsg PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) | 867c349dbc7Sjsg PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) | 868c349dbc7Sjsg PACKET3_UNMAP_QUEUES_NUM_QUEUES(1)); 869c349dbc7Sjsg amdgpu_ring_write(kiq_ring, 870c349dbc7Sjsg PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index)); 871c349dbc7Sjsg 872c349dbc7Sjsg if (action == PREEMPT_QUEUES_NO_UNMAP) { 873c349dbc7Sjsg amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr)); 874c349dbc7Sjsg amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr)); 875c349dbc7Sjsg amdgpu_ring_write(kiq_ring, seq); 876c349dbc7Sjsg } else { 877c349dbc7Sjsg amdgpu_ring_write(kiq_ring, 0); 878c349dbc7Sjsg amdgpu_ring_write(kiq_ring, 0); 879c349dbc7Sjsg amdgpu_ring_write(kiq_ring, 0); 880c349dbc7Sjsg } 881c349dbc7Sjsg } 882c349dbc7Sjsg 883c349dbc7Sjsg static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring, 884c349dbc7Sjsg struct amdgpu_ring *ring, 885c349dbc7Sjsg u64 addr, 886c349dbc7Sjsg u64 seq) 887c349dbc7Sjsg { 888c349dbc7Sjsg uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 889c349dbc7Sjsg 890c349dbc7Sjsg amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5)); 891c349dbc7Sjsg amdgpu_ring_write(kiq_ring, 892c349dbc7Sjsg PACKET3_QUERY_STATUS_CONTEXT_ID(0) | 893c349dbc7Sjsg PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) | 894c349dbc7Sjsg PACKET3_QUERY_STATUS_COMMAND(2)); 895c349dbc7Sjsg /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 896c349dbc7Sjsg amdgpu_ring_write(kiq_ring, 897c349dbc7Sjsg PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) | 898c349dbc7Sjsg PACKET3_QUERY_STATUS_ENG_SEL(eng_sel)); 899c349dbc7Sjsg amdgpu_ring_write(kiq_ring, lower_32_bits(addr)); 900c349dbc7Sjsg amdgpu_ring_write(kiq_ring, upper_32_bits(addr)); 901c349dbc7Sjsg amdgpu_ring_write(kiq_ring, lower_32_bits(seq)); 902c349dbc7Sjsg amdgpu_ring_write(kiq_ring, upper_32_bits(seq)); 903c349dbc7Sjsg } 904c349dbc7Sjsg 905c349dbc7Sjsg static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring, 906c349dbc7Sjsg uint16_t pasid, uint32_t flush_type, 907c349dbc7Sjsg bool all_hub) 908c349dbc7Sjsg { 909c349dbc7Sjsg amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0)); 910c349dbc7Sjsg amdgpu_ring_write(kiq_ring, 911c349dbc7Sjsg PACKET3_INVALIDATE_TLBS_DST_SEL(1) | 912c349dbc7Sjsg PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) | 913c349dbc7Sjsg PACKET3_INVALIDATE_TLBS_PASID(pasid) | 914c349dbc7Sjsg PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type)); 915c349dbc7Sjsg } 916c349dbc7Sjsg 917c349dbc7Sjsg static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = { 918c349dbc7Sjsg .kiq_set_resources = gfx_v9_0_kiq_set_resources, 919c349dbc7Sjsg .kiq_map_queues = gfx_v9_0_kiq_map_queues, 920c349dbc7Sjsg .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues, 921c349dbc7Sjsg .kiq_query_status = gfx_v9_0_kiq_query_status, 922c349dbc7Sjsg .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs, 923c349dbc7Sjsg .set_resources_size = 8, 924c349dbc7Sjsg .map_queues_size = 7, 925c349dbc7Sjsg .unmap_queues_size = 6, 926c349dbc7Sjsg .query_status_size = 7, 927c349dbc7Sjsg .invalidate_tlbs_size = 2, 928c349dbc7Sjsg }; 929c349dbc7Sjsg 930c349dbc7Sjsg static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev) 931c349dbc7Sjsg { 932c349dbc7Sjsg adev->gfx.kiq.pmf = &gfx_v9_0_kiq_pm4_funcs; 933c349dbc7Sjsg } 934fb4d8502Sjsg 935fb4d8502Sjsg static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev) 936fb4d8502Sjsg { 937fb4d8502Sjsg switch (adev->asic_type) { 938fb4d8502Sjsg case CHIP_VEGA10: 939fb4d8502Sjsg soc15_program_register_sequence(adev, 940fb4d8502Sjsg golden_settings_gc_9_0, 941fb4d8502Sjsg ARRAY_SIZE(golden_settings_gc_9_0)); 942fb4d8502Sjsg soc15_program_register_sequence(adev, 943fb4d8502Sjsg golden_settings_gc_9_0_vg10, 944fb4d8502Sjsg ARRAY_SIZE(golden_settings_gc_9_0_vg10)); 945fb4d8502Sjsg break; 946fb4d8502Sjsg case CHIP_VEGA12: 947fb4d8502Sjsg soc15_program_register_sequence(adev, 948fb4d8502Sjsg golden_settings_gc_9_2_1, 949fb4d8502Sjsg ARRAY_SIZE(golden_settings_gc_9_2_1)); 950fb4d8502Sjsg soc15_program_register_sequence(adev, 951fb4d8502Sjsg golden_settings_gc_9_2_1_vg12, 952fb4d8502Sjsg ARRAY_SIZE(golden_settings_gc_9_2_1_vg12)); 953fb4d8502Sjsg break; 954fb4d8502Sjsg case CHIP_VEGA20: 955fb4d8502Sjsg soc15_program_register_sequence(adev, 956fb4d8502Sjsg golden_settings_gc_9_0, 957fb4d8502Sjsg ARRAY_SIZE(golden_settings_gc_9_0)); 958fb4d8502Sjsg soc15_program_register_sequence(adev, 959fb4d8502Sjsg golden_settings_gc_9_0_vg20, 960fb4d8502Sjsg ARRAY_SIZE(golden_settings_gc_9_0_vg20)); 961fb4d8502Sjsg break; 962c349dbc7Sjsg case CHIP_ARCTURUS: 963fb4d8502Sjsg soc15_program_register_sequence(adev, 964c349dbc7Sjsg golden_settings_gc_9_4_1_arct, 965c349dbc7Sjsg ARRAY_SIZE(golden_settings_gc_9_4_1_arct)); 966c349dbc7Sjsg break; 967c349dbc7Sjsg case CHIP_RAVEN: 968c349dbc7Sjsg soc15_program_register_sequence(adev, golden_settings_gc_9_1, 969fb4d8502Sjsg ARRAY_SIZE(golden_settings_gc_9_1)); 970ad8b1aafSjsg if (adev->apu_flags & AMD_APU_IS_RAVEN2) 971c349dbc7Sjsg soc15_program_register_sequence(adev, 972c349dbc7Sjsg golden_settings_gc_9_1_rv2, 973c349dbc7Sjsg ARRAY_SIZE(golden_settings_gc_9_1_rv2)); 974c349dbc7Sjsg else 975fb4d8502Sjsg soc15_program_register_sequence(adev, 976fb4d8502Sjsg golden_settings_gc_9_1_rv1, 977fb4d8502Sjsg ARRAY_SIZE(golden_settings_gc_9_1_rv1)); 978fb4d8502Sjsg break; 979c349dbc7Sjsg case CHIP_RENOIR: 980c349dbc7Sjsg soc15_program_register_sequence(adev, 981c349dbc7Sjsg golden_settings_gc_9_1_rn, 982c349dbc7Sjsg ARRAY_SIZE(golden_settings_gc_9_1_rn)); 983c349dbc7Sjsg return; /* for renoir, don't need common goldensetting */ 984fb4d8502Sjsg default: 985fb4d8502Sjsg break; 986fb4d8502Sjsg } 987fb4d8502Sjsg 988c349dbc7Sjsg if (adev->asic_type != CHIP_ARCTURUS) 989fb4d8502Sjsg soc15_program_register_sequence(adev, golden_settings_gc_9_x_common, 990fb4d8502Sjsg (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common)); 991fb4d8502Sjsg } 992fb4d8502Sjsg 993fb4d8502Sjsg static void gfx_v9_0_scratch_init(struct amdgpu_device *adev) 994fb4d8502Sjsg { 995fb4d8502Sjsg adev->gfx.scratch.num_reg = 8; 996fb4d8502Sjsg adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0); 997fb4d8502Sjsg adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1; 998fb4d8502Sjsg } 999fb4d8502Sjsg 1000fb4d8502Sjsg static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel, 1001fb4d8502Sjsg bool wc, uint32_t reg, uint32_t val) 1002fb4d8502Sjsg { 1003fb4d8502Sjsg amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 1004fb4d8502Sjsg amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) | 1005fb4d8502Sjsg WRITE_DATA_DST_SEL(0) | 1006fb4d8502Sjsg (wc ? WR_CONFIRM : 0)); 1007fb4d8502Sjsg amdgpu_ring_write(ring, reg); 1008fb4d8502Sjsg amdgpu_ring_write(ring, 0); 1009fb4d8502Sjsg amdgpu_ring_write(ring, val); 1010fb4d8502Sjsg } 1011fb4d8502Sjsg 1012fb4d8502Sjsg static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, 1013fb4d8502Sjsg int mem_space, int opt, uint32_t addr0, 1014fb4d8502Sjsg uint32_t addr1, uint32_t ref, uint32_t mask, 1015fb4d8502Sjsg uint32_t inv) 1016fb4d8502Sjsg { 1017fb4d8502Sjsg amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 1018fb4d8502Sjsg amdgpu_ring_write(ring, 1019fb4d8502Sjsg /* memory (1) or register (0) */ 1020fb4d8502Sjsg (WAIT_REG_MEM_MEM_SPACE(mem_space) | 1021fb4d8502Sjsg WAIT_REG_MEM_OPERATION(opt) | /* wait */ 1022fb4d8502Sjsg WAIT_REG_MEM_FUNCTION(3) | /* equal */ 1023fb4d8502Sjsg WAIT_REG_MEM_ENGINE(eng_sel))); 1024fb4d8502Sjsg 1025fb4d8502Sjsg if (mem_space) 1026fb4d8502Sjsg BUG_ON(addr0 & 0x3); /* Dword align */ 1027fb4d8502Sjsg amdgpu_ring_write(ring, addr0); 1028fb4d8502Sjsg amdgpu_ring_write(ring, addr1); 1029fb4d8502Sjsg amdgpu_ring_write(ring, ref); 1030fb4d8502Sjsg amdgpu_ring_write(ring, mask); 1031fb4d8502Sjsg amdgpu_ring_write(ring, inv); /* poll interval */ 1032fb4d8502Sjsg } 1033fb4d8502Sjsg 1034fb4d8502Sjsg static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring) 1035fb4d8502Sjsg { 1036fb4d8502Sjsg struct amdgpu_device *adev = ring->adev; 1037fb4d8502Sjsg uint32_t scratch; 1038fb4d8502Sjsg uint32_t tmp = 0; 1039fb4d8502Sjsg unsigned i; 1040fb4d8502Sjsg int r; 1041fb4d8502Sjsg 1042fb4d8502Sjsg r = amdgpu_gfx_scratch_get(adev, &scratch); 1043c349dbc7Sjsg if (r) 1044fb4d8502Sjsg return r; 1045c349dbc7Sjsg 1046fb4d8502Sjsg WREG32(scratch, 0xCAFEDEAD); 1047fb4d8502Sjsg r = amdgpu_ring_alloc(ring, 3); 1048c349dbc7Sjsg if (r) 1049c349dbc7Sjsg goto error_free_scratch; 1050c349dbc7Sjsg 1051fb4d8502Sjsg amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 1052fb4d8502Sjsg amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START)); 1053fb4d8502Sjsg amdgpu_ring_write(ring, 0xDEADBEEF); 1054fb4d8502Sjsg amdgpu_ring_commit(ring); 1055fb4d8502Sjsg 1056fb4d8502Sjsg for (i = 0; i < adev->usec_timeout; i++) { 1057fb4d8502Sjsg tmp = RREG32(scratch); 1058fb4d8502Sjsg if (tmp == 0xDEADBEEF) 1059fb4d8502Sjsg break; 1060c349dbc7Sjsg udelay(1); 1061fb4d8502Sjsg } 1062c349dbc7Sjsg 1063c349dbc7Sjsg if (i >= adev->usec_timeout) 1064c349dbc7Sjsg r = -ETIMEDOUT; 1065c349dbc7Sjsg 1066c349dbc7Sjsg error_free_scratch: 1067fb4d8502Sjsg amdgpu_gfx_scratch_free(adev, scratch); 1068fb4d8502Sjsg return r; 1069fb4d8502Sjsg } 1070fb4d8502Sjsg 1071fb4d8502Sjsg static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) 1072fb4d8502Sjsg { 1073fb4d8502Sjsg struct amdgpu_device *adev = ring->adev; 1074fb4d8502Sjsg struct amdgpu_ib ib; 1075fb4d8502Sjsg struct dma_fence *f = NULL; 1076fb4d8502Sjsg 1077fb4d8502Sjsg unsigned index; 1078fb4d8502Sjsg uint64_t gpu_addr; 1079fb4d8502Sjsg uint32_t tmp; 1080fb4d8502Sjsg long r; 1081fb4d8502Sjsg 1082fb4d8502Sjsg r = amdgpu_device_wb_get(adev, &index); 1083c349dbc7Sjsg if (r) 1084fb4d8502Sjsg return r; 1085fb4d8502Sjsg 1086fb4d8502Sjsg gpu_addr = adev->wb.gpu_addr + (index * 4); 1087fb4d8502Sjsg adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD); 1088fb4d8502Sjsg memset(&ib, 0, sizeof(ib)); 1089ad8b1aafSjsg r = amdgpu_ib_get(adev, NULL, 16, 1090ad8b1aafSjsg AMDGPU_IB_POOL_DIRECT, &ib); 1091c349dbc7Sjsg if (r) 1092fb4d8502Sjsg goto err1; 1093c349dbc7Sjsg 1094fb4d8502Sjsg ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3); 1095fb4d8502Sjsg ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM; 1096fb4d8502Sjsg ib.ptr[2] = lower_32_bits(gpu_addr); 1097fb4d8502Sjsg ib.ptr[3] = upper_32_bits(gpu_addr); 1098fb4d8502Sjsg ib.ptr[4] = 0xDEADBEEF; 1099fb4d8502Sjsg ib.length_dw = 5; 1100fb4d8502Sjsg 1101fb4d8502Sjsg r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 1102fb4d8502Sjsg if (r) 1103fb4d8502Sjsg goto err2; 1104fb4d8502Sjsg 1105fb4d8502Sjsg r = dma_fence_wait_timeout(f, false, timeout); 1106fb4d8502Sjsg if (r == 0) { 1107fb4d8502Sjsg r = -ETIMEDOUT; 1108fb4d8502Sjsg goto err2; 1109fb4d8502Sjsg } else if (r < 0) { 1110fb4d8502Sjsg goto err2; 1111fb4d8502Sjsg } 1112fb4d8502Sjsg 1113fb4d8502Sjsg tmp = adev->wb.wb[index]; 1114c349dbc7Sjsg if (tmp == 0xDEADBEEF) 1115fb4d8502Sjsg r = 0; 1116c349dbc7Sjsg else 1117fb4d8502Sjsg r = -EINVAL; 1118fb4d8502Sjsg 1119fb4d8502Sjsg err2: 1120fb4d8502Sjsg amdgpu_ib_free(adev, &ib, NULL); 1121fb4d8502Sjsg dma_fence_put(f); 1122fb4d8502Sjsg err1: 1123fb4d8502Sjsg amdgpu_device_wb_free(adev, index); 1124fb4d8502Sjsg return r; 1125fb4d8502Sjsg } 1126fb4d8502Sjsg 1127fb4d8502Sjsg 1128fb4d8502Sjsg static void gfx_v9_0_free_microcode(struct amdgpu_device *adev) 1129fb4d8502Sjsg { 1130fb4d8502Sjsg release_firmware(adev->gfx.pfp_fw); 1131fb4d8502Sjsg adev->gfx.pfp_fw = NULL; 1132fb4d8502Sjsg release_firmware(adev->gfx.me_fw); 1133fb4d8502Sjsg adev->gfx.me_fw = NULL; 1134fb4d8502Sjsg release_firmware(adev->gfx.ce_fw); 1135fb4d8502Sjsg adev->gfx.ce_fw = NULL; 1136fb4d8502Sjsg release_firmware(adev->gfx.rlc_fw); 1137fb4d8502Sjsg adev->gfx.rlc_fw = NULL; 1138fb4d8502Sjsg release_firmware(adev->gfx.mec_fw); 1139fb4d8502Sjsg adev->gfx.mec_fw = NULL; 1140fb4d8502Sjsg release_firmware(adev->gfx.mec2_fw); 1141fb4d8502Sjsg adev->gfx.mec2_fw = NULL; 1142fb4d8502Sjsg 1143fb4d8502Sjsg kfree(adev->gfx.rlc.register_list_format); 1144fb4d8502Sjsg } 1145fb4d8502Sjsg 1146fb4d8502Sjsg static void gfx_v9_0_init_rlc_ext_microcode(struct amdgpu_device *adev) 1147fb4d8502Sjsg { 1148fb4d8502Sjsg const struct rlc_firmware_header_v2_1 *rlc_hdr; 1149fb4d8502Sjsg 1150fb4d8502Sjsg rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data; 1151fb4d8502Sjsg adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver); 1152fb4d8502Sjsg adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver); 1153fb4d8502Sjsg adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes); 1154fb4d8502Sjsg adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes); 1155fb4d8502Sjsg adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver); 1156fb4d8502Sjsg adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver); 1157fb4d8502Sjsg adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes); 1158fb4d8502Sjsg adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes); 1159fb4d8502Sjsg adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver); 1160fb4d8502Sjsg adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver); 1161fb4d8502Sjsg adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes); 1162fb4d8502Sjsg adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes); 1163fb4d8502Sjsg adev->gfx.rlc.reg_list_format_direct_reg_list_length = 1164fb4d8502Sjsg le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length); 1165fb4d8502Sjsg } 1166fb4d8502Sjsg 1167c349dbc7Sjsg static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev) 1168fb4d8502Sjsg { 1169c349dbc7Sjsg adev->gfx.me_fw_write_wait = false; 1170c349dbc7Sjsg adev->gfx.mec_fw_write_wait = false; 1171c349dbc7Sjsg 1172c349dbc7Sjsg if ((adev->asic_type != CHIP_ARCTURUS) && 1173c349dbc7Sjsg ((adev->gfx.mec_fw_version < 0x000001a5) || 1174c349dbc7Sjsg (adev->gfx.mec_feature_version < 46) || 1175c349dbc7Sjsg (adev->gfx.pfp_fw_version < 0x000000b7) || 1176c349dbc7Sjsg (adev->gfx.pfp_feature_version < 46))) 1177c349dbc7Sjsg DRM_WARN_ONCE("CP firmware version too old, please update!"); 1178c349dbc7Sjsg 1179c349dbc7Sjsg switch (adev->asic_type) { 1180c349dbc7Sjsg case CHIP_VEGA10: 1181c349dbc7Sjsg if ((adev->gfx.me_fw_version >= 0x0000009c) && 1182c349dbc7Sjsg (adev->gfx.me_feature_version >= 42) && 1183c349dbc7Sjsg (adev->gfx.pfp_fw_version >= 0x000000b1) && 1184c349dbc7Sjsg (adev->gfx.pfp_feature_version >= 42)) 1185c349dbc7Sjsg adev->gfx.me_fw_write_wait = true; 1186c349dbc7Sjsg 1187c349dbc7Sjsg if ((adev->gfx.mec_fw_version >= 0x00000193) && 1188c349dbc7Sjsg (adev->gfx.mec_feature_version >= 42)) 1189c349dbc7Sjsg adev->gfx.mec_fw_write_wait = true; 1190c349dbc7Sjsg break; 1191c349dbc7Sjsg case CHIP_VEGA12: 1192c349dbc7Sjsg if ((adev->gfx.me_fw_version >= 0x0000009c) && 1193c349dbc7Sjsg (adev->gfx.me_feature_version >= 44) && 1194c349dbc7Sjsg (adev->gfx.pfp_fw_version >= 0x000000b2) && 1195c349dbc7Sjsg (adev->gfx.pfp_feature_version >= 44)) 1196c349dbc7Sjsg adev->gfx.me_fw_write_wait = true; 1197c349dbc7Sjsg 1198c349dbc7Sjsg if ((adev->gfx.mec_fw_version >= 0x00000196) && 1199c349dbc7Sjsg (adev->gfx.mec_feature_version >= 44)) 1200c349dbc7Sjsg adev->gfx.mec_fw_write_wait = true; 1201c349dbc7Sjsg break; 1202c349dbc7Sjsg case CHIP_VEGA20: 1203c349dbc7Sjsg if ((adev->gfx.me_fw_version >= 0x0000009c) && 1204c349dbc7Sjsg (adev->gfx.me_feature_version >= 44) && 1205c349dbc7Sjsg (adev->gfx.pfp_fw_version >= 0x000000b2) && 1206c349dbc7Sjsg (adev->gfx.pfp_feature_version >= 44)) 1207c349dbc7Sjsg adev->gfx.me_fw_write_wait = true; 1208c349dbc7Sjsg 1209c349dbc7Sjsg if ((adev->gfx.mec_fw_version >= 0x00000197) && 1210c349dbc7Sjsg (adev->gfx.mec_feature_version >= 44)) 1211c349dbc7Sjsg adev->gfx.mec_fw_write_wait = true; 1212c349dbc7Sjsg break; 1213c349dbc7Sjsg case CHIP_RAVEN: 1214c349dbc7Sjsg if ((adev->gfx.me_fw_version >= 0x0000009c) && 1215c349dbc7Sjsg (adev->gfx.me_feature_version >= 42) && 1216c349dbc7Sjsg (adev->gfx.pfp_fw_version >= 0x000000b1) && 1217c349dbc7Sjsg (adev->gfx.pfp_feature_version >= 42)) 1218c349dbc7Sjsg adev->gfx.me_fw_write_wait = true; 1219c349dbc7Sjsg 1220c349dbc7Sjsg if ((adev->gfx.mec_fw_version >= 0x00000192) && 1221c349dbc7Sjsg (adev->gfx.mec_feature_version >= 42)) 1222c349dbc7Sjsg adev->gfx.mec_fw_write_wait = true; 1223c349dbc7Sjsg break; 1224c349dbc7Sjsg default: 1225c349dbc7Sjsg adev->gfx.me_fw_write_wait = true; 1226c349dbc7Sjsg adev->gfx.mec_fw_write_wait = true; 1227c349dbc7Sjsg break; 1228c349dbc7Sjsg } 1229c349dbc7Sjsg } 1230c349dbc7Sjsg 1231c349dbc7Sjsg struct amdgpu_gfxoff_quirk { 1232c349dbc7Sjsg u16 chip_vendor; 1233c349dbc7Sjsg u16 chip_device; 1234c349dbc7Sjsg u16 subsys_vendor; 1235c349dbc7Sjsg u16 subsys_device; 1236c349dbc7Sjsg u8 revision; 1237c349dbc7Sjsg }; 1238c349dbc7Sjsg 1239c349dbc7Sjsg static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = { 1240c349dbc7Sjsg /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */ 1241c349dbc7Sjsg { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 }, 1242c349dbc7Sjsg /* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */ 1243c349dbc7Sjsg { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 }, 1244c349dbc7Sjsg /* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */ 1245c349dbc7Sjsg { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 }, 1246c349dbc7Sjsg { 0, 0, 0, 0, 0 }, 1247c349dbc7Sjsg }; 1248c349dbc7Sjsg 1249c349dbc7Sjsg static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev) 1250c349dbc7Sjsg { 1251c349dbc7Sjsg const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list; 1252c349dbc7Sjsg 1253c349dbc7Sjsg while (p && p->chip_device != 0) { 1254c349dbc7Sjsg if (pdev->vendor == p->chip_vendor && 1255c349dbc7Sjsg pdev->device == p->chip_device && 1256c349dbc7Sjsg pdev->subsystem_vendor == p->subsys_vendor && 1257c349dbc7Sjsg pdev->subsystem_device == p->subsys_device && 1258c349dbc7Sjsg pdev->revision == p->revision) { 1259c349dbc7Sjsg return true; 1260c349dbc7Sjsg } 1261c349dbc7Sjsg ++p; 1262c349dbc7Sjsg } 1263c349dbc7Sjsg return false; 1264c349dbc7Sjsg } 1265c349dbc7Sjsg 1266c349dbc7Sjsg static bool is_raven_kicker(struct amdgpu_device *adev) 1267c349dbc7Sjsg { 1268c349dbc7Sjsg if (adev->pm.fw_version >= 0x41e2b) 1269c349dbc7Sjsg return true; 1270c349dbc7Sjsg else 1271c349dbc7Sjsg return false; 1272c349dbc7Sjsg } 1273c349dbc7Sjsg 1274*1c23f5d9Sjsg static bool check_if_enlarge_doorbell_range(struct amdgpu_device *adev) 1275*1c23f5d9Sjsg { 1276*1c23f5d9Sjsg if ((adev->asic_type == CHIP_RENOIR) && 1277*1c23f5d9Sjsg (adev->gfx.me_fw_version >= 0x000000a5) && 1278*1c23f5d9Sjsg (adev->gfx.me_feature_version >= 52)) 1279*1c23f5d9Sjsg return true; 1280*1c23f5d9Sjsg else 1281*1c23f5d9Sjsg return false; 1282*1c23f5d9Sjsg } 1283*1c23f5d9Sjsg 1284c349dbc7Sjsg static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev) 1285c349dbc7Sjsg { 1286c349dbc7Sjsg if (gfx_v9_0_should_disable_gfxoff(adev->pdev)) 1287c349dbc7Sjsg adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1288c349dbc7Sjsg 1289c349dbc7Sjsg switch (adev->asic_type) { 1290c349dbc7Sjsg case CHIP_VEGA10: 1291c349dbc7Sjsg case CHIP_VEGA12: 1292c349dbc7Sjsg case CHIP_VEGA20: 1293c349dbc7Sjsg break; 1294c349dbc7Sjsg case CHIP_RAVEN: 1295ad8b1aafSjsg if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) || 1296ad8b1aafSjsg (adev->apu_flags & AMD_APU_IS_PICASSO)) && 1297c349dbc7Sjsg ((!is_raven_kicker(adev) && 1298c349dbc7Sjsg adev->gfx.rlc_fw_version < 531) || 1299c349dbc7Sjsg (adev->gfx.rlc_feature_version < 1) || 1300c349dbc7Sjsg !adev->gfx.rlc.is_rlc_v2_1)) 1301c349dbc7Sjsg adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1302c349dbc7Sjsg 1303c349dbc7Sjsg if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1304c349dbc7Sjsg adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1305c349dbc7Sjsg AMD_PG_SUPPORT_CP | 1306c349dbc7Sjsg AMD_PG_SUPPORT_RLC_SMU_HS; 1307c349dbc7Sjsg break; 1308c349dbc7Sjsg case CHIP_RENOIR: 1309c349dbc7Sjsg if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1310c349dbc7Sjsg adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1311c349dbc7Sjsg AMD_PG_SUPPORT_CP | 1312c349dbc7Sjsg AMD_PG_SUPPORT_RLC_SMU_HS; 1313c349dbc7Sjsg break; 1314c349dbc7Sjsg default: 1315c349dbc7Sjsg break; 1316c349dbc7Sjsg } 1317c349dbc7Sjsg } 1318c349dbc7Sjsg 1319c349dbc7Sjsg static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev, 1320c349dbc7Sjsg const char *chip_name) 1321c349dbc7Sjsg { 1322fb4d8502Sjsg char fw_name[30]; 1323fb4d8502Sjsg int err; 1324fb4d8502Sjsg struct amdgpu_firmware_info *info = NULL; 1325fb4d8502Sjsg const struct common_firmware_header *header = NULL; 1326fb4d8502Sjsg const struct gfx_firmware_header_v1_0 *cp_hdr; 1327fb4d8502Sjsg 1328fb4d8502Sjsg snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name); 1329fb4d8502Sjsg err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev); 1330fb4d8502Sjsg if (err) 1331fb4d8502Sjsg goto out; 1332fb4d8502Sjsg err = amdgpu_ucode_validate(adev->gfx.pfp_fw); 1333fb4d8502Sjsg if (err) 1334fb4d8502Sjsg goto out; 1335fb4d8502Sjsg cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data; 1336fb4d8502Sjsg adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1337fb4d8502Sjsg adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1338fb4d8502Sjsg 1339fb4d8502Sjsg snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name); 1340fb4d8502Sjsg err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev); 1341fb4d8502Sjsg if (err) 1342fb4d8502Sjsg goto out; 1343fb4d8502Sjsg err = amdgpu_ucode_validate(adev->gfx.me_fw); 1344fb4d8502Sjsg if (err) 1345fb4d8502Sjsg goto out; 1346fb4d8502Sjsg cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data; 1347fb4d8502Sjsg adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1348fb4d8502Sjsg adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1349fb4d8502Sjsg 1350fb4d8502Sjsg snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name); 1351fb4d8502Sjsg err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev); 1352fb4d8502Sjsg if (err) 1353fb4d8502Sjsg goto out; 1354fb4d8502Sjsg err = amdgpu_ucode_validate(adev->gfx.ce_fw); 1355fb4d8502Sjsg if (err) 1356fb4d8502Sjsg goto out; 1357fb4d8502Sjsg cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data; 1358fb4d8502Sjsg adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1359fb4d8502Sjsg adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1360fb4d8502Sjsg 1361c349dbc7Sjsg if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1362c349dbc7Sjsg info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP]; 1363c349dbc7Sjsg info->ucode_id = AMDGPU_UCODE_ID_CP_PFP; 1364c349dbc7Sjsg info->fw = adev->gfx.pfp_fw; 1365c349dbc7Sjsg header = (const struct common_firmware_header *)info->fw->data; 1366c349dbc7Sjsg adev->firmware.fw_size += 1367c349dbc7Sjsg roundup2(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1368c349dbc7Sjsg 1369c349dbc7Sjsg info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME]; 1370c349dbc7Sjsg info->ucode_id = AMDGPU_UCODE_ID_CP_ME; 1371c349dbc7Sjsg info->fw = adev->gfx.me_fw; 1372c349dbc7Sjsg header = (const struct common_firmware_header *)info->fw->data; 1373c349dbc7Sjsg adev->firmware.fw_size += 1374c349dbc7Sjsg roundup2(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1375c349dbc7Sjsg 1376c349dbc7Sjsg info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE]; 1377c349dbc7Sjsg info->ucode_id = AMDGPU_UCODE_ID_CP_CE; 1378c349dbc7Sjsg info->fw = adev->gfx.ce_fw; 1379c349dbc7Sjsg header = (const struct common_firmware_header *)info->fw->data; 1380c349dbc7Sjsg adev->firmware.fw_size += 1381c349dbc7Sjsg roundup2(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1382c349dbc7Sjsg } 1383c349dbc7Sjsg 1384c349dbc7Sjsg out: 1385c349dbc7Sjsg if (err) { 1386c349dbc7Sjsg dev_err(adev->dev, 1387c349dbc7Sjsg "gfx9: Failed to load firmware \"%s\"\n", 1388c349dbc7Sjsg fw_name); 1389c349dbc7Sjsg release_firmware(adev->gfx.pfp_fw); 1390c349dbc7Sjsg adev->gfx.pfp_fw = NULL; 1391c349dbc7Sjsg release_firmware(adev->gfx.me_fw); 1392c349dbc7Sjsg adev->gfx.me_fw = NULL; 1393c349dbc7Sjsg release_firmware(adev->gfx.ce_fw); 1394c349dbc7Sjsg adev->gfx.ce_fw = NULL; 1395c349dbc7Sjsg } 1396c349dbc7Sjsg return err; 1397c349dbc7Sjsg } 1398c349dbc7Sjsg 1399c349dbc7Sjsg static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev, 1400c349dbc7Sjsg const char *chip_name) 1401c349dbc7Sjsg { 1402c349dbc7Sjsg char fw_name[30]; 1403c349dbc7Sjsg int err; 1404c349dbc7Sjsg struct amdgpu_firmware_info *info = NULL; 1405c349dbc7Sjsg const struct common_firmware_header *header = NULL; 1406c349dbc7Sjsg const struct rlc_firmware_header_v2_0 *rlc_hdr; 1407c349dbc7Sjsg unsigned int *tmp = NULL; 1408c349dbc7Sjsg unsigned int i = 0; 1409c349dbc7Sjsg uint16_t version_major; 1410c349dbc7Sjsg uint16_t version_minor; 1411c349dbc7Sjsg uint32_t smu_version; 1412c349dbc7Sjsg 1413c349dbc7Sjsg /* 1414c349dbc7Sjsg * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin 1415c349dbc7Sjsg * instead of picasso_rlc.bin. 1416c349dbc7Sjsg * Judgment method: 1417c349dbc7Sjsg * PCO AM4: revision >= 0xC8 && revision <= 0xCF 1418c349dbc7Sjsg * or revision >= 0xD8 && revision <= 0xDF 1419c349dbc7Sjsg * otherwise is PCO FP5 1420c349dbc7Sjsg */ 1421c349dbc7Sjsg if (!strcmp(chip_name, "picasso") && 1422c349dbc7Sjsg (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) || 1423c349dbc7Sjsg ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF)))) 1424c349dbc7Sjsg snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc_am4.bin", chip_name); 1425c349dbc7Sjsg else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) && 1426c349dbc7Sjsg (smu_version >= 0x41e2b)) 1427c349dbc7Sjsg /** 1428c349dbc7Sjsg *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly. 1429c349dbc7Sjsg */ 1430c349dbc7Sjsg snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_kicker_rlc.bin", chip_name); 1431c349dbc7Sjsg else 1432fb4d8502Sjsg snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name); 1433fb4d8502Sjsg err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev); 1434fb4d8502Sjsg if (err) 1435fb4d8502Sjsg goto out; 1436fb4d8502Sjsg err = amdgpu_ucode_validate(adev->gfx.rlc_fw); 1437fb4d8502Sjsg rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 1438fb4d8502Sjsg 1439fb4d8502Sjsg version_major = le16_to_cpu(rlc_hdr->header.header_version_major); 1440fb4d8502Sjsg version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); 1441fb4d8502Sjsg if (version_major == 2 && version_minor == 1) 1442fb4d8502Sjsg adev->gfx.rlc.is_rlc_v2_1 = true; 1443fb4d8502Sjsg 1444fb4d8502Sjsg adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version); 1445fb4d8502Sjsg adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version); 1446fb4d8502Sjsg adev->gfx.rlc.save_and_restore_offset = 1447fb4d8502Sjsg le32_to_cpu(rlc_hdr->save_and_restore_offset); 1448fb4d8502Sjsg adev->gfx.rlc.clear_state_descriptor_offset = 1449fb4d8502Sjsg le32_to_cpu(rlc_hdr->clear_state_descriptor_offset); 1450fb4d8502Sjsg adev->gfx.rlc.avail_scratch_ram_locations = 1451fb4d8502Sjsg le32_to_cpu(rlc_hdr->avail_scratch_ram_locations); 1452fb4d8502Sjsg adev->gfx.rlc.reg_restore_list_size = 1453fb4d8502Sjsg le32_to_cpu(rlc_hdr->reg_restore_list_size); 1454fb4d8502Sjsg adev->gfx.rlc.reg_list_format_start = 1455fb4d8502Sjsg le32_to_cpu(rlc_hdr->reg_list_format_start); 1456fb4d8502Sjsg adev->gfx.rlc.reg_list_format_separate_start = 1457fb4d8502Sjsg le32_to_cpu(rlc_hdr->reg_list_format_separate_start); 1458fb4d8502Sjsg adev->gfx.rlc.starting_offsets_start = 1459fb4d8502Sjsg le32_to_cpu(rlc_hdr->starting_offsets_start); 1460fb4d8502Sjsg adev->gfx.rlc.reg_list_format_size_bytes = 1461fb4d8502Sjsg le32_to_cpu(rlc_hdr->reg_list_format_size_bytes); 1462fb4d8502Sjsg adev->gfx.rlc.reg_list_size_bytes = 1463fb4d8502Sjsg le32_to_cpu(rlc_hdr->reg_list_size_bytes); 1464fb4d8502Sjsg adev->gfx.rlc.register_list_format = 1465fb4d8502Sjsg kmalloc(adev->gfx.rlc.reg_list_format_size_bytes + 1466fb4d8502Sjsg adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL); 1467fb4d8502Sjsg if (!adev->gfx.rlc.register_list_format) { 1468fb4d8502Sjsg err = -ENOMEM; 1469fb4d8502Sjsg goto out; 1470fb4d8502Sjsg } 1471fb4d8502Sjsg 1472fb4d8502Sjsg tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1473fb4d8502Sjsg le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes)); 1474c349dbc7Sjsg for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++) 1475fb4d8502Sjsg adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]); 1476fb4d8502Sjsg 1477fb4d8502Sjsg adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i; 1478fb4d8502Sjsg 1479fb4d8502Sjsg tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1480fb4d8502Sjsg le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes)); 1481c349dbc7Sjsg for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++) 1482fb4d8502Sjsg adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]); 1483fb4d8502Sjsg 1484fb4d8502Sjsg if (adev->gfx.rlc.is_rlc_v2_1) 1485fb4d8502Sjsg gfx_v9_0_init_rlc_ext_microcode(adev); 1486fb4d8502Sjsg 1487c349dbc7Sjsg if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1488c349dbc7Sjsg info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G]; 1489c349dbc7Sjsg info->ucode_id = AMDGPU_UCODE_ID_RLC_G; 1490c349dbc7Sjsg info->fw = adev->gfx.rlc_fw; 1491c349dbc7Sjsg header = (const struct common_firmware_header *)info->fw->data; 1492c349dbc7Sjsg adev->firmware.fw_size += 1493c349dbc7Sjsg roundup2(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1494c349dbc7Sjsg 1495c349dbc7Sjsg if (adev->gfx.rlc.is_rlc_v2_1 && 1496c349dbc7Sjsg adev->gfx.rlc.save_restore_list_cntl_size_bytes && 1497c349dbc7Sjsg adev->gfx.rlc.save_restore_list_gpm_size_bytes && 1498c349dbc7Sjsg adev->gfx.rlc.save_restore_list_srm_size_bytes) { 1499c349dbc7Sjsg info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL]; 1500c349dbc7Sjsg info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL; 1501c349dbc7Sjsg info->fw = adev->gfx.rlc_fw; 1502c349dbc7Sjsg adev->firmware.fw_size += 1503c349dbc7Sjsg roundup2(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE); 1504c349dbc7Sjsg 1505c349dbc7Sjsg info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM]; 1506c349dbc7Sjsg info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM; 1507c349dbc7Sjsg info->fw = adev->gfx.rlc_fw; 1508c349dbc7Sjsg adev->firmware.fw_size += 1509c349dbc7Sjsg roundup2(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE); 1510c349dbc7Sjsg 1511c349dbc7Sjsg info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM]; 1512c349dbc7Sjsg info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM; 1513c349dbc7Sjsg info->fw = adev->gfx.rlc_fw; 1514c349dbc7Sjsg adev->firmware.fw_size += 1515c349dbc7Sjsg roundup2(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE); 1516c349dbc7Sjsg } 1517c349dbc7Sjsg } 1518c349dbc7Sjsg 1519c349dbc7Sjsg out: 1520c349dbc7Sjsg if (err) { 1521c349dbc7Sjsg dev_err(adev->dev, 1522c349dbc7Sjsg "gfx9: Failed to load firmware \"%s\"\n", 1523c349dbc7Sjsg fw_name); 1524c349dbc7Sjsg release_firmware(adev->gfx.rlc_fw); 1525c349dbc7Sjsg adev->gfx.rlc_fw = NULL; 1526c349dbc7Sjsg } 1527c349dbc7Sjsg return err; 1528c349dbc7Sjsg } 1529c349dbc7Sjsg 1530c349dbc7Sjsg static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev, 1531c349dbc7Sjsg const char *chip_name) 1532c349dbc7Sjsg { 1533c349dbc7Sjsg char fw_name[30]; 1534c349dbc7Sjsg int err; 1535c349dbc7Sjsg struct amdgpu_firmware_info *info = NULL; 1536c349dbc7Sjsg const struct common_firmware_header *header = NULL; 1537c349dbc7Sjsg const struct gfx_firmware_header_v1_0 *cp_hdr; 1538c349dbc7Sjsg 1539fb4d8502Sjsg snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name); 1540fb4d8502Sjsg err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev); 1541fb4d8502Sjsg if (err) 1542fb4d8502Sjsg goto out; 1543fb4d8502Sjsg err = amdgpu_ucode_validate(adev->gfx.mec_fw); 1544fb4d8502Sjsg if (err) 1545fb4d8502Sjsg goto out; 1546fb4d8502Sjsg cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1547fb4d8502Sjsg adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1548fb4d8502Sjsg adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1549fb4d8502Sjsg 1550fb4d8502Sjsg 1551fb4d8502Sjsg snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name); 1552fb4d8502Sjsg err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev); 1553fb4d8502Sjsg if (!err) { 1554fb4d8502Sjsg err = amdgpu_ucode_validate(adev->gfx.mec2_fw); 1555fb4d8502Sjsg if (err) 1556fb4d8502Sjsg goto out; 1557fb4d8502Sjsg cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1558fb4d8502Sjsg adev->gfx.mec2_fw->data; 1559fb4d8502Sjsg adev->gfx.mec2_fw_version = 1560fb4d8502Sjsg le32_to_cpu(cp_hdr->header.ucode_version); 1561fb4d8502Sjsg adev->gfx.mec2_feature_version = 1562fb4d8502Sjsg le32_to_cpu(cp_hdr->ucode_feature_version); 1563fb4d8502Sjsg } else { 1564fb4d8502Sjsg err = 0; 1565fb4d8502Sjsg adev->gfx.mec2_fw = NULL; 1566fb4d8502Sjsg } 1567fb4d8502Sjsg 1568fb4d8502Sjsg if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1569fb4d8502Sjsg info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1]; 1570fb4d8502Sjsg info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1; 1571fb4d8502Sjsg info->fw = adev->gfx.mec_fw; 1572fb4d8502Sjsg header = (const struct common_firmware_header *)info->fw->data; 1573fb4d8502Sjsg cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; 1574fb4d8502Sjsg adev->firmware.fw_size += 1575fb4d8502Sjsg roundup2(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1576fb4d8502Sjsg 1577fb4d8502Sjsg info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT]; 1578fb4d8502Sjsg info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT; 1579fb4d8502Sjsg info->fw = adev->gfx.mec_fw; 1580fb4d8502Sjsg adev->firmware.fw_size += 1581fb4d8502Sjsg roundup2(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1582fb4d8502Sjsg 1583fb4d8502Sjsg if (adev->gfx.mec2_fw) { 1584fb4d8502Sjsg info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2]; 1585fb4d8502Sjsg info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2; 1586fb4d8502Sjsg info->fw = adev->gfx.mec2_fw; 1587fb4d8502Sjsg header = (const struct common_firmware_header *)info->fw->data; 1588fb4d8502Sjsg cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; 1589fb4d8502Sjsg adev->firmware.fw_size += 1590fb4d8502Sjsg roundup2(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1591c349dbc7Sjsg 1592c349dbc7Sjsg /* TODO: Determine if MEC2 JT FW loading can be removed 1593c349dbc7Sjsg for all GFX V9 asic and above */ 1594c349dbc7Sjsg if (adev->asic_type != CHIP_ARCTURUS && 1595c349dbc7Sjsg adev->asic_type != CHIP_RENOIR) { 1596fb4d8502Sjsg info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT]; 1597fb4d8502Sjsg info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT; 1598fb4d8502Sjsg info->fw = adev->gfx.mec2_fw; 1599fb4d8502Sjsg adev->firmware.fw_size += 1600c349dbc7Sjsg roundup2(le32_to_cpu(cp_hdr->jt_size) * 4, 1601c349dbc7Sjsg PAGE_SIZE); 1602fb4d8502Sjsg } 1603c349dbc7Sjsg } 1604fb4d8502Sjsg } 1605fb4d8502Sjsg 1606fb4d8502Sjsg out: 1607c349dbc7Sjsg gfx_v9_0_check_if_need_gfxoff(adev); 1608c349dbc7Sjsg gfx_v9_0_check_fw_write_wait(adev); 1609fb4d8502Sjsg if (err) { 1610fb4d8502Sjsg dev_err(adev->dev, 1611fb4d8502Sjsg "gfx9: Failed to load firmware \"%s\"\n", 1612fb4d8502Sjsg fw_name); 1613fb4d8502Sjsg release_firmware(adev->gfx.mec_fw); 1614fb4d8502Sjsg adev->gfx.mec_fw = NULL; 1615fb4d8502Sjsg release_firmware(adev->gfx.mec2_fw); 1616fb4d8502Sjsg adev->gfx.mec2_fw = NULL; 1617fb4d8502Sjsg } 1618fb4d8502Sjsg return err; 1619fb4d8502Sjsg } 1620fb4d8502Sjsg 1621c349dbc7Sjsg static int gfx_v9_0_init_microcode(struct amdgpu_device *adev) 1622c349dbc7Sjsg { 1623c349dbc7Sjsg const char *chip_name; 1624c349dbc7Sjsg int r; 1625c349dbc7Sjsg 1626c349dbc7Sjsg DRM_DEBUG("\n"); 1627c349dbc7Sjsg 1628c349dbc7Sjsg switch (adev->asic_type) { 1629c349dbc7Sjsg case CHIP_VEGA10: 1630c349dbc7Sjsg chip_name = "vega10"; 1631c349dbc7Sjsg break; 1632c349dbc7Sjsg case CHIP_VEGA12: 1633c349dbc7Sjsg chip_name = "vega12"; 1634c349dbc7Sjsg break; 1635c349dbc7Sjsg case CHIP_VEGA20: 1636c349dbc7Sjsg chip_name = "vega20"; 1637c349dbc7Sjsg break; 1638c349dbc7Sjsg case CHIP_RAVEN: 1639ad8b1aafSjsg if (adev->apu_flags & AMD_APU_IS_RAVEN2) 1640c349dbc7Sjsg chip_name = "raven2"; 1641ad8b1aafSjsg else if (adev->apu_flags & AMD_APU_IS_PICASSO) 1642c349dbc7Sjsg chip_name = "picasso"; 1643c349dbc7Sjsg else 1644c349dbc7Sjsg chip_name = "raven"; 1645c349dbc7Sjsg break; 1646c349dbc7Sjsg case CHIP_ARCTURUS: 1647c349dbc7Sjsg chip_name = "arcturus"; 1648c349dbc7Sjsg break; 1649c349dbc7Sjsg case CHIP_RENOIR: 1650ad8b1aafSjsg if (adev->apu_flags & AMD_APU_IS_RENOIR) 1651c349dbc7Sjsg chip_name = "renoir"; 1652ad8b1aafSjsg else 1653ad8b1aafSjsg chip_name = "green_sardine"; 1654c349dbc7Sjsg break; 1655c349dbc7Sjsg default: 1656c349dbc7Sjsg BUG(); 1657c349dbc7Sjsg } 1658c349dbc7Sjsg 1659c349dbc7Sjsg /* No CPG in Arcturus */ 1660c349dbc7Sjsg if (adev->asic_type != CHIP_ARCTURUS) { 1661c349dbc7Sjsg r = gfx_v9_0_init_cp_gfx_microcode(adev, chip_name); 1662c349dbc7Sjsg if (r) 1663c349dbc7Sjsg return r; 1664c349dbc7Sjsg } 1665c349dbc7Sjsg 1666c349dbc7Sjsg r = gfx_v9_0_init_rlc_microcode(adev, chip_name); 1667c349dbc7Sjsg if (r) 1668c349dbc7Sjsg return r; 1669c349dbc7Sjsg 1670c349dbc7Sjsg r = gfx_v9_0_init_cp_compute_microcode(adev, chip_name); 1671c349dbc7Sjsg if (r) 1672c349dbc7Sjsg return r; 1673c349dbc7Sjsg 1674c349dbc7Sjsg return r; 1675c349dbc7Sjsg } 1676c349dbc7Sjsg 1677fb4d8502Sjsg static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev) 1678fb4d8502Sjsg { 1679fb4d8502Sjsg u32 count = 0; 1680fb4d8502Sjsg const struct cs_section_def *sect = NULL; 1681fb4d8502Sjsg const struct cs_extent_def *ext = NULL; 1682fb4d8502Sjsg 1683fb4d8502Sjsg /* begin clear state */ 1684fb4d8502Sjsg count += 2; 1685fb4d8502Sjsg /* context control state */ 1686fb4d8502Sjsg count += 3; 1687fb4d8502Sjsg 1688fb4d8502Sjsg for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 1689fb4d8502Sjsg for (ext = sect->section; ext->extent != NULL; ++ext) { 1690fb4d8502Sjsg if (sect->id == SECT_CONTEXT) 1691fb4d8502Sjsg count += 2 + ext->reg_count; 1692fb4d8502Sjsg else 1693fb4d8502Sjsg return 0; 1694fb4d8502Sjsg } 1695fb4d8502Sjsg } 1696fb4d8502Sjsg 1697fb4d8502Sjsg /* end clear state */ 1698fb4d8502Sjsg count += 2; 1699fb4d8502Sjsg /* clear state */ 1700fb4d8502Sjsg count += 2; 1701fb4d8502Sjsg 1702fb4d8502Sjsg return count; 1703fb4d8502Sjsg } 1704fb4d8502Sjsg 1705fb4d8502Sjsg static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev, 1706fb4d8502Sjsg volatile u32 *buffer) 1707fb4d8502Sjsg { 1708fb4d8502Sjsg u32 count = 0, i; 1709fb4d8502Sjsg const struct cs_section_def *sect = NULL; 1710fb4d8502Sjsg const struct cs_extent_def *ext = NULL; 1711fb4d8502Sjsg 1712fb4d8502Sjsg if (adev->gfx.rlc.cs_data == NULL) 1713fb4d8502Sjsg return; 1714fb4d8502Sjsg if (buffer == NULL) 1715fb4d8502Sjsg return; 1716fb4d8502Sjsg 1717fb4d8502Sjsg buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1718fb4d8502Sjsg buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 1719fb4d8502Sjsg 1720fb4d8502Sjsg buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 1721fb4d8502Sjsg buffer[count++] = cpu_to_le32(0x80000000); 1722fb4d8502Sjsg buffer[count++] = cpu_to_le32(0x80000000); 1723fb4d8502Sjsg 1724fb4d8502Sjsg for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) { 1725fb4d8502Sjsg for (ext = sect->section; ext->extent != NULL; ++ext) { 1726fb4d8502Sjsg if (sect->id == SECT_CONTEXT) { 1727fb4d8502Sjsg buffer[count++] = 1728fb4d8502Sjsg cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); 1729fb4d8502Sjsg buffer[count++] = cpu_to_le32(ext->reg_index - 1730fb4d8502Sjsg PACKET3_SET_CONTEXT_REG_START); 1731fb4d8502Sjsg for (i = 0; i < ext->reg_count; i++) 1732fb4d8502Sjsg buffer[count++] = cpu_to_le32(ext->extent[i]); 1733fb4d8502Sjsg } else { 1734fb4d8502Sjsg return; 1735fb4d8502Sjsg } 1736fb4d8502Sjsg } 1737fb4d8502Sjsg } 1738fb4d8502Sjsg 1739fb4d8502Sjsg buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1740fb4d8502Sjsg buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE); 1741fb4d8502Sjsg 1742fb4d8502Sjsg buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0)); 1743fb4d8502Sjsg buffer[count++] = cpu_to_le32(0); 1744fb4d8502Sjsg } 1745fb4d8502Sjsg 1746c349dbc7Sjsg static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev) 1747c349dbc7Sjsg { 1748c349dbc7Sjsg struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info; 1749c349dbc7Sjsg uint32_t pg_always_on_cu_num = 2; 1750c349dbc7Sjsg uint32_t always_on_cu_num; 1751c349dbc7Sjsg uint32_t i, j, k; 1752c349dbc7Sjsg uint32_t mask, cu_bitmap, counter; 1753c349dbc7Sjsg 1754c349dbc7Sjsg if (adev->flags & AMD_IS_APU) 1755c349dbc7Sjsg always_on_cu_num = 4; 1756c349dbc7Sjsg else if (adev->asic_type == CHIP_VEGA12) 1757c349dbc7Sjsg always_on_cu_num = 8; 1758c349dbc7Sjsg else 1759c349dbc7Sjsg always_on_cu_num = 12; 1760c349dbc7Sjsg 1761c349dbc7Sjsg mutex_lock(&adev->grbm_idx_mutex); 1762c349dbc7Sjsg for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 1763c349dbc7Sjsg for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 1764c349dbc7Sjsg mask = 1; 1765c349dbc7Sjsg cu_bitmap = 0; 1766c349dbc7Sjsg counter = 0; 1767c349dbc7Sjsg gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 1768c349dbc7Sjsg 1769c349dbc7Sjsg for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 1770c349dbc7Sjsg if (cu_info->bitmap[i][j] & mask) { 1771c349dbc7Sjsg if (counter == pg_always_on_cu_num) 1772c349dbc7Sjsg WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap); 1773c349dbc7Sjsg if (counter < always_on_cu_num) 1774c349dbc7Sjsg cu_bitmap |= mask; 1775c349dbc7Sjsg else 1776c349dbc7Sjsg break; 1777c349dbc7Sjsg counter++; 1778c349dbc7Sjsg } 1779c349dbc7Sjsg mask <<= 1; 1780c349dbc7Sjsg } 1781c349dbc7Sjsg 1782c349dbc7Sjsg WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap); 1783c349dbc7Sjsg cu_info->ao_cu_bitmap[i][j] = cu_bitmap; 1784c349dbc7Sjsg } 1785c349dbc7Sjsg } 1786c349dbc7Sjsg gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1787c349dbc7Sjsg mutex_unlock(&adev->grbm_idx_mutex); 1788c349dbc7Sjsg } 1789c349dbc7Sjsg 1790fb4d8502Sjsg static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev) 1791fb4d8502Sjsg { 1792fb4d8502Sjsg uint32_t data; 1793fb4d8502Sjsg 1794fb4d8502Sjsg /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1795fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1796fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7); 1797fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1798fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16)); 1799fb4d8502Sjsg 1800fb4d8502Sjsg /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1801fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1802fb4d8502Sjsg 1803fb4d8502Sjsg /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1804fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500); 1805fb4d8502Sjsg 1806fb4d8502Sjsg mutex_lock(&adev->grbm_idx_mutex); 1807fb4d8502Sjsg /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1808fb4d8502Sjsg gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1809fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1810fb4d8502Sjsg 1811fb4d8502Sjsg /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1812fb4d8502Sjsg data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1813fb4d8502Sjsg data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1814fb4d8502Sjsg data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1815fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1816fb4d8502Sjsg 1817fb4d8502Sjsg /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1818fb4d8502Sjsg data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1819fb4d8502Sjsg data &= 0x0000FFFF; 1820fb4d8502Sjsg data |= 0x00C00000; 1821fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1822fb4d8502Sjsg 1823c349dbc7Sjsg /* 1824c349dbc7Sjsg * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven), 1825c349dbc7Sjsg * programmed in gfx_v9_0_init_always_on_cu_mask() 1826c349dbc7Sjsg */ 1827fb4d8502Sjsg 1828fb4d8502Sjsg /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1829fb4d8502Sjsg * but used for RLC_LB_CNTL configuration */ 1830fb4d8502Sjsg data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1831fb4d8502Sjsg data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1832fb4d8502Sjsg data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1833fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1834fb4d8502Sjsg mutex_unlock(&adev->grbm_idx_mutex); 1835c349dbc7Sjsg 1836c349dbc7Sjsg gfx_v9_0_init_always_on_cu_mask(adev); 1837c349dbc7Sjsg } 1838c349dbc7Sjsg 1839c349dbc7Sjsg static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev) 1840c349dbc7Sjsg { 1841c349dbc7Sjsg uint32_t data; 1842c349dbc7Sjsg 1843c349dbc7Sjsg /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1844c349dbc7Sjsg WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1845c349dbc7Sjsg WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8); 1846c349dbc7Sjsg WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1847c349dbc7Sjsg WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16)); 1848c349dbc7Sjsg 1849c349dbc7Sjsg /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1850c349dbc7Sjsg WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1851c349dbc7Sjsg 1852c349dbc7Sjsg /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1853c349dbc7Sjsg WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800); 1854c349dbc7Sjsg 1855c349dbc7Sjsg mutex_lock(&adev->grbm_idx_mutex); 1856c349dbc7Sjsg /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1857c349dbc7Sjsg gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1858c349dbc7Sjsg WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1859c349dbc7Sjsg 1860c349dbc7Sjsg /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1861c349dbc7Sjsg data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1862c349dbc7Sjsg data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1863c349dbc7Sjsg data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1864c349dbc7Sjsg WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1865c349dbc7Sjsg 1866c349dbc7Sjsg /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1867c349dbc7Sjsg data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1868c349dbc7Sjsg data &= 0x0000FFFF; 1869c349dbc7Sjsg data |= 0x00C00000; 1870c349dbc7Sjsg WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1871c349dbc7Sjsg 1872c349dbc7Sjsg /* 1873c349dbc7Sjsg * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON), 1874c349dbc7Sjsg * programmed in gfx_v9_0_init_always_on_cu_mask() 1875c349dbc7Sjsg */ 1876c349dbc7Sjsg 1877c349dbc7Sjsg /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1878c349dbc7Sjsg * but used for RLC_LB_CNTL configuration */ 1879c349dbc7Sjsg data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1880c349dbc7Sjsg data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1881c349dbc7Sjsg data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1882c349dbc7Sjsg WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1883c349dbc7Sjsg mutex_unlock(&adev->grbm_idx_mutex); 1884c349dbc7Sjsg 1885c349dbc7Sjsg gfx_v9_0_init_always_on_cu_mask(adev); 1886fb4d8502Sjsg } 1887fb4d8502Sjsg 1888fb4d8502Sjsg static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable) 1889fb4d8502Sjsg { 1890fb4d8502Sjsg WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0); 1891fb4d8502Sjsg } 1892fb4d8502Sjsg 1893c349dbc7Sjsg static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev) 1894fb4d8502Sjsg { 1895c349dbc7Sjsg return 5; 1896fb4d8502Sjsg } 1897fb4d8502Sjsg 1898fb4d8502Sjsg static int gfx_v9_0_rlc_init(struct amdgpu_device *adev) 1899fb4d8502Sjsg { 1900fb4d8502Sjsg const struct cs_section_def *cs_data; 1901fb4d8502Sjsg int r; 1902fb4d8502Sjsg 1903fb4d8502Sjsg adev->gfx.rlc.cs_data = gfx9_cs_data; 1904fb4d8502Sjsg 1905fb4d8502Sjsg cs_data = adev->gfx.rlc.cs_data; 1906fb4d8502Sjsg 1907fb4d8502Sjsg if (cs_data) { 1908c349dbc7Sjsg /* init clear state block */ 1909c349dbc7Sjsg r = amdgpu_gfx_rlc_init_csb(adev); 1910c349dbc7Sjsg if (r) 1911fb4d8502Sjsg return r; 1912fb4d8502Sjsg } 1913fb4d8502Sjsg 1914ad8b1aafSjsg if (adev->flags & AMD_IS_APU) { 1915fb4d8502Sjsg /* TODO: double check the cp_table_size for RV */ 1916fb4d8502Sjsg adev->gfx.rlc.cp_table_size = roundup2(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */ 1917c349dbc7Sjsg r = amdgpu_gfx_rlc_init_cpt(adev); 1918c349dbc7Sjsg if (r) 1919fb4d8502Sjsg return r; 1920fb4d8502Sjsg } 1921fb4d8502Sjsg 1922c349dbc7Sjsg switch (adev->asic_type) { 1923c349dbc7Sjsg case CHIP_RAVEN: 1924fb4d8502Sjsg gfx_v9_0_init_lbpw(adev); 1925c349dbc7Sjsg break; 1926c349dbc7Sjsg case CHIP_VEGA20: 1927c349dbc7Sjsg gfx_v9_4_init_lbpw(adev); 1928c349dbc7Sjsg break; 1929c349dbc7Sjsg default: 1930c349dbc7Sjsg break; 1931fb4d8502Sjsg } 1932fb4d8502Sjsg 1933c349dbc7Sjsg /* init spm vmid with 0xf */ 1934c349dbc7Sjsg if (adev->gfx.rlc.funcs->update_spm_vmid) 1935c349dbc7Sjsg adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf); 1936c349dbc7Sjsg 1937fb4d8502Sjsg return 0; 1938fb4d8502Sjsg } 1939fb4d8502Sjsg 1940fb4d8502Sjsg static void gfx_v9_0_mec_fini(struct amdgpu_device *adev) 1941fb4d8502Sjsg { 1942fb4d8502Sjsg amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); 1943fb4d8502Sjsg amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL); 1944fb4d8502Sjsg } 1945fb4d8502Sjsg 1946fb4d8502Sjsg static int gfx_v9_0_mec_init(struct amdgpu_device *adev) 1947fb4d8502Sjsg { 1948fb4d8502Sjsg int r; 1949fb4d8502Sjsg u32 *hpd; 1950fb4d8502Sjsg const __le32 *fw_data; 1951fb4d8502Sjsg unsigned fw_size; 1952fb4d8502Sjsg u32 *fw; 1953fb4d8502Sjsg size_t mec_hpd_size; 1954fb4d8502Sjsg 1955fb4d8502Sjsg const struct gfx_firmware_header_v1_0 *mec_hdr; 1956fb4d8502Sjsg 1957fb4d8502Sjsg bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 1958fb4d8502Sjsg 1959fb4d8502Sjsg /* take ownership of the relevant compute queues */ 1960fb4d8502Sjsg amdgpu_gfx_compute_queue_acquire(adev); 1961fb4d8502Sjsg mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE; 1962ad8b1aafSjsg if (mec_hpd_size) { 1963fb4d8502Sjsg r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, 1964c349dbc7Sjsg AMDGPU_GEM_DOMAIN_VRAM, 1965fb4d8502Sjsg &adev->gfx.mec.hpd_eop_obj, 1966fb4d8502Sjsg &adev->gfx.mec.hpd_eop_gpu_addr, 1967fb4d8502Sjsg (void **)&hpd); 1968fb4d8502Sjsg if (r) { 1969fb4d8502Sjsg dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); 1970fb4d8502Sjsg gfx_v9_0_mec_fini(adev); 1971fb4d8502Sjsg return r; 1972fb4d8502Sjsg } 1973fb4d8502Sjsg 1974c349dbc7Sjsg memset(hpd, 0, mec_hpd_size); 1975fb4d8502Sjsg 1976fb4d8502Sjsg amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); 1977fb4d8502Sjsg amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); 1978ad8b1aafSjsg } 1979fb4d8502Sjsg 1980fb4d8502Sjsg mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1981fb4d8502Sjsg 1982fb4d8502Sjsg fw_data = (const __le32 *) 1983fb4d8502Sjsg (adev->gfx.mec_fw->data + 1984fb4d8502Sjsg le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 1985ad8b1aafSjsg fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes); 1986fb4d8502Sjsg 1987fb4d8502Sjsg r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes, 1988fb4d8502Sjsg PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 1989fb4d8502Sjsg &adev->gfx.mec.mec_fw_obj, 1990fb4d8502Sjsg &adev->gfx.mec.mec_fw_gpu_addr, 1991fb4d8502Sjsg (void **)&fw); 1992fb4d8502Sjsg if (r) { 1993fb4d8502Sjsg dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r); 1994fb4d8502Sjsg gfx_v9_0_mec_fini(adev); 1995fb4d8502Sjsg return r; 1996fb4d8502Sjsg } 1997fb4d8502Sjsg 1998fb4d8502Sjsg memcpy(fw, fw_data, fw_size); 1999fb4d8502Sjsg 2000fb4d8502Sjsg amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); 2001fb4d8502Sjsg amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); 2002fb4d8502Sjsg 2003fb4d8502Sjsg return 0; 2004fb4d8502Sjsg } 2005fb4d8502Sjsg 2006fb4d8502Sjsg static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address) 2007fb4d8502Sjsg { 2008c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX, 2009fb4d8502Sjsg (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 2010fb4d8502Sjsg (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 2011fb4d8502Sjsg (address << SQ_IND_INDEX__INDEX__SHIFT) | 2012fb4d8502Sjsg (SQ_IND_INDEX__FORCE_READ_MASK)); 2013fb4d8502Sjsg return RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 2014fb4d8502Sjsg } 2015fb4d8502Sjsg 2016fb4d8502Sjsg static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd, 2017fb4d8502Sjsg uint32_t wave, uint32_t thread, 2018fb4d8502Sjsg uint32_t regno, uint32_t num, uint32_t *out) 2019fb4d8502Sjsg { 2020c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX, 2021fb4d8502Sjsg (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 2022fb4d8502Sjsg (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 2023fb4d8502Sjsg (regno << SQ_IND_INDEX__INDEX__SHIFT) | 2024fb4d8502Sjsg (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) | 2025fb4d8502Sjsg (SQ_IND_INDEX__FORCE_READ_MASK) | 2026fb4d8502Sjsg (SQ_IND_INDEX__AUTO_INCR_MASK)); 2027fb4d8502Sjsg while (num--) 2028fb4d8502Sjsg *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 2029fb4d8502Sjsg } 2030fb4d8502Sjsg 2031fb4d8502Sjsg static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields) 2032fb4d8502Sjsg { 2033fb4d8502Sjsg /* type 1 wave data */ 2034fb4d8502Sjsg dst[(*no_fields)++] = 1; 2035fb4d8502Sjsg dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS); 2036fb4d8502Sjsg dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO); 2037fb4d8502Sjsg dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI); 2038fb4d8502Sjsg dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO); 2039fb4d8502Sjsg dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI); 2040fb4d8502Sjsg dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID); 2041fb4d8502Sjsg dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0); 2042fb4d8502Sjsg dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1); 2043fb4d8502Sjsg dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC); 2044fb4d8502Sjsg dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC); 2045fb4d8502Sjsg dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS); 2046fb4d8502Sjsg dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS); 2047fb4d8502Sjsg dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0); 2048fb4d8502Sjsg dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0); 2049fb4d8502Sjsg } 2050fb4d8502Sjsg 2051fb4d8502Sjsg static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd, 2052fb4d8502Sjsg uint32_t wave, uint32_t start, 2053fb4d8502Sjsg uint32_t size, uint32_t *dst) 2054fb4d8502Sjsg { 2055fb4d8502Sjsg wave_read_regs( 2056fb4d8502Sjsg adev, simd, wave, 0, 2057fb4d8502Sjsg start + SQIND_WAVE_SGPRS_OFFSET, size, dst); 2058fb4d8502Sjsg } 2059fb4d8502Sjsg 2060fb4d8502Sjsg static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd, 2061fb4d8502Sjsg uint32_t wave, uint32_t thread, 2062fb4d8502Sjsg uint32_t start, uint32_t size, 2063fb4d8502Sjsg uint32_t *dst) 2064fb4d8502Sjsg { 2065fb4d8502Sjsg wave_read_regs( 2066fb4d8502Sjsg adev, simd, wave, thread, 2067fb4d8502Sjsg start + SQIND_WAVE_VGPRS_OFFSET, size, dst); 2068fb4d8502Sjsg } 2069fb4d8502Sjsg 2070fb4d8502Sjsg static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev, 2071c349dbc7Sjsg u32 me, u32 pipe, u32 q, u32 vm) 2072fb4d8502Sjsg { 2073c349dbc7Sjsg soc15_grbm_select(adev, me, pipe, q, vm); 2074fb4d8502Sjsg } 2075fb4d8502Sjsg 2076fb4d8502Sjsg static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = { 2077fb4d8502Sjsg .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter, 2078fb4d8502Sjsg .select_se_sh = &gfx_v9_0_select_se_sh, 2079fb4d8502Sjsg .read_wave_data = &gfx_v9_0_read_wave_data, 2080fb4d8502Sjsg .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs, 2081fb4d8502Sjsg .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs, 2082c349dbc7Sjsg .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q, 2083c349dbc7Sjsg .ras_error_inject = &gfx_v9_0_ras_error_inject, 2084c349dbc7Sjsg .query_ras_error_count = &gfx_v9_0_query_ras_error_count, 2085c349dbc7Sjsg .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count, 2086c349dbc7Sjsg }; 2087c349dbc7Sjsg 2088c349dbc7Sjsg static const struct amdgpu_gfx_funcs gfx_v9_4_gfx_funcs = { 2089c349dbc7Sjsg .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter, 2090c349dbc7Sjsg .select_se_sh = &gfx_v9_0_select_se_sh, 2091c349dbc7Sjsg .read_wave_data = &gfx_v9_0_read_wave_data, 2092c349dbc7Sjsg .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs, 2093c349dbc7Sjsg .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs, 2094c349dbc7Sjsg .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q, 2095c349dbc7Sjsg .ras_error_inject = &gfx_v9_4_ras_error_inject, 2096c349dbc7Sjsg .query_ras_error_count = &gfx_v9_4_query_ras_error_count, 2097c349dbc7Sjsg .reset_ras_error_count = &gfx_v9_4_reset_ras_error_count, 2098ad8b1aafSjsg .query_ras_error_status = &gfx_v9_4_query_ras_error_status, 2099fb4d8502Sjsg }; 2100fb4d8502Sjsg 2101fb4d8502Sjsg static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev) 2102fb4d8502Sjsg { 2103fb4d8502Sjsg u32 gb_addr_config; 2104fb4d8502Sjsg int err; 2105fb4d8502Sjsg 2106fb4d8502Sjsg adev->gfx.funcs = &gfx_v9_0_gfx_funcs; 2107fb4d8502Sjsg 2108fb4d8502Sjsg switch (adev->asic_type) { 2109fb4d8502Sjsg case CHIP_VEGA10: 2110fb4d8502Sjsg adev->gfx.config.max_hw_contexts = 8; 2111fb4d8502Sjsg adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2112fb4d8502Sjsg adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2113fb4d8502Sjsg adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2114fb4d8502Sjsg adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2115fb4d8502Sjsg gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN; 2116fb4d8502Sjsg break; 2117fb4d8502Sjsg case CHIP_VEGA12: 2118fb4d8502Sjsg adev->gfx.config.max_hw_contexts = 8; 2119fb4d8502Sjsg adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2120fb4d8502Sjsg adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2121fb4d8502Sjsg adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2122fb4d8502Sjsg adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2123fb4d8502Sjsg gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN; 2124fb4d8502Sjsg DRM_INFO("fix gfx.config for vega12\n"); 2125fb4d8502Sjsg break; 2126fb4d8502Sjsg case CHIP_VEGA20: 2127fb4d8502Sjsg adev->gfx.config.max_hw_contexts = 8; 2128fb4d8502Sjsg adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2129fb4d8502Sjsg adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2130fb4d8502Sjsg adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2131fb4d8502Sjsg adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2132fb4d8502Sjsg gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2133fb4d8502Sjsg gb_addr_config &= ~0xf3e777ff; 2134fb4d8502Sjsg gb_addr_config |= 0x22014042; 2135fb4d8502Sjsg /* check vbios table if gpu info is not available */ 2136fb4d8502Sjsg err = amdgpu_atomfirmware_get_gfx_info(adev); 2137fb4d8502Sjsg if (err) 2138fb4d8502Sjsg return err; 2139fb4d8502Sjsg break; 2140fb4d8502Sjsg case CHIP_RAVEN: 2141fb4d8502Sjsg adev->gfx.config.max_hw_contexts = 8; 2142fb4d8502Sjsg adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2143fb4d8502Sjsg adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2144fb4d8502Sjsg adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2145fb4d8502Sjsg adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2146ad8b1aafSjsg if (adev->apu_flags & AMD_APU_IS_RAVEN2) 2147c349dbc7Sjsg gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN; 2148c349dbc7Sjsg else 2149fb4d8502Sjsg gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN; 2150fb4d8502Sjsg break; 2151c349dbc7Sjsg case CHIP_ARCTURUS: 2152c349dbc7Sjsg adev->gfx.funcs = &gfx_v9_4_gfx_funcs; 2153c349dbc7Sjsg adev->gfx.config.max_hw_contexts = 8; 2154c349dbc7Sjsg adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2155c349dbc7Sjsg adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2156c349dbc7Sjsg adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2157c349dbc7Sjsg adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2158c349dbc7Sjsg gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2159c349dbc7Sjsg gb_addr_config &= ~0xf3e777ff; 2160c349dbc7Sjsg gb_addr_config |= 0x22014042; 2161c349dbc7Sjsg break; 2162c349dbc7Sjsg case CHIP_RENOIR: 2163c349dbc7Sjsg adev->gfx.config.max_hw_contexts = 8; 2164c349dbc7Sjsg adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2165c349dbc7Sjsg adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2166c349dbc7Sjsg adev->gfx.config.sc_hiz_tile_fifo_size = 0x80; 2167c349dbc7Sjsg adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2168c349dbc7Sjsg gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2169c349dbc7Sjsg gb_addr_config &= ~0xf3e777ff; 2170c349dbc7Sjsg gb_addr_config |= 0x22010042; 2171c349dbc7Sjsg break; 2172fb4d8502Sjsg default: 2173fb4d8502Sjsg BUG(); 2174fb4d8502Sjsg break; 2175fb4d8502Sjsg } 2176fb4d8502Sjsg 2177fb4d8502Sjsg adev->gfx.config.gb_addr_config = gb_addr_config; 2178fb4d8502Sjsg 2179fb4d8502Sjsg adev->gfx.config.gb_addr_config_fields.num_pipes = 1 << 2180fb4d8502Sjsg REG_GET_FIELD( 2181fb4d8502Sjsg adev->gfx.config.gb_addr_config, 2182fb4d8502Sjsg GB_ADDR_CONFIG, 2183fb4d8502Sjsg NUM_PIPES); 2184fb4d8502Sjsg 2185fb4d8502Sjsg adev->gfx.config.max_tile_pipes = 2186fb4d8502Sjsg adev->gfx.config.gb_addr_config_fields.num_pipes; 2187fb4d8502Sjsg 2188fb4d8502Sjsg adev->gfx.config.gb_addr_config_fields.num_banks = 1 << 2189fb4d8502Sjsg REG_GET_FIELD( 2190fb4d8502Sjsg adev->gfx.config.gb_addr_config, 2191fb4d8502Sjsg GB_ADDR_CONFIG, 2192fb4d8502Sjsg NUM_BANKS); 2193fb4d8502Sjsg adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 << 2194fb4d8502Sjsg REG_GET_FIELD( 2195fb4d8502Sjsg adev->gfx.config.gb_addr_config, 2196fb4d8502Sjsg GB_ADDR_CONFIG, 2197fb4d8502Sjsg MAX_COMPRESSED_FRAGS); 2198fb4d8502Sjsg adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 << 2199fb4d8502Sjsg REG_GET_FIELD( 2200fb4d8502Sjsg adev->gfx.config.gb_addr_config, 2201fb4d8502Sjsg GB_ADDR_CONFIG, 2202fb4d8502Sjsg NUM_RB_PER_SE); 2203fb4d8502Sjsg adev->gfx.config.gb_addr_config_fields.num_se = 1 << 2204fb4d8502Sjsg REG_GET_FIELD( 2205fb4d8502Sjsg adev->gfx.config.gb_addr_config, 2206fb4d8502Sjsg GB_ADDR_CONFIG, 2207fb4d8502Sjsg NUM_SHADER_ENGINES); 2208fb4d8502Sjsg adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 + 2209fb4d8502Sjsg REG_GET_FIELD( 2210fb4d8502Sjsg adev->gfx.config.gb_addr_config, 2211fb4d8502Sjsg GB_ADDR_CONFIG, 2212fb4d8502Sjsg PIPE_INTERLEAVE_SIZE)); 2213fb4d8502Sjsg 2214fb4d8502Sjsg return 0; 2215fb4d8502Sjsg } 2216fb4d8502Sjsg 2217fb4d8502Sjsg static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, 2218fb4d8502Sjsg int mec, int pipe, int queue) 2219fb4d8502Sjsg { 2220fb4d8502Sjsg unsigned irq_type; 2221fb4d8502Sjsg struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id]; 2222ad8b1aafSjsg unsigned int hw_prio; 2223fb4d8502Sjsg 2224fb4d8502Sjsg ring = &adev->gfx.compute_ring[ring_id]; 2225fb4d8502Sjsg 2226fb4d8502Sjsg /* mec0 is me1 */ 2227fb4d8502Sjsg ring->me = mec + 1; 2228fb4d8502Sjsg ring->pipe = pipe; 2229fb4d8502Sjsg ring->queue = queue; 2230fb4d8502Sjsg 2231fb4d8502Sjsg ring->ring_obj = NULL; 2232fb4d8502Sjsg ring->use_doorbell = true; 2233c349dbc7Sjsg ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1; 2234fb4d8502Sjsg ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr 2235fb4d8502Sjsg + (ring_id * GFX9_MEC_HPD_SIZE); 2236fb4d8502Sjsg snprintf(ring->name, sizeof(ring->name), "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue); 2237fb4d8502Sjsg 2238fb4d8502Sjsg irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 2239fb4d8502Sjsg + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) 2240fb4d8502Sjsg + ring->pipe; 2241ad8b1aafSjsg hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring->pipe, 2242ad8b1aafSjsg ring->queue) ? 2243ad8b1aafSjsg AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL; 2244fb4d8502Sjsg /* type-2 packets are deprecated on MEC, use type-3 instead */ 2245ad8b1aafSjsg return amdgpu_ring_init(adev, ring, 1024, 2246ad8b1aafSjsg &adev->gfx.eop_irq, irq_type, hw_prio); 2247fb4d8502Sjsg } 2248fb4d8502Sjsg 2249fb4d8502Sjsg static int gfx_v9_0_sw_init(void *handle) 2250fb4d8502Sjsg { 2251fb4d8502Sjsg int i, j, k, r, ring_id; 2252fb4d8502Sjsg struct amdgpu_ring *ring; 2253fb4d8502Sjsg struct amdgpu_kiq *kiq; 2254fb4d8502Sjsg struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2255fb4d8502Sjsg 2256fb4d8502Sjsg switch (adev->asic_type) { 2257fb4d8502Sjsg case CHIP_VEGA10: 2258fb4d8502Sjsg case CHIP_VEGA12: 2259fb4d8502Sjsg case CHIP_VEGA20: 2260fb4d8502Sjsg case CHIP_RAVEN: 2261c349dbc7Sjsg case CHIP_ARCTURUS: 2262c349dbc7Sjsg case CHIP_RENOIR: 2263fb4d8502Sjsg adev->gfx.mec.num_mec = 2; 2264fb4d8502Sjsg break; 2265fb4d8502Sjsg default: 2266fb4d8502Sjsg adev->gfx.mec.num_mec = 1; 2267fb4d8502Sjsg break; 2268fb4d8502Sjsg } 2269fb4d8502Sjsg 2270fb4d8502Sjsg adev->gfx.mec.num_pipe_per_mec = 4; 2271fb4d8502Sjsg adev->gfx.mec.num_queue_per_pipe = 8; 2272fb4d8502Sjsg 2273fb4d8502Sjsg /* EOP Event */ 2274fb4d8502Sjsg r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq); 2275fb4d8502Sjsg if (r) 2276fb4d8502Sjsg return r; 2277fb4d8502Sjsg 2278fb4d8502Sjsg /* Privileged reg */ 2279fb4d8502Sjsg r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT, 2280fb4d8502Sjsg &adev->gfx.priv_reg_irq); 2281fb4d8502Sjsg if (r) 2282fb4d8502Sjsg return r; 2283fb4d8502Sjsg 2284fb4d8502Sjsg /* Privileged inst */ 2285fb4d8502Sjsg r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT, 2286fb4d8502Sjsg &adev->gfx.priv_inst_irq); 2287fb4d8502Sjsg if (r) 2288fb4d8502Sjsg return r; 2289fb4d8502Sjsg 2290c349dbc7Sjsg /* ECC error */ 2291c349dbc7Sjsg r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR, 2292c349dbc7Sjsg &adev->gfx.cp_ecc_error_irq); 2293c349dbc7Sjsg if (r) 2294c349dbc7Sjsg return r; 2295c349dbc7Sjsg 2296c349dbc7Sjsg /* FUE error */ 2297c349dbc7Sjsg r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR, 2298c349dbc7Sjsg &adev->gfx.cp_ecc_error_irq); 2299c349dbc7Sjsg if (r) 2300c349dbc7Sjsg return r; 2301c349dbc7Sjsg 2302fb4d8502Sjsg adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; 2303fb4d8502Sjsg 2304fb4d8502Sjsg gfx_v9_0_scratch_init(adev); 2305fb4d8502Sjsg 2306fb4d8502Sjsg r = gfx_v9_0_init_microcode(adev); 2307fb4d8502Sjsg if (r) { 2308fb4d8502Sjsg DRM_ERROR("Failed to load gfx firmware!\n"); 2309fb4d8502Sjsg return r; 2310fb4d8502Sjsg } 2311fb4d8502Sjsg 2312c349dbc7Sjsg r = adev->gfx.rlc.funcs->init(adev); 2313fb4d8502Sjsg if (r) { 2314fb4d8502Sjsg DRM_ERROR("Failed to init rlc BOs!\n"); 2315fb4d8502Sjsg return r; 2316fb4d8502Sjsg } 2317fb4d8502Sjsg 2318fb4d8502Sjsg r = gfx_v9_0_mec_init(adev); 2319fb4d8502Sjsg if (r) { 2320fb4d8502Sjsg DRM_ERROR("Failed to init MEC BOs!\n"); 2321fb4d8502Sjsg return r; 2322fb4d8502Sjsg } 2323fb4d8502Sjsg 2324fb4d8502Sjsg /* set up the gfx ring */ 2325fb4d8502Sjsg for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 2326fb4d8502Sjsg ring = &adev->gfx.gfx_ring[i]; 2327fb4d8502Sjsg ring->ring_obj = NULL; 2328fb4d8502Sjsg if (!i) 2329fb4d8502Sjsg snprintf(ring->name, sizeof(ring->name), "gfx"); 2330fb4d8502Sjsg else 2331fb4d8502Sjsg snprintf(ring->name, sizeof(ring->name), "gfx_%d", i); 2332fb4d8502Sjsg ring->use_doorbell = true; 2333c349dbc7Sjsg ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1; 2334fb4d8502Sjsg r = amdgpu_ring_init(adev, ring, 1024, 2335ad8b1aafSjsg &adev->gfx.eop_irq, 2336ad8b1aafSjsg AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, 2337ad8b1aafSjsg AMDGPU_RING_PRIO_DEFAULT); 2338fb4d8502Sjsg if (r) 2339fb4d8502Sjsg return r; 2340fb4d8502Sjsg } 2341fb4d8502Sjsg 2342fb4d8502Sjsg /* set up the compute queues - allocate horizontally across pipes */ 2343fb4d8502Sjsg ring_id = 0; 2344fb4d8502Sjsg for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 2345fb4d8502Sjsg for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 2346fb4d8502Sjsg for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 2347fb4d8502Sjsg if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j)) 2348fb4d8502Sjsg continue; 2349fb4d8502Sjsg 2350fb4d8502Sjsg r = gfx_v9_0_compute_ring_init(adev, 2351fb4d8502Sjsg ring_id, 2352fb4d8502Sjsg i, k, j); 2353fb4d8502Sjsg if (r) 2354fb4d8502Sjsg return r; 2355fb4d8502Sjsg 2356fb4d8502Sjsg ring_id++; 2357fb4d8502Sjsg } 2358fb4d8502Sjsg } 2359fb4d8502Sjsg } 2360fb4d8502Sjsg 2361fb4d8502Sjsg r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE); 2362fb4d8502Sjsg if (r) { 2363fb4d8502Sjsg DRM_ERROR("Failed to init KIQ BOs!\n"); 2364fb4d8502Sjsg return r; 2365fb4d8502Sjsg } 2366fb4d8502Sjsg 2367fb4d8502Sjsg kiq = &adev->gfx.kiq; 2368fb4d8502Sjsg r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq); 2369fb4d8502Sjsg if (r) 2370fb4d8502Sjsg return r; 2371fb4d8502Sjsg 2372fb4d8502Sjsg /* create MQD for all compute queues as wel as KIQ for SRIOV case */ 2373c349dbc7Sjsg r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation)); 2374fb4d8502Sjsg if (r) 2375fb4d8502Sjsg return r; 2376fb4d8502Sjsg 2377fb4d8502Sjsg adev->gfx.ce_ram_size = 0x8000; 2378fb4d8502Sjsg 2379fb4d8502Sjsg r = gfx_v9_0_gpu_early_init(adev); 2380fb4d8502Sjsg if (r) 2381fb4d8502Sjsg return r; 2382fb4d8502Sjsg 2383fb4d8502Sjsg return 0; 2384fb4d8502Sjsg } 2385fb4d8502Sjsg 2386fb4d8502Sjsg 2387fb4d8502Sjsg static int gfx_v9_0_sw_fini(void *handle) 2388fb4d8502Sjsg { 2389fb4d8502Sjsg int i; 2390fb4d8502Sjsg struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2391fb4d8502Sjsg 2392c349dbc7Sjsg amdgpu_gfx_ras_fini(adev); 2393fb4d8502Sjsg 2394fb4d8502Sjsg for (i = 0; i < adev->gfx.num_gfx_rings; i++) 2395fb4d8502Sjsg amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); 2396fb4d8502Sjsg for (i = 0; i < adev->gfx.num_compute_rings; i++) 2397fb4d8502Sjsg amdgpu_ring_fini(&adev->gfx.compute_ring[i]); 2398fb4d8502Sjsg 2399c349dbc7Sjsg amdgpu_gfx_mqd_sw_fini(adev); 2400c349dbc7Sjsg amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring); 2401fb4d8502Sjsg amdgpu_gfx_kiq_fini(adev); 2402fb4d8502Sjsg 2403fb4d8502Sjsg gfx_v9_0_mec_fini(adev); 2404c349dbc7Sjsg amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj); 2405ad8b1aafSjsg if (adev->flags & AMD_IS_APU) { 2406fb4d8502Sjsg amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, 2407fb4d8502Sjsg &adev->gfx.rlc.cp_table_gpu_addr, 2408fb4d8502Sjsg (void **)&adev->gfx.rlc.cp_table_ptr); 2409fb4d8502Sjsg } 2410fb4d8502Sjsg gfx_v9_0_free_microcode(adev); 2411fb4d8502Sjsg 2412fb4d8502Sjsg return 0; 2413fb4d8502Sjsg } 2414fb4d8502Sjsg 2415fb4d8502Sjsg 2416fb4d8502Sjsg static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev) 2417fb4d8502Sjsg { 2418fb4d8502Sjsg /* TODO */ 2419fb4d8502Sjsg } 2420fb4d8502Sjsg 2421ad8b1aafSjsg void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, 2422ad8b1aafSjsg u32 instance) 2423fb4d8502Sjsg { 2424fb4d8502Sjsg u32 data; 2425fb4d8502Sjsg 2426fb4d8502Sjsg if (instance == 0xffffffff) 2427fb4d8502Sjsg data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1); 2428fb4d8502Sjsg else 2429fb4d8502Sjsg data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance); 2430fb4d8502Sjsg 2431fb4d8502Sjsg if (se_num == 0xffffffff) 2432fb4d8502Sjsg data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1); 2433fb4d8502Sjsg else 2434fb4d8502Sjsg data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num); 2435fb4d8502Sjsg 2436fb4d8502Sjsg if (sh_num == 0xffffffff) 2437fb4d8502Sjsg data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1); 2438fb4d8502Sjsg else 2439fb4d8502Sjsg data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num); 2440fb4d8502Sjsg 2441c349dbc7Sjsg WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data); 2442fb4d8502Sjsg } 2443fb4d8502Sjsg 2444fb4d8502Sjsg static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev) 2445fb4d8502Sjsg { 2446fb4d8502Sjsg u32 data, mask; 2447fb4d8502Sjsg 2448fb4d8502Sjsg data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE); 2449fb4d8502Sjsg data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE); 2450fb4d8502Sjsg 2451fb4d8502Sjsg data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK; 2452fb4d8502Sjsg data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT; 2453fb4d8502Sjsg 2454fb4d8502Sjsg mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se / 2455fb4d8502Sjsg adev->gfx.config.max_sh_per_se); 2456fb4d8502Sjsg 2457fb4d8502Sjsg return (~data) & mask; 2458fb4d8502Sjsg } 2459fb4d8502Sjsg 2460fb4d8502Sjsg static void gfx_v9_0_setup_rb(struct amdgpu_device *adev) 2461fb4d8502Sjsg { 2462fb4d8502Sjsg int i, j; 2463fb4d8502Sjsg u32 data; 2464fb4d8502Sjsg u32 active_rbs = 0; 2465fb4d8502Sjsg u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se / 2466fb4d8502Sjsg adev->gfx.config.max_sh_per_se; 2467fb4d8502Sjsg 2468fb4d8502Sjsg mutex_lock(&adev->grbm_idx_mutex); 2469fb4d8502Sjsg for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2470fb4d8502Sjsg for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2471fb4d8502Sjsg gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 2472fb4d8502Sjsg data = gfx_v9_0_get_rb_active_bitmap(adev); 2473fb4d8502Sjsg active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) * 2474fb4d8502Sjsg rb_bitmap_width_per_sh); 2475fb4d8502Sjsg } 2476fb4d8502Sjsg } 2477fb4d8502Sjsg gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 2478fb4d8502Sjsg mutex_unlock(&adev->grbm_idx_mutex); 2479fb4d8502Sjsg 2480fb4d8502Sjsg adev->gfx.config.backend_enable_mask = active_rbs; 2481fb4d8502Sjsg adev->gfx.config.num_rbs = hweight32(active_rbs); 2482fb4d8502Sjsg } 2483fb4d8502Sjsg 2484fb4d8502Sjsg #define DEFAULT_SH_MEM_BASES (0x6000) 2485fb4d8502Sjsg static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev) 2486fb4d8502Sjsg { 2487fb4d8502Sjsg int i; 2488fb4d8502Sjsg uint32_t sh_mem_config; 2489fb4d8502Sjsg uint32_t sh_mem_bases; 2490fb4d8502Sjsg 2491fb4d8502Sjsg /* 2492fb4d8502Sjsg * Configure apertures: 2493fb4d8502Sjsg * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) 2494fb4d8502Sjsg * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) 2495fb4d8502Sjsg * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) 2496fb4d8502Sjsg */ 2497fb4d8502Sjsg sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16); 2498fb4d8502Sjsg 2499fb4d8502Sjsg sh_mem_config = SH_MEM_ADDRESS_MODE_64 | 2500fb4d8502Sjsg SH_MEM_ALIGNMENT_MODE_UNALIGNED << 2501fb4d8502Sjsg SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT; 2502fb4d8502Sjsg 2503fb4d8502Sjsg mutex_lock(&adev->srbm_mutex); 2504ad8b1aafSjsg for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2505fb4d8502Sjsg soc15_grbm_select(adev, 0, 0, 0, i); 2506fb4d8502Sjsg /* CP and shaders */ 2507c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config); 2508c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases); 2509fb4d8502Sjsg } 2510fb4d8502Sjsg soc15_grbm_select(adev, 0, 0, 0, 0); 2511fb4d8502Sjsg mutex_unlock(&adev->srbm_mutex); 2512c349dbc7Sjsg 2513c349dbc7Sjsg /* Initialize all compute VMIDs to have no GDS, GWS, or OA 2514c349dbc7Sjsg acccess. These should be enabled by FW for target VMIDs. */ 2515ad8b1aafSjsg for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2516c349dbc7Sjsg WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0); 2517c349dbc7Sjsg WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0); 2518c349dbc7Sjsg WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0); 2519c349dbc7Sjsg WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0); 2520c349dbc7Sjsg } 2521fb4d8502Sjsg } 2522fb4d8502Sjsg 2523c349dbc7Sjsg static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev) 2524c349dbc7Sjsg { 2525c349dbc7Sjsg int vmid; 2526c349dbc7Sjsg 2527c349dbc7Sjsg /* 2528c349dbc7Sjsg * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA 2529c349dbc7Sjsg * access. Compute VMIDs should be enabled by FW for target VMIDs, 2530c349dbc7Sjsg * the driver can enable them for graphics. VMID0 should maintain 2531c349dbc7Sjsg * access so that HWS firmware can save/restore entries. 2532c349dbc7Sjsg */ 2533c349dbc7Sjsg for (vmid = 1; vmid < 16; vmid++) { 2534c349dbc7Sjsg WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0); 2535c349dbc7Sjsg WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0); 2536c349dbc7Sjsg WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0); 2537c349dbc7Sjsg WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0); 2538c349dbc7Sjsg } 2539c349dbc7Sjsg } 2540c349dbc7Sjsg 2541c349dbc7Sjsg static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev) 2542c349dbc7Sjsg { 2543c349dbc7Sjsg uint32_t tmp; 2544c349dbc7Sjsg 2545c349dbc7Sjsg switch (adev->asic_type) { 2546c349dbc7Sjsg case CHIP_ARCTURUS: 2547c349dbc7Sjsg tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG); 2548c349dbc7Sjsg tmp = REG_SET_FIELD(tmp, SQ_CONFIG, 2549c349dbc7Sjsg DISABLE_BARRIER_WAITCNT, 1); 2550c349dbc7Sjsg WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp); 2551c349dbc7Sjsg break; 2552c349dbc7Sjsg default: 2553c349dbc7Sjsg break; 2554ad8b1aafSjsg } 2555c349dbc7Sjsg } 2556c349dbc7Sjsg 2557c349dbc7Sjsg static void gfx_v9_0_constants_init(struct amdgpu_device *adev) 2558fb4d8502Sjsg { 2559fb4d8502Sjsg u32 tmp; 2560fb4d8502Sjsg int i; 2561fb4d8502Sjsg 2562c349dbc7Sjsg WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff); 2563fb4d8502Sjsg 2564fb4d8502Sjsg gfx_v9_0_tiling_mode_table_init(adev); 2565fb4d8502Sjsg 2566fb4d8502Sjsg gfx_v9_0_setup_rb(adev); 2567fb4d8502Sjsg gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info); 2568fb4d8502Sjsg adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2); 2569fb4d8502Sjsg 2570fb4d8502Sjsg /* XXX SH_MEM regs */ 2571fb4d8502Sjsg /* where to put LDS, scratch, GPUVM in FSA64 space */ 2572fb4d8502Sjsg mutex_lock(&adev->srbm_mutex); 2573c349dbc7Sjsg for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids; i++) { 2574fb4d8502Sjsg soc15_grbm_select(adev, 0, 0, 0, i); 2575fb4d8502Sjsg /* CP and shaders */ 2576fb4d8502Sjsg if (i == 0) { 2577fb4d8502Sjsg tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2578fb4d8502Sjsg SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2579c349dbc7Sjsg tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2580ad8b1aafSjsg !!adev->gmc.noretry); 2581c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2582c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0); 2583fb4d8502Sjsg } else { 2584fb4d8502Sjsg tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2585fb4d8502Sjsg SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2586c349dbc7Sjsg tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2587ad8b1aafSjsg !!adev->gmc.noretry); 2588c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2589fb4d8502Sjsg tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE, 2590fb4d8502Sjsg (adev->gmc.private_aperture_start >> 48)); 2591fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE, 2592fb4d8502Sjsg (adev->gmc.shared_aperture_start >> 48)); 2593c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp); 2594fb4d8502Sjsg } 2595fb4d8502Sjsg } 2596fb4d8502Sjsg soc15_grbm_select(adev, 0, 0, 0, 0); 2597fb4d8502Sjsg 2598fb4d8502Sjsg mutex_unlock(&adev->srbm_mutex); 2599fb4d8502Sjsg 2600fb4d8502Sjsg gfx_v9_0_init_compute_vmid(adev); 2601c349dbc7Sjsg gfx_v9_0_init_gds_vmid(adev); 2602c349dbc7Sjsg gfx_v9_0_init_sq_config(adev); 2603fb4d8502Sjsg } 2604fb4d8502Sjsg 2605fb4d8502Sjsg static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev) 2606fb4d8502Sjsg { 2607fb4d8502Sjsg u32 i, j, k; 2608fb4d8502Sjsg u32 mask; 2609fb4d8502Sjsg 2610fb4d8502Sjsg mutex_lock(&adev->grbm_idx_mutex); 2611fb4d8502Sjsg for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2612fb4d8502Sjsg for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2613fb4d8502Sjsg gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 2614fb4d8502Sjsg for (k = 0; k < adev->usec_timeout; k++) { 2615fb4d8502Sjsg if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0) 2616fb4d8502Sjsg break; 2617fb4d8502Sjsg udelay(1); 2618fb4d8502Sjsg } 2619fb4d8502Sjsg if (k == adev->usec_timeout) { 2620fb4d8502Sjsg gfx_v9_0_select_se_sh(adev, 0xffffffff, 2621fb4d8502Sjsg 0xffffffff, 0xffffffff); 2622fb4d8502Sjsg mutex_unlock(&adev->grbm_idx_mutex); 2623fb4d8502Sjsg DRM_INFO("Timeout wait for RLC serdes %u,%u\n", 2624fb4d8502Sjsg i, j); 2625fb4d8502Sjsg return; 2626fb4d8502Sjsg } 2627fb4d8502Sjsg } 2628fb4d8502Sjsg } 2629fb4d8502Sjsg gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 2630fb4d8502Sjsg mutex_unlock(&adev->grbm_idx_mutex); 2631fb4d8502Sjsg 2632fb4d8502Sjsg mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK | 2633fb4d8502Sjsg RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK | 2634fb4d8502Sjsg RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK | 2635fb4d8502Sjsg RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK; 2636fb4d8502Sjsg for (k = 0; k < adev->usec_timeout; k++) { 2637fb4d8502Sjsg if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0) 2638fb4d8502Sjsg break; 2639fb4d8502Sjsg udelay(1); 2640fb4d8502Sjsg } 2641fb4d8502Sjsg } 2642fb4d8502Sjsg 2643fb4d8502Sjsg static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev, 2644fb4d8502Sjsg bool enable) 2645fb4d8502Sjsg { 2646fb4d8502Sjsg u32 tmp = RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0); 2647fb4d8502Sjsg 2648fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0); 2649fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0); 2650fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0); 2651fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0); 2652fb4d8502Sjsg 2653fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp); 2654fb4d8502Sjsg } 2655fb4d8502Sjsg 2656fb4d8502Sjsg static void gfx_v9_0_init_csb(struct amdgpu_device *adev) 2657fb4d8502Sjsg { 2658c349dbc7Sjsg adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); 2659fb4d8502Sjsg /* csib */ 2660c349dbc7Sjsg WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI), 2661fb4d8502Sjsg adev->gfx.rlc.clear_state_gpu_addr >> 32); 2662c349dbc7Sjsg WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO), 2663fb4d8502Sjsg adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); 2664c349dbc7Sjsg WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH), 2665fb4d8502Sjsg adev->gfx.rlc.clear_state_size); 2666fb4d8502Sjsg } 2667fb4d8502Sjsg 2668fb4d8502Sjsg static void gfx_v9_1_parse_ind_reg_list(int *register_list_format, 2669fb4d8502Sjsg int indirect_offset, 2670fb4d8502Sjsg int list_size, 2671fb4d8502Sjsg int *unique_indirect_regs, 2672fb4d8502Sjsg int unique_indirect_reg_count, 2673fb4d8502Sjsg int *indirect_start_offsets, 2674fb4d8502Sjsg int *indirect_start_offsets_count, 2675fb4d8502Sjsg int max_start_offsets_count) 2676fb4d8502Sjsg { 2677fb4d8502Sjsg int idx; 2678fb4d8502Sjsg 2679fb4d8502Sjsg for (; indirect_offset < list_size; indirect_offset++) { 2680fb4d8502Sjsg WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count); 2681fb4d8502Sjsg indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset; 2682fb4d8502Sjsg *indirect_start_offsets_count = *indirect_start_offsets_count + 1; 2683fb4d8502Sjsg 2684fb4d8502Sjsg while (register_list_format[indirect_offset] != 0xFFFFFFFF) { 2685fb4d8502Sjsg indirect_offset += 2; 2686fb4d8502Sjsg 2687fb4d8502Sjsg /* look for the matching indice */ 2688fb4d8502Sjsg for (idx = 0; idx < unique_indirect_reg_count; idx++) { 2689fb4d8502Sjsg if (unique_indirect_regs[idx] == 2690fb4d8502Sjsg register_list_format[indirect_offset] || 2691fb4d8502Sjsg !unique_indirect_regs[idx]) 2692fb4d8502Sjsg break; 2693fb4d8502Sjsg } 2694fb4d8502Sjsg 2695fb4d8502Sjsg BUG_ON(idx >= unique_indirect_reg_count); 2696fb4d8502Sjsg 2697fb4d8502Sjsg if (!unique_indirect_regs[idx]) 2698fb4d8502Sjsg unique_indirect_regs[idx] = register_list_format[indirect_offset]; 2699fb4d8502Sjsg 2700fb4d8502Sjsg indirect_offset++; 2701fb4d8502Sjsg } 2702fb4d8502Sjsg } 2703fb4d8502Sjsg } 2704fb4d8502Sjsg 2705fb4d8502Sjsg static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev) 2706fb4d8502Sjsg { 2707fb4d8502Sjsg int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2708fb4d8502Sjsg int unique_indirect_reg_count = 0; 2709fb4d8502Sjsg 2710fb4d8502Sjsg int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2711fb4d8502Sjsg int indirect_start_offsets_count = 0; 2712fb4d8502Sjsg 2713fb4d8502Sjsg int list_size = 0; 2714fb4d8502Sjsg int i = 0, j = 0; 2715fb4d8502Sjsg u32 tmp = 0; 2716fb4d8502Sjsg 2717fb4d8502Sjsg u32 *register_list_format = 2718c349dbc7Sjsg kmemdup(adev->gfx.rlc.register_list_format, 2719c349dbc7Sjsg adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL); 2720fb4d8502Sjsg if (!register_list_format) 2721fb4d8502Sjsg return -ENOMEM; 2722fb4d8502Sjsg 2723fb4d8502Sjsg /* setup unique_indirect_regs array and indirect_start_offsets array */ 2724fb4d8502Sjsg unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs); 2725fb4d8502Sjsg gfx_v9_1_parse_ind_reg_list(register_list_format, 2726fb4d8502Sjsg adev->gfx.rlc.reg_list_format_direct_reg_list_length, 2727fb4d8502Sjsg adev->gfx.rlc.reg_list_format_size_bytes >> 2, 2728fb4d8502Sjsg unique_indirect_regs, 2729fb4d8502Sjsg unique_indirect_reg_count, 2730fb4d8502Sjsg indirect_start_offsets, 2731fb4d8502Sjsg &indirect_start_offsets_count, 2732fb4d8502Sjsg ARRAY_SIZE(indirect_start_offsets)); 2733fb4d8502Sjsg 2734fb4d8502Sjsg /* enable auto inc in case it is disabled */ 2735fb4d8502Sjsg tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL)); 2736fb4d8502Sjsg tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK; 2737fb4d8502Sjsg WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp); 2738fb4d8502Sjsg 2739fb4d8502Sjsg /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */ 2740fb4d8502Sjsg WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR), 2741fb4d8502Sjsg RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET); 2742fb4d8502Sjsg for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++) 2743fb4d8502Sjsg WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA), 2744fb4d8502Sjsg adev->gfx.rlc.register_restore[i]); 2745fb4d8502Sjsg 2746fb4d8502Sjsg /* load indirect register */ 2747fb4d8502Sjsg WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2748fb4d8502Sjsg adev->gfx.rlc.reg_list_format_start); 2749fb4d8502Sjsg 2750fb4d8502Sjsg /* direct register portion */ 2751fb4d8502Sjsg for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++) 2752fb4d8502Sjsg WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2753fb4d8502Sjsg register_list_format[i]); 2754fb4d8502Sjsg 2755fb4d8502Sjsg /* indirect register portion */ 2756fb4d8502Sjsg while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) { 2757fb4d8502Sjsg if (register_list_format[i] == 0xFFFFFFFF) { 2758fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2759fb4d8502Sjsg continue; 2760fb4d8502Sjsg } 2761fb4d8502Sjsg 2762fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2763fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2764fb4d8502Sjsg 2765fb4d8502Sjsg for (j = 0; j < unique_indirect_reg_count; j++) { 2766fb4d8502Sjsg if (register_list_format[i] == unique_indirect_regs[j]) { 2767fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j); 2768fb4d8502Sjsg break; 2769fb4d8502Sjsg } 2770fb4d8502Sjsg } 2771fb4d8502Sjsg 2772fb4d8502Sjsg BUG_ON(j >= unique_indirect_reg_count); 2773fb4d8502Sjsg 2774fb4d8502Sjsg i++; 2775fb4d8502Sjsg } 2776fb4d8502Sjsg 2777fb4d8502Sjsg /* set save/restore list size */ 2778fb4d8502Sjsg list_size = adev->gfx.rlc.reg_list_size_bytes >> 2; 2779fb4d8502Sjsg list_size = list_size >> 1; 2780fb4d8502Sjsg WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2781fb4d8502Sjsg adev->gfx.rlc.reg_restore_list_size); 2782fb4d8502Sjsg WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size); 2783fb4d8502Sjsg 2784fb4d8502Sjsg /* write the starting offsets to RLC scratch ram */ 2785fb4d8502Sjsg WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2786fb4d8502Sjsg adev->gfx.rlc.starting_offsets_start); 2787fb4d8502Sjsg for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++) 2788fb4d8502Sjsg WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2789fb4d8502Sjsg indirect_start_offsets[i]); 2790fb4d8502Sjsg 2791fb4d8502Sjsg /* load unique indirect regs*/ 2792fb4d8502Sjsg for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) { 2793fb4d8502Sjsg if (unique_indirect_regs[i] != 0) { 2794fb4d8502Sjsg WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0) 2795fb4d8502Sjsg + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i], 2796fb4d8502Sjsg unique_indirect_regs[i] & 0x3FFFF); 2797fb4d8502Sjsg 2798fb4d8502Sjsg WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0) 2799fb4d8502Sjsg + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i], 2800fb4d8502Sjsg unique_indirect_regs[i] >> 20); 2801fb4d8502Sjsg } 2802fb4d8502Sjsg } 2803fb4d8502Sjsg 2804fb4d8502Sjsg kfree(register_list_format); 2805fb4d8502Sjsg return 0; 2806fb4d8502Sjsg } 2807fb4d8502Sjsg 2808fb4d8502Sjsg static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev) 2809fb4d8502Sjsg { 2810fb4d8502Sjsg WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1); 2811fb4d8502Sjsg } 2812fb4d8502Sjsg 2813fb4d8502Sjsg static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev, 2814fb4d8502Sjsg bool enable) 2815fb4d8502Sjsg { 2816fb4d8502Sjsg uint32_t data = 0; 2817fb4d8502Sjsg uint32_t default_data = 0; 2818fb4d8502Sjsg 2819fb4d8502Sjsg default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS)); 2820ad8b1aafSjsg if (enable) { 2821fb4d8502Sjsg /* enable GFXIP control over CGPG */ 2822fb4d8502Sjsg data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2823fb4d8502Sjsg if(default_data != data) 2824fb4d8502Sjsg WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2825fb4d8502Sjsg 2826fb4d8502Sjsg /* update status */ 2827fb4d8502Sjsg data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK; 2828fb4d8502Sjsg data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT); 2829fb4d8502Sjsg if(default_data != data) 2830fb4d8502Sjsg WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2831fb4d8502Sjsg } else { 2832fb4d8502Sjsg /* restore GFXIP control over GCPG */ 2833fb4d8502Sjsg data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2834fb4d8502Sjsg if(default_data != data) 2835fb4d8502Sjsg WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2836fb4d8502Sjsg } 2837fb4d8502Sjsg } 2838fb4d8502Sjsg 2839fb4d8502Sjsg static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev) 2840fb4d8502Sjsg { 2841fb4d8502Sjsg uint32_t data = 0; 2842fb4d8502Sjsg 2843fb4d8502Sjsg if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 2844fb4d8502Sjsg AMD_PG_SUPPORT_GFX_SMG | 2845fb4d8502Sjsg AMD_PG_SUPPORT_GFX_DMG)) { 2846fb4d8502Sjsg /* init IDLE_POLL_COUNT = 60 */ 2847fb4d8502Sjsg data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL)); 2848fb4d8502Sjsg data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK; 2849fb4d8502Sjsg data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 2850fb4d8502Sjsg WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data); 2851fb4d8502Sjsg 2852fb4d8502Sjsg /* init RLC PG Delay */ 2853fb4d8502Sjsg data = 0; 2854fb4d8502Sjsg data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT); 2855fb4d8502Sjsg data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT); 2856fb4d8502Sjsg data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT); 2857fb4d8502Sjsg data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT); 2858fb4d8502Sjsg WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data); 2859fb4d8502Sjsg 2860fb4d8502Sjsg data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2)); 2861fb4d8502Sjsg data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK; 2862fb4d8502Sjsg data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT); 2863fb4d8502Sjsg WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data); 2864fb4d8502Sjsg 2865fb4d8502Sjsg data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3)); 2866fb4d8502Sjsg data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK; 2867fb4d8502Sjsg data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT); 2868fb4d8502Sjsg WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data); 2869fb4d8502Sjsg 2870fb4d8502Sjsg data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL)); 2871fb4d8502Sjsg data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK; 2872fb4d8502Sjsg 2873fb4d8502Sjsg /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */ 2874fb4d8502Sjsg data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT); 2875fb4d8502Sjsg WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data); 2876ad8b1aafSjsg if (adev->asic_type != CHIP_RENOIR) 2877fb4d8502Sjsg pwr_10_0_gfxip_control_over_cgpg(adev, true); 2878fb4d8502Sjsg } 2879fb4d8502Sjsg } 2880fb4d8502Sjsg 2881fb4d8502Sjsg static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev, 2882fb4d8502Sjsg bool enable) 2883fb4d8502Sjsg { 2884fb4d8502Sjsg uint32_t data = 0; 2885fb4d8502Sjsg uint32_t default_data = 0; 2886fb4d8502Sjsg 2887fb4d8502Sjsg default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2888fb4d8502Sjsg data = REG_SET_FIELD(data, RLC_PG_CNTL, 2889fb4d8502Sjsg SMU_CLK_SLOWDOWN_ON_PU_ENABLE, 2890fb4d8502Sjsg enable ? 1 : 0); 2891fb4d8502Sjsg if (default_data != data) 2892fb4d8502Sjsg WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2893fb4d8502Sjsg } 2894fb4d8502Sjsg 2895fb4d8502Sjsg static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev, 2896fb4d8502Sjsg bool enable) 2897fb4d8502Sjsg { 2898fb4d8502Sjsg uint32_t data = 0; 2899fb4d8502Sjsg uint32_t default_data = 0; 2900fb4d8502Sjsg 2901fb4d8502Sjsg default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2902fb4d8502Sjsg data = REG_SET_FIELD(data, RLC_PG_CNTL, 2903fb4d8502Sjsg SMU_CLK_SLOWDOWN_ON_PD_ENABLE, 2904fb4d8502Sjsg enable ? 1 : 0); 2905fb4d8502Sjsg if(default_data != data) 2906fb4d8502Sjsg WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2907fb4d8502Sjsg } 2908fb4d8502Sjsg 2909fb4d8502Sjsg static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev, 2910fb4d8502Sjsg bool enable) 2911fb4d8502Sjsg { 2912fb4d8502Sjsg uint32_t data = 0; 2913fb4d8502Sjsg uint32_t default_data = 0; 2914fb4d8502Sjsg 2915fb4d8502Sjsg default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2916fb4d8502Sjsg data = REG_SET_FIELD(data, RLC_PG_CNTL, 2917fb4d8502Sjsg CP_PG_DISABLE, 2918fb4d8502Sjsg enable ? 0 : 1); 2919fb4d8502Sjsg if(default_data != data) 2920fb4d8502Sjsg WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2921fb4d8502Sjsg } 2922fb4d8502Sjsg 2923fb4d8502Sjsg static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev, 2924fb4d8502Sjsg bool enable) 2925fb4d8502Sjsg { 2926fb4d8502Sjsg uint32_t data, default_data; 2927fb4d8502Sjsg 2928fb4d8502Sjsg default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2929fb4d8502Sjsg data = REG_SET_FIELD(data, RLC_PG_CNTL, 2930fb4d8502Sjsg GFX_POWER_GATING_ENABLE, 2931fb4d8502Sjsg enable ? 1 : 0); 2932fb4d8502Sjsg if(default_data != data) 2933fb4d8502Sjsg WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2934fb4d8502Sjsg } 2935fb4d8502Sjsg 2936fb4d8502Sjsg static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev, 2937fb4d8502Sjsg bool enable) 2938fb4d8502Sjsg { 2939fb4d8502Sjsg uint32_t data, default_data; 2940fb4d8502Sjsg 2941fb4d8502Sjsg default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2942fb4d8502Sjsg data = REG_SET_FIELD(data, RLC_PG_CNTL, 2943fb4d8502Sjsg GFX_PIPELINE_PG_ENABLE, 2944fb4d8502Sjsg enable ? 1 : 0); 2945fb4d8502Sjsg if(default_data != data) 2946fb4d8502Sjsg WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2947fb4d8502Sjsg 2948fb4d8502Sjsg if (!enable) 2949fb4d8502Sjsg /* read any GFX register to wake up GFX */ 2950fb4d8502Sjsg data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL)); 2951fb4d8502Sjsg } 2952fb4d8502Sjsg 2953fb4d8502Sjsg static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev, 2954fb4d8502Sjsg bool enable) 2955fb4d8502Sjsg { 2956fb4d8502Sjsg uint32_t data, default_data; 2957fb4d8502Sjsg 2958fb4d8502Sjsg default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2959fb4d8502Sjsg data = REG_SET_FIELD(data, RLC_PG_CNTL, 2960fb4d8502Sjsg STATIC_PER_CU_PG_ENABLE, 2961fb4d8502Sjsg enable ? 1 : 0); 2962fb4d8502Sjsg if(default_data != data) 2963fb4d8502Sjsg WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2964fb4d8502Sjsg } 2965fb4d8502Sjsg 2966fb4d8502Sjsg static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev, 2967fb4d8502Sjsg bool enable) 2968fb4d8502Sjsg { 2969fb4d8502Sjsg uint32_t data, default_data; 2970fb4d8502Sjsg 2971fb4d8502Sjsg default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2972fb4d8502Sjsg data = REG_SET_FIELD(data, RLC_PG_CNTL, 2973fb4d8502Sjsg DYN_PER_CU_PG_ENABLE, 2974fb4d8502Sjsg enable ? 1 : 0); 2975fb4d8502Sjsg if(default_data != data) 2976fb4d8502Sjsg WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2977fb4d8502Sjsg } 2978fb4d8502Sjsg 2979fb4d8502Sjsg static void gfx_v9_0_init_pg(struct amdgpu_device *adev) 2980fb4d8502Sjsg { 2981fb4d8502Sjsg gfx_v9_0_init_csb(adev); 2982fb4d8502Sjsg 2983fb4d8502Sjsg /* 2984fb4d8502Sjsg * Rlc save restore list is workable since v2_1. 2985fb4d8502Sjsg * And it's needed by gfxoff feature. 2986fb4d8502Sjsg */ 2987fb4d8502Sjsg if (adev->gfx.rlc.is_rlc_v2_1) { 2988c349dbc7Sjsg if (adev->asic_type == CHIP_VEGA12 || 2989ad8b1aafSjsg (adev->apu_flags & AMD_APU_IS_RAVEN2)) 2990fb4d8502Sjsg gfx_v9_1_init_rlc_save_restore_list(adev); 2991fb4d8502Sjsg gfx_v9_0_enable_save_restore_machine(adev); 2992fb4d8502Sjsg } 2993fb4d8502Sjsg 2994fb4d8502Sjsg if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 2995fb4d8502Sjsg AMD_PG_SUPPORT_GFX_SMG | 2996fb4d8502Sjsg AMD_PG_SUPPORT_GFX_DMG | 2997fb4d8502Sjsg AMD_PG_SUPPORT_CP | 2998fb4d8502Sjsg AMD_PG_SUPPORT_GDS | 2999fb4d8502Sjsg AMD_PG_SUPPORT_RLC_SMU_HS)) { 3000fb4d8502Sjsg WREG32(mmRLC_JUMP_TABLE_RESTORE, 3001fb4d8502Sjsg adev->gfx.rlc.cp_table_gpu_addr >> 8); 3002fb4d8502Sjsg gfx_v9_0_init_gfx_power_gating(adev); 3003fb4d8502Sjsg } 3004fb4d8502Sjsg } 3005fb4d8502Sjsg 3006fb4d8502Sjsg void gfx_v9_0_rlc_stop(struct amdgpu_device *adev) 3007fb4d8502Sjsg { 3008fb4d8502Sjsg WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0); 3009fb4d8502Sjsg gfx_v9_0_enable_gui_idle_interrupt(adev, false); 3010fb4d8502Sjsg gfx_v9_0_wait_for_rlc_serdes(adev); 3011fb4d8502Sjsg } 3012fb4d8502Sjsg 3013fb4d8502Sjsg static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev) 3014fb4d8502Sjsg { 3015fb4d8502Sjsg WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 3016fb4d8502Sjsg udelay(50); 3017fb4d8502Sjsg WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0); 3018fb4d8502Sjsg udelay(50); 3019fb4d8502Sjsg } 3020fb4d8502Sjsg 3021fb4d8502Sjsg static void gfx_v9_0_rlc_start(struct amdgpu_device *adev) 3022fb4d8502Sjsg { 3023fb4d8502Sjsg #ifdef AMDGPU_RLC_DEBUG_RETRY 3024fb4d8502Sjsg u32 rlc_ucode_ver; 3025fb4d8502Sjsg #endif 3026fb4d8502Sjsg 3027fb4d8502Sjsg WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1); 3028fb4d8502Sjsg udelay(50); 3029fb4d8502Sjsg 3030fb4d8502Sjsg /* carrizo do enable cp interrupt after cp inited */ 3031fb4d8502Sjsg if (!(adev->flags & AMD_IS_APU)) { 3032fb4d8502Sjsg gfx_v9_0_enable_gui_idle_interrupt(adev, true); 3033fb4d8502Sjsg udelay(50); 3034fb4d8502Sjsg } 3035fb4d8502Sjsg 3036fb4d8502Sjsg #ifdef AMDGPU_RLC_DEBUG_RETRY 3037fb4d8502Sjsg /* RLC_GPM_GENERAL_6 : RLC Ucode version */ 3038fb4d8502Sjsg rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6); 3039fb4d8502Sjsg if(rlc_ucode_ver == 0x108) { 3040fb4d8502Sjsg DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n", 3041fb4d8502Sjsg rlc_ucode_ver, adev->gfx.rlc_fw_version); 3042fb4d8502Sjsg /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles, 3043fb4d8502Sjsg * default is 0x9C4 to create a 100us interval */ 3044fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4); 3045fb4d8502Sjsg /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr 3046fb4d8502Sjsg * to disable the page fault retry interrupts, default is 3047fb4d8502Sjsg * 0x100 (256) */ 3048fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100); 3049fb4d8502Sjsg } 3050fb4d8502Sjsg #endif 3051fb4d8502Sjsg } 3052fb4d8502Sjsg 3053fb4d8502Sjsg static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev) 3054fb4d8502Sjsg { 3055fb4d8502Sjsg const struct rlc_firmware_header_v2_0 *hdr; 3056fb4d8502Sjsg const __le32 *fw_data; 3057fb4d8502Sjsg unsigned i, fw_size; 3058fb4d8502Sjsg 3059fb4d8502Sjsg if (!adev->gfx.rlc_fw) 3060fb4d8502Sjsg return -EINVAL; 3061fb4d8502Sjsg 3062fb4d8502Sjsg hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 3063fb4d8502Sjsg amdgpu_ucode_print_rlc_hdr(&hdr->header); 3064fb4d8502Sjsg 3065fb4d8502Sjsg fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 3066fb4d8502Sjsg le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 3067fb4d8502Sjsg fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 3068fb4d8502Sjsg 3069fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, 3070fb4d8502Sjsg RLCG_UCODE_LOADING_START_ADDRESS); 3071fb4d8502Sjsg for (i = 0; i < fw_size; i++) 3072fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++)); 3073fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version); 3074fb4d8502Sjsg 3075fb4d8502Sjsg return 0; 3076fb4d8502Sjsg } 3077fb4d8502Sjsg 3078fb4d8502Sjsg static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev) 3079fb4d8502Sjsg { 3080fb4d8502Sjsg int r; 3081fb4d8502Sjsg 3082fb4d8502Sjsg if (amdgpu_sriov_vf(adev)) { 3083fb4d8502Sjsg gfx_v9_0_init_csb(adev); 3084fb4d8502Sjsg return 0; 3085fb4d8502Sjsg } 3086fb4d8502Sjsg 3087c349dbc7Sjsg adev->gfx.rlc.funcs->stop(adev); 3088fb4d8502Sjsg 3089fb4d8502Sjsg /* disable CG */ 3090fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0); 3091fb4d8502Sjsg 3092fb4d8502Sjsg gfx_v9_0_init_pg(adev); 3093fb4d8502Sjsg 3094fb4d8502Sjsg if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 3095fb4d8502Sjsg /* legacy rlc firmware loading */ 3096fb4d8502Sjsg r = gfx_v9_0_rlc_load_microcode(adev); 3097fb4d8502Sjsg if (r) 3098fb4d8502Sjsg return r; 3099fb4d8502Sjsg } 3100fb4d8502Sjsg 3101c349dbc7Sjsg switch (adev->asic_type) { 3102c349dbc7Sjsg case CHIP_RAVEN: 3103c349dbc7Sjsg if (amdgpu_lbpw == 0) 3104c349dbc7Sjsg gfx_v9_0_enable_lbpw(adev, false); 3105c349dbc7Sjsg else 3106c349dbc7Sjsg gfx_v9_0_enable_lbpw(adev, true); 3107c349dbc7Sjsg break; 3108c349dbc7Sjsg case CHIP_VEGA20: 3109c349dbc7Sjsg if (amdgpu_lbpw > 0) 3110fb4d8502Sjsg gfx_v9_0_enable_lbpw(adev, true); 3111fb4d8502Sjsg else 3112fb4d8502Sjsg gfx_v9_0_enable_lbpw(adev, false); 3113c349dbc7Sjsg break; 3114c349dbc7Sjsg default: 3115c349dbc7Sjsg break; 3116fb4d8502Sjsg } 3117fb4d8502Sjsg 3118c349dbc7Sjsg adev->gfx.rlc.funcs->start(adev); 3119fb4d8502Sjsg 3120fb4d8502Sjsg return 0; 3121fb4d8502Sjsg } 3122fb4d8502Sjsg 3123fb4d8502Sjsg static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) 3124fb4d8502Sjsg { 3125fb4d8502Sjsg u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL); 3126fb4d8502Sjsg 3127fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1); 3128fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1); 3129fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1); 3130c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp); 3131fb4d8502Sjsg udelay(50); 3132fb4d8502Sjsg } 3133fb4d8502Sjsg 3134fb4d8502Sjsg static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev) 3135fb4d8502Sjsg { 3136fb4d8502Sjsg const struct gfx_firmware_header_v1_0 *pfp_hdr; 3137fb4d8502Sjsg const struct gfx_firmware_header_v1_0 *ce_hdr; 3138fb4d8502Sjsg const struct gfx_firmware_header_v1_0 *me_hdr; 3139fb4d8502Sjsg const __le32 *fw_data; 3140fb4d8502Sjsg unsigned i, fw_size; 3141fb4d8502Sjsg 3142fb4d8502Sjsg if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw) 3143fb4d8502Sjsg return -EINVAL; 3144fb4d8502Sjsg 3145fb4d8502Sjsg pfp_hdr = (const struct gfx_firmware_header_v1_0 *) 3146fb4d8502Sjsg adev->gfx.pfp_fw->data; 3147fb4d8502Sjsg ce_hdr = (const struct gfx_firmware_header_v1_0 *) 3148fb4d8502Sjsg adev->gfx.ce_fw->data; 3149fb4d8502Sjsg me_hdr = (const struct gfx_firmware_header_v1_0 *) 3150fb4d8502Sjsg adev->gfx.me_fw->data; 3151fb4d8502Sjsg 3152fb4d8502Sjsg amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header); 3153fb4d8502Sjsg amdgpu_ucode_print_gfx_hdr(&ce_hdr->header); 3154fb4d8502Sjsg amdgpu_ucode_print_gfx_hdr(&me_hdr->header); 3155fb4d8502Sjsg 3156fb4d8502Sjsg gfx_v9_0_cp_gfx_enable(adev, false); 3157fb4d8502Sjsg 3158fb4d8502Sjsg /* PFP */ 3159fb4d8502Sjsg fw_data = (const __le32 *) 3160fb4d8502Sjsg (adev->gfx.pfp_fw->data + 3161fb4d8502Sjsg le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); 3162fb4d8502Sjsg fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4; 3163fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0); 3164fb4d8502Sjsg for (i = 0; i < fw_size; i++) 3165fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++)); 3166fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version); 3167fb4d8502Sjsg 3168fb4d8502Sjsg /* CE */ 3169fb4d8502Sjsg fw_data = (const __le32 *) 3170fb4d8502Sjsg (adev->gfx.ce_fw->data + 3171fb4d8502Sjsg le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); 3172fb4d8502Sjsg fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4; 3173fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0); 3174fb4d8502Sjsg for (i = 0; i < fw_size; i++) 3175fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++)); 3176fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version); 3177fb4d8502Sjsg 3178fb4d8502Sjsg /* ME */ 3179fb4d8502Sjsg fw_data = (const __le32 *) 3180fb4d8502Sjsg (adev->gfx.me_fw->data + 3181fb4d8502Sjsg le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); 3182fb4d8502Sjsg fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4; 3183fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0); 3184fb4d8502Sjsg for (i = 0; i < fw_size; i++) 3185fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++)); 3186fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version); 3187fb4d8502Sjsg 3188fb4d8502Sjsg return 0; 3189fb4d8502Sjsg } 3190fb4d8502Sjsg 3191fb4d8502Sjsg static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev) 3192fb4d8502Sjsg { 3193fb4d8502Sjsg struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0]; 3194fb4d8502Sjsg const struct cs_section_def *sect = NULL; 3195fb4d8502Sjsg const struct cs_extent_def *ext = NULL; 3196fb4d8502Sjsg int r, i, tmp; 3197fb4d8502Sjsg 3198fb4d8502Sjsg /* init the CP */ 3199fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1); 3200fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1); 3201fb4d8502Sjsg 3202fb4d8502Sjsg gfx_v9_0_cp_gfx_enable(adev, true); 3203fb4d8502Sjsg 3204fb4d8502Sjsg r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3); 3205fb4d8502Sjsg if (r) { 3206fb4d8502Sjsg DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r); 3207fb4d8502Sjsg return r; 3208fb4d8502Sjsg } 3209fb4d8502Sjsg 3210fb4d8502Sjsg amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3211fb4d8502Sjsg amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 3212fb4d8502Sjsg 3213fb4d8502Sjsg amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 3214fb4d8502Sjsg amdgpu_ring_write(ring, 0x80000000); 3215fb4d8502Sjsg amdgpu_ring_write(ring, 0x80000000); 3216fb4d8502Sjsg 3217fb4d8502Sjsg for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 3218fb4d8502Sjsg for (ext = sect->section; ext->extent != NULL; ++ext) { 3219fb4d8502Sjsg if (sect->id == SECT_CONTEXT) { 3220fb4d8502Sjsg amdgpu_ring_write(ring, 3221fb4d8502Sjsg PACKET3(PACKET3_SET_CONTEXT_REG, 3222fb4d8502Sjsg ext->reg_count)); 3223fb4d8502Sjsg amdgpu_ring_write(ring, 3224fb4d8502Sjsg ext->reg_index - PACKET3_SET_CONTEXT_REG_START); 3225fb4d8502Sjsg for (i = 0; i < ext->reg_count; i++) 3226fb4d8502Sjsg amdgpu_ring_write(ring, ext->extent[i]); 3227fb4d8502Sjsg } 3228fb4d8502Sjsg } 3229fb4d8502Sjsg } 3230fb4d8502Sjsg 3231fb4d8502Sjsg amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3232fb4d8502Sjsg amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 3233fb4d8502Sjsg 3234fb4d8502Sjsg amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3235fb4d8502Sjsg amdgpu_ring_write(ring, 0); 3236fb4d8502Sjsg 3237fb4d8502Sjsg amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 3238fb4d8502Sjsg amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 3239fb4d8502Sjsg amdgpu_ring_write(ring, 0x8000); 3240fb4d8502Sjsg amdgpu_ring_write(ring, 0x8000); 3241fb4d8502Sjsg 3242fb4d8502Sjsg amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1)); 3243fb4d8502Sjsg tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE | 3244fb4d8502Sjsg (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START)); 3245fb4d8502Sjsg amdgpu_ring_write(ring, tmp); 3246fb4d8502Sjsg amdgpu_ring_write(ring, 0); 3247fb4d8502Sjsg 3248fb4d8502Sjsg amdgpu_ring_commit(ring); 3249fb4d8502Sjsg 3250fb4d8502Sjsg return 0; 3251fb4d8502Sjsg } 3252fb4d8502Sjsg 3253fb4d8502Sjsg static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev) 3254fb4d8502Sjsg { 3255fb4d8502Sjsg struct amdgpu_ring *ring; 3256fb4d8502Sjsg u32 tmp; 3257fb4d8502Sjsg u32 rb_bufsz; 3258fb4d8502Sjsg u64 rb_addr, rptr_addr, wptr_gpu_addr; 3259fb4d8502Sjsg 3260fb4d8502Sjsg /* Set the write pointer delay */ 3261fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0); 3262fb4d8502Sjsg 3263fb4d8502Sjsg /* set the RB to use vmid 0 */ 3264fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0); 3265fb4d8502Sjsg 3266fb4d8502Sjsg /* Set ring buffer size */ 3267fb4d8502Sjsg ring = &adev->gfx.gfx_ring[0]; 3268fb4d8502Sjsg rb_bufsz = order_base_2(ring->ring_size / 8); 3269fb4d8502Sjsg tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz); 3270fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2); 3271fb4d8502Sjsg #ifdef __BIG_ENDIAN 3272fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1); 3273fb4d8502Sjsg #endif 3274fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3275fb4d8502Sjsg 3276fb4d8502Sjsg /* Initialize the ring buffer's write pointers */ 3277fb4d8502Sjsg ring->wptr = 0; 3278fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 3279fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 3280fb4d8502Sjsg 3281fb4d8502Sjsg /* set the wb address wether it's enabled or not */ 3282fb4d8502Sjsg rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); 3283fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr)); 3284fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK); 3285fb4d8502Sjsg 3286fb4d8502Sjsg wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 3287fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr)); 3288fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr)); 3289fb4d8502Sjsg 3290fb4d8502Sjsg mdelay(1); 3291fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3292fb4d8502Sjsg 3293fb4d8502Sjsg rb_addr = ring->gpu_addr >> 8; 3294fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr); 3295fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr)); 3296fb4d8502Sjsg 3297fb4d8502Sjsg tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL); 3298fb4d8502Sjsg if (ring->use_doorbell) { 3299fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3300fb4d8502Sjsg DOORBELL_OFFSET, ring->doorbell_index); 3301fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3302fb4d8502Sjsg DOORBELL_EN, 1); 3303fb4d8502Sjsg } else { 3304fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0); 3305fb4d8502Sjsg } 3306fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp); 3307fb4d8502Sjsg 3308fb4d8502Sjsg tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER, 3309fb4d8502Sjsg DOORBELL_RANGE_LOWER, ring->doorbell_index); 3310fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp); 3311fb4d8502Sjsg 3312fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER, 3313fb4d8502Sjsg CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK); 3314fb4d8502Sjsg 3315fb4d8502Sjsg 3316fb4d8502Sjsg /* start the ring */ 3317fb4d8502Sjsg gfx_v9_0_cp_gfx_start(adev); 3318c349dbc7Sjsg ring->sched.ready = true; 3319fb4d8502Sjsg 3320fb4d8502Sjsg return 0; 3321fb4d8502Sjsg } 3322fb4d8502Sjsg 3323fb4d8502Sjsg static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable) 3324fb4d8502Sjsg { 3325fb4d8502Sjsg if (enable) { 3326c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0); 3327fb4d8502Sjsg } else { 3328c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 3329fb4d8502Sjsg (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK)); 3330c349dbc7Sjsg adev->gfx.kiq.ring.sched.ready = false; 3331fb4d8502Sjsg } 3332fb4d8502Sjsg udelay(50); 3333fb4d8502Sjsg } 3334fb4d8502Sjsg 3335fb4d8502Sjsg static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev) 3336fb4d8502Sjsg { 3337fb4d8502Sjsg const struct gfx_firmware_header_v1_0 *mec_hdr; 3338fb4d8502Sjsg const __le32 *fw_data; 3339fb4d8502Sjsg unsigned i; 3340fb4d8502Sjsg u32 tmp; 3341fb4d8502Sjsg 3342fb4d8502Sjsg if (!adev->gfx.mec_fw) 3343fb4d8502Sjsg return -EINVAL; 3344fb4d8502Sjsg 3345fb4d8502Sjsg gfx_v9_0_cp_compute_enable(adev, false); 3346fb4d8502Sjsg 3347fb4d8502Sjsg mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 3348fb4d8502Sjsg amdgpu_ucode_print_gfx_hdr(&mec_hdr->header); 3349fb4d8502Sjsg 3350fb4d8502Sjsg fw_data = (const __le32 *) 3351fb4d8502Sjsg (adev->gfx.mec_fw->data + 3352fb4d8502Sjsg le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 3353fb4d8502Sjsg tmp = 0; 3354fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); 3355fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 3356fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp); 3357fb4d8502Sjsg 3358fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO, 3359fb4d8502Sjsg adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000); 3360fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI, 3361fb4d8502Sjsg upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr)); 3362fb4d8502Sjsg 3363fb4d8502Sjsg /* MEC1 */ 3364fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3365fb4d8502Sjsg mec_hdr->jt_offset); 3366fb4d8502Sjsg for (i = 0; i < mec_hdr->jt_size; i++) 3367fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA, 3368fb4d8502Sjsg le32_to_cpup(fw_data + mec_hdr->jt_offset + i)); 3369fb4d8502Sjsg 3370fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3371fb4d8502Sjsg adev->gfx.mec_fw_version); 3372fb4d8502Sjsg /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */ 3373fb4d8502Sjsg 3374fb4d8502Sjsg return 0; 3375fb4d8502Sjsg } 3376fb4d8502Sjsg 3377fb4d8502Sjsg /* KIQ functions */ 3378fb4d8502Sjsg static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring) 3379fb4d8502Sjsg { 3380fb4d8502Sjsg uint32_t tmp; 3381fb4d8502Sjsg struct amdgpu_device *adev = ring->adev; 3382fb4d8502Sjsg 3383fb4d8502Sjsg /* tell RLC which is KIQ queue */ 3384fb4d8502Sjsg tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS); 3385fb4d8502Sjsg tmp &= 0xffffff00; 3386fb4d8502Sjsg tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); 3387c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp); 3388fb4d8502Sjsg tmp |= 0x80; 3389c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp); 3390fb4d8502Sjsg } 3391fb4d8502Sjsg 3392c349dbc7Sjsg static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd) 3393fb4d8502Sjsg { 3394c349dbc7Sjsg struct amdgpu_device *adev = ring->adev; 3395fb4d8502Sjsg 3396c349dbc7Sjsg if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 3397ad8b1aafSjsg if (amdgpu_gfx_is_high_priority_compute_queue(adev, 3398ad8b1aafSjsg ring->pipe, 3399ad8b1aafSjsg ring->queue)) { 3400c349dbc7Sjsg mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH; 3401c349dbc7Sjsg mqd->cp_hqd_queue_priority = 3402c349dbc7Sjsg AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM; 3403fb4d8502Sjsg } 3404fb4d8502Sjsg } 3405fb4d8502Sjsg } 3406fb4d8502Sjsg 3407fb4d8502Sjsg static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring) 3408fb4d8502Sjsg { 3409fb4d8502Sjsg struct amdgpu_device *adev = ring->adev; 3410fb4d8502Sjsg struct v9_mqd *mqd = ring->mqd_ptr; 3411fb4d8502Sjsg uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr; 3412fb4d8502Sjsg uint32_t tmp; 3413fb4d8502Sjsg 3414fb4d8502Sjsg mqd->header = 0xC0310800; 3415fb4d8502Sjsg mqd->compute_pipelinestat_enable = 0x00000001; 3416fb4d8502Sjsg mqd->compute_static_thread_mgmt_se0 = 0xffffffff; 3417fb4d8502Sjsg mqd->compute_static_thread_mgmt_se1 = 0xffffffff; 3418fb4d8502Sjsg mqd->compute_static_thread_mgmt_se2 = 0xffffffff; 3419fb4d8502Sjsg mqd->compute_static_thread_mgmt_se3 = 0xffffffff; 3420c349dbc7Sjsg mqd->compute_static_thread_mgmt_se4 = 0xffffffff; 3421c349dbc7Sjsg mqd->compute_static_thread_mgmt_se5 = 0xffffffff; 3422c349dbc7Sjsg mqd->compute_static_thread_mgmt_se6 = 0xffffffff; 3423c349dbc7Sjsg mqd->compute_static_thread_mgmt_se7 = 0xffffffff; 3424fb4d8502Sjsg mqd->compute_misc_reserved = 0x00000003; 3425fb4d8502Sjsg 3426fb4d8502Sjsg mqd->dynamic_cu_mask_addr_lo = 3427fb4d8502Sjsg lower_32_bits(ring->mqd_gpu_addr 3428fb4d8502Sjsg + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3429fb4d8502Sjsg mqd->dynamic_cu_mask_addr_hi = 3430fb4d8502Sjsg upper_32_bits(ring->mqd_gpu_addr 3431fb4d8502Sjsg + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3432fb4d8502Sjsg 3433fb4d8502Sjsg eop_base_addr = ring->eop_gpu_addr >> 8; 3434fb4d8502Sjsg mqd->cp_hqd_eop_base_addr_lo = eop_base_addr; 3435fb4d8502Sjsg mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); 3436fb4d8502Sjsg 3437fb4d8502Sjsg /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3438fb4d8502Sjsg tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL); 3439fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE, 3440fb4d8502Sjsg (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1)); 3441fb4d8502Sjsg 3442fb4d8502Sjsg mqd->cp_hqd_eop_control = tmp; 3443fb4d8502Sjsg 3444fb4d8502Sjsg /* enable doorbell? */ 3445fb4d8502Sjsg tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL); 3446fb4d8502Sjsg 3447fb4d8502Sjsg if (ring->use_doorbell) { 3448fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3449fb4d8502Sjsg DOORBELL_OFFSET, ring->doorbell_index); 3450fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3451fb4d8502Sjsg DOORBELL_EN, 1); 3452fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3453fb4d8502Sjsg DOORBELL_SOURCE, 0); 3454fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3455fb4d8502Sjsg DOORBELL_HIT, 0); 3456fb4d8502Sjsg } else { 3457fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3458fb4d8502Sjsg DOORBELL_EN, 0); 3459fb4d8502Sjsg } 3460fb4d8502Sjsg 3461fb4d8502Sjsg mqd->cp_hqd_pq_doorbell_control = tmp; 3462fb4d8502Sjsg 3463fb4d8502Sjsg /* disable the queue if it's active */ 3464fb4d8502Sjsg ring->wptr = 0; 3465fb4d8502Sjsg mqd->cp_hqd_dequeue_request = 0; 3466fb4d8502Sjsg mqd->cp_hqd_pq_rptr = 0; 3467fb4d8502Sjsg mqd->cp_hqd_pq_wptr_lo = 0; 3468fb4d8502Sjsg mqd->cp_hqd_pq_wptr_hi = 0; 3469fb4d8502Sjsg 3470fb4d8502Sjsg /* set the pointer to the MQD */ 3471fb4d8502Sjsg mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc; 3472fb4d8502Sjsg mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr); 3473fb4d8502Sjsg 3474fb4d8502Sjsg /* set MQD vmid to 0 */ 3475fb4d8502Sjsg tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL); 3476fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); 3477fb4d8502Sjsg mqd->cp_mqd_control = tmp; 3478fb4d8502Sjsg 3479fb4d8502Sjsg /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3480fb4d8502Sjsg hqd_gpu_addr = ring->gpu_addr >> 8; 3481fb4d8502Sjsg mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; 3482fb4d8502Sjsg mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 3483fb4d8502Sjsg 3484fb4d8502Sjsg /* set up the HQD, this is similar to CP_RB0_CNTL */ 3485fb4d8502Sjsg tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL); 3486fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE, 3487fb4d8502Sjsg (order_base_2(ring->ring_size / 4) - 1)); 3488fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE, 3489fb4d8502Sjsg ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8)); 3490fb4d8502Sjsg #ifdef __BIG_ENDIAN 3491fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1); 3492fb4d8502Sjsg #endif 3493fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0); 3494fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0); 3495fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1); 3496fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1); 3497fb4d8502Sjsg mqd->cp_hqd_pq_control = tmp; 3498fb4d8502Sjsg 3499fb4d8502Sjsg /* set the wb address whether it's enabled or not */ 3500fb4d8502Sjsg wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); 3501fb4d8502Sjsg mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; 3502fb4d8502Sjsg mqd->cp_hqd_pq_rptr_report_addr_hi = 3503fb4d8502Sjsg upper_32_bits(wb_gpu_addr) & 0xffff; 3504fb4d8502Sjsg 3505fb4d8502Sjsg /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3506fb4d8502Sjsg wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 3507fb4d8502Sjsg mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 3508fb4d8502Sjsg mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 3509fb4d8502Sjsg 3510fb4d8502Sjsg tmp = 0; 3511fb4d8502Sjsg /* enable the doorbell if requested */ 3512fb4d8502Sjsg if (ring->use_doorbell) { 3513fb4d8502Sjsg tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL); 3514fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3515fb4d8502Sjsg DOORBELL_OFFSET, ring->doorbell_index); 3516fb4d8502Sjsg 3517fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3518fb4d8502Sjsg DOORBELL_EN, 1); 3519fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3520fb4d8502Sjsg DOORBELL_SOURCE, 0); 3521fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3522fb4d8502Sjsg DOORBELL_HIT, 0); 3523fb4d8502Sjsg } 3524fb4d8502Sjsg 3525fb4d8502Sjsg mqd->cp_hqd_pq_doorbell_control = tmp; 3526fb4d8502Sjsg 3527fb4d8502Sjsg /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3528fb4d8502Sjsg ring->wptr = 0; 3529fb4d8502Sjsg mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR); 3530fb4d8502Sjsg 3531fb4d8502Sjsg /* set the vmid for the queue */ 3532fb4d8502Sjsg mqd->cp_hqd_vmid = 0; 3533fb4d8502Sjsg 3534fb4d8502Sjsg tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE); 3535fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53); 3536fb4d8502Sjsg mqd->cp_hqd_persistent_state = tmp; 3537fb4d8502Sjsg 3538fb4d8502Sjsg /* set MIN_IB_AVAIL_SIZE */ 3539fb4d8502Sjsg tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL); 3540fb4d8502Sjsg tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3); 3541fb4d8502Sjsg mqd->cp_hqd_ib_control = tmp; 3542fb4d8502Sjsg 3543c349dbc7Sjsg /* set static priority for a queue/ring */ 3544c349dbc7Sjsg gfx_v9_0_mqd_set_priority(ring, mqd); 3545c349dbc7Sjsg mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM); 3546c349dbc7Sjsg 3547c349dbc7Sjsg /* map_queues packet doesn't need activate the queue, 3548c349dbc7Sjsg * so only kiq need set this field. 3549c349dbc7Sjsg */ 3550c349dbc7Sjsg if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 3551fb4d8502Sjsg mqd->cp_hqd_active = 1; 3552fb4d8502Sjsg 3553fb4d8502Sjsg return 0; 3554fb4d8502Sjsg } 3555fb4d8502Sjsg 3556fb4d8502Sjsg static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring) 3557fb4d8502Sjsg { 3558fb4d8502Sjsg struct amdgpu_device *adev = ring->adev; 3559fb4d8502Sjsg struct v9_mqd *mqd = ring->mqd_ptr; 3560fb4d8502Sjsg int j; 3561fb4d8502Sjsg 3562fb4d8502Sjsg /* disable wptr polling */ 3563fb4d8502Sjsg WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 3564fb4d8502Sjsg 3565c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR, 3566fb4d8502Sjsg mqd->cp_hqd_eop_base_addr_lo); 3567c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI, 3568fb4d8502Sjsg mqd->cp_hqd_eop_base_addr_hi); 3569fb4d8502Sjsg 3570fb4d8502Sjsg /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3571c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL, 3572fb4d8502Sjsg mqd->cp_hqd_eop_control); 3573fb4d8502Sjsg 3574fb4d8502Sjsg /* enable doorbell? */ 3575c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3576fb4d8502Sjsg mqd->cp_hqd_pq_doorbell_control); 3577fb4d8502Sjsg 3578fb4d8502Sjsg /* disable the queue if it's active */ 3579fb4d8502Sjsg if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3580c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3581fb4d8502Sjsg for (j = 0; j < adev->usec_timeout; j++) { 3582fb4d8502Sjsg if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3583fb4d8502Sjsg break; 3584fb4d8502Sjsg udelay(1); 3585fb4d8502Sjsg } 3586c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3587fb4d8502Sjsg mqd->cp_hqd_dequeue_request); 3588c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 3589fb4d8502Sjsg mqd->cp_hqd_pq_rptr); 3590c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3591fb4d8502Sjsg mqd->cp_hqd_pq_wptr_lo); 3592c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3593fb4d8502Sjsg mqd->cp_hqd_pq_wptr_hi); 3594fb4d8502Sjsg } 3595fb4d8502Sjsg 3596fb4d8502Sjsg /* set the pointer to the MQD */ 3597c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR, 3598fb4d8502Sjsg mqd->cp_mqd_base_addr_lo); 3599c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI, 3600fb4d8502Sjsg mqd->cp_mqd_base_addr_hi); 3601fb4d8502Sjsg 3602fb4d8502Sjsg /* set MQD vmid to 0 */ 3603c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL, 3604fb4d8502Sjsg mqd->cp_mqd_control); 3605fb4d8502Sjsg 3606fb4d8502Sjsg /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3607c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE, 3608fb4d8502Sjsg mqd->cp_hqd_pq_base_lo); 3609c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI, 3610fb4d8502Sjsg mqd->cp_hqd_pq_base_hi); 3611fb4d8502Sjsg 3612fb4d8502Sjsg /* set up the HQD, this is similar to CP_RB0_CNTL */ 3613c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL, 3614fb4d8502Sjsg mqd->cp_hqd_pq_control); 3615fb4d8502Sjsg 3616fb4d8502Sjsg /* set the wb address whether it's enabled or not */ 3617c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR, 3618fb4d8502Sjsg mqd->cp_hqd_pq_rptr_report_addr_lo); 3619c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI, 3620fb4d8502Sjsg mqd->cp_hqd_pq_rptr_report_addr_hi); 3621fb4d8502Sjsg 3622fb4d8502Sjsg /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3623c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR, 3624fb4d8502Sjsg mqd->cp_hqd_pq_wptr_poll_addr_lo); 3625c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI, 3626fb4d8502Sjsg mqd->cp_hqd_pq_wptr_poll_addr_hi); 3627fb4d8502Sjsg 3628fb4d8502Sjsg /* enable the doorbell if requested */ 3629fb4d8502Sjsg if (ring->use_doorbell) { 3630fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER, 3631c349dbc7Sjsg (adev->doorbell_index.kiq * 2) << 2); 3632*1c23f5d9Sjsg /* If GC has entered CGPG, ringing doorbell > first page 3633*1c23f5d9Sjsg * doesn't wakeup GC. Enlarge CP_MEC_DOORBELL_RANGE_UPPER to 3634*1c23f5d9Sjsg * workaround this issue. And this change has to align with firmware 3635*1c23f5d9Sjsg * update. 3636*1c23f5d9Sjsg */ 3637*1c23f5d9Sjsg if (check_if_enlarge_doorbell_range(adev)) 3638*1c23f5d9Sjsg WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER, 3639*1c23f5d9Sjsg (adev->doorbell.size - 4)); 3640*1c23f5d9Sjsg else 3641fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER, 3642c349dbc7Sjsg (adev->doorbell_index.userqueue_end * 2) << 2); 3643fb4d8502Sjsg } 3644fb4d8502Sjsg 3645c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3646fb4d8502Sjsg mqd->cp_hqd_pq_doorbell_control); 3647fb4d8502Sjsg 3648fb4d8502Sjsg /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3649c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3650fb4d8502Sjsg mqd->cp_hqd_pq_wptr_lo); 3651c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3652fb4d8502Sjsg mqd->cp_hqd_pq_wptr_hi); 3653fb4d8502Sjsg 3654fb4d8502Sjsg /* set the vmid for the queue */ 3655c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid); 3656fb4d8502Sjsg 3657c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 3658fb4d8502Sjsg mqd->cp_hqd_persistent_state); 3659fb4d8502Sjsg 3660fb4d8502Sjsg /* activate the queue */ 3661c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 3662fb4d8502Sjsg mqd->cp_hqd_active); 3663fb4d8502Sjsg 3664fb4d8502Sjsg if (ring->use_doorbell) 3665fb4d8502Sjsg WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1); 3666fb4d8502Sjsg 3667fb4d8502Sjsg return 0; 3668fb4d8502Sjsg } 3669fb4d8502Sjsg 3670fb4d8502Sjsg static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring) 3671fb4d8502Sjsg { 3672fb4d8502Sjsg struct amdgpu_device *adev = ring->adev; 3673fb4d8502Sjsg int j; 3674fb4d8502Sjsg 3675fb4d8502Sjsg /* disable the queue if it's active */ 3676fb4d8502Sjsg if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3677fb4d8502Sjsg 3678c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3679fb4d8502Sjsg 3680fb4d8502Sjsg for (j = 0; j < adev->usec_timeout; j++) { 3681fb4d8502Sjsg if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3682fb4d8502Sjsg break; 3683fb4d8502Sjsg udelay(1); 3684fb4d8502Sjsg } 3685fb4d8502Sjsg 3686fb4d8502Sjsg if (j == AMDGPU_MAX_USEC_TIMEOUT) { 3687fb4d8502Sjsg DRM_DEBUG("KIQ dequeue request failed.\n"); 3688fb4d8502Sjsg 3689fb4d8502Sjsg /* Manual disable if dequeue request times out */ 3690c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0); 3691fb4d8502Sjsg } 3692fb4d8502Sjsg 3693c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3694fb4d8502Sjsg 0); 3695fb4d8502Sjsg } 3696fb4d8502Sjsg 3697c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0); 3698c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0); 3699c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0); 3700c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000); 3701c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0); 3702c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0); 3703c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0); 3704c349dbc7Sjsg WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0); 3705fb4d8502Sjsg 3706fb4d8502Sjsg return 0; 3707fb4d8502Sjsg } 3708fb4d8502Sjsg 3709fb4d8502Sjsg static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring) 3710fb4d8502Sjsg { 3711fb4d8502Sjsg struct amdgpu_device *adev = ring->adev; 3712fb4d8502Sjsg struct v9_mqd *mqd = ring->mqd_ptr; 3713fb4d8502Sjsg int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS; 3714fb4d8502Sjsg 3715fb4d8502Sjsg gfx_v9_0_kiq_setting(ring); 3716fb4d8502Sjsg 3717ad8b1aafSjsg if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ 3718fb4d8502Sjsg /* reset MQD to a clean status */ 3719fb4d8502Sjsg if (adev->gfx.mec.mqd_backup[mqd_idx]) 3720fb4d8502Sjsg memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 3721fb4d8502Sjsg 3722fb4d8502Sjsg /* reset ring buffer */ 3723fb4d8502Sjsg ring->wptr = 0; 3724fb4d8502Sjsg amdgpu_ring_clear_ring(ring); 3725fb4d8502Sjsg 3726fb4d8502Sjsg mutex_lock(&adev->srbm_mutex); 3727fb4d8502Sjsg soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3728fb4d8502Sjsg gfx_v9_0_kiq_init_register(ring); 3729fb4d8502Sjsg soc15_grbm_select(adev, 0, 0, 0, 0); 3730fb4d8502Sjsg mutex_unlock(&adev->srbm_mutex); 3731fb4d8502Sjsg } else { 3732fb4d8502Sjsg memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3733fb4d8502Sjsg ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3734fb4d8502Sjsg ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3735fb4d8502Sjsg mutex_lock(&adev->srbm_mutex); 3736fb4d8502Sjsg soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3737fb4d8502Sjsg gfx_v9_0_mqd_init(ring); 3738fb4d8502Sjsg gfx_v9_0_kiq_init_register(ring); 3739fb4d8502Sjsg soc15_grbm_select(adev, 0, 0, 0, 0); 3740fb4d8502Sjsg mutex_unlock(&adev->srbm_mutex); 3741fb4d8502Sjsg 3742fb4d8502Sjsg if (adev->gfx.mec.mqd_backup[mqd_idx]) 3743fb4d8502Sjsg memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); 3744fb4d8502Sjsg } 3745fb4d8502Sjsg 3746fb4d8502Sjsg return 0; 3747fb4d8502Sjsg } 3748fb4d8502Sjsg 3749fb4d8502Sjsg static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring) 3750fb4d8502Sjsg { 3751fb4d8502Sjsg struct amdgpu_device *adev = ring->adev; 3752fb4d8502Sjsg struct v9_mqd *mqd = ring->mqd_ptr; 3753fb4d8502Sjsg int mqd_idx = ring - &adev->gfx.compute_ring[0]; 3754fb4d8502Sjsg 3755ad8b1aafSjsg if (!amdgpu_in_reset(adev) && !adev->in_suspend) { 3756fb4d8502Sjsg memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3757fb4d8502Sjsg ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3758fb4d8502Sjsg ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3759fb4d8502Sjsg mutex_lock(&adev->srbm_mutex); 3760fb4d8502Sjsg soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3761fb4d8502Sjsg gfx_v9_0_mqd_init(ring); 3762fb4d8502Sjsg soc15_grbm_select(adev, 0, 0, 0, 0); 3763fb4d8502Sjsg mutex_unlock(&adev->srbm_mutex); 3764fb4d8502Sjsg 3765fb4d8502Sjsg if (adev->gfx.mec.mqd_backup[mqd_idx]) 3766fb4d8502Sjsg memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); 3767ad8b1aafSjsg } else if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ 3768fb4d8502Sjsg /* reset MQD to a clean status */ 3769fb4d8502Sjsg if (adev->gfx.mec.mqd_backup[mqd_idx]) 3770fb4d8502Sjsg memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 3771fb4d8502Sjsg 3772fb4d8502Sjsg /* reset ring buffer */ 3773fb4d8502Sjsg ring->wptr = 0; 3774c349dbc7Sjsg atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], 0); 3775fb4d8502Sjsg amdgpu_ring_clear_ring(ring); 3776fb4d8502Sjsg } else { 3777fb4d8502Sjsg amdgpu_ring_clear_ring(ring); 3778fb4d8502Sjsg } 3779fb4d8502Sjsg 3780fb4d8502Sjsg return 0; 3781fb4d8502Sjsg } 3782fb4d8502Sjsg 3783fb4d8502Sjsg static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev) 3784fb4d8502Sjsg { 3785c349dbc7Sjsg struct amdgpu_ring *ring; 3786c349dbc7Sjsg int r; 3787fb4d8502Sjsg 3788fb4d8502Sjsg ring = &adev->gfx.kiq.ring; 3789fb4d8502Sjsg 3790fb4d8502Sjsg r = amdgpu_bo_reserve(ring->mqd_obj, false); 3791fb4d8502Sjsg if (unlikely(r != 0)) 3792c349dbc7Sjsg return r; 3793fb4d8502Sjsg 3794fb4d8502Sjsg r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 3795c349dbc7Sjsg if (unlikely(r != 0)) 3796c349dbc7Sjsg return r; 3797c349dbc7Sjsg 3798c349dbc7Sjsg gfx_v9_0_kiq_init_queue(ring); 3799fb4d8502Sjsg amdgpu_bo_kunmap(ring->mqd_obj); 3800fb4d8502Sjsg ring->mqd_ptr = NULL; 3801fb4d8502Sjsg amdgpu_bo_unreserve(ring->mqd_obj); 3802c349dbc7Sjsg ring->sched.ready = true; 3803c349dbc7Sjsg return 0; 3804c349dbc7Sjsg } 3805c349dbc7Sjsg 3806c349dbc7Sjsg static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev) 3807c349dbc7Sjsg { 3808c349dbc7Sjsg struct amdgpu_ring *ring = NULL; 3809c349dbc7Sjsg int r = 0, i; 3810c349dbc7Sjsg 3811c349dbc7Sjsg gfx_v9_0_cp_compute_enable(adev, true); 3812fb4d8502Sjsg 3813fb4d8502Sjsg for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3814fb4d8502Sjsg ring = &adev->gfx.compute_ring[i]; 3815fb4d8502Sjsg 3816fb4d8502Sjsg r = amdgpu_bo_reserve(ring->mqd_obj, false); 3817fb4d8502Sjsg if (unlikely(r != 0)) 3818fb4d8502Sjsg goto done; 3819fb4d8502Sjsg r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 3820fb4d8502Sjsg if (!r) { 3821fb4d8502Sjsg r = gfx_v9_0_kcq_init_queue(ring); 3822fb4d8502Sjsg amdgpu_bo_kunmap(ring->mqd_obj); 3823fb4d8502Sjsg ring->mqd_ptr = NULL; 3824fb4d8502Sjsg } 3825fb4d8502Sjsg amdgpu_bo_unreserve(ring->mqd_obj); 3826fb4d8502Sjsg if (r) 3827fb4d8502Sjsg goto done; 3828fb4d8502Sjsg } 3829fb4d8502Sjsg 3830c349dbc7Sjsg r = amdgpu_gfx_enable_kcq(adev); 3831fb4d8502Sjsg done: 3832fb4d8502Sjsg return r; 3833fb4d8502Sjsg } 3834fb4d8502Sjsg 3835fb4d8502Sjsg static int gfx_v9_0_cp_resume(struct amdgpu_device *adev) 3836fb4d8502Sjsg { 3837fb4d8502Sjsg int r, i; 3838fb4d8502Sjsg struct amdgpu_ring *ring; 3839fb4d8502Sjsg 3840fb4d8502Sjsg if (!(adev->flags & AMD_IS_APU)) 3841fb4d8502Sjsg gfx_v9_0_enable_gui_idle_interrupt(adev, false); 3842fb4d8502Sjsg 3843fb4d8502Sjsg if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 3844c349dbc7Sjsg if (adev->asic_type != CHIP_ARCTURUS) { 3845fb4d8502Sjsg /* legacy firmware loading */ 3846fb4d8502Sjsg r = gfx_v9_0_cp_gfx_load_microcode(adev); 3847fb4d8502Sjsg if (r) 3848fb4d8502Sjsg return r; 3849c349dbc7Sjsg } 3850fb4d8502Sjsg 3851fb4d8502Sjsg r = gfx_v9_0_cp_compute_load_microcode(adev); 3852fb4d8502Sjsg if (r) 3853fb4d8502Sjsg return r; 3854fb4d8502Sjsg } 3855fb4d8502Sjsg 3856fb4d8502Sjsg r = gfx_v9_0_kiq_resume(adev); 3857fb4d8502Sjsg if (r) 3858fb4d8502Sjsg return r; 3859fb4d8502Sjsg 3860c349dbc7Sjsg if (adev->asic_type != CHIP_ARCTURUS) { 3861c349dbc7Sjsg r = gfx_v9_0_cp_gfx_resume(adev); 3862c349dbc7Sjsg if (r) 3863fb4d8502Sjsg return r; 3864fb4d8502Sjsg } 3865fb4d8502Sjsg 3866c349dbc7Sjsg r = gfx_v9_0_kcq_resume(adev); 3867fb4d8502Sjsg if (r) 3868c349dbc7Sjsg return r; 3869c349dbc7Sjsg 3870c349dbc7Sjsg if (adev->asic_type != CHIP_ARCTURUS) { 3871c349dbc7Sjsg ring = &adev->gfx.gfx_ring[0]; 3872c349dbc7Sjsg r = amdgpu_ring_test_helper(ring); 3873c349dbc7Sjsg if (r) 3874c349dbc7Sjsg return r; 3875c349dbc7Sjsg } 3876fb4d8502Sjsg 3877fb4d8502Sjsg for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3878fb4d8502Sjsg ring = &adev->gfx.compute_ring[i]; 3879c349dbc7Sjsg amdgpu_ring_test_helper(ring); 3880fb4d8502Sjsg } 3881fb4d8502Sjsg 3882fb4d8502Sjsg gfx_v9_0_enable_gui_idle_interrupt(adev, true); 3883fb4d8502Sjsg 3884fb4d8502Sjsg return 0; 3885fb4d8502Sjsg } 3886fb4d8502Sjsg 3887c349dbc7Sjsg static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev) 3888c349dbc7Sjsg { 3889c349dbc7Sjsg u32 tmp; 3890c349dbc7Sjsg 3891c349dbc7Sjsg if (adev->asic_type != CHIP_ARCTURUS) 3892c349dbc7Sjsg return; 3893c349dbc7Sjsg 3894c349dbc7Sjsg tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG); 3895c349dbc7Sjsg tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH, 3896c349dbc7Sjsg adev->df.hash_status.hash_64k); 3897c349dbc7Sjsg tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH, 3898c349dbc7Sjsg adev->df.hash_status.hash_2m); 3899c349dbc7Sjsg tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH, 3900c349dbc7Sjsg adev->df.hash_status.hash_1g); 3901c349dbc7Sjsg WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp); 3902c349dbc7Sjsg } 3903c349dbc7Sjsg 3904fb4d8502Sjsg static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable) 3905fb4d8502Sjsg { 3906c349dbc7Sjsg if (adev->asic_type != CHIP_ARCTURUS) 3907fb4d8502Sjsg gfx_v9_0_cp_gfx_enable(adev, enable); 3908fb4d8502Sjsg gfx_v9_0_cp_compute_enable(adev, enable); 3909fb4d8502Sjsg } 3910fb4d8502Sjsg 3911fb4d8502Sjsg static int gfx_v9_0_hw_init(void *handle) 3912fb4d8502Sjsg { 3913fb4d8502Sjsg int r; 3914fb4d8502Sjsg struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3915fb4d8502Sjsg 3916c349dbc7Sjsg if (!amdgpu_sriov_vf(adev)) 3917fb4d8502Sjsg gfx_v9_0_init_golden_registers(adev); 3918fb4d8502Sjsg 3919c349dbc7Sjsg gfx_v9_0_constants_init(adev); 3920fb4d8502Sjsg 3921c349dbc7Sjsg gfx_v9_0_init_tcp_config(adev); 3922fb4d8502Sjsg 3923c349dbc7Sjsg r = adev->gfx.rlc.funcs->resume(adev); 3924fb4d8502Sjsg if (r) 3925fb4d8502Sjsg return r; 3926fb4d8502Sjsg 3927fb4d8502Sjsg r = gfx_v9_0_cp_resume(adev); 3928fb4d8502Sjsg if (r) 3929fb4d8502Sjsg return r; 3930fb4d8502Sjsg 3931fb4d8502Sjsg return r; 3932fb4d8502Sjsg } 3933fb4d8502Sjsg 3934fb4d8502Sjsg static int gfx_v9_0_hw_fini(void *handle) 3935fb4d8502Sjsg { 3936fb4d8502Sjsg struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3937fb4d8502Sjsg 3938c349dbc7Sjsg amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); 3939fb4d8502Sjsg amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 3940fb4d8502Sjsg amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 3941fb4d8502Sjsg 3942c349dbc7Sjsg /* DF freeze and kcq disable will fail */ 3943c349dbc7Sjsg if (!amdgpu_ras_intr_triggered()) 3944fb4d8502Sjsg /* disable KCQ to avoid CPC touch memory not valid anymore */ 3945c349dbc7Sjsg amdgpu_gfx_disable_kcq(adev); 3946fb4d8502Sjsg 3947fb4d8502Sjsg if (amdgpu_sriov_vf(adev)) { 3948fb4d8502Sjsg gfx_v9_0_cp_gfx_enable(adev, false); 3949fb4d8502Sjsg /* must disable polling for SRIOV when hw finished, otherwise 3950fb4d8502Sjsg * CPC engine may still keep fetching WB address which is already 3951fb4d8502Sjsg * invalid after sw finished and trigger DMAR reading error in 3952fb4d8502Sjsg * hypervisor side. 3953fb4d8502Sjsg */ 3954fb4d8502Sjsg WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 3955fb4d8502Sjsg return 0; 3956fb4d8502Sjsg } 3957fb4d8502Sjsg 3958fb4d8502Sjsg /* Use deinitialize sequence from CAIL when unbinding device from driver, 3959fb4d8502Sjsg * otherwise KIQ is hanging when binding back 3960fb4d8502Sjsg */ 3961ad8b1aafSjsg if (!amdgpu_in_reset(adev) && !adev->in_suspend) { 3962fb4d8502Sjsg mutex_lock(&adev->srbm_mutex); 3963fb4d8502Sjsg soc15_grbm_select(adev, adev->gfx.kiq.ring.me, 3964fb4d8502Sjsg adev->gfx.kiq.ring.pipe, 3965fb4d8502Sjsg adev->gfx.kiq.ring.queue, 0); 3966fb4d8502Sjsg gfx_v9_0_kiq_fini_register(&adev->gfx.kiq.ring); 3967fb4d8502Sjsg soc15_grbm_select(adev, 0, 0, 0, 0); 3968fb4d8502Sjsg mutex_unlock(&adev->srbm_mutex); 3969fb4d8502Sjsg } 3970fb4d8502Sjsg 3971fb4d8502Sjsg gfx_v9_0_cp_enable(adev, false); 3972c349dbc7Sjsg adev->gfx.rlc.funcs->stop(adev); 3973fb4d8502Sjsg 3974fb4d8502Sjsg return 0; 3975fb4d8502Sjsg } 3976fb4d8502Sjsg 3977fb4d8502Sjsg static int gfx_v9_0_suspend(void *handle) 3978fb4d8502Sjsg { 3979c349dbc7Sjsg return gfx_v9_0_hw_fini(handle); 3980fb4d8502Sjsg } 3981fb4d8502Sjsg 3982fb4d8502Sjsg static int gfx_v9_0_resume(void *handle) 3983fb4d8502Sjsg { 3984c349dbc7Sjsg return gfx_v9_0_hw_init(handle); 3985fb4d8502Sjsg } 3986fb4d8502Sjsg 3987fb4d8502Sjsg static bool gfx_v9_0_is_idle(void *handle) 3988fb4d8502Sjsg { 3989fb4d8502Sjsg struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3990fb4d8502Sjsg 3991fb4d8502Sjsg if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS), 3992fb4d8502Sjsg GRBM_STATUS, GUI_ACTIVE)) 3993fb4d8502Sjsg return false; 3994fb4d8502Sjsg else 3995fb4d8502Sjsg return true; 3996fb4d8502Sjsg } 3997fb4d8502Sjsg 3998fb4d8502Sjsg static int gfx_v9_0_wait_for_idle(void *handle) 3999fb4d8502Sjsg { 4000fb4d8502Sjsg unsigned i; 4001fb4d8502Sjsg struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4002fb4d8502Sjsg 4003fb4d8502Sjsg for (i = 0; i < adev->usec_timeout; i++) { 4004fb4d8502Sjsg if (gfx_v9_0_is_idle(handle)) 4005fb4d8502Sjsg return 0; 4006fb4d8502Sjsg udelay(1); 4007fb4d8502Sjsg } 4008fb4d8502Sjsg return -ETIMEDOUT; 4009fb4d8502Sjsg } 4010fb4d8502Sjsg 4011fb4d8502Sjsg static int gfx_v9_0_soft_reset(void *handle) 4012fb4d8502Sjsg { 4013fb4d8502Sjsg u32 grbm_soft_reset = 0; 4014fb4d8502Sjsg u32 tmp; 4015fb4d8502Sjsg struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4016fb4d8502Sjsg 4017fb4d8502Sjsg /* GRBM_STATUS */ 4018fb4d8502Sjsg tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS); 4019fb4d8502Sjsg if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK | 4020fb4d8502Sjsg GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK | 4021fb4d8502Sjsg GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK | 4022fb4d8502Sjsg GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK | 4023fb4d8502Sjsg GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK | 4024fb4d8502Sjsg GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) { 4025fb4d8502Sjsg grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4026fb4d8502Sjsg GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 4027fb4d8502Sjsg grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4028fb4d8502Sjsg GRBM_SOFT_RESET, SOFT_RESET_GFX, 1); 4029fb4d8502Sjsg } 4030fb4d8502Sjsg 4031fb4d8502Sjsg if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) { 4032fb4d8502Sjsg grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4033fb4d8502Sjsg GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 4034fb4d8502Sjsg } 4035fb4d8502Sjsg 4036fb4d8502Sjsg /* GRBM_STATUS2 */ 4037fb4d8502Sjsg tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2); 4038fb4d8502Sjsg if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY)) 4039fb4d8502Sjsg grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4040fb4d8502Sjsg GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 4041fb4d8502Sjsg 4042fb4d8502Sjsg 4043fb4d8502Sjsg if (grbm_soft_reset) { 4044fb4d8502Sjsg /* stop the rlc */ 4045c349dbc7Sjsg adev->gfx.rlc.funcs->stop(adev); 4046fb4d8502Sjsg 4047c349dbc7Sjsg if (adev->asic_type != CHIP_ARCTURUS) 4048fb4d8502Sjsg /* Disable GFX parsing/prefetching */ 4049fb4d8502Sjsg gfx_v9_0_cp_gfx_enable(adev, false); 4050fb4d8502Sjsg 4051fb4d8502Sjsg /* Disable MEC parsing/prefetching */ 4052fb4d8502Sjsg gfx_v9_0_cp_compute_enable(adev, false); 4053fb4d8502Sjsg 4054fb4d8502Sjsg if (grbm_soft_reset) { 4055fb4d8502Sjsg tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4056fb4d8502Sjsg tmp |= grbm_soft_reset; 4057fb4d8502Sjsg dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 4058fb4d8502Sjsg WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 4059fb4d8502Sjsg tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4060fb4d8502Sjsg 4061fb4d8502Sjsg udelay(50); 4062fb4d8502Sjsg 4063fb4d8502Sjsg tmp &= ~grbm_soft_reset; 4064fb4d8502Sjsg WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 4065fb4d8502Sjsg tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4066fb4d8502Sjsg } 4067fb4d8502Sjsg 4068fb4d8502Sjsg /* Wait a little for things to settle down */ 4069fb4d8502Sjsg udelay(50); 4070fb4d8502Sjsg } 4071fb4d8502Sjsg return 0; 4072fb4d8502Sjsg } 4073fb4d8502Sjsg 4074c349dbc7Sjsg static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev) 4075c349dbc7Sjsg { 4076c349dbc7Sjsg signed long r, cnt = 0; 4077c349dbc7Sjsg unsigned long flags; 4078ad8b1aafSjsg uint32_t seq, reg_val_offs = 0; 4079ad8b1aafSjsg uint64_t value = 0; 4080c349dbc7Sjsg struct amdgpu_kiq *kiq = &adev->gfx.kiq; 4081c349dbc7Sjsg struct amdgpu_ring *ring = &kiq->ring; 4082c349dbc7Sjsg 4083c349dbc7Sjsg BUG_ON(!ring->funcs->emit_rreg); 4084c349dbc7Sjsg 4085c349dbc7Sjsg spin_lock_irqsave(&kiq->ring_lock, flags); 4086ad8b1aafSjsg if (amdgpu_device_wb_get(adev, ®_val_offs)) { 4087ad8b1aafSjsg pr_err("critical bug! too many kiq readers\n"); 4088ad8b1aafSjsg goto failed_unlock; 4089ad8b1aafSjsg } 4090c349dbc7Sjsg amdgpu_ring_alloc(ring, 32); 4091c349dbc7Sjsg amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 4092c349dbc7Sjsg amdgpu_ring_write(ring, 9 | /* src: register*/ 4093c349dbc7Sjsg (5 << 8) | /* dst: memory */ 4094c349dbc7Sjsg (1 << 16) | /* count sel */ 4095c349dbc7Sjsg (1 << 20)); /* write confirm */ 4096c349dbc7Sjsg amdgpu_ring_write(ring, 0); 4097c349dbc7Sjsg amdgpu_ring_write(ring, 0); 4098c349dbc7Sjsg amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 4099ad8b1aafSjsg reg_val_offs * 4)); 4100c349dbc7Sjsg amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 4101ad8b1aafSjsg reg_val_offs * 4)); 4102ad8b1aafSjsg r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 4103ad8b1aafSjsg if (r) 4104ad8b1aafSjsg goto failed_undo; 4105ad8b1aafSjsg 4106c349dbc7Sjsg amdgpu_ring_commit(ring); 4107c349dbc7Sjsg spin_unlock_irqrestore(&kiq->ring_lock, flags); 4108c349dbc7Sjsg 4109c349dbc7Sjsg r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 4110c349dbc7Sjsg 4111c349dbc7Sjsg /* don't wait anymore for gpu reset case because this way may 4112c349dbc7Sjsg * block gpu_recover() routine forever, e.g. this virt_kiq_rreg 4113c349dbc7Sjsg * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will 4114c349dbc7Sjsg * never return if we keep waiting in virt_kiq_rreg, which cause 4115c349dbc7Sjsg * gpu_recover() hang there. 4116c349dbc7Sjsg * 4117c349dbc7Sjsg * also don't wait anymore for IRQ context 4118c349dbc7Sjsg * */ 4119ad8b1aafSjsg if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt())) 4120c349dbc7Sjsg goto failed_kiq_read; 4121c349dbc7Sjsg 4122c349dbc7Sjsg might_sleep(); 4123c349dbc7Sjsg while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 4124c349dbc7Sjsg drm_msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 4125c349dbc7Sjsg r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 4126c349dbc7Sjsg } 4127c349dbc7Sjsg 4128c349dbc7Sjsg if (cnt > MAX_KIQ_REG_TRY) 4129c349dbc7Sjsg goto failed_kiq_read; 4130c349dbc7Sjsg 4131ad8b1aafSjsg mb(); 4132ad8b1aafSjsg value = (uint64_t)adev->wb.wb[reg_val_offs] | 4133ad8b1aafSjsg (uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL; 4134ad8b1aafSjsg amdgpu_device_wb_free(adev, reg_val_offs); 4135ad8b1aafSjsg return value; 4136c349dbc7Sjsg 4137ad8b1aafSjsg failed_undo: 4138ad8b1aafSjsg amdgpu_ring_undo(ring); 4139ad8b1aafSjsg failed_unlock: 4140ad8b1aafSjsg spin_unlock_irqrestore(&kiq->ring_lock, flags); 4141c349dbc7Sjsg failed_kiq_read: 4142ad8b1aafSjsg if (reg_val_offs) 4143ad8b1aafSjsg amdgpu_device_wb_free(adev, reg_val_offs); 4144c349dbc7Sjsg pr_err("failed to read gpu clock\n"); 4145c349dbc7Sjsg return ~0; 4146c349dbc7Sjsg } 4147c349dbc7Sjsg 4148fb4d8502Sjsg static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev) 4149fb4d8502Sjsg { 4150fb4d8502Sjsg uint64_t clock; 4151fb4d8502Sjsg 4152c349dbc7Sjsg amdgpu_gfx_off_ctrl(adev, false); 4153fb4d8502Sjsg mutex_lock(&adev->gfx.gpu_clock_mutex); 4154c349dbc7Sjsg if (adev->asic_type == CHIP_VEGA10 && amdgpu_sriov_runtime(adev)) { 4155c349dbc7Sjsg clock = gfx_v9_0_kiq_read_clock(adev); 4156c349dbc7Sjsg } else { 4157fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1); 4158fb4d8502Sjsg clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) | 4159fb4d8502Sjsg ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 4160c349dbc7Sjsg } 4161fb4d8502Sjsg mutex_unlock(&adev->gfx.gpu_clock_mutex); 4162c349dbc7Sjsg amdgpu_gfx_off_ctrl(adev, true); 4163fb4d8502Sjsg return clock; 4164fb4d8502Sjsg } 4165fb4d8502Sjsg 4166fb4d8502Sjsg static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring, 4167fb4d8502Sjsg uint32_t vmid, 4168fb4d8502Sjsg uint32_t gds_base, uint32_t gds_size, 4169fb4d8502Sjsg uint32_t gws_base, uint32_t gws_size, 4170fb4d8502Sjsg uint32_t oa_base, uint32_t oa_size) 4171fb4d8502Sjsg { 4172fb4d8502Sjsg struct amdgpu_device *adev = ring->adev; 4173fb4d8502Sjsg 4174fb4d8502Sjsg /* GDS Base */ 4175fb4d8502Sjsg gfx_v9_0_write_data_to_reg(ring, 0, false, 4176fb4d8502Sjsg SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid, 4177fb4d8502Sjsg gds_base); 4178fb4d8502Sjsg 4179fb4d8502Sjsg /* GDS Size */ 4180fb4d8502Sjsg gfx_v9_0_write_data_to_reg(ring, 0, false, 4181fb4d8502Sjsg SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid, 4182fb4d8502Sjsg gds_size); 4183fb4d8502Sjsg 4184fb4d8502Sjsg /* GWS */ 4185fb4d8502Sjsg gfx_v9_0_write_data_to_reg(ring, 0, false, 4186fb4d8502Sjsg SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid, 4187fb4d8502Sjsg gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base); 4188fb4d8502Sjsg 4189fb4d8502Sjsg /* OA */ 4190fb4d8502Sjsg gfx_v9_0_write_data_to_reg(ring, 0, false, 4191fb4d8502Sjsg SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid, 4192fb4d8502Sjsg (1 << (oa_size + oa_base)) - (1 << oa_base)); 4193fb4d8502Sjsg } 4194fb4d8502Sjsg 4195c349dbc7Sjsg static const u32 vgpr_init_compute_shader[] = 4196c349dbc7Sjsg { 4197c349dbc7Sjsg 0xb07c0000, 0xbe8000ff, 4198c349dbc7Sjsg 0x000000f8, 0xbf110800, 4199c349dbc7Sjsg 0x7e000280, 0x7e020280, 4200c349dbc7Sjsg 0x7e040280, 0x7e060280, 4201c349dbc7Sjsg 0x7e080280, 0x7e0a0280, 4202c349dbc7Sjsg 0x7e0c0280, 0x7e0e0280, 4203c349dbc7Sjsg 0x80808800, 0xbe803200, 4204c349dbc7Sjsg 0xbf84fff5, 0xbf9c0000, 4205c349dbc7Sjsg 0xd28c0001, 0x0001007f, 4206c349dbc7Sjsg 0xd28d0001, 0x0002027e, 4207c349dbc7Sjsg 0x10020288, 0xb8810904, 4208c349dbc7Sjsg 0xb7814000, 0xd1196a01, 4209c349dbc7Sjsg 0x00000301, 0xbe800087, 4210c349dbc7Sjsg 0xbefc00c1, 0xd89c4000, 4211c349dbc7Sjsg 0x00020201, 0xd89cc080, 4212c349dbc7Sjsg 0x00040401, 0x320202ff, 4213c349dbc7Sjsg 0x00000800, 0x80808100, 4214c349dbc7Sjsg 0xbf84fff8, 0x7e020280, 4215c349dbc7Sjsg 0xbf810000, 0x00000000, 4216c349dbc7Sjsg }; 4217c349dbc7Sjsg 4218c349dbc7Sjsg static const u32 sgpr_init_compute_shader[] = 4219c349dbc7Sjsg { 4220c349dbc7Sjsg 0xb07c0000, 0xbe8000ff, 4221c349dbc7Sjsg 0x0000005f, 0xbee50080, 4222c349dbc7Sjsg 0xbe812c65, 0xbe822c65, 4223c349dbc7Sjsg 0xbe832c65, 0xbe842c65, 4224c349dbc7Sjsg 0xbe852c65, 0xb77c0005, 4225c349dbc7Sjsg 0x80808500, 0xbf84fff8, 4226c349dbc7Sjsg 0xbe800080, 0xbf810000, 4227c349dbc7Sjsg }; 4228c349dbc7Sjsg 4229c349dbc7Sjsg static const u32 vgpr_init_compute_shader_arcturus[] = { 4230c349dbc7Sjsg 0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080, 4231c349dbc7Sjsg 0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080, 4232c349dbc7Sjsg 0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080, 4233c349dbc7Sjsg 0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080, 4234c349dbc7Sjsg 0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080, 4235c349dbc7Sjsg 0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080, 4236c349dbc7Sjsg 0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080, 4237c349dbc7Sjsg 0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080, 4238c349dbc7Sjsg 0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080, 4239c349dbc7Sjsg 0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080, 4240c349dbc7Sjsg 0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080, 4241c349dbc7Sjsg 0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080, 4242c349dbc7Sjsg 0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080, 4243c349dbc7Sjsg 0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080, 4244c349dbc7Sjsg 0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080, 4245c349dbc7Sjsg 0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080, 4246c349dbc7Sjsg 0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080, 4247c349dbc7Sjsg 0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080, 4248c349dbc7Sjsg 0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080, 4249c349dbc7Sjsg 0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080, 4250c349dbc7Sjsg 0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080, 4251c349dbc7Sjsg 0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080, 4252c349dbc7Sjsg 0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080, 4253c349dbc7Sjsg 0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080, 4254c349dbc7Sjsg 0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080, 4255c349dbc7Sjsg 0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080, 4256c349dbc7Sjsg 0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080, 4257c349dbc7Sjsg 0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080, 4258c349dbc7Sjsg 0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080, 4259c349dbc7Sjsg 0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080, 4260c349dbc7Sjsg 0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080, 4261c349dbc7Sjsg 0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080, 4262c349dbc7Sjsg 0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080, 4263c349dbc7Sjsg 0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080, 4264c349dbc7Sjsg 0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080, 4265c349dbc7Sjsg 0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080, 4266c349dbc7Sjsg 0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080, 4267c349dbc7Sjsg 0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080, 4268c349dbc7Sjsg 0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080, 4269c349dbc7Sjsg 0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080, 4270c349dbc7Sjsg 0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080, 4271c349dbc7Sjsg 0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080, 4272c349dbc7Sjsg 0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080, 4273c349dbc7Sjsg 0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080, 4274c349dbc7Sjsg 0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080, 4275c349dbc7Sjsg 0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080, 4276c349dbc7Sjsg 0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080, 4277c349dbc7Sjsg 0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080, 4278c349dbc7Sjsg 0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080, 4279c349dbc7Sjsg 0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080, 4280c349dbc7Sjsg 0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080, 4281c349dbc7Sjsg 0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080, 4282c349dbc7Sjsg 0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080, 4283c349dbc7Sjsg 0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080, 4284c349dbc7Sjsg 0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080, 4285c349dbc7Sjsg 0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080, 4286c349dbc7Sjsg 0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080, 4287c349dbc7Sjsg 0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080, 4288c349dbc7Sjsg 0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080, 4289c349dbc7Sjsg 0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080, 4290c349dbc7Sjsg 0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080, 4291c349dbc7Sjsg 0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080, 4292c349dbc7Sjsg 0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080, 4293c349dbc7Sjsg 0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080, 4294c349dbc7Sjsg 0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080, 4295c349dbc7Sjsg 0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080, 4296c349dbc7Sjsg 0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080, 4297c349dbc7Sjsg 0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080, 4298c349dbc7Sjsg 0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080, 4299c349dbc7Sjsg 0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080, 4300c349dbc7Sjsg 0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080, 4301c349dbc7Sjsg 0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080, 4302c349dbc7Sjsg 0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080, 4303c349dbc7Sjsg 0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080, 4304c349dbc7Sjsg 0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080, 4305c349dbc7Sjsg 0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080, 4306c349dbc7Sjsg 0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080, 4307c349dbc7Sjsg 0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080, 4308c349dbc7Sjsg 0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080, 4309c349dbc7Sjsg 0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080, 4310c349dbc7Sjsg 0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080, 4311c349dbc7Sjsg 0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080, 4312c349dbc7Sjsg 0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080, 4313c349dbc7Sjsg 0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080, 4314c349dbc7Sjsg 0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080, 4315c349dbc7Sjsg 0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a, 4316c349dbc7Sjsg 0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280, 4317c349dbc7Sjsg 0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000, 4318c349dbc7Sjsg 0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904, 4319c349dbc7Sjsg 0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000, 4320c349dbc7Sjsg 0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a, 4321c349dbc7Sjsg 0xbf84fff8, 0xbf810000, 4322c349dbc7Sjsg }; 4323c349dbc7Sjsg 4324c349dbc7Sjsg /* When below register arrays changed, please update gpr_reg_size, 4325c349dbc7Sjsg and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds, 4326c349dbc7Sjsg to cover all gfx9 ASICs */ 4327c349dbc7Sjsg static const struct soc15_reg_entry vgpr_init_regs[] = { 4328c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4329c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4330c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 }, 4331c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4332c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f }, 4333c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 4334c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 4335c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 4336c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 4337c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 4338c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, 4339c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, 4340c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, 4341c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, 4342c349dbc7Sjsg }; 4343c349dbc7Sjsg 4344c349dbc7Sjsg static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = { 4345c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4346c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4347c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 }, 4348c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4349c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf }, 4350c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 4351c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 4352c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 4353c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 4354c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 4355c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, 4356c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, 4357c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, 4358c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, 4359c349dbc7Sjsg }; 4360c349dbc7Sjsg 4361c349dbc7Sjsg static const struct soc15_reg_entry sgpr1_init_regs[] = { 4362c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4363c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4364c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 }, 4365c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4366c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */ 4367c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4368c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff }, 4369c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff }, 4370c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff }, 4371c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff }, 4372c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff }, 4373c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff }, 4374c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff }, 4375c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff }, 4376c349dbc7Sjsg }; 4377c349dbc7Sjsg 4378c349dbc7Sjsg static const struct soc15_reg_entry sgpr2_init_regs[] = { 4379c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4380c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4381c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 }, 4382c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4383c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */ 4384c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4385c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 }, 4386c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 }, 4387c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 }, 4388c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 }, 4389c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 }, 4390c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 }, 4391c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 }, 4392c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 }, 4393c349dbc7Sjsg }; 4394c349dbc7Sjsg 4395c349dbc7Sjsg static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = { 4396c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1}, 4397c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1}, 4398c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1}, 4399c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1}, 4400c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1}, 4401c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1}, 4402c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1}, 4403c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1}, 4404c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1}, 4405c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1}, 4406c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1}, 4407c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1}, 4408c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1}, 4409c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6}, 4410c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16}, 4411c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16}, 4412c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16}, 4413c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16}, 4414c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16}, 4415c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16}, 4416c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16}, 4417c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16}, 4418c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6}, 4419c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16}, 4420c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16}, 4421c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1}, 4422c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1}, 4423c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32}, 4424c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32}, 4425c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72}, 4426c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16}, 4427c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2}, 4428c349dbc7Sjsg { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6}, 4429c349dbc7Sjsg }; 4430c349dbc7Sjsg 4431c349dbc7Sjsg static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev) 4432c349dbc7Sjsg { 4433c349dbc7Sjsg struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4434c349dbc7Sjsg int i, r; 4435c349dbc7Sjsg 4436c349dbc7Sjsg /* only support when RAS is enabled */ 4437c349dbc7Sjsg if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4438c349dbc7Sjsg return 0; 4439c349dbc7Sjsg 4440c349dbc7Sjsg r = amdgpu_ring_alloc(ring, 7); 4441c349dbc7Sjsg if (r) { 4442c349dbc7Sjsg DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n", 4443c349dbc7Sjsg ring->name, r); 4444c349dbc7Sjsg return r; 4445c349dbc7Sjsg } 4446c349dbc7Sjsg 4447c349dbc7Sjsg WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000); 4448c349dbc7Sjsg WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size); 4449c349dbc7Sjsg 4450c349dbc7Sjsg amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5)); 4451c349dbc7Sjsg amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC | 4452c349dbc7Sjsg PACKET3_DMA_DATA_DST_SEL(1) | 4453c349dbc7Sjsg PACKET3_DMA_DATA_SRC_SEL(2) | 4454c349dbc7Sjsg PACKET3_DMA_DATA_ENGINE(0))); 4455c349dbc7Sjsg amdgpu_ring_write(ring, 0); 4456c349dbc7Sjsg amdgpu_ring_write(ring, 0); 4457c349dbc7Sjsg amdgpu_ring_write(ring, 0); 4458c349dbc7Sjsg amdgpu_ring_write(ring, 0); 4459c349dbc7Sjsg amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT | 4460c349dbc7Sjsg adev->gds.gds_size); 4461c349dbc7Sjsg 4462c349dbc7Sjsg amdgpu_ring_commit(ring); 4463c349dbc7Sjsg 4464c349dbc7Sjsg for (i = 0; i < adev->usec_timeout; i++) { 4465c349dbc7Sjsg if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring)) 4466c349dbc7Sjsg break; 4467c349dbc7Sjsg udelay(1); 4468c349dbc7Sjsg } 4469c349dbc7Sjsg 4470c349dbc7Sjsg if (i >= adev->usec_timeout) 4471c349dbc7Sjsg r = -ETIMEDOUT; 4472c349dbc7Sjsg 4473c349dbc7Sjsg WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000); 4474c349dbc7Sjsg 4475c349dbc7Sjsg return r; 4476c349dbc7Sjsg } 4477c349dbc7Sjsg 4478c349dbc7Sjsg static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev) 4479c349dbc7Sjsg { 4480c349dbc7Sjsg struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4481c349dbc7Sjsg struct amdgpu_ib ib; 4482c349dbc7Sjsg struct dma_fence *f = NULL; 4483c349dbc7Sjsg int r, i; 4484c349dbc7Sjsg unsigned total_size, vgpr_offset, sgpr_offset; 4485c349dbc7Sjsg u64 gpu_addr; 4486c349dbc7Sjsg 4487c349dbc7Sjsg int compute_dim_x = adev->gfx.config.max_shader_engines * 4488c349dbc7Sjsg adev->gfx.config.max_cu_per_sh * 4489c349dbc7Sjsg adev->gfx.config.max_sh_per_se; 4490c349dbc7Sjsg int sgpr_work_group_size = 5; 4491c349dbc7Sjsg int gpr_reg_size = adev->gfx.config.max_shader_engines + 6; 4492c349dbc7Sjsg int vgpr_init_shader_size; 4493c349dbc7Sjsg const u32 *vgpr_init_shader_ptr; 4494c349dbc7Sjsg const struct soc15_reg_entry *vgpr_init_regs_ptr; 4495c349dbc7Sjsg 4496c349dbc7Sjsg /* only support when RAS is enabled */ 4497c349dbc7Sjsg if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4498c349dbc7Sjsg return 0; 4499c349dbc7Sjsg 4500c349dbc7Sjsg /* bail if the compute ring is not ready */ 4501c349dbc7Sjsg if (!ring->sched.ready) 4502c349dbc7Sjsg return 0; 4503c349dbc7Sjsg 4504c349dbc7Sjsg if (adev->asic_type == CHIP_ARCTURUS) { 4505c349dbc7Sjsg vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus; 4506c349dbc7Sjsg vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus); 4507c349dbc7Sjsg vgpr_init_regs_ptr = vgpr_init_regs_arcturus; 4508c349dbc7Sjsg } else { 4509c349dbc7Sjsg vgpr_init_shader_ptr = vgpr_init_compute_shader; 4510c349dbc7Sjsg vgpr_init_shader_size = sizeof(vgpr_init_compute_shader); 4511c349dbc7Sjsg vgpr_init_regs_ptr = vgpr_init_regs; 4512c349dbc7Sjsg } 4513c349dbc7Sjsg 4514c349dbc7Sjsg total_size = 4515c349dbc7Sjsg (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */ 4516c349dbc7Sjsg total_size += 4517c349dbc7Sjsg (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */ 4518c349dbc7Sjsg total_size += 4519c349dbc7Sjsg (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */ 4520c349dbc7Sjsg total_size = roundup2(total_size, 256); 4521c349dbc7Sjsg vgpr_offset = total_size; 4522c349dbc7Sjsg total_size += roundup2(vgpr_init_shader_size, 256); 4523c349dbc7Sjsg sgpr_offset = total_size; 4524c349dbc7Sjsg total_size += sizeof(sgpr_init_compute_shader); 4525c349dbc7Sjsg 4526c349dbc7Sjsg /* allocate an indirect buffer to put the commands in */ 4527c349dbc7Sjsg memset(&ib, 0, sizeof(ib)); 4528ad8b1aafSjsg r = amdgpu_ib_get(adev, NULL, total_size, 4529ad8b1aafSjsg AMDGPU_IB_POOL_DIRECT, &ib); 4530c349dbc7Sjsg if (r) { 4531c349dbc7Sjsg DRM_ERROR("amdgpu: failed to get ib (%d).\n", r); 4532c349dbc7Sjsg return r; 4533c349dbc7Sjsg } 4534c349dbc7Sjsg 4535c349dbc7Sjsg /* load the compute shaders */ 4536c349dbc7Sjsg for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++) 4537c349dbc7Sjsg ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i]; 4538c349dbc7Sjsg 4539c349dbc7Sjsg for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++) 4540c349dbc7Sjsg ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i]; 4541c349dbc7Sjsg 4542c349dbc7Sjsg /* init the ib length to 0 */ 4543c349dbc7Sjsg ib.length_dw = 0; 4544c349dbc7Sjsg 4545c349dbc7Sjsg /* VGPR */ 4546c349dbc7Sjsg /* write the register state for the compute dispatch */ 4547c349dbc7Sjsg for (i = 0; i < gpr_reg_size; i++) { 4548c349dbc7Sjsg ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4549c349dbc7Sjsg ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i]) 4550c349dbc7Sjsg - PACKET3_SET_SH_REG_START; 4551c349dbc7Sjsg ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value; 4552c349dbc7Sjsg } 4553c349dbc7Sjsg /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4554c349dbc7Sjsg gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8; 4555c349dbc7Sjsg ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4556c349dbc7Sjsg ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4557c349dbc7Sjsg - PACKET3_SET_SH_REG_START; 4558c349dbc7Sjsg ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4559c349dbc7Sjsg ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4560c349dbc7Sjsg 4561c349dbc7Sjsg /* write dispatch packet */ 4562c349dbc7Sjsg ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4563c349dbc7Sjsg ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */ 4564c349dbc7Sjsg ib.ptr[ib.length_dw++] = 1; /* y */ 4565c349dbc7Sjsg ib.ptr[ib.length_dw++] = 1; /* z */ 4566c349dbc7Sjsg ib.ptr[ib.length_dw++] = 4567c349dbc7Sjsg REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4568c349dbc7Sjsg 4569c349dbc7Sjsg /* write CS partial flush packet */ 4570c349dbc7Sjsg ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4571c349dbc7Sjsg ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4572c349dbc7Sjsg 4573c349dbc7Sjsg /* SGPR1 */ 4574c349dbc7Sjsg /* write the register state for the compute dispatch */ 4575c349dbc7Sjsg for (i = 0; i < gpr_reg_size; i++) { 4576c349dbc7Sjsg ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4577c349dbc7Sjsg ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i]) 4578c349dbc7Sjsg - PACKET3_SET_SH_REG_START; 4579c349dbc7Sjsg ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value; 4580c349dbc7Sjsg } 4581c349dbc7Sjsg /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4582c349dbc7Sjsg gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4583c349dbc7Sjsg ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4584c349dbc7Sjsg ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4585c349dbc7Sjsg - PACKET3_SET_SH_REG_START; 4586c349dbc7Sjsg ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4587c349dbc7Sjsg ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4588c349dbc7Sjsg 4589c349dbc7Sjsg /* write dispatch packet */ 4590c349dbc7Sjsg ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4591c349dbc7Sjsg ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ 4592c349dbc7Sjsg ib.ptr[ib.length_dw++] = 1; /* y */ 4593c349dbc7Sjsg ib.ptr[ib.length_dw++] = 1; /* z */ 4594c349dbc7Sjsg ib.ptr[ib.length_dw++] = 4595c349dbc7Sjsg REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4596c349dbc7Sjsg 4597c349dbc7Sjsg /* write CS partial flush packet */ 4598c349dbc7Sjsg ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4599c349dbc7Sjsg ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4600c349dbc7Sjsg 4601c349dbc7Sjsg /* SGPR2 */ 4602c349dbc7Sjsg /* write the register state for the compute dispatch */ 4603c349dbc7Sjsg for (i = 0; i < gpr_reg_size; i++) { 4604c349dbc7Sjsg ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4605c349dbc7Sjsg ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i]) 4606c349dbc7Sjsg - PACKET3_SET_SH_REG_START; 4607c349dbc7Sjsg ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value; 4608c349dbc7Sjsg } 4609c349dbc7Sjsg /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4610c349dbc7Sjsg gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4611c349dbc7Sjsg ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4612c349dbc7Sjsg ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4613c349dbc7Sjsg - PACKET3_SET_SH_REG_START; 4614c349dbc7Sjsg ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4615c349dbc7Sjsg ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4616c349dbc7Sjsg 4617c349dbc7Sjsg /* write dispatch packet */ 4618c349dbc7Sjsg ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4619c349dbc7Sjsg ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ 4620c349dbc7Sjsg ib.ptr[ib.length_dw++] = 1; /* y */ 4621c349dbc7Sjsg ib.ptr[ib.length_dw++] = 1; /* z */ 4622c349dbc7Sjsg ib.ptr[ib.length_dw++] = 4623c349dbc7Sjsg REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4624c349dbc7Sjsg 4625c349dbc7Sjsg /* write CS partial flush packet */ 4626c349dbc7Sjsg ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4627c349dbc7Sjsg ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4628c349dbc7Sjsg 4629c349dbc7Sjsg /* shedule the ib on the ring */ 4630c349dbc7Sjsg r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 4631c349dbc7Sjsg if (r) { 4632c349dbc7Sjsg DRM_ERROR("amdgpu: ib submit failed (%d).\n", r); 4633c349dbc7Sjsg goto fail; 4634c349dbc7Sjsg } 4635c349dbc7Sjsg 4636c349dbc7Sjsg /* wait for the GPU to finish processing the IB */ 4637c349dbc7Sjsg r = dma_fence_wait(f, false); 4638c349dbc7Sjsg if (r) { 4639c349dbc7Sjsg DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); 4640c349dbc7Sjsg goto fail; 4641c349dbc7Sjsg } 4642c349dbc7Sjsg 4643c349dbc7Sjsg fail: 4644c349dbc7Sjsg amdgpu_ib_free(adev, &ib, NULL); 4645c349dbc7Sjsg dma_fence_put(f); 4646c349dbc7Sjsg 4647c349dbc7Sjsg return r; 4648c349dbc7Sjsg } 4649c349dbc7Sjsg 4650fb4d8502Sjsg static int gfx_v9_0_early_init(void *handle) 4651fb4d8502Sjsg { 4652fb4d8502Sjsg struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4653fb4d8502Sjsg 4654c349dbc7Sjsg if (adev->asic_type == CHIP_ARCTURUS) 4655c349dbc7Sjsg adev->gfx.num_gfx_rings = 0; 4656c349dbc7Sjsg else 4657fb4d8502Sjsg adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS; 4658ad8b1aafSjsg adev->gfx.num_compute_rings = amdgpu_num_kcq; 4659c349dbc7Sjsg gfx_v9_0_set_kiq_pm4_funcs(adev); 4660fb4d8502Sjsg gfx_v9_0_set_ring_funcs(adev); 4661fb4d8502Sjsg gfx_v9_0_set_irq_funcs(adev); 4662fb4d8502Sjsg gfx_v9_0_set_gds_init(adev); 4663fb4d8502Sjsg gfx_v9_0_set_rlc_funcs(adev); 4664fb4d8502Sjsg 4665fb4d8502Sjsg return 0; 4666fb4d8502Sjsg } 4667fb4d8502Sjsg 4668c349dbc7Sjsg static int gfx_v9_0_ecc_late_init(void *handle) 4669c349dbc7Sjsg { 4670c349dbc7Sjsg struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4671c349dbc7Sjsg int r; 4672c349dbc7Sjsg 4673c349dbc7Sjsg /* 4674c349dbc7Sjsg * Temp workaround to fix the issue that CP firmware fails to 4675c349dbc7Sjsg * update read pointer when CPDMA is writing clearing operation 4676c349dbc7Sjsg * to GDS in suspend/resume sequence on several cards. So just 4677c349dbc7Sjsg * limit this operation in cold boot sequence. 4678c349dbc7Sjsg */ 4679c349dbc7Sjsg if (!adev->in_suspend) { 4680c349dbc7Sjsg r = gfx_v9_0_do_edc_gds_workarounds(adev); 4681c349dbc7Sjsg if (r) 4682c349dbc7Sjsg return r; 4683c349dbc7Sjsg } 4684c349dbc7Sjsg 4685c349dbc7Sjsg /* requires IBs so do in late init after IB pool is initialized */ 4686c349dbc7Sjsg r = gfx_v9_0_do_edc_gpr_workarounds(adev); 4687c349dbc7Sjsg if (r) 4688c349dbc7Sjsg return r; 4689c349dbc7Sjsg 4690c349dbc7Sjsg if (adev->gfx.funcs && 4691c349dbc7Sjsg adev->gfx.funcs->reset_ras_error_count) 4692c349dbc7Sjsg adev->gfx.funcs->reset_ras_error_count(adev); 4693c349dbc7Sjsg 4694c349dbc7Sjsg r = amdgpu_gfx_ras_late_init(adev); 4695c349dbc7Sjsg if (r) 4696c349dbc7Sjsg return r; 4697c349dbc7Sjsg 4698c349dbc7Sjsg return 0; 4699c349dbc7Sjsg } 4700c349dbc7Sjsg 4701fb4d8502Sjsg static int gfx_v9_0_late_init(void *handle) 4702fb4d8502Sjsg { 4703fb4d8502Sjsg struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4704fb4d8502Sjsg int r; 4705fb4d8502Sjsg 4706fb4d8502Sjsg r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); 4707fb4d8502Sjsg if (r) 4708fb4d8502Sjsg return r; 4709fb4d8502Sjsg 4710fb4d8502Sjsg r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); 4711fb4d8502Sjsg if (r) 4712fb4d8502Sjsg return r; 4713fb4d8502Sjsg 4714c349dbc7Sjsg r = gfx_v9_0_ecc_late_init(handle); 4715c349dbc7Sjsg if (r) 4716c349dbc7Sjsg return r; 4717c349dbc7Sjsg 4718fb4d8502Sjsg return 0; 4719fb4d8502Sjsg } 4720fb4d8502Sjsg 4721c349dbc7Sjsg static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev) 4722fb4d8502Sjsg { 4723c349dbc7Sjsg uint32_t rlc_setting; 4724fb4d8502Sjsg 4725fb4d8502Sjsg /* if RLC is not enabled, do nothing */ 4726fb4d8502Sjsg rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL); 4727fb4d8502Sjsg if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK)) 4728c349dbc7Sjsg return false; 4729fb4d8502Sjsg 4730c349dbc7Sjsg return true; 4731c349dbc7Sjsg } 4732c349dbc7Sjsg 4733c349dbc7Sjsg static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev) 4734c349dbc7Sjsg { 4735c349dbc7Sjsg uint32_t data; 4736c349dbc7Sjsg unsigned i; 4737c349dbc7Sjsg 4738fb4d8502Sjsg data = RLC_SAFE_MODE__CMD_MASK; 4739fb4d8502Sjsg data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT); 4740fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4741fb4d8502Sjsg 4742fb4d8502Sjsg /* wait for RLC_SAFE_MODE */ 4743fb4d8502Sjsg for (i = 0; i < adev->usec_timeout; i++) { 4744fb4d8502Sjsg if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD)) 4745fb4d8502Sjsg break; 4746fb4d8502Sjsg udelay(1); 4747fb4d8502Sjsg } 4748fb4d8502Sjsg } 4749fb4d8502Sjsg 4750c349dbc7Sjsg static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev) 4751fb4d8502Sjsg { 4752c349dbc7Sjsg uint32_t data; 4753fb4d8502Sjsg 4754fb4d8502Sjsg data = RLC_SAFE_MODE__CMD_MASK; 4755fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4756fb4d8502Sjsg } 4757fb4d8502Sjsg 4758fb4d8502Sjsg static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev, 4759fb4d8502Sjsg bool enable) 4760fb4d8502Sjsg { 4761c349dbc7Sjsg amdgpu_gfx_rlc_enter_safe_mode(adev); 4762fb4d8502Sjsg 4763fb4d8502Sjsg if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) { 4764fb4d8502Sjsg gfx_v9_0_enable_gfx_cg_power_gating(adev, true); 4765fb4d8502Sjsg if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4766fb4d8502Sjsg gfx_v9_0_enable_gfx_pipeline_powergating(adev, true); 4767fb4d8502Sjsg } else { 4768fb4d8502Sjsg gfx_v9_0_enable_gfx_cg_power_gating(adev, false); 4769c349dbc7Sjsg if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4770fb4d8502Sjsg gfx_v9_0_enable_gfx_pipeline_powergating(adev, false); 4771fb4d8502Sjsg } 4772fb4d8502Sjsg 4773c349dbc7Sjsg amdgpu_gfx_rlc_exit_safe_mode(adev); 4774fb4d8502Sjsg } 4775fb4d8502Sjsg 4776fb4d8502Sjsg static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev, 4777fb4d8502Sjsg bool enable) 4778fb4d8502Sjsg { 4779fb4d8502Sjsg /* TODO: double check if we need to perform under safe mode */ 4780fb4d8502Sjsg /* gfx_v9_0_enter_rlc_safe_mode(adev); */ 4781fb4d8502Sjsg 4782fb4d8502Sjsg if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable) 4783fb4d8502Sjsg gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true); 4784fb4d8502Sjsg else 4785fb4d8502Sjsg gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false); 4786fb4d8502Sjsg 4787fb4d8502Sjsg if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable) 4788fb4d8502Sjsg gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true); 4789fb4d8502Sjsg else 4790fb4d8502Sjsg gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false); 4791fb4d8502Sjsg 4792fb4d8502Sjsg /* gfx_v9_0_exit_rlc_safe_mode(adev); */ 4793fb4d8502Sjsg } 4794fb4d8502Sjsg 4795fb4d8502Sjsg static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, 4796fb4d8502Sjsg bool enable) 4797fb4d8502Sjsg { 4798fb4d8502Sjsg uint32_t data, def; 4799fb4d8502Sjsg 4800c349dbc7Sjsg amdgpu_gfx_rlc_enter_safe_mode(adev); 4801c349dbc7Sjsg 4802fb4d8502Sjsg /* It is disabled by HW by default */ 4803fb4d8502Sjsg if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) { 4804fb4d8502Sjsg /* 1 - RLC_CGTT_MGCG_OVERRIDE */ 4805fb4d8502Sjsg def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4806fb4d8502Sjsg 4807fb4d8502Sjsg if (adev->asic_type != CHIP_VEGA12) 4808fb4d8502Sjsg data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 4809fb4d8502Sjsg 4810fb4d8502Sjsg data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 4811fb4d8502Sjsg RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 4812fb4d8502Sjsg RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 4813fb4d8502Sjsg 4814fb4d8502Sjsg /* only for Vega10 & Raven1 */ 4815fb4d8502Sjsg data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK; 4816fb4d8502Sjsg 4817fb4d8502Sjsg if (def != data) 4818fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4819fb4d8502Sjsg 4820fb4d8502Sjsg /* MGLS is a global flag to control all MGLS in GFX */ 4821fb4d8502Sjsg if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { 4822fb4d8502Sjsg /* 2 - RLC memory Light sleep */ 4823fb4d8502Sjsg if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) { 4824fb4d8502Sjsg def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4825fb4d8502Sjsg data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 4826fb4d8502Sjsg if (def != data) 4827fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 4828fb4d8502Sjsg } 4829fb4d8502Sjsg /* 3 - CP memory Light sleep */ 4830fb4d8502Sjsg if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) { 4831fb4d8502Sjsg def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 4832fb4d8502Sjsg data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 4833fb4d8502Sjsg if (def != data) 4834fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 4835fb4d8502Sjsg } 4836fb4d8502Sjsg } 4837fb4d8502Sjsg } else { 4838fb4d8502Sjsg /* 1 - MGCG_OVERRIDE */ 4839fb4d8502Sjsg def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4840fb4d8502Sjsg 4841fb4d8502Sjsg if (adev->asic_type != CHIP_VEGA12) 4842fb4d8502Sjsg data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 4843fb4d8502Sjsg 4844fb4d8502Sjsg data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK | 4845fb4d8502Sjsg RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 4846fb4d8502Sjsg RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 4847fb4d8502Sjsg RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 4848fb4d8502Sjsg 4849fb4d8502Sjsg if (def != data) 4850fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4851fb4d8502Sjsg 4852fb4d8502Sjsg /* 2 - disable MGLS in RLC */ 4853fb4d8502Sjsg data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4854fb4d8502Sjsg if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) { 4855fb4d8502Sjsg data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 4856fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 4857fb4d8502Sjsg } 4858fb4d8502Sjsg 4859fb4d8502Sjsg /* 3 - disable MGLS in CP */ 4860fb4d8502Sjsg data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 4861fb4d8502Sjsg if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) { 4862fb4d8502Sjsg data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 4863fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 4864fb4d8502Sjsg } 4865fb4d8502Sjsg } 4866c349dbc7Sjsg 4867c349dbc7Sjsg amdgpu_gfx_rlc_exit_safe_mode(adev); 4868fb4d8502Sjsg } 4869fb4d8502Sjsg 4870fb4d8502Sjsg static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev, 4871fb4d8502Sjsg bool enable) 4872fb4d8502Sjsg { 4873fb4d8502Sjsg uint32_t data, def; 4874fb4d8502Sjsg 4875c349dbc7Sjsg if (adev->asic_type == CHIP_ARCTURUS) 4876c349dbc7Sjsg return; 4877c349dbc7Sjsg 4878c349dbc7Sjsg amdgpu_gfx_rlc_enter_safe_mode(adev); 4879fb4d8502Sjsg 4880fb4d8502Sjsg /* Enable 3D CGCG/CGLS */ 4881ad8b1aafSjsg if (enable) { 4882fb4d8502Sjsg /* write cmd to clear cgcg/cgls ov */ 4883fb4d8502Sjsg def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4884fb4d8502Sjsg /* unset CGCG override */ 4885fb4d8502Sjsg data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK; 4886fb4d8502Sjsg /* update CGCG and CGLS override bits */ 4887fb4d8502Sjsg if (def != data) 4888fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4889fb4d8502Sjsg 4890fb4d8502Sjsg /* enable 3Dcgcg FSM(0x0000363f) */ 4891fb4d8502Sjsg def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 4892fb4d8502Sjsg 4893ad8b1aafSjsg if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) 4894fb4d8502Sjsg data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4895fb4d8502Sjsg RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK; 4896ad8b1aafSjsg else 4897ad8b1aafSjsg data = 0x0 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT; 4898ad8b1aafSjsg 4899fb4d8502Sjsg if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) 4900fb4d8502Sjsg data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 4901fb4d8502Sjsg RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK; 4902fb4d8502Sjsg if (def != data) 4903fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 4904fb4d8502Sjsg 4905fb4d8502Sjsg /* set IDLE_POLL_COUNT(0x00900100) */ 4906fb4d8502Sjsg def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 4907fb4d8502Sjsg data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 4908fb4d8502Sjsg (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 4909fb4d8502Sjsg if (def != data) 4910fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 4911fb4d8502Sjsg } else { 4912fb4d8502Sjsg /* Disable CGCG/CGLS */ 4913fb4d8502Sjsg def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 4914fb4d8502Sjsg /* disable cgcg, cgls should be disabled */ 4915fb4d8502Sjsg data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK | 4916fb4d8502Sjsg RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK); 4917fb4d8502Sjsg /* disable cgcg and cgls in FSM */ 4918fb4d8502Sjsg if (def != data) 4919fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 4920fb4d8502Sjsg } 4921fb4d8502Sjsg 4922c349dbc7Sjsg amdgpu_gfx_rlc_exit_safe_mode(adev); 4923fb4d8502Sjsg } 4924fb4d8502Sjsg 4925fb4d8502Sjsg static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev, 4926fb4d8502Sjsg bool enable) 4927fb4d8502Sjsg { 4928fb4d8502Sjsg uint32_t def, data; 4929fb4d8502Sjsg 4930c349dbc7Sjsg amdgpu_gfx_rlc_enter_safe_mode(adev); 4931fb4d8502Sjsg 4932fb4d8502Sjsg if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) { 4933fb4d8502Sjsg def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4934fb4d8502Sjsg /* unset CGCG override */ 4935fb4d8502Sjsg data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK; 4936fb4d8502Sjsg if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 4937fb4d8502Sjsg data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 4938fb4d8502Sjsg else 4939fb4d8502Sjsg data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 4940fb4d8502Sjsg /* update CGCG and CGLS override bits */ 4941fb4d8502Sjsg if (def != data) 4942fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4943fb4d8502Sjsg 4944fb4d8502Sjsg /* enable cgcg FSM(0x0000363F) */ 4945fb4d8502Sjsg def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 4946fb4d8502Sjsg 4947c349dbc7Sjsg if (adev->asic_type == CHIP_ARCTURUS) 4948c349dbc7Sjsg data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4949c349dbc7Sjsg RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 4950c349dbc7Sjsg else 4951fb4d8502Sjsg data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4952fb4d8502Sjsg RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 4953fb4d8502Sjsg if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 4954fb4d8502Sjsg data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 4955fb4d8502Sjsg RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 4956fb4d8502Sjsg if (def != data) 4957fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 4958fb4d8502Sjsg 4959fb4d8502Sjsg /* set IDLE_POLL_COUNT(0x00900100) */ 4960fb4d8502Sjsg def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 4961fb4d8502Sjsg data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 4962fb4d8502Sjsg (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 4963fb4d8502Sjsg if (def != data) 4964fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 4965fb4d8502Sjsg } else { 4966fb4d8502Sjsg def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 4967fb4d8502Sjsg /* reset CGCG/CGLS bits */ 4968fb4d8502Sjsg data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK); 4969fb4d8502Sjsg /* disable cgcg and cgls in FSM */ 4970fb4d8502Sjsg if (def != data) 4971fb4d8502Sjsg WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 4972fb4d8502Sjsg } 4973fb4d8502Sjsg 4974c349dbc7Sjsg amdgpu_gfx_rlc_exit_safe_mode(adev); 4975fb4d8502Sjsg } 4976fb4d8502Sjsg 4977fb4d8502Sjsg static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev, 4978fb4d8502Sjsg bool enable) 4979fb4d8502Sjsg { 4980fb4d8502Sjsg if (enable) { 4981fb4d8502Sjsg /* CGCG/CGLS should be enabled after MGCG/MGLS 4982fb4d8502Sjsg * === MGCG + MGLS === 4983fb4d8502Sjsg */ 4984fb4d8502Sjsg gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 4985fb4d8502Sjsg /* === CGCG /CGLS for GFX 3D Only === */ 4986fb4d8502Sjsg gfx_v9_0_update_3d_clock_gating(adev, enable); 4987fb4d8502Sjsg /* === CGCG + CGLS === */ 4988fb4d8502Sjsg gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 4989fb4d8502Sjsg } else { 4990fb4d8502Sjsg /* CGCG/CGLS should be disabled before MGCG/MGLS 4991fb4d8502Sjsg * === CGCG + CGLS === 4992fb4d8502Sjsg */ 4993fb4d8502Sjsg gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 4994fb4d8502Sjsg /* === CGCG /CGLS for GFX 3D Only === */ 4995fb4d8502Sjsg gfx_v9_0_update_3d_clock_gating(adev, enable); 4996fb4d8502Sjsg /* === MGCG + MGLS === */ 4997fb4d8502Sjsg gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 4998fb4d8502Sjsg } 4999fb4d8502Sjsg return 0; 5000fb4d8502Sjsg } 5001fb4d8502Sjsg 5002c349dbc7Sjsg static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid) 5003c349dbc7Sjsg { 5004ad8b1aafSjsg u32 reg, data; 5005c349dbc7Sjsg 5006ad8b1aafSjsg reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL); 5007ad8b1aafSjsg if (amdgpu_sriov_is_pp_one_vf(adev)) 5008ad8b1aafSjsg data = RREG32_NO_KIQ(reg); 5009ad8b1aafSjsg else 5010ad8b1aafSjsg data = RREG32(reg); 5011c349dbc7Sjsg 5012c349dbc7Sjsg data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK; 5013c349dbc7Sjsg data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT; 5014c349dbc7Sjsg 5015ad8b1aafSjsg if (amdgpu_sriov_is_pp_one_vf(adev)) 5016ad8b1aafSjsg WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data); 5017ad8b1aafSjsg else 5018c349dbc7Sjsg WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data); 5019c349dbc7Sjsg } 5020c349dbc7Sjsg 5021c349dbc7Sjsg static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev, 5022c349dbc7Sjsg uint32_t offset, 5023c349dbc7Sjsg struct soc15_reg_rlcg *entries, int arr_size) 5024c349dbc7Sjsg { 5025c349dbc7Sjsg int i; 5026c349dbc7Sjsg uint32_t reg; 5027c349dbc7Sjsg 5028c349dbc7Sjsg if (!entries) 5029c349dbc7Sjsg return false; 5030c349dbc7Sjsg 5031c349dbc7Sjsg for (i = 0; i < arr_size; i++) { 5032c349dbc7Sjsg const struct soc15_reg_rlcg *entry; 5033c349dbc7Sjsg 5034c349dbc7Sjsg entry = &entries[i]; 5035c349dbc7Sjsg reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg; 5036c349dbc7Sjsg if (offset == reg) 5037c349dbc7Sjsg return true; 5038c349dbc7Sjsg } 5039c349dbc7Sjsg 5040c349dbc7Sjsg return false; 5041c349dbc7Sjsg } 5042c349dbc7Sjsg 5043c349dbc7Sjsg static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset) 5044c349dbc7Sjsg { 5045c349dbc7Sjsg return gfx_v9_0_check_rlcg_range(adev, offset, 5046c349dbc7Sjsg (void *)rlcg_access_gc_9_0, 5047c349dbc7Sjsg ARRAY_SIZE(rlcg_access_gc_9_0)); 5048c349dbc7Sjsg } 5049c349dbc7Sjsg 5050fb4d8502Sjsg static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = { 5051c349dbc7Sjsg .is_rlc_enabled = gfx_v9_0_is_rlc_enabled, 5052c349dbc7Sjsg .set_safe_mode = gfx_v9_0_set_safe_mode, 5053c349dbc7Sjsg .unset_safe_mode = gfx_v9_0_unset_safe_mode, 5054c349dbc7Sjsg .init = gfx_v9_0_rlc_init, 5055c349dbc7Sjsg .get_csb_size = gfx_v9_0_get_csb_size, 5056c349dbc7Sjsg .get_csb_buffer = gfx_v9_0_get_csb_buffer, 5057c349dbc7Sjsg .get_cp_table_num = gfx_v9_0_cp_jump_table_num, 5058c349dbc7Sjsg .resume = gfx_v9_0_rlc_resume, 5059c349dbc7Sjsg .stop = gfx_v9_0_rlc_stop, 5060c349dbc7Sjsg .reset = gfx_v9_0_rlc_reset, 5061c349dbc7Sjsg .start = gfx_v9_0_rlc_start, 5062c349dbc7Sjsg .update_spm_vmid = gfx_v9_0_update_spm_vmid, 5063c349dbc7Sjsg .rlcg_wreg = gfx_v9_0_rlcg_wreg, 5064c349dbc7Sjsg .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range, 5065fb4d8502Sjsg }; 5066fb4d8502Sjsg 5067fb4d8502Sjsg static int gfx_v9_0_set_powergating_state(void *handle, 5068fb4d8502Sjsg enum amd_powergating_state state) 5069fb4d8502Sjsg { 5070fb4d8502Sjsg struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5071c349dbc7Sjsg bool enable = (state == AMD_PG_STATE_GATE); 5072fb4d8502Sjsg 5073fb4d8502Sjsg switch (adev->asic_type) { 5074fb4d8502Sjsg case CHIP_RAVEN: 5075c349dbc7Sjsg case CHIP_RENOIR: 5076c349dbc7Sjsg if (!enable) 5077c349dbc7Sjsg amdgpu_gfx_off_ctrl(adev, false); 5078c349dbc7Sjsg 5079fb4d8502Sjsg if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) { 5080fb4d8502Sjsg gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true); 5081fb4d8502Sjsg gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true); 5082fb4d8502Sjsg } else { 5083fb4d8502Sjsg gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false); 5084fb4d8502Sjsg gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false); 5085fb4d8502Sjsg } 5086fb4d8502Sjsg 5087fb4d8502Sjsg if (adev->pg_flags & AMD_PG_SUPPORT_CP) 5088fb4d8502Sjsg gfx_v9_0_enable_cp_power_gating(adev, true); 5089fb4d8502Sjsg else 5090fb4d8502Sjsg gfx_v9_0_enable_cp_power_gating(adev, false); 5091fb4d8502Sjsg 5092fb4d8502Sjsg /* update gfx cgpg state */ 5093fb4d8502Sjsg gfx_v9_0_update_gfx_cg_power_gating(adev, enable); 5094fb4d8502Sjsg 5095fb4d8502Sjsg /* update mgcg state */ 5096fb4d8502Sjsg gfx_v9_0_update_gfx_mg_power_gating(adev, enable); 5097fb4d8502Sjsg 5098c349dbc7Sjsg if (enable) 5099c349dbc7Sjsg amdgpu_gfx_off_ctrl(adev, true); 5100fb4d8502Sjsg break; 5101fb4d8502Sjsg case CHIP_VEGA12: 5102c349dbc7Sjsg amdgpu_gfx_off_ctrl(adev, enable); 5103fb4d8502Sjsg break; 5104fb4d8502Sjsg default: 5105fb4d8502Sjsg break; 5106fb4d8502Sjsg } 5107fb4d8502Sjsg 5108fb4d8502Sjsg return 0; 5109fb4d8502Sjsg } 5110fb4d8502Sjsg 5111fb4d8502Sjsg static int gfx_v9_0_set_clockgating_state(void *handle, 5112fb4d8502Sjsg enum amd_clockgating_state state) 5113fb4d8502Sjsg { 5114fb4d8502Sjsg struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5115fb4d8502Sjsg 5116fb4d8502Sjsg if (amdgpu_sriov_vf(adev)) 5117fb4d8502Sjsg return 0; 5118fb4d8502Sjsg 5119fb4d8502Sjsg switch (adev->asic_type) { 5120fb4d8502Sjsg case CHIP_VEGA10: 5121fb4d8502Sjsg case CHIP_VEGA12: 5122fb4d8502Sjsg case CHIP_VEGA20: 5123fb4d8502Sjsg case CHIP_RAVEN: 5124c349dbc7Sjsg case CHIP_ARCTURUS: 5125c349dbc7Sjsg case CHIP_RENOIR: 5126fb4d8502Sjsg gfx_v9_0_update_gfx_clock_gating(adev, 5127c349dbc7Sjsg state == AMD_CG_STATE_GATE); 5128fb4d8502Sjsg break; 5129fb4d8502Sjsg default: 5130fb4d8502Sjsg break; 5131fb4d8502Sjsg } 5132fb4d8502Sjsg return 0; 5133fb4d8502Sjsg } 5134fb4d8502Sjsg 5135fb4d8502Sjsg static void gfx_v9_0_get_clockgating_state(void *handle, u32 *flags) 5136fb4d8502Sjsg { 5137fb4d8502Sjsg struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5138fb4d8502Sjsg int data; 5139fb4d8502Sjsg 5140fb4d8502Sjsg if (amdgpu_sriov_vf(adev)) 5141fb4d8502Sjsg *flags = 0; 5142fb4d8502Sjsg 5143fb4d8502Sjsg /* AMD_CG_SUPPORT_GFX_MGCG */ 5144c349dbc7Sjsg data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE)); 5145fb4d8502Sjsg if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK)) 5146fb4d8502Sjsg *flags |= AMD_CG_SUPPORT_GFX_MGCG; 5147fb4d8502Sjsg 5148fb4d8502Sjsg /* AMD_CG_SUPPORT_GFX_CGCG */ 5149c349dbc7Sjsg data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL)); 5150fb4d8502Sjsg if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK) 5151fb4d8502Sjsg *flags |= AMD_CG_SUPPORT_GFX_CGCG; 5152fb4d8502Sjsg 5153fb4d8502Sjsg /* AMD_CG_SUPPORT_GFX_CGLS */ 5154fb4d8502Sjsg if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK) 5155fb4d8502Sjsg *flags |= AMD_CG_SUPPORT_GFX_CGLS; 5156fb4d8502Sjsg 5157fb4d8502Sjsg /* AMD_CG_SUPPORT_GFX_RLC_LS */ 5158c349dbc7Sjsg data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL)); 5159fb4d8502Sjsg if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) 5160fb4d8502Sjsg *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS; 5161fb4d8502Sjsg 5162fb4d8502Sjsg /* AMD_CG_SUPPORT_GFX_CP_LS */ 5163c349dbc7Sjsg data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL)); 5164fb4d8502Sjsg if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) 5165fb4d8502Sjsg *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS; 5166fb4d8502Sjsg 5167c349dbc7Sjsg if (adev->asic_type != CHIP_ARCTURUS) { 5168fb4d8502Sjsg /* AMD_CG_SUPPORT_GFX_3D_CGCG */ 5169c349dbc7Sjsg data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D)); 5170fb4d8502Sjsg if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK) 5171fb4d8502Sjsg *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG; 5172fb4d8502Sjsg 5173fb4d8502Sjsg /* AMD_CG_SUPPORT_GFX_3D_CGLS */ 5174fb4d8502Sjsg if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK) 5175fb4d8502Sjsg *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS; 5176fb4d8502Sjsg } 5177c349dbc7Sjsg } 5178fb4d8502Sjsg 5179fb4d8502Sjsg static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring) 5180fb4d8502Sjsg { 5181fb4d8502Sjsg return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 is 32bit rptr*/ 5182fb4d8502Sjsg } 5183fb4d8502Sjsg 5184fb4d8502Sjsg static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring) 5185fb4d8502Sjsg { 5186fb4d8502Sjsg struct amdgpu_device *adev = ring->adev; 5187fb4d8502Sjsg u64 wptr; 5188fb4d8502Sjsg 5189fb4d8502Sjsg /* XXX check if swapping is necessary on BE */ 5190fb4d8502Sjsg if (ring->use_doorbell) { 5191fb4d8502Sjsg wptr = atomic64_read((atomic64_t *)&adev->wb.wb[ring->wptr_offs]); 5192fb4d8502Sjsg } else { 5193fb4d8502Sjsg wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR); 5194fb4d8502Sjsg wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32; 5195fb4d8502Sjsg } 5196fb4d8502Sjsg 5197fb4d8502Sjsg return wptr; 5198fb4d8502Sjsg } 5199fb4d8502Sjsg 5200fb4d8502Sjsg static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring) 5201fb4d8502Sjsg { 5202fb4d8502Sjsg struct amdgpu_device *adev = ring->adev; 5203fb4d8502Sjsg 5204fb4d8502Sjsg if (ring->use_doorbell) { 5205fb4d8502Sjsg /* XXX check if swapping is necessary on BE */ 5206fb4d8502Sjsg atomic64_set((atomic64_t*)&adev->wb.wb[ring->wptr_offs], ring->wptr); 5207fb4d8502Sjsg WDOORBELL64(ring->doorbell_index, ring->wptr); 5208fb4d8502Sjsg } else { 5209fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 5210fb4d8502Sjsg WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 5211fb4d8502Sjsg } 5212fb4d8502Sjsg } 5213fb4d8502Sjsg 5214fb4d8502Sjsg static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) 5215fb4d8502Sjsg { 5216fb4d8502Sjsg struct amdgpu_device *adev = ring->adev; 5217fb4d8502Sjsg u32 ref_and_mask, reg_mem_engine; 5218c349dbc7Sjsg const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg; 5219fb4d8502Sjsg 5220fb4d8502Sjsg if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 5221fb4d8502Sjsg switch (ring->me) { 5222fb4d8502Sjsg case 1: 5223fb4d8502Sjsg ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe; 5224fb4d8502Sjsg break; 5225fb4d8502Sjsg case 2: 5226fb4d8502Sjsg ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe; 5227fb4d8502Sjsg break; 5228fb4d8502Sjsg default: 5229fb4d8502Sjsg return; 5230fb4d8502Sjsg } 5231fb4d8502Sjsg reg_mem_engine = 0; 5232fb4d8502Sjsg } else { 5233fb4d8502Sjsg ref_and_mask = nbio_hf_reg->ref_and_mask_cp0; 5234fb4d8502Sjsg reg_mem_engine = 1; /* pfp */ 5235fb4d8502Sjsg } 5236fb4d8502Sjsg 5237fb4d8502Sjsg gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1, 5238c349dbc7Sjsg adev->nbio.funcs->get_hdp_flush_req_offset(adev), 5239c349dbc7Sjsg adev->nbio.funcs->get_hdp_flush_done_offset(adev), 5240fb4d8502Sjsg ref_and_mask, ref_and_mask, 0x20); 5241fb4d8502Sjsg } 5242fb4d8502Sjsg 5243fb4d8502Sjsg static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, 5244c349dbc7Sjsg struct amdgpu_job *job, 5245fb4d8502Sjsg struct amdgpu_ib *ib, 5246c349dbc7Sjsg uint32_t flags) 5247fb4d8502Sjsg { 5248c349dbc7Sjsg unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5249fb4d8502Sjsg u32 header, control = 0; 5250fb4d8502Sjsg 5251fb4d8502Sjsg if (ib->flags & AMDGPU_IB_FLAG_CE) 5252fb4d8502Sjsg header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 5253fb4d8502Sjsg else 5254fb4d8502Sjsg header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 5255fb4d8502Sjsg 5256fb4d8502Sjsg control |= ib->length_dw | (vmid << 24); 5257fb4d8502Sjsg 5258fb4d8502Sjsg if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) { 5259fb4d8502Sjsg control |= INDIRECT_BUFFER_PRE_ENB(1); 5260fb4d8502Sjsg 5261c349dbc7Sjsg if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid) 5262fb4d8502Sjsg gfx_v9_0_ring_emit_de_meta(ring); 5263fb4d8502Sjsg } 5264fb4d8502Sjsg 5265fb4d8502Sjsg amdgpu_ring_write(ring, header); 5266fb4d8502Sjsg BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 5267fb4d8502Sjsg amdgpu_ring_write(ring, 5268fb4d8502Sjsg #ifdef __BIG_ENDIAN 5269fb4d8502Sjsg (2 << 0) | 5270fb4d8502Sjsg #endif 5271fb4d8502Sjsg lower_32_bits(ib->gpu_addr)); 5272fb4d8502Sjsg amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5273fb4d8502Sjsg amdgpu_ring_write(ring, control); 5274fb4d8502Sjsg } 5275fb4d8502Sjsg 5276fb4d8502Sjsg static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring, 5277c349dbc7Sjsg struct amdgpu_job *job, 5278fb4d8502Sjsg struct amdgpu_ib *ib, 5279c349dbc7Sjsg uint32_t flags) 5280fb4d8502Sjsg { 5281c349dbc7Sjsg unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5282fb4d8502Sjsg u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); 5283fb4d8502Sjsg 5284c349dbc7Sjsg /* Currently, there is a high possibility to get wave ID mismatch 5285c349dbc7Sjsg * between ME and GDS, leading to a hw deadlock, because ME generates 5286c349dbc7Sjsg * different wave IDs than the GDS expects. This situation happens 5287c349dbc7Sjsg * randomly when at least 5 compute pipes use GDS ordered append. 5288c349dbc7Sjsg * The wave IDs generated by ME are also wrong after suspend/resume. 5289c349dbc7Sjsg * Those are probably bugs somewhere else in the kernel driver. 5290c349dbc7Sjsg * 5291c349dbc7Sjsg * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and 5292c349dbc7Sjsg * GDS to 0 for this ring (me/pipe). 5293c349dbc7Sjsg */ 5294c349dbc7Sjsg if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { 5295c349dbc7Sjsg amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 5296c349dbc7Sjsg amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID); 5297c349dbc7Sjsg amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); 5298c349dbc7Sjsg } 5299c349dbc7Sjsg 5300fb4d8502Sjsg amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 5301fb4d8502Sjsg BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 5302fb4d8502Sjsg amdgpu_ring_write(ring, 5303fb4d8502Sjsg #ifdef __BIG_ENDIAN 5304fb4d8502Sjsg (2 << 0) | 5305fb4d8502Sjsg #endif 5306fb4d8502Sjsg lower_32_bits(ib->gpu_addr)); 5307fb4d8502Sjsg amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5308fb4d8502Sjsg amdgpu_ring_write(ring, control); 5309fb4d8502Sjsg } 5310fb4d8502Sjsg 5311fb4d8502Sjsg static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, 5312fb4d8502Sjsg u64 seq, unsigned flags) 5313fb4d8502Sjsg { 5314fb4d8502Sjsg bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 5315fb4d8502Sjsg bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 5316fb4d8502Sjsg bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY; 5317fb4d8502Sjsg 5318fb4d8502Sjsg /* RELEASE_MEM - flush caches, send int */ 5319fb4d8502Sjsg amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6)); 5320fb4d8502Sjsg amdgpu_ring_write(ring, ((writeback ? (EOP_TC_WB_ACTION_EN | 5321fb4d8502Sjsg EOP_TC_NC_ACTION_EN) : 5322fb4d8502Sjsg (EOP_TCL1_ACTION_EN | 5323fb4d8502Sjsg EOP_TC_ACTION_EN | 5324fb4d8502Sjsg EOP_TC_WB_ACTION_EN | 5325fb4d8502Sjsg EOP_TC_MD_ACTION_EN)) | 5326fb4d8502Sjsg EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 5327fb4d8502Sjsg EVENT_INDEX(5))); 5328fb4d8502Sjsg amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0)); 5329fb4d8502Sjsg 5330fb4d8502Sjsg /* 5331fb4d8502Sjsg * the address should be Qword aligned if 64bit write, Dword 5332fb4d8502Sjsg * aligned if only send 32bit data low (discard data high) 5333fb4d8502Sjsg */ 5334fb4d8502Sjsg if (write64bit) 5335fb4d8502Sjsg BUG_ON(addr & 0x7); 5336fb4d8502Sjsg else 5337fb4d8502Sjsg BUG_ON(addr & 0x3); 5338fb4d8502Sjsg amdgpu_ring_write(ring, lower_32_bits(addr)); 5339fb4d8502Sjsg amdgpu_ring_write(ring, upper_32_bits(addr)); 5340fb4d8502Sjsg amdgpu_ring_write(ring, lower_32_bits(seq)); 5341fb4d8502Sjsg amdgpu_ring_write(ring, upper_32_bits(seq)); 5342fb4d8502Sjsg amdgpu_ring_write(ring, 0); 5343fb4d8502Sjsg } 5344fb4d8502Sjsg 5345fb4d8502Sjsg static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) 5346fb4d8502Sjsg { 5347fb4d8502Sjsg int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5348fb4d8502Sjsg uint32_t seq = ring->fence_drv.sync_seq; 5349fb4d8502Sjsg uint64_t addr = ring->fence_drv.gpu_addr; 5350fb4d8502Sjsg 5351fb4d8502Sjsg gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0, 5352fb4d8502Sjsg lower_32_bits(addr), upper_32_bits(addr), 5353fb4d8502Sjsg seq, 0xffffffff, 4); 5354fb4d8502Sjsg } 5355fb4d8502Sjsg 5356fb4d8502Sjsg static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 5357fb4d8502Sjsg unsigned vmid, uint64_t pd_addr) 5358fb4d8502Sjsg { 5359fb4d8502Sjsg amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 5360fb4d8502Sjsg 5361fb4d8502Sjsg /* compute doesn't have PFP */ 5362fb4d8502Sjsg if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) { 5363fb4d8502Sjsg /* sync PFP to ME, otherwise we might get invalid PFP reads */ 5364fb4d8502Sjsg amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 5365fb4d8502Sjsg amdgpu_ring_write(ring, 0x0); 5366fb4d8502Sjsg } 5367fb4d8502Sjsg } 5368fb4d8502Sjsg 5369fb4d8502Sjsg static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring) 5370fb4d8502Sjsg { 5371fb4d8502Sjsg return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 hardware is 32bit rptr */ 5372fb4d8502Sjsg } 5373fb4d8502Sjsg 5374fb4d8502Sjsg static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring) 5375fb4d8502Sjsg { 5376fb4d8502Sjsg u64 wptr; 5377fb4d8502Sjsg 5378fb4d8502Sjsg /* XXX check if swapping is necessary on BE */ 5379fb4d8502Sjsg if (ring->use_doorbell) 5380fb4d8502Sjsg wptr = atomic64_read((atomic64_t *)&ring->adev->wb.wb[ring->wptr_offs]); 5381fb4d8502Sjsg else 5382fb4d8502Sjsg BUG(); 5383fb4d8502Sjsg return wptr; 5384fb4d8502Sjsg } 5385fb4d8502Sjsg 5386fb4d8502Sjsg static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring) 5387fb4d8502Sjsg { 5388fb4d8502Sjsg struct amdgpu_device *adev = ring->adev; 5389fb4d8502Sjsg 5390fb4d8502Sjsg /* XXX check if swapping is necessary on BE */ 5391fb4d8502Sjsg if (ring->use_doorbell) { 5392fb4d8502Sjsg atomic64_set((atomic64_t*)&adev->wb.wb[ring->wptr_offs], ring->wptr); 5393fb4d8502Sjsg WDOORBELL64(ring->doorbell_index, ring->wptr); 5394fb4d8502Sjsg } else{ 5395fb4d8502Sjsg BUG(); /* only DOORBELL method supported on gfx9 now */ 5396fb4d8502Sjsg } 5397fb4d8502Sjsg } 5398fb4d8502Sjsg 5399fb4d8502Sjsg static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, 5400fb4d8502Sjsg u64 seq, unsigned int flags) 5401fb4d8502Sjsg { 5402fb4d8502Sjsg struct amdgpu_device *adev = ring->adev; 5403fb4d8502Sjsg 5404fb4d8502Sjsg /* we only allocate 32bit for each seq wb address */ 5405fb4d8502Sjsg BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); 5406fb4d8502Sjsg 5407fb4d8502Sjsg /* write fence seq to the "addr" */ 5408fb4d8502Sjsg amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5409fb4d8502Sjsg amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5410fb4d8502Sjsg WRITE_DATA_DST_SEL(5) | WR_CONFIRM)); 5411fb4d8502Sjsg amdgpu_ring_write(ring, lower_32_bits(addr)); 5412fb4d8502Sjsg amdgpu_ring_write(ring, upper_32_bits(addr)); 5413fb4d8502Sjsg amdgpu_ring_write(ring, lower_32_bits(seq)); 5414fb4d8502Sjsg 5415fb4d8502Sjsg if (flags & AMDGPU_FENCE_FLAG_INT) { 5416fb4d8502Sjsg /* set register to trigger INT */ 5417fb4d8502Sjsg amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5418fb4d8502Sjsg amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5419fb4d8502Sjsg WRITE_DATA_DST_SEL(0) | WR_CONFIRM)); 5420fb4d8502Sjsg amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS)); 5421fb4d8502Sjsg amdgpu_ring_write(ring, 0); 5422fb4d8502Sjsg amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */ 5423fb4d8502Sjsg } 5424fb4d8502Sjsg } 5425fb4d8502Sjsg 5426fb4d8502Sjsg static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring) 5427fb4d8502Sjsg { 5428fb4d8502Sjsg amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 5429fb4d8502Sjsg amdgpu_ring_write(ring, 0); 5430fb4d8502Sjsg } 5431fb4d8502Sjsg 5432fb4d8502Sjsg static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring) 5433fb4d8502Sjsg { 5434fb4d8502Sjsg struct v9_ce_ib_state ce_payload = {0}; 5435fb4d8502Sjsg uint64_t csa_addr; 5436fb4d8502Sjsg int cnt; 5437fb4d8502Sjsg 5438fb4d8502Sjsg cnt = (sizeof(ce_payload) >> 2) + 4 - 2; 5439fb4d8502Sjsg csa_addr = amdgpu_csa_vaddr(ring->adev); 5440fb4d8502Sjsg 5441fb4d8502Sjsg amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5442fb4d8502Sjsg amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) | 5443fb4d8502Sjsg WRITE_DATA_DST_SEL(8) | 5444fb4d8502Sjsg WR_CONFIRM) | 5445fb4d8502Sjsg WRITE_DATA_CACHE_POLICY(0)); 5446fb4d8502Sjsg amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload))); 5447fb4d8502Sjsg amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload))); 5448fb4d8502Sjsg amdgpu_ring_write_multiple(ring, (void *)&ce_payload, sizeof(ce_payload) >> 2); 5449fb4d8502Sjsg } 5450fb4d8502Sjsg 5451fb4d8502Sjsg static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring) 5452fb4d8502Sjsg { 5453fb4d8502Sjsg struct v9_de_ib_state de_payload = {0}; 5454fb4d8502Sjsg uint64_t csa_addr, gds_addr; 5455fb4d8502Sjsg int cnt; 5456fb4d8502Sjsg 5457fb4d8502Sjsg csa_addr = amdgpu_csa_vaddr(ring->adev); 5458fb4d8502Sjsg gds_addr = csa_addr + 4096; 5459fb4d8502Sjsg de_payload.gds_backup_addrlo = lower_32_bits(gds_addr); 5460fb4d8502Sjsg de_payload.gds_backup_addrhi = upper_32_bits(gds_addr); 5461fb4d8502Sjsg 5462fb4d8502Sjsg cnt = (sizeof(de_payload) >> 2) + 4 - 2; 5463fb4d8502Sjsg amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5464fb4d8502Sjsg amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | 5465fb4d8502Sjsg WRITE_DATA_DST_SEL(8) | 5466fb4d8502Sjsg WR_CONFIRM) | 5467fb4d8502Sjsg WRITE_DATA_CACHE_POLICY(0)); 5468fb4d8502Sjsg amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload))); 5469fb4d8502Sjsg amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload))); 5470fb4d8502Sjsg amdgpu_ring_write_multiple(ring, (void *)&de_payload, sizeof(de_payload) >> 2); 5471fb4d8502Sjsg } 5472fb4d8502Sjsg 5473ad8b1aafSjsg static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, 5474ad8b1aafSjsg bool secure) 5475fb4d8502Sjsg { 5476ad8b1aafSjsg uint32_t v = secure ? FRAME_TMZ : 0; 5477ad8b1aafSjsg 5478fb4d8502Sjsg amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0)); 5479ad8b1aafSjsg amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1)); 5480fb4d8502Sjsg } 5481fb4d8502Sjsg 5482fb4d8502Sjsg static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags) 5483fb4d8502Sjsg { 5484fb4d8502Sjsg uint32_t dw2 = 0; 5485fb4d8502Sjsg 5486fb4d8502Sjsg if (amdgpu_sriov_vf(ring->adev)) 5487fb4d8502Sjsg gfx_v9_0_ring_emit_ce_meta(ring); 5488fb4d8502Sjsg 5489fb4d8502Sjsg dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */ 5490fb4d8502Sjsg if (flags & AMDGPU_HAVE_CTX_SWITCH) { 5491fb4d8502Sjsg /* set load_global_config & load_global_uconfig */ 5492fb4d8502Sjsg dw2 |= 0x8001; 5493fb4d8502Sjsg /* set load_cs_sh_regs */ 5494fb4d8502Sjsg dw2 |= 0x01000000; 5495fb4d8502Sjsg /* set load_per_context_state & load_gfx_sh_regs for GFX */ 5496fb4d8502Sjsg dw2 |= 0x10002; 5497fb4d8502Sjsg 5498fb4d8502Sjsg /* set load_ce_ram if preamble presented */ 5499fb4d8502Sjsg if (AMDGPU_PREAMBLE_IB_PRESENT & flags) 5500fb4d8502Sjsg dw2 |= 0x10000000; 5501fb4d8502Sjsg } else { 5502fb4d8502Sjsg /* still load_ce_ram if this is the first time preamble presented 5503fb4d8502Sjsg * although there is no context switch happens. 5504fb4d8502Sjsg */ 5505fb4d8502Sjsg if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags) 5506fb4d8502Sjsg dw2 |= 0x10000000; 5507fb4d8502Sjsg } 5508fb4d8502Sjsg 5509fb4d8502Sjsg amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 5510fb4d8502Sjsg amdgpu_ring_write(ring, dw2); 5511fb4d8502Sjsg amdgpu_ring_write(ring, 0); 5512fb4d8502Sjsg } 5513fb4d8502Sjsg 5514fb4d8502Sjsg static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring) 5515fb4d8502Sjsg { 5516fb4d8502Sjsg unsigned ret; 5517fb4d8502Sjsg amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3)); 5518fb4d8502Sjsg amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr)); 5519fb4d8502Sjsg amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr)); 5520fb4d8502Sjsg amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */ 5521fb4d8502Sjsg ret = ring->wptr & ring->buf_mask; 5522fb4d8502Sjsg amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */ 5523fb4d8502Sjsg return ret; 5524fb4d8502Sjsg } 5525fb4d8502Sjsg 5526fb4d8502Sjsg static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset) 5527fb4d8502Sjsg { 5528fb4d8502Sjsg unsigned cur; 5529fb4d8502Sjsg BUG_ON(offset > ring->buf_mask); 5530fb4d8502Sjsg BUG_ON(ring->ring[offset] != 0x55aa55aa); 5531fb4d8502Sjsg 5532fb4d8502Sjsg cur = (ring->wptr & ring->buf_mask) - 1; 5533fb4d8502Sjsg if (likely(cur > offset)) 5534fb4d8502Sjsg ring->ring[offset] = cur - offset; 5535fb4d8502Sjsg else 5536fb4d8502Sjsg ring->ring[offset] = (ring->ring_size>>2) - offset + cur; 5537fb4d8502Sjsg } 5538fb4d8502Sjsg 5539ad8b1aafSjsg static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg, 5540ad8b1aafSjsg uint32_t reg_val_offs) 5541fb4d8502Sjsg { 5542fb4d8502Sjsg struct amdgpu_device *adev = ring->adev; 5543fb4d8502Sjsg 5544fb4d8502Sjsg amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 5545fb4d8502Sjsg amdgpu_ring_write(ring, 0 | /* src: register*/ 5546fb4d8502Sjsg (5 << 8) | /* dst: memory */ 5547fb4d8502Sjsg (1 << 20)); /* write confirm */ 5548fb4d8502Sjsg amdgpu_ring_write(ring, reg); 5549fb4d8502Sjsg amdgpu_ring_write(ring, 0); 5550fb4d8502Sjsg amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 5551ad8b1aafSjsg reg_val_offs * 4)); 5552fb4d8502Sjsg amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 5553ad8b1aafSjsg reg_val_offs * 4)); 5554fb4d8502Sjsg } 5555fb4d8502Sjsg 5556fb4d8502Sjsg static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 5557fb4d8502Sjsg uint32_t val) 5558fb4d8502Sjsg { 5559fb4d8502Sjsg uint32_t cmd = 0; 5560fb4d8502Sjsg 5561fb4d8502Sjsg switch (ring->funcs->type) { 5562fb4d8502Sjsg case AMDGPU_RING_TYPE_GFX: 5563fb4d8502Sjsg cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM; 5564fb4d8502Sjsg break; 5565fb4d8502Sjsg case AMDGPU_RING_TYPE_KIQ: 5566fb4d8502Sjsg cmd = (1 << 16); /* no inc addr */ 5567fb4d8502Sjsg break; 5568fb4d8502Sjsg default: 5569fb4d8502Sjsg cmd = WR_CONFIRM; 5570fb4d8502Sjsg break; 5571fb4d8502Sjsg } 5572fb4d8502Sjsg amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5573fb4d8502Sjsg amdgpu_ring_write(ring, cmd); 5574fb4d8502Sjsg amdgpu_ring_write(ring, reg); 5575fb4d8502Sjsg amdgpu_ring_write(ring, 0); 5576fb4d8502Sjsg amdgpu_ring_write(ring, val); 5577fb4d8502Sjsg } 5578fb4d8502Sjsg 5579fb4d8502Sjsg static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, 5580fb4d8502Sjsg uint32_t val, uint32_t mask) 5581fb4d8502Sjsg { 5582fb4d8502Sjsg gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20); 5583fb4d8502Sjsg } 5584fb4d8502Sjsg 5585fb4d8502Sjsg static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring, 5586fb4d8502Sjsg uint32_t reg0, uint32_t reg1, 5587fb4d8502Sjsg uint32_t ref, uint32_t mask) 5588fb4d8502Sjsg { 5589fb4d8502Sjsg int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5590c349dbc7Sjsg struct amdgpu_device *adev = ring->adev; 5591c349dbc7Sjsg bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ? 5592c349dbc7Sjsg adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait; 5593fb4d8502Sjsg 5594c349dbc7Sjsg if (fw_version_ok) 5595fb4d8502Sjsg gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1, 5596fb4d8502Sjsg ref, mask, 0x20); 5597fb4d8502Sjsg else 5598fb4d8502Sjsg amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1, 5599fb4d8502Sjsg ref, mask); 5600fb4d8502Sjsg } 5601fb4d8502Sjsg 5602c349dbc7Sjsg static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid) 5603c349dbc7Sjsg { 5604c349dbc7Sjsg struct amdgpu_device *adev = ring->adev; 5605c349dbc7Sjsg uint32_t value = 0; 5606c349dbc7Sjsg 5607c349dbc7Sjsg value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03); 5608c349dbc7Sjsg value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01); 5609c349dbc7Sjsg value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1); 5610c349dbc7Sjsg value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid); 5611c349dbc7Sjsg WREG32_SOC15(GC, 0, mmSQ_CMD, value); 5612c349dbc7Sjsg } 5613c349dbc7Sjsg 5614fb4d8502Sjsg static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev, 5615fb4d8502Sjsg enum amdgpu_interrupt_state state) 5616fb4d8502Sjsg { 5617fb4d8502Sjsg switch (state) { 5618fb4d8502Sjsg case AMDGPU_IRQ_STATE_DISABLE: 5619fb4d8502Sjsg case AMDGPU_IRQ_STATE_ENABLE: 5620fb4d8502Sjsg WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5621fb4d8502Sjsg TIME_STAMP_INT_ENABLE, 5622fb4d8502Sjsg state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5623fb4d8502Sjsg break; 5624fb4d8502Sjsg default: 5625fb4d8502Sjsg break; 5626fb4d8502Sjsg } 5627fb4d8502Sjsg } 5628fb4d8502Sjsg 5629fb4d8502Sjsg static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev, 5630fb4d8502Sjsg int me, int pipe, 5631fb4d8502Sjsg enum amdgpu_interrupt_state state) 5632fb4d8502Sjsg { 5633fb4d8502Sjsg u32 mec_int_cntl, mec_int_cntl_reg; 5634fb4d8502Sjsg 5635fb4d8502Sjsg /* 5636fb4d8502Sjsg * amdgpu controls only the first MEC. That's why this function only 5637fb4d8502Sjsg * handles the setting of interrupts for this specific MEC. All other 5638fb4d8502Sjsg * pipes' interrupts are set by amdkfd. 5639fb4d8502Sjsg */ 5640fb4d8502Sjsg 5641fb4d8502Sjsg if (me == 1) { 5642fb4d8502Sjsg switch (pipe) { 5643fb4d8502Sjsg case 0: 5644fb4d8502Sjsg mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL); 5645fb4d8502Sjsg break; 5646fb4d8502Sjsg case 1: 5647fb4d8502Sjsg mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL); 5648fb4d8502Sjsg break; 5649fb4d8502Sjsg case 2: 5650fb4d8502Sjsg mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL); 5651fb4d8502Sjsg break; 5652fb4d8502Sjsg case 3: 5653fb4d8502Sjsg mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL); 5654fb4d8502Sjsg break; 5655fb4d8502Sjsg default: 5656fb4d8502Sjsg DRM_DEBUG("invalid pipe %d\n", pipe); 5657fb4d8502Sjsg return; 5658fb4d8502Sjsg } 5659fb4d8502Sjsg } else { 5660fb4d8502Sjsg DRM_DEBUG("invalid me %d\n", me); 5661fb4d8502Sjsg return; 5662fb4d8502Sjsg } 5663fb4d8502Sjsg 5664fb4d8502Sjsg switch (state) { 5665fb4d8502Sjsg case AMDGPU_IRQ_STATE_DISABLE: 5666fb4d8502Sjsg mec_int_cntl = RREG32(mec_int_cntl_reg); 5667fb4d8502Sjsg mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 5668fb4d8502Sjsg TIME_STAMP_INT_ENABLE, 0); 5669fb4d8502Sjsg WREG32(mec_int_cntl_reg, mec_int_cntl); 5670fb4d8502Sjsg break; 5671fb4d8502Sjsg case AMDGPU_IRQ_STATE_ENABLE: 5672fb4d8502Sjsg mec_int_cntl = RREG32(mec_int_cntl_reg); 5673fb4d8502Sjsg mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 5674fb4d8502Sjsg TIME_STAMP_INT_ENABLE, 1); 5675fb4d8502Sjsg WREG32(mec_int_cntl_reg, mec_int_cntl); 5676fb4d8502Sjsg break; 5677fb4d8502Sjsg default: 5678fb4d8502Sjsg break; 5679fb4d8502Sjsg } 5680fb4d8502Sjsg } 5681fb4d8502Sjsg 5682fb4d8502Sjsg static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev, 5683fb4d8502Sjsg struct amdgpu_irq_src *source, 5684fb4d8502Sjsg unsigned type, 5685fb4d8502Sjsg enum amdgpu_interrupt_state state) 5686fb4d8502Sjsg { 5687fb4d8502Sjsg switch (state) { 5688fb4d8502Sjsg case AMDGPU_IRQ_STATE_DISABLE: 5689fb4d8502Sjsg case AMDGPU_IRQ_STATE_ENABLE: 5690fb4d8502Sjsg WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5691fb4d8502Sjsg PRIV_REG_INT_ENABLE, 5692fb4d8502Sjsg state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5693fb4d8502Sjsg break; 5694fb4d8502Sjsg default: 5695fb4d8502Sjsg break; 5696fb4d8502Sjsg } 5697fb4d8502Sjsg 5698fb4d8502Sjsg return 0; 5699fb4d8502Sjsg } 5700fb4d8502Sjsg 5701fb4d8502Sjsg static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev, 5702fb4d8502Sjsg struct amdgpu_irq_src *source, 5703fb4d8502Sjsg unsigned type, 5704fb4d8502Sjsg enum amdgpu_interrupt_state state) 5705fb4d8502Sjsg { 5706fb4d8502Sjsg switch (state) { 5707fb4d8502Sjsg case AMDGPU_IRQ_STATE_DISABLE: 5708fb4d8502Sjsg case AMDGPU_IRQ_STATE_ENABLE: 5709fb4d8502Sjsg WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5710fb4d8502Sjsg PRIV_INSTR_INT_ENABLE, 5711fb4d8502Sjsg state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5712fb4d8502Sjsg default: 5713fb4d8502Sjsg break; 5714fb4d8502Sjsg } 5715fb4d8502Sjsg 5716fb4d8502Sjsg return 0; 5717fb4d8502Sjsg } 5718fb4d8502Sjsg 5719c349dbc7Sjsg #define ENABLE_ECC_ON_ME_PIPE(me, pipe) \ 5720c349dbc7Sjsg WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 5721c349dbc7Sjsg CP_ECC_ERROR_INT_ENABLE, 1) 5722c349dbc7Sjsg 5723c349dbc7Sjsg #define DISABLE_ECC_ON_ME_PIPE(me, pipe) \ 5724c349dbc7Sjsg WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 5725c349dbc7Sjsg CP_ECC_ERROR_INT_ENABLE, 0) 5726c349dbc7Sjsg 5727c349dbc7Sjsg static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev, 5728c349dbc7Sjsg struct amdgpu_irq_src *source, 5729c349dbc7Sjsg unsigned type, 5730c349dbc7Sjsg enum amdgpu_interrupt_state state) 5731c349dbc7Sjsg { 5732c349dbc7Sjsg switch (state) { 5733c349dbc7Sjsg case AMDGPU_IRQ_STATE_DISABLE: 5734c349dbc7Sjsg WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5735c349dbc7Sjsg CP_ECC_ERROR_INT_ENABLE, 0); 5736c349dbc7Sjsg DISABLE_ECC_ON_ME_PIPE(1, 0); 5737c349dbc7Sjsg DISABLE_ECC_ON_ME_PIPE(1, 1); 5738c349dbc7Sjsg DISABLE_ECC_ON_ME_PIPE(1, 2); 5739c349dbc7Sjsg DISABLE_ECC_ON_ME_PIPE(1, 3); 5740c349dbc7Sjsg break; 5741c349dbc7Sjsg 5742c349dbc7Sjsg case AMDGPU_IRQ_STATE_ENABLE: 5743c349dbc7Sjsg WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5744c349dbc7Sjsg CP_ECC_ERROR_INT_ENABLE, 1); 5745c349dbc7Sjsg ENABLE_ECC_ON_ME_PIPE(1, 0); 5746c349dbc7Sjsg ENABLE_ECC_ON_ME_PIPE(1, 1); 5747c349dbc7Sjsg ENABLE_ECC_ON_ME_PIPE(1, 2); 5748c349dbc7Sjsg ENABLE_ECC_ON_ME_PIPE(1, 3); 5749c349dbc7Sjsg break; 5750c349dbc7Sjsg default: 5751c349dbc7Sjsg break; 5752c349dbc7Sjsg } 5753c349dbc7Sjsg 5754c349dbc7Sjsg return 0; 5755c349dbc7Sjsg } 5756c349dbc7Sjsg 5757c349dbc7Sjsg 5758fb4d8502Sjsg static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev, 5759fb4d8502Sjsg struct amdgpu_irq_src *src, 5760fb4d8502Sjsg unsigned type, 5761fb4d8502Sjsg enum amdgpu_interrupt_state state) 5762fb4d8502Sjsg { 5763fb4d8502Sjsg switch (type) { 5764c349dbc7Sjsg case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP: 5765fb4d8502Sjsg gfx_v9_0_set_gfx_eop_interrupt_state(adev, state); 5766fb4d8502Sjsg break; 5767fb4d8502Sjsg case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP: 5768fb4d8502Sjsg gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state); 5769fb4d8502Sjsg break; 5770fb4d8502Sjsg case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP: 5771fb4d8502Sjsg gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state); 5772fb4d8502Sjsg break; 5773fb4d8502Sjsg case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP: 5774fb4d8502Sjsg gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state); 5775fb4d8502Sjsg break; 5776fb4d8502Sjsg case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP: 5777fb4d8502Sjsg gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state); 5778fb4d8502Sjsg break; 5779fb4d8502Sjsg case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP: 5780fb4d8502Sjsg gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state); 5781fb4d8502Sjsg break; 5782fb4d8502Sjsg case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP: 5783fb4d8502Sjsg gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state); 5784fb4d8502Sjsg break; 5785fb4d8502Sjsg case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP: 5786fb4d8502Sjsg gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state); 5787fb4d8502Sjsg break; 5788fb4d8502Sjsg case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP: 5789fb4d8502Sjsg gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state); 5790fb4d8502Sjsg break; 5791fb4d8502Sjsg default: 5792fb4d8502Sjsg break; 5793fb4d8502Sjsg } 5794fb4d8502Sjsg return 0; 5795fb4d8502Sjsg } 5796fb4d8502Sjsg 5797fb4d8502Sjsg static int gfx_v9_0_eop_irq(struct amdgpu_device *adev, 5798fb4d8502Sjsg struct amdgpu_irq_src *source, 5799fb4d8502Sjsg struct amdgpu_iv_entry *entry) 5800fb4d8502Sjsg { 5801fb4d8502Sjsg int i; 5802fb4d8502Sjsg u8 me_id, pipe_id, queue_id; 5803fb4d8502Sjsg struct amdgpu_ring *ring; 5804fb4d8502Sjsg 5805fb4d8502Sjsg DRM_DEBUG("IH: CP EOP\n"); 5806fb4d8502Sjsg me_id = (entry->ring_id & 0x0c) >> 2; 5807fb4d8502Sjsg pipe_id = (entry->ring_id & 0x03) >> 0; 5808fb4d8502Sjsg queue_id = (entry->ring_id & 0x70) >> 4; 5809fb4d8502Sjsg 5810fb4d8502Sjsg switch (me_id) { 5811fb4d8502Sjsg case 0: 5812fb4d8502Sjsg amdgpu_fence_process(&adev->gfx.gfx_ring[0]); 5813fb4d8502Sjsg break; 5814fb4d8502Sjsg case 1: 5815fb4d8502Sjsg case 2: 5816fb4d8502Sjsg for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5817fb4d8502Sjsg ring = &adev->gfx.compute_ring[i]; 5818fb4d8502Sjsg /* Per-queue interrupt is supported for MEC starting from VI. 5819fb4d8502Sjsg * The interrupt can only be enabled/disabled per pipe instead of per queue. 5820fb4d8502Sjsg */ 5821fb4d8502Sjsg if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id)) 5822fb4d8502Sjsg amdgpu_fence_process(ring); 5823fb4d8502Sjsg } 5824fb4d8502Sjsg break; 5825fb4d8502Sjsg } 5826fb4d8502Sjsg return 0; 5827fb4d8502Sjsg } 5828fb4d8502Sjsg 5829c349dbc7Sjsg static void gfx_v9_0_fault(struct amdgpu_device *adev, 5830c349dbc7Sjsg struct amdgpu_iv_entry *entry) 5831c349dbc7Sjsg { 5832c349dbc7Sjsg u8 me_id, pipe_id, queue_id; 5833c349dbc7Sjsg struct amdgpu_ring *ring; 5834c349dbc7Sjsg int i; 5835c349dbc7Sjsg 5836c349dbc7Sjsg me_id = (entry->ring_id & 0x0c) >> 2; 5837c349dbc7Sjsg pipe_id = (entry->ring_id & 0x03) >> 0; 5838c349dbc7Sjsg queue_id = (entry->ring_id & 0x70) >> 4; 5839c349dbc7Sjsg 5840c349dbc7Sjsg switch (me_id) { 5841c349dbc7Sjsg case 0: 5842c349dbc7Sjsg drm_sched_fault(&adev->gfx.gfx_ring[0].sched); 5843c349dbc7Sjsg break; 5844c349dbc7Sjsg case 1: 5845c349dbc7Sjsg case 2: 5846c349dbc7Sjsg for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5847c349dbc7Sjsg ring = &adev->gfx.compute_ring[i]; 5848c349dbc7Sjsg if (ring->me == me_id && ring->pipe == pipe_id && 5849c349dbc7Sjsg ring->queue == queue_id) 5850c349dbc7Sjsg drm_sched_fault(&ring->sched); 5851c349dbc7Sjsg } 5852c349dbc7Sjsg break; 5853c349dbc7Sjsg } 5854c349dbc7Sjsg } 5855c349dbc7Sjsg 5856fb4d8502Sjsg static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev, 5857fb4d8502Sjsg struct amdgpu_irq_src *source, 5858fb4d8502Sjsg struct amdgpu_iv_entry *entry) 5859fb4d8502Sjsg { 5860fb4d8502Sjsg DRM_ERROR("Illegal register access in command stream\n"); 5861c349dbc7Sjsg gfx_v9_0_fault(adev, entry); 5862fb4d8502Sjsg return 0; 5863fb4d8502Sjsg } 5864fb4d8502Sjsg 5865fb4d8502Sjsg static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev, 5866fb4d8502Sjsg struct amdgpu_irq_src *source, 5867fb4d8502Sjsg struct amdgpu_iv_entry *entry) 5868fb4d8502Sjsg { 5869fb4d8502Sjsg DRM_ERROR("Illegal instruction in command stream\n"); 5870c349dbc7Sjsg gfx_v9_0_fault(adev, entry); 5871fb4d8502Sjsg return 0; 5872fb4d8502Sjsg } 5873fb4d8502Sjsg 5874c349dbc7Sjsg 5875c349dbc7Sjsg static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = { 5876c349dbc7Sjsg { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 5877c349dbc7Sjsg SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT), 5878c349dbc7Sjsg SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT) 5879c349dbc7Sjsg }, 5880c349dbc7Sjsg { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 5881c349dbc7Sjsg SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT), 5882c349dbc7Sjsg SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT) 5883c349dbc7Sjsg }, 5884c349dbc7Sjsg { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 5885c349dbc7Sjsg SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1), 5886c349dbc7Sjsg 0, 0 5887c349dbc7Sjsg }, 5888c349dbc7Sjsg { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 5889c349dbc7Sjsg SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2), 5890c349dbc7Sjsg 0, 0 5891c349dbc7Sjsg }, 5892c349dbc7Sjsg { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 5893c349dbc7Sjsg SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT), 5894c349dbc7Sjsg SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT) 5895c349dbc7Sjsg }, 5896c349dbc7Sjsg { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 5897c349dbc7Sjsg SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT), 5898c349dbc7Sjsg 0, 0 5899c349dbc7Sjsg }, 5900c349dbc7Sjsg { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 5901c349dbc7Sjsg SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT), 5902c349dbc7Sjsg SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT) 5903c349dbc7Sjsg }, 5904c349dbc7Sjsg { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 5905c349dbc7Sjsg SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT), 5906c349dbc7Sjsg SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT) 5907c349dbc7Sjsg }, 5908c349dbc7Sjsg { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 5909c349dbc7Sjsg SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1), 5910c349dbc7Sjsg 0, 0 5911c349dbc7Sjsg }, 5912c349dbc7Sjsg { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 5913c349dbc7Sjsg SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1), 5914c349dbc7Sjsg 0, 0 5915c349dbc7Sjsg }, 5916c349dbc7Sjsg { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 5917c349dbc7Sjsg SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1), 5918c349dbc7Sjsg 0, 0 5919c349dbc7Sjsg }, 5920c349dbc7Sjsg { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 5921c349dbc7Sjsg SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC), 5922c349dbc7Sjsg SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED) 5923c349dbc7Sjsg }, 5924c349dbc7Sjsg { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 5925c349dbc7Sjsg SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED), 5926c349dbc7Sjsg 0, 0 5927c349dbc7Sjsg }, 5928c349dbc7Sjsg { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 5929c349dbc7Sjsg SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC), 5930c349dbc7Sjsg SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED) 5931c349dbc7Sjsg }, 5932c349dbc7Sjsg { "GDS_OA_PHY_PHY_CMD_RAM_MEM", 5933c349dbc7Sjsg SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 5934c349dbc7Sjsg SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC), 5935c349dbc7Sjsg SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED) 5936c349dbc7Sjsg }, 5937c349dbc7Sjsg { "GDS_OA_PHY_PHY_DATA_RAM_MEM", 5938c349dbc7Sjsg SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 5939c349dbc7Sjsg SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED), 5940c349dbc7Sjsg 0, 0 5941c349dbc7Sjsg }, 5942c349dbc7Sjsg { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM", 5943c349dbc7Sjsg SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 5944c349dbc7Sjsg SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC), 5945c349dbc7Sjsg SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED) 5946c349dbc7Sjsg }, 5947c349dbc7Sjsg { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM", 5948c349dbc7Sjsg SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 5949c349dbc7Sjsg SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC), 5950c349dbc7Sjsg SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED) 5951c349dbc7Sjsg }, 5952c349dbc7Sjsg { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM", 5953c349dbc7Sjsg SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 5954c349dbc7Sjsg SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC), 5955c349dbc7Sjsg SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED) 5956c349dbc7Sjsg }, 5957c349dbc7Sjsg { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM", 5958c349dbc7Sjsg SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 5959c349dbc7Sjsg SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC), 5960c349dbc7Sjsg SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED) 5961c349dbc7Sjsg }, 5962c349dbc7Sjsg { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 5963c349dbc7Sjsg SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT), 5964c349dbc7Sjsg 0, 0 5965c349dbc7Sjsg }, 5966c349dbc7Sjsg { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5967c349dbc7Sjsg SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT), 5968c349dbc7Sjsg SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT) 5969c349dbc7Sjsg }, 5970c349dbc7Sjsg { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5971c349dbc7Sjsg SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT), 5972c349dbc7Sjsg 0, 0 5973c349dbc7Sjsg }, 5974c349dbc7Sjsg { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5975c349dbc7Sjsg SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT), 5976c349dbc7Sjsg 0, 0 5977c349dbc7Sjsg }, 5978c349dbc7Sjsg { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5979c349dbc7Sjsg SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT), 5980c349dbc7Sjsg 0, 0 5981c349dbc7Sjsg }, 5982c349dbc7Sjsg { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5983c349dbc7Sjsg SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT), 5984c349dbc7Sjsg 0, 0 5985c349dbc7Sjsg }, 5986c349dbc7Sjsg { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 5987c349dbc7Sjsg SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT), 5988c349dbc7Sjsg 0, 0 5989c349dbc7Sjsg }, 5990c349dbc7Sjsg { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 5991c349dbc7Sjsg SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT), 5992c349dbc7Sjsg 0, 0 5993c349dbc7Sjsg }, 5994c349dbc7Sjsg { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5995c349dbc7Sjsg SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT), 5996c349dbc7Sjsg SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT) 5997c349dbc7Sjsg }, 5998c349dbc7Sjsg { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5999c349dbc7Sjsg SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT), 6000c349dbc7Sjsg SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT) 6001c349dbc7Sjsg }, 6002c349dbc7Sjsg { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6003c349dbc7Sjsg SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT), 6004c349dbc7Sjsg SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT) 6005c349dbc7Sjsg }, 6006c349dbc7Sjsg { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6007c349dbc7Sjsg SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT), 6008c349dbc7Sjsg SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT) 6009c349dbc7Sjsg }, 6010c349dbc7Sjsg { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6011c349dbc7Sjsg SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT), 6012c349dbc7Sjsg SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT) 6013c349dbc7Sjsg }, 6014c349dbc7Sjsg { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6015c349dbc7Sjsg SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT), 6016c349dbc7Sjsg 0, 0 6017c349dbc7Sjsg }, 6018c349dbc7Sjsg { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6019c349dbc7Sjsg SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT), 6020c349dbc7Sjsg 0, 0 6021c349dbc7Sjsg }, 6022c349dbc7Sjsg { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6023c349dbc7Sjsg SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT), 6024c349dbc7Sjsg 0, 0 6025c349dbc7Sjsg }, 6026c349dbc7Sjsg { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6027c349dbc7Sjsg SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT), 6028c349dbc7Sjsg 0, 0 6029c349dbc7Sjsg }, 6030c349dbc7Sjsg { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6031c349dbc7Sjsg SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT), 6032c349dbc7Sjsg 0, 0 6033c349dbc7Sjsg }, 6034c349dbc7Sjsg { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6035c349dbc7Sjsg SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT), 6036c349dbc7Sjsg 0, 0 6037c349dbc7Sjsg }, 6038c349dbc7Sjsg { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6039c349dbc7Sjsg SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT), 6040c349dbc7Sjsg 0, 0 6041c349dbc7Sjsg }, 6042c349dbc7Sjsg { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6043c349dbc7Sjsg SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT), 6044c349dbc7Sjsg 0, 0 6045c349dbc7Sjsg }, 6046c349dbc7Sjsg { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6047c349dbc7Sjsg SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT), 6048c349dbc7Sjsg 0, 0 6049c349dbc7Sjsg }, 6050c349dbc7Sjsg { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6051c349dbc7Sjsg SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT), 6052c349dbc7Sjsg 0, 0 6053c349dbc7Sjsg }, 6054c349dbc7Sjsg { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6055c349dbc7Sjsg SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT), 6056c349dbc7Sjsg 0, 0 6057c349dbc7Sjsg }, 6058c349dbc7Sjsg { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6059c349dbc7Sjsg SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT), 6060c349dbc7Sjsg 0, 0 6061c349dbc7Sjsg }, 6062c349dbc7Sjsg { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6063c349dbc7Sjsg SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT), 6064c349dbc7Sjsg 0, 0 6065c349dbc7Sjsg }, 6066c349dbc7Sjsg { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 6067c349dbc7Sjsg SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT), 6068c349dbc7Sjsg 0, 0 6069c349dbc7Sjsg }, 6070c349dbc7Sjsg { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6071c349dbc7Sjsg SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT), 6072c349dbc7Sjsg SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT) 6073c349dbc7Sjsg }, 6074c349dbc7Sjsg { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6075c349dbc7Sjsg SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT), 6076c349dbc7Sjsg SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT) 6077c349dbc7Sjsg }, 6078c349dbc7Sjsg { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6079c349dbc7Sjsg SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT), 6080c349dbc7Sjsg 0, 0 6081c349dbc7Sjsg }, 6082c349dbc7Sjsg { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6083c349dbc7Sjsg SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT), 6084c349dbc7Sjsg 0, 0 6085c349dbc7Sjsg }, 6086c349dbc7Sjsg { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6087c349dbc7Sjsg SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT), 6088c349dbc7Sjsg 0, 0 6089c349dbc7Sjsg }, 6090c349dbc7Sjsg { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6091c349dbc7Sjsg SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT), 6092c349dbc7Sjsg SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT) 6093c349dbc7Sjsg }, 6094c349dbc7Sjsg { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6095c349dbc7Sjsg SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT), 6096c349dbc7Sjsg SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT) 6097c349dbc7Sjsg }, 6098c349dbc7Sjsg { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6099c349dbc7Sjsg SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT), 6100c349dbc7Sjsg SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT) 6101c349dbc7Sjsg }, 6102c349dbc7Sjsg { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6103c349dbc7Sjsg SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT), 6104c349dbc7Sjsg SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT) 6105c349dbc7Sjsg }, 6106c349dbc7Sjsg { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6107c349dbc7Sjsg SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT), 6108c349dbc7Sjsg 0, 0 6109c349dbc7Sjsg }, 6110c349dbc7Sjsg { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6111c349dbc7Sjsg SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT), 6112c349dbc7Sjsg SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT) 6113c349dbc7Sjsg }, 6114c349dbc7Sjsg { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6115c349dbc7Sjsg SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT), 6116c349dbc7Sjsg SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT) 6117c349dbc7Sjsg }, 6118c349dbc7Sjsg { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6119c349dbc7Sjsg SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT), 6120c349dbc7Sjsg SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT) 6121c349dbc7Sjsg }, 6122c349dbc7Sjsg { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6123c349dbc7Sjsg SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT), 6124c349dbc7Sjsg SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT) 6125c349dbc7Sjsg }, 6126c349dbc7Sjsg { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6127c349dbc7Sjsg SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT), 6128c349dbc7Sjsg SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT) 6129c349dbc7Sjsg }, 6130c349dbc7Sjsg { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6131c349dbc7Sjsg SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT), 6132c349dbc7Sjsg SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT) 6133c349dbc7Sjsg }, 6134c349dbc7Sjsg { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6135c349dbc7Sjsg SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT), 6136c349dbc7Sjsg SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT) 6137c349dbc7Sjsg }, 6138c349dbc7Sjsg { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6139c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT), 6140c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT) 6141c349dbc7Sjsg }, 6142c349dbc7Sjsg { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6143c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT), 6144c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT) 6145c349dbc7Sjsg }, 6146c349dbc7Sjsg { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6147c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT), 6148c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT) 6149c349dbc7Sjsg }, 6150c349dbc7Sjsg { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6151c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT), 6152c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT) 6153c349dbc7Sjsg }, 6154c349dbc7Sjsg { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6155c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT), 6156c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT) 6157c349dbc7Sjsg }, 6158c349dbc7Sjsg { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6159c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT), 6160c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT) 6161c349dbc7Sjsg }, 6162c349dbc7Sjsg { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6163c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT), 6164c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT) 6165c349dbc7Sjsg }, 6166c349dbc7Sjsg { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6167c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT), 6168c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT) 6169c349dbc7Sjsg }, 6170c349dbc7Sjsg { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6171c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT), 6172c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT) 6173c349dbc7Sjsg }, 6174c349dbc7Sjsg { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6175c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT), 6176c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT) 6177c349dbc7Sjsg }, 6178c349dbc7Sjsg { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6179c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT), 6180c349dbc7Sjsg 0, 0 6181c349dbc7Sjsg }, 6182c349dbc7Sjsg { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6183c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT), 6184c349dbc7Sjsg 0, 0 6185c349dbc7Sjsg }, 6186c349dbc7Sjsg { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6187c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT), 6188c349dbc7Sjsg 0, 0 6189c349dbc7Sjsg }, 6190c349dbc7Sjsg { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6191c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT), 6192c349dbc7Sjsg 0, 0 6193c349dbc7Sjsg }, 6194c349dbc7Sjsg { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6195c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT), 6196c349dbc7Sjsg 0, 0 6197c349dbc7Sjsg }, 6198c349dbc7Sjsg { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6199c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT), 6200c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT) 6201c349dbc7Sjsg }, 6202c349dbc7Sjsg { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6203c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT), 6204c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT) 6205c349dbc7Sjsg }, 6206c349dbc7Sjsg { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6207c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT), 6208c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT) 6209c349dbc7Sjsg }, 6210c349dbc7Sjsg { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6211c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT), 6212c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT) 6213c349dbc7Sjsg }, 6214c349dbc7Sjsg { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6215c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT), 6216c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT) 6217c349dbc7Sjsg }, 6218c349dbc7Sjsg { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6219c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT), 6220c349dbc7Sjsg 0, 0 6221c349dbc7Sjsg }, 6222c349dbc7Sjsg { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6223c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT), 6224c349dbc7Sjsg 0, 0 6225c349dbc7Sjsg }, 6226c349dbc7Sjsg { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6227c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT), 6228c349dbc7Sjsg 0, 0 6229c349dbc7Sjsg }, 6230c349dbc7Sjsg { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6231c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT), 6232c349dbc7Sjsg 0, 0 6233c349dbc7Sjsg }, 6234c349dbc7Sjsg { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6235c349dbc7Sjsg SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT), 6236c349dbc7Sjsg 0, 0 6237c349dbc7Sjsg }, 6238c349dbc7Sjsg { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6239c349dbc7Sjsg SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT), 6240c349dbc7Sjsg SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT) 6241c349dbc7Sjsg }, 6242c349dbc7Sjsg { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6243c349dbc7Sjsg SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT), 6244c349dbc7Sjsg SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT) 6245c349dbc7Sjsg }, 6246c349dbc7Sjsg { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6247c349dbc7Sjsg SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT), 6248c349dbc7Sjsg SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT) 6249c349dbc7Sjsg }, 6250c349dbc7Sjsg { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6251c349dbc7Sjsg SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT), 6252c349dbc7Sjsg SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT) 6253c349dbc7Sjsg }, 6254c349dbc7Sjsg { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6255c349dbc7Sjsg SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT), 6256c349dbc7Sjsg SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT) 6257c349dbc7Sjsg }, 6258c349dbc7Sjsg { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6259c349dbc7Sjsg SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT), 6260c349dbc7Sjsg 0, 0 6261c349dbc7Sjsg }, 6262c349dbc7Sjsg { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6263c349dbc7Sjsg SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT), 6264c349dbc7Sjsg 0, 0 6265c349dbc7Sjsg }, 6266c349dbc7Sjsg { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6267c349dbc7Sjsg SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT), 6268c349dbc7Sjsg 0, 0 6269c349dbc7Sjsg }, 6270c349dbc7Sjsg { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6271c349dbc7Sjsg SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT), 6272c349dbc7Sjsg 0, 0 6273c349dbc7Sjsg }, 6274c349dbc7Sjsg { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6275c349dbc7Sjsg SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT), 6276c349dbc7Sjsg 0, 0 6277c349dbc7Sjsg }, 6278c349dbc7Sjsg { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6279c349dbc7Sjsg SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT), 6280c349dbc7Sjsg SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT) 6281c349dbc7Sjsg }, 6282c349dbc7Sjsg { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6283c349dbc7Sjsg SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT), 6284c349dbc7Sjsg SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT) 6285c349dbc7Sjsg }, 6286c349dbc7Sjsg { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6287c349dbc7Sjsg SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT), 6288c349dbc7Sjsg SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT) 6289c349dbc7Sjsg }, 6290c349dbc7Sjsg { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6291c349dbc7Sjsg SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT), 6292c349dbc7Sjsg 0, 0 6293c349dbc7Sjsg }, 6294c349dbc7Sjsg { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6295c349dbc7Sjsg SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT), 6296c349dbc7Sjsg 0, 0 6297c349dbc7Sjsg }, 6298c349dbc7Sjsg { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6299c349dbc7Sjsg SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT), 6300c349dbc7Sjsg 0, 0 6301c349dbc7Sjsg }, 6302c349dbc7Sjsg { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6303c349dbc7Sjsg SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT), 6304c349dbc7Sjsg 0, 0 6305c349dbc7Sjsg }, 6306c349dbc7Sjsg { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6307c349dbc7Sjsg SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT), 6308c349dbc7Sjsg 0, 0 6309c349dbc7Sjsg }, 6310c349dbc7Sjsg { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6311c349dbc7Sjsg SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT), 6312c349dbc7Sjsg 0, 0 6313c349dbc7Sjsg } 6314c349dbc7Sjsg }; 6315c349dbc7Sjsg 6316c349dbc7Sjsg static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 6317c349dbc7Sjsg void *inject_if) 6318fb4d8502Sjsg { 6319c349dbc7Sjsg struct ras_inject_if *info = (struct ras_inject_if *)inject_if; 6320c349dbc7Sjsg int ret; 6321c349dbc7Sjsg struct ta_ras_trigger_error_input block_info = { 0 }; 6322fb4d8502Sjsg 6323c349dbc7Sjsg if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6324c349dbc7Sjsg return -EINVAL; 6325fb4d8502Sjsg 6326c349dbc7Sjsg if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks)) 6327c349dbc7Sjsg return -EINVAL; 6328fb4d8502Sjsg 6329c349dbc7Sjsg if (!ras_gfx_subblocks[info->head.sub_block_index].name) 6330c349dbc7Sjsg return -EPERM; 6331fb4d8502Sjsg 6332c349dbc7Sjsg if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type & 6333c349dbc7Sjsg info->head.type)) { 6334c349dbc7Sjsg DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n", 6335c349dbc7Sjsg ras_gfx_subblocks[info->head.sub_block_index].name, 6336c349dbc7Sjsg info->head.type); 6337c349dbc7Sjsg return -EPERM; 6338fb4d8502Sjsg } 6339c349dbc7Sjsg 6340c349dbc7Sjsg if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type & 6341c349dbc7Sjsg info->head.type)) { 6342c349dbc7Sjsg DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n", 6343c349dbc7Sjsg ras_gfx_subblocks[info->head.sub_block_index].name, 6344c349dbc7Sjsg info->head.type); 6345c349dbc7Sjsg return -EPERM; 6346fb4d8502Sjsg } 6347c349dbc7Sjsg 6348c349dbc7Sjsg block_info.block_id = amdgpu_ras_block_to_ta(info->head.block); 6349c349dbc7Sjsg block_info.sub_block_index = 6350c349dbc7Sjsg ras_gfx_subblocks[info->head.sub_block_index].ta_subblock; 6351c349dbc7Sjsg block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type); 6352c349dbc7Sjsg block_info.address = info->address; 6353c349dbc7Sjsg block_info.value = info->value; 6354c349dbc7Sjsg 6355c349dbc7Sjsg mutex_lock(&adev->grbm_idx_mutex); 6356c349dbc7Sjsg ret = psp_ras_trigger_error(&adev->psp, &block_info); 6357c349dbc7Sjsg mutex_unlock(&adev->grbm_idx_mutex); 6358c349dbc7Sjsg 6359c349dbc7Sjsg return ret; 6360c349dbc7Sjsg } 6361c349dbc7Sjsg 6362c349dbc7Sjsg static const char *vml2_mems[] = { 6363c349dbc7Sjsg "UTC_VML2_BANK_CACHE_0_BIGK_MEM0", 6364c349dbc7Sjsg "UTC_VML2_BANK_CACHE_0_BIGK_MEM1", 6365c349dbc7Sjsg "UTC_VML2_BANK_CACHE_0_4K_MEM0", 6366c349dbc7Sjsg "UTC_VML2_BANK_CACHE_0_4K_MEM1", 6367c349dbc7Sjsg "UTC_VML2_BANK_CACHE_1_BIGK_MEM0", 6368c349dbc7Sjsg "UTC_VML2_BANK_CACHE_1_BIGK_MEM1", 6369c349dbc7Sjsg "UTC_VML2_BANK_CACHE_1_4K_MEM0", 6370c349dbc7Sjsg "UTC_VML2_BANK_CACHE_1_4K_MEM1", 6371c349dbc7Sjsg "UTC_VML2_BANK_CACHE_2_BIGK_MEM0", 6372c349dbc7Sjsg "UTC_VML2_BANK_CACHE_2_BIGK_MEM1", 6373c349dbc7Sjsg "UTC_VML2_BANK_CACHE_2_4K_MEM0", 6374c349dbc7Sjsg "UTC_VML2_BANK_CACHE_2_4K_MEM1", 6375c349dbc7Sjsg "UTC_VML2_BANK_CACHE_3_BIGK_MEM0", 6376c349dbc7Sjsg "UTC_VML2_BANK_CACHE_3_BIGK_MEM1", 6377c349dbc7Sjsg "UTC_VML2_BANK_CACHE_3_4K_MEM0", 6378c349dbc7Sjsg "UTC_VML2_BANK_CACHE_3_4K_MEM1", 6379c349dbc7Sjsg }; 6380c349dbc7Sjsg 6381c349dbc7Sjsg static const char *vml2_walker_mems[] = { 6382c349dbc7Sjsg "UTC_VML2_CACHE_PDE0_MEM0", 6383c349dbc7Sjsg "UTC_VML2_CACHE_PDE0_MEM1", 6384c349dbc7Sjsg "UTC_VML2_CACHE_PDE1_MEM0", 6385c349dbc7Sjsg "UTC_VML2_CACHE_PDE1_MEM1", 6386c349dbc7Sjsg "UTC_VML2_CACHE_PDE2_MEM0", 6387c349dbc7Sjsg "UTC_VML2_CACHE_PDE2_MEM1", 6388c349dbc7Sjsg "UTC_VML2_RDIF_LOG_FIFO", 6389c349dbc7Sjsg }; 6390c349dbc7Sjsg 6391c349dbc7Sjsg static const char *atc_l2_cache_2m_mems[] = { 6392c349dbc7Sjsg "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM", 6393c349dbc7Sjsg "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM", 6394c349dbc7Sjsg "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM", 6395c349dbc7Sjsg "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM", 6396c349dbc7Sjsg }; 6397c349dbc7Sjsg 6398c349dbc7Sjsg static const char *atc_l2_cache_4k_mems[] = { 6399c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0", 6400c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1", 6401c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2", 6402c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3", 6403c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4", 6404c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5", 6405c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6", 6406c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7", 6407c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0", 6408c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1", 6409c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2", 6410c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3", 6411c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4", 6412c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5", 6413c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6", 6414c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7", 6415c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0", 6416c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1", 6417c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2", 6418c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3", 6419c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4", 6420c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5", 6421c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6", 6422c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7", 6423c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0", 6424c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1", 6425c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2", 6426c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3", 6427c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4", 6428c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5", 6429c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6", 6430c349dbc7Sjsg "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7", 6431c349dbc7Sjsg }; 6432c349dbc7Sjsg 6433c349dbc7Sjsg static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev, 6434c349dbc7Sjsg struct ras_err_data *err_data) 6435c349dbc7Sjsg { 6436c349dbc7Sjsg uint32_t i, data; 6437c349dbc7Sjsg uint32_t sec_count, ded_count; 6438c349dbc7Sjsg 6439c349dbc7Sjsg WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6440c349dbc7Sjsg WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 6441c349dbc7Sjsg WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6442c349dbc7Sjsg WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 6443c349dbc7Sjsg WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6444c349dbc7Sjsg WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 6445c349dbc7Sjsg WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6446c349dbc7Sjsg WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 6447c349dbc7Sjsg 6448c349dbc7Sjsg for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { 6449c349dbc7Sjsg WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 6450c349dbc7Sjsg data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 6451c349dbc7Sjsg 6452c349dbc7Sjsg sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT); 6453c349dbc7Sjsg if (sec_count) { 6454ad8b1aafSjsg dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6455ad8b1aafSjsg "SEC %d\n", i, vml2_mems[i], sec_count); 6456c349dbc7Sjsg err_data->ce_count += sec_count; 6457c349dbc7Sjsg } 6458c349dbc7Sjsg 6459c349dbc7Sjsg ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT); 6460c349dbc7Sjsg if (ded_count) { 6461ad8b1aafSjsg dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6462ad8b1aafSjsg "DED %d\n", i, vml2_mems[i], ded_count); 6463c349dbc7Sjsg err_data->ue_count += ded_count; 6464c349dbc7Sjsg } 6465c349dbc7Sjsg } 6466c349dbc7Sjsg 6467c349dbc7Sjsg for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { 6468c349dbc7Sjsg WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 6469c349dbc7Sjsg data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 6470c349dbc7Sjsg 6471c349dbc7Sjsg sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6472c349dbc7Sjsg SEC_COUNT); 6473c349dbc7Sjsg if (sec_count) { 6474ad8b1aafSjsg dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6475ad8b1aafSjsg "SEC %d\n", i, vml2_walker_mems[i], sec_count); 6476c349dbc7Sjsg err_data->ce_count += sec_count; 6477c349dbc7Sjsg } 6478c349dbc7Sjsg 6479c349dbc7Sjsg ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6480c349dbc7Sjsg DED_COUNT); 6481c349dbc7Sjsg if (ded_count) { 6482ad8b1aafSjsg dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6483ad8b1aafSjsg "DED %d\n", i, vml2_walker_mems[i], ded_count); 6484c349dbc7Sjsg err_data->ue_count += ded_count; 6485c349dbc7Sjsg } 6486c349dbc7Sjsg } 6487c349dbc7Sjsg 6488c349dbc7Sjsg for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { 6489c349dbc7Sjsg WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 6490c349dbc7Sjsg data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 6491c349dbc7Sjsg 6492c349dbc7Sjsg sec_count = (data & 0x00006000L) >> 0xd; 6493c349dbc7Sjsg if (sec_count) { 6494ad8b1aafSjsg dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6495ad8b1aafSjsg "SEC %d\n", i, atc_l2_cache_2m_mems[i], 6496ad8b1aafSjsg sec_count); 6497c349dbc7Sjsg err_data->ce_count += sec_count; 6498c349dbc7Sjsg } 6499c349dbc7Sjsg } 6500c349dbc7Sjsg 6501c349dbc7Sjsg for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { 6502c349dbc7Sjsg WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 6503c349dbc7Sjsg data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 6504c349dbc7Sjsg 6505c349dbc7Sjsg sec_count = (data & 0x00006000L) >> 0xd; 6506c349dbc7Sjsg if (sec_count) { 6507ad8b1aafSjsg dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6508ad8b1aafSjsg "SEC %d\n", i, atc_l2_cache_4k_mems[i], 6509ad8b1aafSjsg sec_count); 6510c349dbc7Sjsg err_data->ce_count += sec_count; 6511c349dbc7Sjsg } 6512c349dbc7Sjsg 6513c349dbc7Sjsg ded_count = (data & 0x00018000L) >> 0xf; 6514c349dbc7Sjsg if (ded_count) { 6515ad8b1aafSjsg dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6516ad8b1aafSjsg "DED %d\n", i, atc_l2_cache_4k_mems[i], 6517ad8b1aafSjsg ded_count); 6518c349dbc7Sjsg err_data->ue_count += ded_count; 6519c349dbc7Sjsg } 6520c349dbc7Sjsg } 6521c349dbc7Sjsg 6522c349dbc7Sjsg WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6523c349dbc7Sjsg WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6524c349dbc7Sjsg WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6525c349dbc7Sjsg WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6526c349dbc7Sjsg 6527fb4d8502Sjsg return 0; 6528fb4d8502Sjsg } 6529fb4d8502Sjsg 6530ad8b1aafSjsg static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev, 6531ad8b1aafSjsg const struct soc15_reg_entry *reg, 6532c349dbc7Sjsg uint32_t se_id, uint32_t inst_id, uint32_t value, 6533c349dbc7Sjsg uint32_t *sec_count, uint32_t *ded_count) 6534fb4d8502Sjsg { 6535c349dbc7Sjsg uint32_t i; 6536c349dbc7Sjsg uint32_t sec_cnt, ded_cnt; 6537fb4d8502Sjsg 6538c349dbc7Sjsg for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) { 6539c349dbc7Sjsg if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset || 6540c349dbc7Sjsg gfx_v9_0_ras_fields[i].seg != reg->seg || 6541c349dbc7Sjsg gfx_v9_0_ras_fields[i].inst != reg->inst) 6542c349dbc7Sjsg continue; 6543fb4d8502Sjsg 6544c349dbc7Sjsg sec_cnt = (value & 6545c349dbc7Sjsg gfx_v9_0_ras_fields[i].sec_count_mask) >> 6546c349dbc7Sjsg gfx_v9_0_ras_fields[i].sec_count_shift; 6547c349dbc7Sjsg if (sec_cnt) { 6548ad8b1aafSjsg dev_info(adev->dev, "GFX SubBlock %s, " 6549ad8b1aafSjsg "Instance[%d][%d], SEC %d\n", 6550c349dbc7Sjsg gfx_v9_0_ras_fields[i].name, 6551c349dbc7Sjsg se_id, inst_id, 6552c349dbc7Sjsg sec_cnt); 6553c349dbc7Sjsg *sec_count += sec_cnt; 6554c349dbc7Sjsg } 6555c349dbc7Sjsg 6556c349dbc7Sjsg ded_cnt = (value & 6557c349dbc7Sjsg gfx_v9_0_ras_fields[i].ded_count_mask) >> 6558c349dbc7Sjsg gfx_v9_0_ras_fields[i].ded_count_shift; 6559c349dbc7Sjsg if (ded_cnt) { 6560ad8b1aafSjsg dev_info(adev->dev, "GFX SubBlock %s, " 6561ad8b1aafSjsg "Instance[%d][%d], DED %d\n", 6562c349dbc7Sjsg gfx_v9_0_ras_fields[i].name, 6563c349dbc7Sjsg se_id, inst_id, 6564c349dbc7Sjsg ded_cnt); 6565c349dbc7Sjsg *ded_count += ded_cnt; 6566c349dbc7Sjsg } 6567c349dbc7Sjsg } 6568c349dbc7Sjsg 6569c349dbc7Sjsg return 0; 6570c349dbc7Sjsg } 6571c349dbc7Sjsg 6572c349dbc7Sjsg static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev) 6573c349dbc7Sjsg { 6574c349dbc7Sjsg int i, j, k; 6575c349dbc7Sjsg 6576c349dbc7Sjsg if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6577c349dbc7Sjsg return; 6578c349dbc7Sjsg 6579c349dbc7Sjsg /* read back registers to clear the counters */ 6580c349dbc7Sjsg mutex_lock(&adev->grbm_idx_mutex); 6581c349dbc7Sjsg for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { 6582c349dbc7Sjsg for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { 6583c349dbc7Sjsg for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { 6584c349dbc7Sjsg gfx_v9_0_select_se_sh(adev, j, 0x0, k); 6585c349dbc7Sjsg RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); 6586c349dbc7Sjsg } 6587c349dbc7Sjsg } 6588c349dbc7Sjsg } 6589c349dbc7Sjsg WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000); 6590c349dbc7Sjsg mutex_unlock(&adev->grbm_idx_mutex); 6591c349dbc7Sjsg 6592c349dbc7Sjsg WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6593c349dbc7Sjsg WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 6594c349dbc7Sjsg WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6595c349dbc7Sjsg WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 6596c349dbc7Sjsg WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6597c349dbc7Sjsg WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 6598c349dbc7Sjsg WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6599c349dbc7Sjsg WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 6600c349dbc7Sjsg 6601c349dbc7Sjsg for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { 6602c349dbc7Sjsg WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 6603c349dbc7Sjsg RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 6604c349dbc7Sjsg } 6605c349dbc7Sjsg 6606c349dbc7Sjsg for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { 6607c349dbc7Sjsg WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 6608c349dbc7Sjsg RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 6609c349dbc7Sjsg } 6610c349dbc7Sjsg 6611c349dbc7Sjsg for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { 6612c349dbc7Sjsg WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 6613c349dbc7Sjsg RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 6614c349dbc7Sjsg } 6615c349dbc7Sjsg 6616c349dbc7Sjsg for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { 6617c349dbc7Sjsg WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 6618c349dbc7Sjsg RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 6619c349dbc7Sjsg } 6620c349dbc7Sjsg 6621c349dbc7Sjsg WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6622c349dbc7Sjsg WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6623c349dbc7Sjsg WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6624c349dbc7Sjsg WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6625c349dbc7Sjsg } 6626c349dbc7Sjsg 6627c349dbc7Sjsg static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 6628c349dbc7Sjsg void *ras_error_status) 6629c349dbc7Sjsg { 6630c349dbc7Sjsg struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; 6631c349dbc7Sjsg uint32_t sec_count = 0, ded_count = 0; 6632c349dbc7Sjsg uint32_t i, j, k; 6633c349dbc7Sjsg uint32_t reg_value; 6634c349dbc7Sjsg 6635c349dbc7Sjsg if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6636c349dbc7Sjsg return -EINVAL; 6637c349dbc7Sjsg 6638c349dbc7Sjsg err_data->ue_count = 0; 6639c349dbc7Sjsg err_data->ce_count = 0; 6640c349dbc7Sjsg 6641c349dbc7Sjsg mutex_lock(&adev->grbm_idx_mutex); 6642c349dbc7Sjsg 6643c349dbc7Sjsg for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { 6644c349dbc7Sjsg for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { 6645c349dbc7Sjsg for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { 6646c349dbc7Sjsg gfx_v9_0_select_se_sh(adev, j, 0, k); 6647c349dbc7Sjsg reg_value = 6648c349dbc7Sjsg RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); 6649c349dbc7Sjsg if (reg_value) 6650ad8b1aafSjsg gfx_v9_0_ras_error_count(adev, 6651ad8b1aafSjsg &gfx_v9_0_edc_counter_regs[i], 6652c349dbc7Sjsg j, k, reg_value, 6653c349dbc7Sjsg &sec_count, &ded_count); 6654c349dbc7Sjsg } 6655c349dbc7Sjsg } 6656c349dbc7Sjsg } 6657c349dbc7Sjsg 6658c349dbc7Sjsg err_data->ce_count += sec_count; 6659c349dbc7Sjsg err_data->ue_count += ded_count; 6660c349dbc7Sjsg 6661c349dbc7Sjsg gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 6662c349dbc7Sjsg mutex_unlock(&adev->grbm_idx_mutex); 6663c349dbc7Sjsg 6664c349dbc7Sjsg gfx_v9_0_query_utc_edc_status(adev, err_data); 6665c349dbc7Sjsg 6666fb4d8502Sjsg return 0; 6667fb4d8502Sjsg } 6668fb4d8502Sjsg 6669ad8b1aafSjsg static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring) 6670ad8b1aafSjsg { 6671ad8b1aafSjsg const unsigned int cp_coher_cntl = 6672ad8b1aafSjsg PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) | 6673ad8b1aafSjsg PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) | 6674ad8b1aafSjsg PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) | 6675ad8b1aafSjsg PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) | 6676ad8b1aafSjsg PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1); 6677ad8b1aafSjsg 6678ad8b1aafSjsg /* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */ 6679ad8b1aafSjsg amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5)); 6680ad8b1aafSjsg amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */ 6681ad8b1aafSjsg amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */ 6682ad8b1aafSjsg amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */ 6683ad8b1aafSjsg amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ 6684ad8b1aafSjsg amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */ 6685ad8b1aafSjsg amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */ 6686ad8b1aafSjsg } 6687ad8b1aafSjsg 6688fb4d8502Sjsg static const struct amd_ip_funcs gfx_v9_0_ip_funcs = { 6689fb4d8502Sjsg .name = "gfx_v9_0", 6690fb4d8502Sjsg .early_init = gfx_v9_0_early_init, 6691fb4d8502Sjsg .late_init = gfx_v9_0_late_init, 6692fb4d8502Sjsg .sw_init = gfx_v9_0_sw_init, 6693fb4d8502Sjsg .sw_fini = gfx_v9_0_sw_fini, 6694fb4d8502Sjsg .hw_init = gfx_v9_0_hw_init, 6695fb4d8502Sjsg .hw_fini = gfx_v9_0_hw_fini, 6696fb4d8502Sjsg .suspend = gfx_v9_0_suspend, 6697fb4d8502Sjsg .resume = gfx_v9_0_resume, 6698fb4d8502Sjsg .is_idle = gfx_v9_0_is_idle, 6699fb4d8502Sjsg .wait_for_idle = gfx_v9_0_wait_for_idle, 6700fb4d8502Sjsg .soft_reset = gfx_v9_0_soft_reset, 6701fb4d8502Sjsg .set_clockgating_state = gfx_v9_0_set_clockgating_state, 6702fb4d8502Sjsg .set_powergating_state = gfx_v9_0_set_powergating_state, 6703fb4d8502Sjsg .get_clockgating_state = gfx_v9_0_get_clockgating_state, 6704fb4d8502Sjsg }; 6705fb4d8502Sjsg 6706fb4d8502Sjsg static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = { 6707fb4d8502Sjsg .type = AMDGPU_RING_TYPE_GFX, 6708fb4d8502Sjsg .align_mask = 0xff, 6709fb4d8502Sjsg .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6710fb4d8502Sjsg .support_64bit_ptrs = true, 6711c349dbc7Sjsg .vmhub = AMDGPU_GFXHUB_0, 6712fb4d8502Sjsg .get_rptr = gfx_v9_0_ring_get_rptr_gfx, 6713fb4d8502Sjsg .get_wptr = gfx_v9_0_ring_get_wptr_gfx, 6714fb4d8502Sjsg .set_wptr = gfx_v9_0_ring_set_wptr_gfx, 6715fb4d8502Sjsg .emit_frame_size = /* totally 242 maximum if 16 IBs */ 6716fb4d8502Sjsg 5 + /* COND_EXEC */ 6717fb4d8502Sjsg 7 + /* PIPELINE_SYNC */ 6718fb4d8502Sjsg SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6719fb4d8502Sjsg SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6720fb4d8502Sjsg 2 + /* VM_FLUSH */ 6721fb4d8502Sjsg 8 + /* FENCE for VM_FLUSH */ 6722fb4d8502Sjsg 20 + /* GDS switch */ 6723fb4d8502Sjsg 4 + /* double SWITCH_BUFFER, 6724fb4d8502Sjsg the first COND_EXEC jump to the place just 6725fb4d8502Sjsg prior to this double SWITCH_BUFFER */ 6726fb4d8502Sjsg 5 + /* COND_EXEC */ 6727fb4d8502Sjsg 7 + /* HDP_flush */ 6728fb4d8502Sjsg 4 + /* VGT_flush */ 6729fb4d8502Sjsg 14 + /* CE_META */ 6730fb4d8502Sjsg 31 + /* DE_META */ 6731fb4d8502Sjsg 3 + /* CNTX_CTRL */ 6732fb4d8502Sjsg 5 + /* HDP_INVL */ 6733fb4d8502Sjsg 8 + 8 + /* FENCE x2 */ 6734ad8b1aafSjsg 2 + /* SWITCH_BUFFER */ 6735ad8b1aafSjsg 7, /* gfx_v9_0_emit_mem_sync */ 6736fb4d8502Sjsg .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */ 6737fb4d8502Sjsg .emit_ib = gfx_v9_0_ring_emit_ib_gfx, 6738fb4d8502Sjsg .emit_fence = gfx_v9_0_ring_emit_fence, 6739fb4d8502Sjsg .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 6740fb4d8502Sjsg .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 6741fb4d8502Sjsg .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 6742fb4d8502Sjsg .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 6743fb4d8502Sjsg .test_ring = gfx_v9_0_ring_test_ring, 6744fb4d8502Sjsg .test_ib = gfx_v9_0_ring_test_ib, 6745fb4d8502Sjsg .insert_nop = amdgpu_ring_insert_nop, 6746fb4d8502Sjsg .pad_ib = amdgpu_ring_generic_pad_ib, 6747fb4d8502Sjsg .emit_switch_buffer = gfx_v9_ring_emit_sb, 6748fb4d8502Sjsg .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl, 6749fb4d8502Sjsg .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec, 6750fb4d8502Sjsg .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec, 6751ad8b1aafSjsg .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl, 6752fb4d8502Sjsg .emit_wreg = gfx_v9_0_ring_emit_wreg, 6753fb4d8502Sjsg .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6754fb4d8502Sjsg .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6755c349dbc7Sjsg .soft_recovery = gfx_v9_0_ring_soft_recovery, 6756ad8b1aafSjsg .emit_mem_sync = gfx_v9_0_emit_mem_sync, 6757fb4d8502Sjsg }; 6758fb4d8502Sjsg 6759fb4d8502Sjsg static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = { 6760fb4d8502Sjsg .type = AMDGPU_RING_TYPE_COMPUTE, 6761fb4d8502Sjsg .align_mask = 0xff, 6762fb4d8502Sjsg .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6763fb4d8502Sjsg .support_64bit_ptrs = true, 6764c349dbc7Sjsg .vmhub = AMDGPU_GFXHUB_0, 6765fb4d8502Sjsg .get_rptr = gfx_v9_0_ring_get_rptr_compute, 6766fb4d8502Sjsg .get_wptr = gfx_v9_0_ring_get_wptr_compute, 6767fb4d8502Sjsg .set_wptr = gfx_v9_0_ring_set_wptr_compute, 6768fb4d8502Sjsg .emit_frame_size = 6769fb4d8502Sjsg 20 + /* gfx_v9_0_ring_emit_gds_switch */ 6770fb4d8502Sjsg 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 6771fb4d8502Sjsg 5 + /* hdp invalidate */ 6772fb4d8502Sjsg 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 6773fb4d8502Sjsg SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6774fb4d8502Sjsg SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6775fb4d8502Sjsg 2 + /* gfx_v9_0_ring_emit_vm_flush */ 6776ad8b1aafSjsg 8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */ 6777ad8b1aafSjsg 7, /* gfx_v9_0_emit_mem_sync */ 6778c349dbc7Sjsg .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 6779fb4d8502Sjsg .emit_ib = gfx_v9_0_ring_emit_ib_compute, 6780fb4d8502Sjsg .emit_fence = gfx_v9_0_ring_emit_fence, 6781fb4d8502Sjsg .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 6782fb4d8502Sjsg .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 6783fb4d8502Sjsg .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 6784fb4d8502Sjsg .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 6785fb4d8502Sjsg .test_ring = gfx_v9_0_ring_test_ring, 6786fb4d8502Sjsg .test_ib = gfx_v9_0_ring_test_ib, 6787fb4d8502Sjsg .insert_nop = amdgpu_ring_insert_nop, 6788fb4d8502Sjsg .pad_ib = amdgpu_ring_generic_pad_ib, 6789fb4d8502Sjsg .emit_wreg = gfx_v9_0_ring_emit_wreg, 6790fb4d8502Sjsg .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6791fb4d8502Sjsg .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6792ad8b1aafSjsg .emit_mem_sync = gfx_v9_0_emit_mem_sync, 6793fb4d8502Sjsg }; 6794fb4d8502Sjsg 6795fb4d8502Sjsg static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = { 6796fb4d8502Sjsg .type = AMDGPU_RING_TYPE_KIQ, 6797fb4d8502Sjsg .align_mask = 0xff, 6798fb4d8502Sjsg .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6799fb4d8502Sjsg .support_64bit_ptrs = true, 6800c349dbc7Sjsg .vmhub = AMDGPU_GFXHUB_0, 6801fb4d8502Sjsg .get_rptr = gfx_v9_0_ring_get_rptr_compute, 6802fb4d8502Sjsg .get_wptr = gfx_v9_0_ring_get_wptr_compute, 6803fb4d8502Sjsg .set_wptr = gfx_v9_0_ring_set_wptr_compute, 6804fb4d8502Sjsg .emit_frame_size = 6805fb4d8502Sjsg 20 + /* gfx_v9_0_ring_emit_gds_switch */ 6806fb4d8502Sjsg 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 6807fb4d8502Sjsg 5 + /* hdp invalidate */ 6808fb4d8502Sjsg 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 6809fb4d8502Sjsg SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6810fb4d8502Sjsg SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6811fb4d8502Sjsg 2 + /* gfx_v9_0_ring_emit_vm_flush */ 6812fb4d8502Sjsg 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */ 6813c349dbc7Sjsg .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 6814fb4d8502Sjsg .emit_fence = gfx_v9_0_ring_emit_fence_kiq, 6815fb4d8502Sjsg .test_ring = gfx_v9_0_ring_test_ring, 6816fb4d8502Sjsg .insert_nop = amdgpu_ring_insert_nop, 6817fb4d8502Sjsg .pad_ib = amdgpu_ring_generic_pad_ib, 6818fb4d8502Sjsg .emit_rreg = gfx_v9_0_ring_emit_rreg, 6819fb4d8502Sjsg .emit_wreg = gfx_v9_0_ring_emit_wreg, 6820fb4d8502Sjsg .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6821fb4d8502Sjsg .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6822fb4d8502Sjsg }; 6823fb4d8502Sjsg 6824fb4d8502Sjsg static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev) 6825fb4d8502Sjsg { 6826fb4d8502Sjsg int i; 6827fb4d8502Sjsg 6828fb4d8502Sjsg adev->gfx.kiq.ring.funcs = &gfx_v9_0_ring_funcs_kiq; 6829fb4d8502Sjsg 6830fb4d8502Sjsg for (i = 0; i < adev->gfx.num_gfx_rings; i++) 6831fb4d8502Sjsg adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx; 6832fb4d8502Sjsg 6833fb4d8502Sjsg for (i = 0; i < adev->gfx.num_compute_rings; i++) 6834fb4d8502Sjsg adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute; 6835fb4d8502Sjsg } 6836fb4d8502Sjsg 6837fb4d8502Sjsg static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = { 6838fb4d8502Sjsg .set = gfx_v9_0_set_eop_interrupt_state, 6839fb4d8502Sjsg .process = gfx_v9_0_eop_irq, 6840fb4d8502Sjsg }; 6841fb4d8502Sjsg 6842fb4d8502Sjsg static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = { 6843fb4d8502Sjsg .set = gfx_v9_0_set_priv_reg_fault_state, 6844fb4d8502Sjsg .process = gfx_v9_0_priv_reg_irq, 6845fb4d8502Sjsg }; 6846fb4d8502Sjsg 6847fb4d8502Sjsg static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = { 6848fb4d8502Sjsg .set = gfx_v9_0_set_priv_inst_fault_state, 6849fb4d8502Sjsg .process = gfx_v9_0_priv_inst_irq, 6850fb4d8502Sjsg }; 6851fb4d8502Sjsg 6852c349dbc7Sjsg static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = { 6853c349dbc7Sjsg .set = gfx_v9_0_set_cp_ecc_error_state, 6854c349dbc7Sjsg .process = amdgpu_gfx_cp_ecc_error_irq, 6855c349dbc7Sjsg }; 6856c349dbc7Sjsg 6857c349dbc7Sjsg 6858fb4d8502Sjsg static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev) 6859fb4d8502Sjsg { 6860fb4d8502Sjsg adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; 6861fb4d8502Sjsg adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs; 6862fb4d8502Sjsg 6863fb4d8502Sjsg adev->gfx.priv_reg_irq.num_types = 1; 6864fb4d8502Sjsg adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs; 6865fb4d8502Sjsg 6866fb4d8502Sjsg adev->gfx.priv_inst_irq.num_types = 1; 6867fb4d8502Sjsg adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs; 6868fb4d8502Sjsg 6869c349dbc7Sjsg adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/ 6870c349dbc7Sjsg adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs; 6871fb4d8502Sjsg } 6872fb4d8502Sjsg 6873fb4d8502Sjsg static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev) 6874fb4d8502Sjsg { 6875fb4d8502Sjsg switch (adev->asic_type) { 6876fb4d8502Sjsg case CHIP_VEGA10: 6877fb4d8502Sjsg case CHIP_VEGA12: 6878fb4d8502Sjsg case CHIP_VEGA20: 6879fb4d8502Sjsg case CHIP_RAVEN: 6880c349dbc7Sjsg case CHIP_ARCTURUS: 6881c349dbc7Sjsg case CHIP_RENOIR: 6882fb4d8502Sjsg adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs; 6883fb4d8502Sjsg break; 6884fb4d8502Sjsg default: 6885fb4d8502Sjsg break; 6886fb4d8502Sjsg } 6887fb4d8502Sjsg } 6888fb4d8502Sjsg 6889fb4d8502Sjsg static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev) 6890fb4d8502Sjsg { 6891fb4d8502Sjsg /* init asci gds info */ 6892c349dbc7Sjsg switch (adev->asic_type) { 6893c349dbc7Sjsg case CHIP_VEGA10: 6894c349dbc7Sjsg case CHIP_VEGA12: 6895c349dbc7Sjsg case CHIP_VEGA20: 6896c349dbc7Sjsg adev->gds.gds_size = 0x10000; 6897c349dbc7Sjsg break; 6898c349dbc7Sjsg case CHIP_RAVEN: 6899c349dbc7Sjsg case CHIP_ARCTURUS: 6900c349dbc7Sjsg adev->gds.gds_size = 0x1000; 6901c349dbc7Sjsg break; 6902c349dbc7Sjsg default: 6903c349dbc7Sjsg adev->gds.gds_size = 0x10000; 6904c349dbc7Sjsg break; 6905fb4d8502Sjsg } 6906c349dbc7Sjsg 6907c349dbc7Sjsg switch (adev->asic_type) { 6908c349dbc7Sjsg case CHIP_VEGA10: 6909c349dbc7Sjsg case CHIP_VEGA20: 6910c349dbc7Sjsg adev->gds.gds_compute_max_wave_id = 0x7ff; 6911c349dbc7Sjsg break; 6912c349dbc7Sjsg case CHIP_VEGA12: 6913c349dbc7Sjsg adev->gds.gds_compute_max_wave_id = 0x27f; 6914c349dbc7Sjsg break; 6915c349dbc7Sjsg case CHIP_RAVEN: 6916ad8b1aafSjsg if (adev->apu_flags & AMD_APU_IS_RAVEN2) 6917c349dbc7Sjsg adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */ 6918c349dbc7Sjsg else 6919c349dbc7Sjsg adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */ 6920c349dbc7Sjsg break; 6921c349dbc7Sjsg case CHIP_ARCTURUS: 6922c349dbc7Sjsg adev->gds.gds_compute_max_wave_id = 0xfff; 6923c349dbc7Sjsg break; 6924c349dbc7Sjsg default: 6925c349dbc7Sjsg /* this really depends on the chip */ 6926c349dbc7Sjsg adev->gds.gds_compute_max_wave_id = 0x7ff; 6927c349dbc7Sjsg break; 6928c349dbc7Sjsg } 6929c349dbc7Sjsg 6930c349dbc7Sjsg adev->gds.gws_size = 64; 6931c349dbc7Sjsg adev->gds.oa_size = 16; 6932fb4d8502Sjsg } 6933fb4d8502Sjsg 6934fb4d8502Sjsg static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev, 6935fb4d8502Sjsg u32 bitmap) 6936fb4d8502Sjsg { 6937fb4d8502Sjsg u32 data; 6938fb4d8502Sjsg 6939fb4d8502Sjsg if (!bitmap) 6940fb4d8502Sjsg return; 6941fb4d8502Sjsg 6942fb4d8502Sjsg data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 6943fb4d8502Sjsg data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 6944fb4d8502Sjsg 6945fb4d8502Sjsg WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data); 6946fb4d8502Sjsg } 6947fb4d8502Sjsg 6948fb4d8502Sjsg static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev) 6949fb4d8502Sjsg { 6950fb4d8502Sjsg u32 data, mask; 6951fb4d8502Sjsg 6952fb4d8502Sjsg data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG); 6953fb4d8502Sjsg data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG); 6954fb4d8502Sjsg 6955fb4d8502Sjsg data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 6956fb4d8502Sjsg data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 6957fb4d8502Sjsg 6958fb4d8502Sjsg mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh); 6959fb4d8502Sjsg 6960fb4d8502Sjsg return (~data) & mask; 6961fb4d8502Sjsg } 6962fb4d8502Sjsg 6963fb4d8502Sjsg static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 6964fb4d8502Sjsg struct amdgpu_cu_info *cu_info) 6965fb4d8502Sjsg { 6966fb4d8502Sjsg int i, j, k, counter, active_cu_number = 0; 6967fb4d8502Sjsg u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0; 6968c349dbc7Sjsg unsigned disable_masks[4 * 4]; 6969fb4d8502Sjsg 6970fb4d8502Sjsg if (!adev || !cu_info) 6971fb4d8502Sjsg return -EINVAL; 6972fb4d8502Sjsg 6973c349dbc7Sjsg /* 6974c349dbc7Sjsg * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs 6975c349dbc7Sjsg */ 6976c349dbc7Sjsg if (adev->gfx.config.max_shader_engines * 6977c349dbc7Sjsg adev->gfx.config.max_sh_per_se > 16) 6978c349dbc7Sjsg return -EINVAL; 6979c349dbc7Sjsg 6980c349dbc7Sjsg amdgpu_gfx_parse_disable_cu(disable_masks, 6981c349dbc7Sjsg adev->gfx.config.max_shader_engines, 6982c349dbc7Sjsg adev->gfx.config.max_sh_per_se); 6983fb4d8502Sjsg 6984fb4d8502Sjsg mutex_lock(&adev->grbm_idx_mutex); 6985fb4d8502Sjsg for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 6986fb4d8502Sjsg for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 6987fb4d8502Sjsg mask = 1; 6988fb4d8502Sjsg ao_bitmap = 0; 6989fb4d8502Sjsg counter = 0; 6990fb4d8502Sjsg gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 6991fb4d8502Sjsg gfx_v9_0_set_user_cu_inactive_bitmap( 6992c349dbc7Sjsg adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]); 6993fb4d8502Sjsg bitmap = gfx_v9_0_get_cu_active_bitmap(adev); 6994c349dbc7Sjsg 6995c349dbc7Sjsg /* 6996c349dbc7Sjsg * The bitmap(and ao_cu_bitmap) in cu_info structure is 6997c349dbc7Sjsg * 4x4 size array, and it's usually suitable for Vega 6998c349dbc7Sjsg * ASICs which has 4*2 SE/SH layout. 6999c349dbc7Sjsg * But for Arcturus, SE/SH layout is changed to 8*1. 7000c349dbc7Sjsg * To mostly reduce the impact, we make it compatible 7001c349dbc7Sjsg * with current bitmap array as below: 7002c349dbc7Sjsg * SE4,SH0 --> bitmap[0][1] 7003c349dbc7Sjsg * SE5,SH0 --> bitmap[1][1] 7004c349dbc7Sjsg * SE6,SH0 --> bitmap[2][1] 7005c349dbc7Sjsg * SE7,SH0 --> bitmap[3][1] 7006c349dbc7Sjsg */ 7007c349dbc7Sjsg cu_info->bitmap[i % 4][j + i / 4] = bitmap; 7008fb4d8502Sjsg 7009fb4d8502Sjsg for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 7010fb4d8502Sjsg if (bitmap & mask) { 7011fb4d8502Sjsg if (counter < adev->gfx.config.max_cu_per_sh) 7012fb4d8502Sjsg ao_bitmap |= mask; 7013fb4d8502Sjsg counter ++; 7014fb4d8502Sjsg } 7015fb4d8502Sjsg mask <<= 1; 7016fb4d8502Sjsg } 7017fb4d8502Sjsg active_cu_number += counter; 7018fb4d8502Sjsg if (i < 2 && j < 2) 7019fb4d8502Sjsg ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8)); 7020c349dbc7Sjsg cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap; 7021fb4d8502Sjsg } 7022fb4d8502Sjsg } 7023fb4d8502Sjsg gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 7024fb4d8502Sjsg mutex_unlock(&adev->grbm_idx_mutex); 7025fb4d8502Sjsg 7026fb4d8502Sjsg cu_info->number = active_cu_number; 7027fb4d8502Sjsg cu_info->ao_cu_mask = ao_cu_mask; 7028fb4d8502Sjsg cu_info->simd_per_cu = NUM_SIMD_PER_CU; 7029fb4d8502Sjsg 7030fb4d8502Sjsg return 0; 7031fb4d8502Sjsg } 7032fb4d8502Sjsg 7033fb4d8502Sjsg const struct amdgpu_ip_block_version gfx_v9_0_ip_block = 7034fb4d8502Sjsg { 7035fb4d8502Sjsg .type = AMD_IP_BLOCK_TYPE_GFX, 7036fb4d8502Sjsg .major = 9, 7037fb4d8502Sjsg .minor = 0, 7038fb4d8502Sjsg .rev = 0, 7039fb4d8502Sjsg .funcs = &gfx_v9_0_ip_funcs, 7040fb4d8502Sjsg }; 7041