1 // SPDX-License-Identifier: GPL-2.0 2 3 /* 4 * Copyright 2020-2022 HabanaLabs, Ltd. 5 * All Rights Reserved. 6 */ 7 8 #include "gaudi2P.h" 9 #include "gaudi2_masks.h" 10 #include "../include/gaudi2/gaudi2_special_blocks.h" 11 #include "../include/hw_ip/mmu/mmu_general.h" 12 #include "../include/hw_ip/mmu/mmu_v2_0.h" 13 #include "../include/gaudi2/gaudi2_packets.h" 14 #include "../include/gaudi2/gaudi2_reg_map.h" 15 #include "../include/gaudi2/gaudi2_async_ids_map_extended.h" 16 #include "../include/gaudi2/arc/gaudi2_arc_common_packets.h" 17 18 #include <linux/module.h> 19 #include <linux/pci.h> 20 #include <linux/hwmon.h> 21 #include <linux/iommu.h> 22 23 #define GAUDI2_DMA_POOL_BLK_SIZE SZ_256 /* 256 bytes */ 24 25 #define GAUDI2_RESET_TIMEOUT_MSEC 2000 /* 2000ms */ 26 27 #define GAUDI2_RESET_POLL_TIMEOUT_USEC 500000 /* 500ms */ 28 #define GAUDI2_PLDM_HRESET_TIMEOUT_MSEC 25000 /* 25s */ 29 #define GAUDI2_PLDM_SRESET_TIMEOUT_MSEC 25000 /* 25s */ 30 #define GAUDI2_PLDM_RESET_POLL_TIMEOUT_USEC 3000000 /* 3s */ 31 #define GAUDI2_RESET_POLL_CNT 3 32 #define GAUDI2_RESET_WAIT_MSEC 1 /* 1ms */ 33 #define GAUDI2_CPU_RESET_WAIT_MSEC 100 /* 100ms */ 34 #define GAUDI2_PLDM_RESET_WAIT_MSEC 1000 /* 1s */ 35 #define GAUDI2_CB_POOL_CB_CNT 512 36 #define GAUDI2_CB_POOL_CB_SIZE SZ_128K /* 128KB */ 37 #define GAUDI2_MSG_TO_CPU_TIMEOUT_USEC 4000000 /* 4s */ 38 #define GAUDI2_WAIT_FOR_BL_TIMEOUT_USEC 25000000 /* 25s */ 39 #define GAUDI2_TEST_QUEUE_WAIT_USEC 100000 /* 100ms */ 40 #define GAUDI2_PLDM_TEST_QUEUE_WAIT_USEC 1000000 /* 1s */ 41 42 #define GAUDI2_ALLOC_CPU_MEM_RETRY_CNT 3 43 44 /* 45 * since the code already has built-in support for binning of up to MAX_FAULTY_TPCS TPCs 46 * and the code relies on that value (for array size etc..) we define another value 47 * for MAX faulty TPCs which reflects the cluster binning requirements 48 */ 49 #define MAX_CLUSTER_BINNING_FAULTY_TPCS 1 50 #define MAX_FAULTY_XBARS 1 51 #define MAX_FAULTY_EDMAS 1 52 #define MAX_FAULTY_DECODERS 1 53 54 #define GAUDI2_TPC_FULL_MASK 0x1FFFFFF 55 #define GAUDI2_HIF_HMMU_FULL_MASK 0xFFFF 56 #define GAUDI2_DECODER_FULL_MASK 0x3FF 57 58 #define GAUDI2_NA_EVENT_CAUSE 0xFF 59 #define GAUDI2_NUM_OF_QM_ERR_CAUSE 18 60 #define GAUDI2_NUM_OF_LOWER_QM_ERR_CAUSE 25 61 #define GAUDI2_NUM_OF_QM_ARB_ERR_CAUSE 3 62 #define GAUDI2_NUM_OF_ARC_SEI_ERR_CAUSE 14 63 #define GAUDI2_NUM_OF_CPU_SEI_ERR_CAUSE 3 64 #define GAUDI2_NUM_OF_QM_SEI_ERR_CAUSE 2 65 #define GAUDI2_NUM_OF_ROT_ERR_CAUSE 22 66 #define GAUDI2_NUM_OF_TPC_INTR_CAUSE 31 67 #define GAUDI2_NUM_OF_DEC_ERR_CAUSE 25 68 #define GAUDI2_NUM_OF_MME_ERR_CAUSE 16 69 #define GAUDI2_NUM_OF_MME_WAP_ERR_CAUSE 7 70 #define GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE 8 71 #define GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE 19 72 #define GAUDI2_NUM_OF_HBM_SEI_CAUSE 9 73 #define GAUDI2_NUM_OF_SM_SEI_ERR_CAUSE 3 74 #define GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE 3 75 #define GAUDI2_NUM_OF_PMMU_FATAL_ERR_CAUSE 2 76 #define GAUDI2_NUM_OF_HIF_FATAL_ERR_CAUSE 2 77 #define GAUDI2_NUM_OF_AXI_DRAIN_ERR_CAUSE 2 78 #define GAUDI2_NUM_OF_HBM_MC_SPI_CAUSE 5 79 80 #define GAUDI2_MMU_CACHE_INV_TIMEOUT_USEC (MMU_CONFIG_TIMEOUT_USEC * 10) 81 #define GAUDI2_PLDM_MMU_TIMEOUT_USEC (MMU_CONFIG_TIMEOUT_USEC * 200) 82 #define GAUDI2_ARB_WDT_TIMEOUT (0x1000000) 83 84 #define GAUDI2_VDEC_TIMEOUT_USEC 10000 /* 10ms */ 85 #define GAUDI2_PLDM_VDEC_TIMEOUT_USEC (GAUDI2_VDEC_TIMEOUT_USEC * 100) 86 87 #define KDMA_TIMEOUT_USEC USEC_PER_SEC 88 89 #define IS_DMA_IDLE(dma_core_sts0) \ 90 (!((dma_core_sts0) & (DCORE0_EDMA0_CORE_STS0_BUSY_MASK))) 91 92 #define IS_DMA_HALTED(dma_core_sts1) \ 93 ((dma_core_sts1) & (DCORE0_EDMA0_CORE_STS1_IS_HALT_MASK)) 94 95 #define IS_MME_IDLE(mme_arch_sts) (((mme_arch_sts) & MME_ARCH_IDLE_MASK) == MME_ARCH_IDLE_MASK) 96 97 #define IS_TPC_IDLE(tpc_cfg_sts) (((tpc_cfg_sts) & (TPC_IDLE_MASK)) == (TPC_IDLE_MASK)) 98 99 #define IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) \ 100 ((((qm_glbl_sts0) & (QM_IDLE_MASK)) == (QM_IDLE_MASK)) && \ 101 (((qm_glbl_sts1) & (QM_ARC_IDLE_MASK)) == (QM_ARC_IDLE_MASK)) && \ 102 (((qm_cgm_sts) & (CGM_IDLE_MASK)) == (CGM_IDLE_MASK))) 103 104 #define PCIE_DEC_EN_MASK 0x300 105 #define DEC_WORK_STATE_IDLE 0 106 #define DEC_WORK_STATE_PEND 3 107 #define IS_DEC_IDLE(dec_swreg15) \ 108 (((dec_swreg15) & DCORE0_DEC0_CMD_SWREG15_SW_WORK_STATE_MASK) == DEC_WORK_STATE_IDLE || \ 109 ((dec_swreg15) & DCORE0_DEC0_CMD_SWREG15_SW_WORK_STATE_MASK) == DEC_WORK_STATE_PEND) 110 111 /* HBM MMU address scrambling parameters */ 112 #define GAUDI2_HBM_MMU_SCRM_MEM_SIZE SZ_8M 113 #define GAUDI2_HBM_MMU_SCRM_DIV_SHIFT 26 114 #define GAUDI2_HBM_MMU_SCRM_MOD_SHIFT 0 115 #define GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK DRAM_VA_HINT_MASK 116 #define GAUDI2_COMPENSATE_TLB_PAGE_SIZE_FACTOR 16 117 #define MMU_RANGE_INV_VA_LSB_SHIFT 12 118 #define MMU_RANGE_INV_VA_MSB_SHIFT 44 119 #define MMU_RANGE_INV_EN_SHIFT 0 120 #define MMU_RANGE_INV_ASID_EN_SHIFT 1 121 #define MMU_RANGE_INV_ASID_SHIFT 2 122 123 /* The last SPI_SEI cause bit, "burst_fifo_full", is expected to be triggered in PMMU because it has 124 * a 2 entries FIFO, and hence it is not enabled for it. 125 */ 126 #define GAUDI2_PMMU_SPI_SEI_ENABLE_MASK GENMASK(GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE - 2, 0) 127 #define GAUDI2_HMMU_SPI_SEI_ENABLE_MASK GENMASK(GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE - 1, 0) 128 129 #define GAUDI2_MAX_STRING_LEN 64 130 131 #define GAUDI2_VDEC_MSIX_ENTRIES (GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM - \ 132 GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + 1) 133 134 #define ENGINE_ID_DCORE_OFFSET (GAUDI2_DCORE1_ENGINE_ID_EDMA_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0) 135 136 /* RAZWI initiator coordinates */ 137 #define RAZWI_GET_AXUSER_XY(x) \ 138 ((x & 0xF8001FF0) >> 4) 139 140 #define RAZWI_GET_AXUSER_LOW_XY(x) \ 141 ((x & 0x00001FF0) >> 4) 142 143 #define RAZWI_INITIATOR_AXUER_L_X_SHIFT 0 144 #define RAZWI_INITIATOR_AXUER_L_X_MASK 0x1F 145 #define RAZWI_INITIATOR_AXUER_L_Y_SHIFT 5 146 #define RAZWI_INITIATOR_AXUER_L_Y_MASK 0xF 147 148 #define RAZWI_INITIATOR_AXUER_H_X_SHIFT 23 149 #define RAZWI_INITIATOR_AXUER_H_X_MASK 0x1F 150 151 #define RAZWI_INITIATOR_ID_X_Y_LOW(x, y) \ 152 ((((y) & RAZWI_INITIATOR_AXUER_L_Y_MASK) << RAZWI_INITIATOR_AXUER_L_Y_SHIFT) | \ 153 (((x) & RAZWI_INITIATOR_AXUER_L_X_MASK) << RAZWI_INITIATOR_AXUER_L_X_SHIFT)) 154 155 #define RAZWI_INITIATOR_ID_X_HIGH(x) \ 156 (((x) & RAZWI_INITIATOR_AXUER_H_X_MASK) << RAZWI_INITIATOR_AXUER_H_X_SHIFT) 157 158 #define RAZWI_INITIATOR_ID_X_Y(xl, yl, xh) \ 159 (RAZWI_INITIATOR_ID_X_Y_LOW(xl, yl) | RAZWI_INITIATOR_ID_X_HIGH(xh)) 160 161 #define PSOC_RAZWI_ENG_STR_SIZE 128 162 #define PSOC_RAZWI_MAX_ENG_PER_RTR 5 163 164 /* HW scrambles only bits 0-25 */ 165 #define HW_UNSCRAMBLED_BITS_MASK GENMASK_ULL(63, 26) 166 167 struct gaudi2_razwi_info { 168 u32 axuser_xy; 169 u32 rtr_ctrl; 170 u16 eng_id; 171 char *eng_name; 172 }; 173 174 static struct gaudi2_razwi_info common_razwi_info[] = { 175 {RAZWI_INITIATOR_ID_X_Y(2, 4, 0), mmDCORE0_RTR0_CTRL_BASE, 176 GAUDI2_DCORE0_ENGINE_ID_DEC_0, "DEC0"}, 177 {RAZWI_INITIATOR_ID_X_Y(2, 4, 4), mmDCORE0_RTR0_CTRL_BASE, 178 GAUDI2_DCORE0_ENGINE_ID_DEC_1, "DEC1"}, 179 {RAZWI_INITIATOR_ID_X_Y(17, 4, 18), mmDCORE1_RTR7_CTRL_BASE, 180 GAUDI2_DCORE1_ENGINE_ID_DEC_0, "DEC2"}, 181 {RAZWI_INITIATOR_ID_X_Y(17, 4, 14), mmDCORE1_RTR7_CTRL_BASE, 182 GAUDI2_DCORE1_ENGINE_ID_DEC_1, "DEC3"}, 183 {RAZWI_INITIATOR_ID_X_Y(2, 11, 0), mmDCORE2_RTR0_CTRL_BASE, 184 GAUDI2_DCORE2_ENGINE_ID_DEC_0, "DEC4"}, 185 {RAZWI_INITIATOR_ID_X_Y(2, 11, 4), mmDCORE2_RTR0_CTRL_BASE, 186 GAUDI2_DCORE2_ENGINE_ID_DEC_1, "DEC5"}, 187 {RAZWI_INITIATOR_ID_X_Y(17, 11, 18), mmDCORE3_RTR7_CTRL_BASE, 188 GAUDI2_DCORE3_ENGINE_ID_DEC_0, "DEC6"}, 189 {RAZWI_INITIATOR_ID_X_Y(17, 11, 14), mmDCORE3_RTR7_CTRL_BASE, 190 GAUDI2_DCORE3_ENGINE_ID_DEC_1, "DEC7"}, 191 {RAZWI_INITIATOR_ID_X_Y(2, 4, 6), mmDCORE0_RTR0_CTRL_BASE, 192 GAUDI2_PCIE_ENGINE_ID_DEC_0, "DEC8"}, 193 {RAZWI_INITIATOR_ID_X_Y(2, 4, 7), mmDCORE0_RTR0_CTRL_BASE, 194 GAUDI2_PCIE_ENGINE_ID_DEC_0, "DEC9"}, 195 {RAZWI_INITIATOR_ID_X_Y(3, 4, 2), mmDCORE0_RTR1_CTRL_BASE, 196 GAUDI2_DCORE0_ENGINE_ID_TPC_0, "TPC0"}, 197 {RAZWI_INITIATOR_ID_X_Y(3, 4, 4), mmDCORE0_RTR1_CTRL_BASE, 198 GAUDI2_DCORE0_ENGINE_ID_TPC_1, "TPC1"}, 199 {RAZWI_INITIATOR_ID_X_Y(4, 4, 2), mmDCORE0_RTR2_CTRL_BASE, 200 GAUDI2_DCORE0_ENGINE_ID_TPC_2, "TPC2"}, 201 {RAZWI_INITIATOR_ID_X_Y(4, 4, 4), mmDCORE0_RTR2_CTRL_BASE, 202 GAUDI2_DCORE0_ENGINE_ID_TPC_3, "TPC3"}, 203 {RAZWI_INITIATOR_ID_X_Y(5, 4, 2), mmDCORE0_RTR3_CTRL_BASE, 204 GAUDI2_DCORE0_ENGINE_ID_TPC_4, "TPC4"}, 205 {RAZWI_INITIATOR_ID_X_Y(5, 4, 4), mmDCORE0_RTR3_CTRL_BASE, 206 GAUDI2_DCORE0_ENGINE_ID_TPC_5, "TPC5"}, 207 {RAZWI_INITIATOR_ID_X_Y(16, 4, 14), mmDCORE1_RTR6_CTRL_BASE, 208 GAUDI2_DCORE1_ENGINE_ID_TPC_0, "TPC6"}, 209 {RAZWI_INITIATOR_ID_X_Y(16, 4, 16), mmDCORE1_RTR6_CTRL_BASE, 210 GAUDI2_DCORE1_ENGINE_ID_TPC_1, "TPC7"}, 211 {RAZWI_INITIATOR_ID_X_Y(15, 4, 14), mmDCORE1_RTR5_CTRL_BASE, 212 GAUDI2_DCORE1_ENGINE_ID_TPC_2, "TPC8"}, 213 {RAZWI_INITIATOR_ID_X_Y(15, 4, 16), mmDCORE1_RTR5_CTRL_BASE, 214 GAUDI2_DCORE1_ENGINE_ID_TPC_3, "TPC9"}, 215 {RAZWI_INITIATOR_ID_X_Y(14, 4, 14), mmDCORE1_RTR4_CTRL_BASE, 216 GAUDI2_DCORE1_ENGINE_ID_TPC_4, "TPC10"}, 217 {RAZWI_INITIATOR_ID_X_Y(14, 4, 16), mmDCORE1_RTR4_CTRL_BASE, 218 GAUDI2_DCORE1_ENGINE_ID_TPC_5, "TPC11"}, 219 {RAZWI_INITIATOR_ID_X_Y(5, 11, 2), mmDCORE2_RTR3_CTRL_BASE, 220 GAUDI2_DCORE2_ENGINE_ID_TPC_0, "TPC12"}, 221 {RAZWI_INITIATOR_ID_X_Y(5, 11, 4), mmDCORE2_RTR3_CTRL_BASE, 222 GAUDI2_DCORE2_ENGINE_ID_TPC_1, "TPC13"}, 223 {RAZWI_INITIATOR_ID_X_Y(4, 11, 2), mmDCORE2_RTR2_CTRL_BASE, 224 GAUDI2_DCORE2_ENGINE_ID_TPC_2, "TPC14"}, 225 {RAZWI_INITIATOR_ID_X_Y(4, 11, 4), mmDCORE2_RTR2_CTRL_BASE, 226 GAUDI2_DCORE2_ENGINE_ID_TPC_3, "TPC15"}, 227 {RAZWI_INITIATOR_ID_X_Y(3, 11, 2), mmDCORE2_RTR1_CTRL_BASE, 228 GAUDI2_DCORE2_ENGINE_ID_TPC_4, "TPC16"}, 229 {RAZWI_INITIATOR_ID_X_Y(3, 11, 4), mmDCORE2_RTR1_CTRL_BASE, 230 GAUDI2_DCORE2_ENGINE_ID_TPC_5, "TPC17"}, 231 {RAZWI_INITIATOR_ID_X_Y(14, 11, 14), mmDCORE3_RTR4_CTRL_BASE, 232 GAUDI2_DCORE3_ENGINE_ID_TPC_0, "TPC18"}, 233 {RAZWI_INITIATOR_ID_X_Y(14, 11, 16), mmDCORE3_RTR4_CTRL_BASE, 234 GAUDI2_DCORE3_ENGINE_ID_TPC_1, "TPC19"}, 235 {RAZWI_INITIATOR_ID_X_Y(15, 11, 14), mmDCORE3_RTR5_CTRL_BASE, 236 GAUDI2_DCORE3_ENGINE_ID_TPC_2, "TPC20"}, 237 {RAZWI_INITIATOR_ID_X_Y(15, 11, 16), mmDCORE3_RTR5_CTRL_BASE, 238 GAUDI2_DCORE3_ENGINE_ID_TPC_3, "TPC21"}, 239 {RAZWI_INITIATOR_ID_X_Y(16, 11, 14), mmDCORE3_RTR6_CTRL_BASE, 240 GAUDI2_DCORE3_ENGINE_ID_TPC_4, "TPC22"}, 241 {RAZWI_INITIATOR_ID_X_Y(16, 11, 16), mmDCORE3_RTR6_CTRL_BASE, 242 GAUDI2_DCORE3_ENGINE_ID_TPC_5, "TPC23"}, 243 {RAZWI_INITIATOR_ID_X_Y(2, 4, 2), mmDCORE0_RTR0_CTRL_BASE, 244 GAUDI2_DCORE3_ENGINE_ID_TPC_5, "TPC24"}, 245 {RAZWI_INITIATOR_ID_X_Y(17, 4, 8), mmDCORE1_RTR7_CTRL_BASE, 246 GAUDI2_ENGINE_ID_NIC0_0, "NIC0"}, 247 {RAZWI_INITIATOR_ID_X_Y(17, 4, 10), mmDCORE1_RTR7_CTRL_BASE, 248 GAUDI2_ENGINE_ID_NIC0_1, "NIC1"}, 249 {RAZWI_INITIATOR_ID_X_Y(17, 4, 12), mmDCORE1_RTR7_CTRL_BASE, 250 GAUDI2_ENGINE_ID_NIC1_0, "NIC2"}, 251 {RAZWI_INITIATOR_ID_X_Y(17, 4, 14), mmDCORE1_RTR7_CTRL_BASE, 252 GAUDI2_ENGINE_ID_NIC1_1, "NIC3"}, 253 {RAZWI_INITIATOR_ID_X_Y(17, 4, 15), mmDCORE1_RTR7_CTRL_BASE, 254 GAUDI2_ENGINE_ID_NIC2_0, "NIC4"}, 255 {RAZWI_INITIATOR_ID_X_Y(2, 11, 2), mmDCORE2_RTR0_CTRL_BASE, 256 GAUDI2_ENGINE_ID_NIC2_1, "NIC5"}, 257 {RAZWI_INITIATOR_ID_X_Y(2, 11, 4), mmDCORE2_RTR0_CTRL_BASE, 258 GAUDI2_ENGINE_ID_NIC3_0, "NIC6"}, 259 {RAZWI_INITIATOR_ID_X_Y(2, 11, 6), mmDCORE2_RTR0_CTRL_BASE, 260 GAUDI2_ENGINE_ID_NIC3_1, "NIC7"}, 261 {RAZWI_INITIATOR_ID_X_Y(2, 11, 8), mmDCORE2_RTR0_CTRL_BASE, 262 GAUDI2_ENGINE_ID_NIC4_0, "NIC8"}, 263 {RAZWI_INITIATOR_ID_X_Y(17, 11, 12), mmDCORE3_RTR7_CTRL_BASE, 264 GAUDI2_ENGINE_ID_NIC4_1, "NIC9"}, 265 {RAZWI_INITIATOR_ID_X_Y(17, 11, 14), mmDCORE3_RTR7_CTRL_BASE, 266 GAUDI2_ENGINE_ID_NIC5_0, "NIC10"}, 267 {RAZWI_INITIATOR_ID_X_Y(17, 11, 16), mmDCORE3_RTR7_CTRL_BASE, 268 GAUDI2_ENGINE_ID_NIC5_1, "NIC11"}, 269 {RAZWI_INITIATOR_ID_X_Y(2, 4, 2), mmDCORE0_RTR0_CTRL_BASE, 270 GAUDI2_ENGINE_ID_PDMA_0, "PDMA0"}, 271 {RAZWI_INITIATOR_ID_X_Y(2, 4, 3), mmDCORE0_RTR0_CTRL_BASE, 272 GAUDI2_ENGINE_ID_PDMA_1, "PDMA1"}, 273 {RAZWI_INITIATOR_ID_X_Y(2, 4, 4), mmDCORE0_RTR0_CTRL_BASE, 274 GAUDI2_ENGINE_ID_SIZE, "PMMU"}, 275 {RAZWI_INITIATOR_ID_X_Y(2, 4, 5), mmDCORE0_RTR0_CTRL_BASE, 276 GAUDI2_ENGINE_ID_SIZE, "PCIE"}, 277 {RAZWI_INITIATOR_ID_X_Y(17, 4, 16), mmDCORE1_RTR7_CTRL_BASE, 278 GAUDI2_ENGINE_ID_ARC_FARM, "ARC_FARM"}, 279 {RAZWI_INITIATOR_ID_X_Y(17, 4, 17), mmDCORE1_RTR7_CTRL_BASE, 280 GAUDI2_ENGINE_ID_KDMA, "KDMA"}, 281 {RAZWI_INITIATOR_ID_X_Y(1, 5, 1), mmSFT0_HBW_RTR_IF1_RTR_CTRL_BASE, 282 GAUDI2_DCORE0_ENGINE_ID_EDMA_0, "EDMA0"}, 283 {RAZWI_INITIATOR_ID_X_Y(1, 5, 1), mmSFT0_HBW_RTR_IF0_RTR_CTRL_BASE, 284 GAUDI2_DCORE0_ENGINE_ID_EDMA_1, "EDMA1"}, 285 {RAZWI_INITIATOR_ID_X_Y(18, 5, 18), mmSFT1_HBW_RTR_IF1_RTR_CTRL_BASE, 286 GAUDI2_DCORE1_ENGINE_ID_EDMA_0, "EDMA2"}, 287 {RAZWI_INITIATOR_ID_X_Y(18, 5, 18), mmSFT1_HBW_RTR_IF0_RTR_CTRL_BASE, 288 GAUDI2_DCORE1_ENGINE_ID_EDMA_1, "EDMA3"}, 289 {RAZWI_INITIATOR_ID_X_Y(1, 10, 1), mmSFT2_HBW_RTR_IF0_RTR_CTRL_BASE, 290 GAUDI2_DCORE2_ENGINE_ID_EDMA_0, "EDMA4"}, 291 {RAZWI_INITIATOR_ID_X_Y(1, 10, 1), mmSFT2_HBW_RTR_IF1_RTR_CTRL_BASE, 292 GAUDI2_DCORE2_ENGINE_ID_EDMA_1, "EDMA5"}, 293 {RAZWI_INITIATOR_ID_X_Y(18, 10, 18), mmSFT2_HBW_RTR_IF0_RTR_CTRL_BASE, 294 GAUDI2_DCORE3_ENGINE_ID_EDMA_0, "EDMA6"}, 295 {RAZWI_INITIATOR_ID_X_Y(18, 10, 18), mmSFT2_HBW_RTR_IF1_RTR_CTRL_BASE, 296 GAUDI2_DCORE3_ENGINE_ID_EDMA_1, "EDMA7"}, 297 {RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE, 298 GAUDI2_ENGINE_ID_SIZE, "HMMU0"}, 299 {RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE, 300 GAUDI2_ENGINE_ID_SIZE, "HMMU1"}, 301 {RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE, 302 GAUDI2_ENGINE_ID_SIZE, "HMMU2"}, 303 {RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE, 304 GAUDI2_ENGINE_ID_SIZE, "HMMU3"}, 305 {RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE, 306 GAUDI2_ENGINE_ID_SIZE, "HMMU4"}, 307 {RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE, 308 GAUDI2_ENGINE_ID_SIZE, "HMMU5"}, 309 {RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE, 310 GAUDI2_ENGINE_ID_SIZE, "HMMU6"}, 311 {RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE, 312 GAUDI2_ENGINE_ID_SIZE, "HMMU7"}, 313 {RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE, 314 GAUDI2_ENGINE_ID_SIZE, "HMMU8"}, 315 {RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE, 316 GAUDI2_ENGINE_ID_SIZE, "HMMU9"}, 317 {RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE, 318 GAUDI2_ENGINE_ID_SIZE, "HMMU10"}, 319 {RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE, 320 GAUDI2_ENGINE_ID_SIZE, "HMMU11"}, 321 {RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE, 322 GAUDI2_ENGINE_ID_SIZE, "HMMU12"}, 323 {RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE, 324 GAUDI2_ENGINE_ID_SIZE, "HMMU13"}, 325 {RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE, 326 GAUDI2_ENGINE_ID_SIZE, "HMMU14"}, 327 {RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE, 328 GAUDI2_ENGINE_ID_SIZE, "HMMU15"}, 329 {RAZWI_INITIATOR_ID_X_Y(2, 11, 2), mmDCORE2_RTR0_CTRL_BASE, 330 GAUDI2_ENGINE_ID_ROT_0, "ROT0"}, 331 {RAZWI_INITIATOR_ID_X_Y(17, 11, 16), mmDCORE3_RTR7_CTRL_BASE, 332 GAUDI2_ENGINE_ID_ROT_1, "ROT1"}, 333 {RAZWI_INITIATOR_ID_X_Y(2, 11, 2), mmDCORE2_RTR0_CTRL_BASE, 334 GAUDI2_ENGINE_ID_PSOC, "CPU"}, 335 {RAZWI_INITIATOR_ID_X_Y(17, 11, 11), mmDCORE3_RTR7_CTRL_BASE, 336 GAUDI2_ENGINE_ID_PSOC, "PSOC"} 337 }; 338 339 static struct gaudi2_razwi_info mme_razwi_info[] = { 340 /* MME X high coordinate is N/A, hence using only low coordinates */ 341 {RAZWI_INITIATOR_ID_X_Y_LOW(7, 4), mmDCORE0_RTR5_CTRL_BASE, 342 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_WAP0"}, 343 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 4), mmDCORE0_RTR7_CTRL_BASE, 344 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_WAP1"}, 345 {RAZWI_INITIATOR_ID_X_Y_LOW(8, 4), mmDCORE0_RTR6_CTRL_BASE, 346 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_CTRL_WR"}, 347 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 4), mmDCORE0_RTR7_CTRL_BASE, 348 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_CTRL_RD"}, 349 {RAZWI_INITIATOR_ID_X_Y_LOW(6, 4), mmDCORE0_RTR4_CTRL_BASE, 350 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE0"}, 351 {RAZWI_INITIATOR_ID_X_Y_LOW(6, 4), mmDCORE0_RTR4_CTRL_BASE, 352 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE1"}, 353 {RAZWI_INITIATOR_ID_X_Y_LOW(7, 4), mmDCORE0_RTR5_CTRL_BASE, 354 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE2"}, 355 {RAZWI_INITIATOR_ID_X_Y_LOW(8, 4), mmDCORE0_RTR6_CTRL_BASE, 356 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE3"}, 357 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 4), mmDCORE0_RTR7_CTRL_BASE, 358 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE4"}, 359 {RAZWI_INITIATOR_ID_X_Y_LOW(12, 4), mmDCORE1_RTR2_CTRL_BASE, 360 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_WAP0"}, 361 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 4), mmDCORE1_RTR0_CTRL_BASE, 362 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_WAP1"}, 363 {RAZWI_INITIATOR_ID_X_Y_LOW(11, 4), mmDCORE1_RTR1_CTRL_BASE, 364 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_CTRL_WR"}, 365 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 4), mmDCORE1_RTR0_CTRL_BASE, 366 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_CTRL_RD"}, 367 {RAZWI_INITIATOR_ID_X_Y_LOW(13, 4), mmDCORE1_RTR3_CTRL_BASE, 368 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE0"}, 369 {RAZWI_INITIATOR_ID_X_Y_LOW(13, 4), mmDCORE1_RTR3_CTRL_BASE, 370 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE1"}, 371 {RAZWI_INITIATOR_ID_X_Y_LOW(12, 4), mmDCORE1_RTR2_CTRL_BASE, 372 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE2"}, 373 {RAZWI_INITIATOR_ID_X_Y_LOW(11, 4), mmDCORE1_RTR1_CTRL_BASE, 374 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE3"}, 375 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 4), mmDCORE1_RTR0_CTRL_BASE, 376 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE4"}, 377 {RAZWI_INITIATOR_ID_X_Y_LOW(7, 11), mmDCORE2_RTR5_CTRL_BASE, 378 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_WAP0"}, 379 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 11), mmDCORE2_RTR7_CTRL_BASE, 380 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_WAP1"}, 381 {RAZWI_INITIATOR_ID_X_Y_LOW(8, 11), mmDCORE2_RTR6_CTRL_BASE, 382 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_CTRL_WR"}, 383 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 11), mmDCORE2_RTR7_CTRL_BASE, 384 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_CTRL_RD"}, 385 {RAZWI_INITIATOR_ID_X_Y_LOW(6, 11), mmDCORE2_RTR4_CTRL_BASE, 386 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE0"}, 387 {RAZWI_INITIATOR_ID_X_Y_LOW(6, 11), mmDCORE2_RTR4_CTRL_BASE, 388 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE1"}, 389 {RAZWI_INITIATOR_ID_X_Y_LOW(7, 11), mmDCORE2_RTR5_CTRL_BASE, 390 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE2"}, 391 {RAZWI_INITIATOR_ID_X_Y_LOW(8, 11), mmDCORE2_RTR6_CTRL_BASE, 392 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE3"}, 393 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 11), mmDCORE2_RTR7_CTRL_BASE, 394 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE4"}, 395 {RAZWI_INITIATOR_ID_X_Y_LOW(12, 11), mmDCORE3_RTR2_CTRL_BASE, 396 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_WAP0"}, 397 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 11), mmDCORE3_RTR0_CTRL_BASE, 398 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_WAP1"}, 399 {RAZWI_INITIATOR_ID_X_Y_LOW(11, 11), mmDCORE3_RTR1_CTRL_BASE, 400 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_CTRL_WR"}, 401 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 11), mmDCORE3_RTR0_CTRL_BASE, 402 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_CTRL_RD"}, 403 {RAZWI_INITIATOR_ID_X_Y_LOW(13, 11), mmDCORE3_RTR3_CTRL_BASE, 404 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE0"}, 405 {RAZWI_INITIATOR_ID_X_Y_LOW(13, 11), mmDCORE3_RTR3_CTRL_BASE, 406 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE1"}, 407 {RAZWI_INITIATOR_ID_X_Y_LOW(12, 11), mmDCORE3_RTR2_CTRL_BASE, 408 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE2"}, 409 {RAZWI_INITIATOR_ID_X_Y_LOW(11, 11), mmDCORE3_RTR1_CTRL_BASE, 410 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE3"}, 411 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 11), mmDCORE3_RTR0_CTRL_BASE, 412 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE4"} 413 }; 414 415 enum hl_pmmu_fatal_cause { 416 LATENCY_RD_OUT_FIFO_OVERRUN, 417 LATENCY_WR_OUT_FIFO_OVERRUN, 418 }; 419 420 enum hl_pcie_drain_ind_cause { 421 LBW_AXI_DRAIN_IND, 422 HBW_AXI_DRAIN_IND 423 }; 424 425 static const u32 cluster_hmmu_hif_enabled_mask[GAUDI2_HBM_NUM] = { 426 [HBM_ID0] = 0xFFFC, 427 [HBM_ID1] = 0xFFCF, 428 [HBM_ID2] = 0xF7F7, 429 [HBM_ID3] = 0x7F7F, 430 [HBM_ID4] = 0xFCFF, 431 [HBM_ID5] = 0xCFFF, 432 }; 433 434 static const u8 xbar_edge_to_hbm_cluster[EDMA_ID_SIZE] = { 435 [0] = HBM_ID0, 436 [1] = HBM_ID1, 437 [2] = HBM_ID4, 438 [3] = HBM_ID5, 439 }; 440 441 static const u8 edma_to_hbm_cluster[EDMA_ID_SIZE] = { 442 [EDMA_ID_DCORE0_INSTANCE0] = HBM_ID0, 443 [EDMA_ID_DCORE0_INSTANCE1] = HBM_ID2, 444 [EDMA_ID_DCORE1_INSTANCE0] = HBM_ID1, 445 [EDMA_ID_DCORE1_INSTANCE1] = HBM_ID3, 446 [EDMA_ID_DCORE2_INSTANCE0] = HBM_ID2, 447 [EDMA_ID_DCORE2_INSTANCE1] = HBM_ID4, 448 [EDMA_ID_DCORE3_INSTANCE0] = HBM_ID3, 449 [EDMA_ID_DCORE3_INSTANCE1] = HBM_ID5, 450 }; 451 452 static const int gaudi2_qman_async_event_id[] = { 453 [GAUDI2_QUEUE_ID_PDMA_0_0] = GAUDI2_EVENT_PDMA0_QM, 454 [GAUDI2_QUEUE_ID_PDMA_0_1] = GAUDI2_EVENT_PDMA0_QM, 455 [GAUDI2_QUEUE_ID_PDMA_0_2] = GAUDI2_EVENT_PDMA0_QM, 456 [GAUDI2_QUEUE_ID_PDMA_0_3] = GAUDI2_EVENT_PDMA0_QM, 457 [GAUDI2_QUEUE_ID_PDMA_1_0] = GAUDI2_EVENT_PDMA1_QM, 458 [GAUDI2_QUEUE_ID_PDMA_1_1] = GAUDI2_EVENT_PDMA1_QM, 459 [GAUDI2_QUEUE_ID_PDMA_1_2] = GAUDI2_EVENT_PDMA1_QM, 460 [GAUDI2_QUEUE_ID_PDMA_1_3] = GAUDI2_EVENT_PDMA1_QM, 461 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0] = GAUDI2_EVENT_HDMA0_QM, 462 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_1] = GAUDI2_EVENT_HDMA0_QM, 463 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_2] = GAUDI2_EVENT_HDMA0_QM, 464 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] = GAUDI2_EVENT_HDMA0_QM, 465 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0] = GAUDI2_EVENT_HDMA1_QM, 466 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_1] = GAUDI2_EVENT_HDMA1_QM, 467 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_2] = GAUDI2_EVENT_HDMA1_QM, 468 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] = GAUDI2_EVENT_HDMA1_QM, 469 [GAUDI2_QUEUE_ID_DCORE0_MME_0_0] = GAUDI2_EVENT_MME0_QM, 470 [GAUDI2_QUEUE_ID_DCORE0_MME_0_1] = GAUDI2_EVENT_MME0_QM, 471 [GAUDI2_QUEUE_ID_DCORE0_MME_0_2] = GAUDI2_EVENT_MME0_QM, 472 [GAUDI2_QUEUE_ID_DCORE0_MME_0_3] = GAUDI2_EVENT_MME0_QM, 473 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_0] = GAUDI2_EVENT_TPC0_QM, 474 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_1] = GAUDI2_EVENT_TPC0_QM, 475 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_2] = GAUDI2_EVENT_TPC0_QM, 476 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] = GAUDI2_EVENT_TPC0_QM, 477 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_0] = GAUDI2_EVENT_TPC1_QM, 478 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_1] = GAUDI2_EVENT_TPC1_QM, 479 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_2] = GAUDI2_EVENT_TPC1_QM, 480 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] = GAUDI2_EVENT_TPC1_QM, 481 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_0] = GAUDI2_EVENT_TPC2_QM, 482 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_1] = GAUDI2_EVENT_TPC2_QM, 483 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_2] = GAUDI2_EVENT_TPC2_QM, 484 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] = GAUDI2_EVENT_TPC2_QM, 485 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_0] = GAUDI2_EVENT_TPC3_QM, 486 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_1] = GAUDI2_EVENT_TPC3_QM, 487 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_2] = GAUDI2_EVENT_TPC3_QM, 488 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] = GAUDI2_EVENT_TPC3_QM, 489 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_0] = GAUDI2_EVENT_TPC4_QM, 490 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_1] = GAUDI2_EVENT_TPC4_QM, 491 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_2] = GAUDI2_EVENT_TPC4_QM, 492 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] = GAUDI2_EVENT_TPC4_QM, 493 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_0] = GAUDI2_EVENT_TPC5_QM, 494 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_1] = GAUDI2_EVENT_TPC5_QM, 495 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_2] = GAUDI2_EVENT_TPC5_QM, 496 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] = GAUDI2_EVENT_TPC5_QM, 497 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_0] = GAUDI2_EVENT_TPC24_QM, 498 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_1] = GAUDI2_EVENT_TPC24_QM, 499 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_2] = GAUDI2_EVENT_TPC24_QM, 500 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] = GAUDI2_EVENT_TPC24_QM, 501 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0] = GAUDI2_EVENT_HDMA2_QM, 502 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_1] = GAUDI2_EVENT_HDMA2_QM, 503 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_2] = GAUDI2_EVENT_HDMA2_QM, 504 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] = GAUDI2_EVENT_HDMA2_QM, 505 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0] = GAUDI2_EVENT_HDMA3_QM, 506 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_1] = GAUDI2_EVENT_HDMA3_QM, 507 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_2] = GAUDI2_EVENT_HDMA3_QM, 508 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] = GAUDI2_EVENT_HDMA3_QM, 509 [GAUDI2_QUEUE_ID_DCORE1_MME_0_0] = GAUDI2_EVENT_MME1_QM, 510 [GAUDI2_QUEUE_ID_DCORE1_MME_0_1] = GAUDI2_EVENT_MME1_QM, 511 [GAUDI2_QUEUE_ID_DCORE1_MME_0_2] = GAUDI2_EVENT_MME1_QM, 512 [GAUDI2_QUEUE_ID_DCORE1_MME_0_3] = GAUDI2_EVENT_MME1_QM, 513 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_0] = GAUDI2_EVENT_TPC6_QM, 514 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_1] = GAUDI2_EVENT_TPC6_QM, 515 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_2] = GAUDI2_EVENT_TPC6_QM, 516 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] = GAUDI2_EVENT_TPC6_QM, 517 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_0] = GAUDI2_EVENT_TPC7_QM, 518 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_1] = GAUDI2_EVENT_TPC7_QM, 519 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_2] = GAUDI2_EVENT_TPC7_QM, 520 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] = GAUDI2_EVENT_TPC7_QM, 521 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_0] = GAUDI2_EVENT_TPC8_QM, 522 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_1] = GAUDI2_EVENT_TPC8_QM, 523 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_2] = GAUDI2_EVENT_TPC8_QM, 524 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] = GAUDI2_EVENT_TPC8_QM, 525 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_0] = GAUDI2_EVENT_TPC9_QM, 526 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_1] = GAUDI2_EVENT_TPC9_QM, 527 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_2] = GAUDI2_EVENT_TPC9_QM, 528 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] = GAUDI2_EVENT_TPC9_QM, 529 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_0] = GAUDI2_EVENT_TPC10_QM, 530 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_1] = GAUDI2_EVENT_TPC10_QM, 531 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_2] = GAUDI2_EVENT_TPC10_QM, 532 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] = GAUDI2_EVENT_TPC10_QM, 533 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_0] = GAUDI2_EVENT_TPC11_QM, 534 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_1] = GAUDI2_EVENT_TPC11_QM, 535 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_2] = GAUDI2_EVENT_TPC11_QM, 536 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] = GAUDI2_EVENT_TPC11_QM, 537 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0] = GAUDI2_EVENT_HDMA4_QM, 538 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_1] = GAUDI2_EVENT_HDMA4_QM, 539 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_2] = GAUDI2_EVENT_HDMA4_QM, 540 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] = GAUDI2_EVENT_HDMA4_QM, 541 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0] = GAUDI2_EVENT_HDMA5_QM, 542 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_1] = GAUDI2_EVENT_HDMA5_QM, 543 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_2] = GAUDI2_EVENT_HDMA5_QM, 544 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] = GAUDI2_EVENT_HDMA5_QM, 545 [GAUDI2_QUEUE_ID_DCORE2_MME_0_0] = GAUDI2_EVENT_MME2_QM, 546 [GAUDI2_QUEUE_ID_DCORE2_MME_0_1] = GAUDI2_EVENT_MME2_QM, 547 [GAUDI2_QUEUE_ID_DCORE2_MME_0_2] = GAUDI2_EVENT_MME2_QM, 548 [GAUDI2_QUEUE_ID_DCORE2_MME_0_3] = GAUDI2_EVENT_MME2_QM, 549 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_0] = GAUDI2_EVENT_TPC12_QM, 550 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_1] = GAUDI2_EVENT_TPC12_QM, 551 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_2] = GAUDI2_EVENT_TPC12_QM, 552 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] = GAUDI2_EVENT_TPC12_QM, 553 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_0] = GAUDI2_EVENT_TPC13_QM, 554 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_1] = GAUDI2_EVENT_TPC13_QM, 555 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_2] = GAUDI2_EVENT_TPC13_QM, 556 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] = GAUDI2_EVENT_TPC13_QM, 557 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_0] = GAUDI2_EVENT_TPC14_QM, 558 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_1] = GAUDI2_EVENT_TPC14_QM, 559 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_2] = GAUDI2_EVENT_TPC14_QM, 560 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] = GAUDI2_EVENT_TPC14_QM, 561 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_0] = GAUDI2_EVENT_TPC15_QM, 562 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_1] = GAUDI2_EVENT_TPC15_QM, 563 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_2] = GAUDI2_EVENT_TPC15_QM, 564 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] = GAUDI2_EVENT_TPC15_QM, 565 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_0] = GAUDI2_EVENT_TPC16_QM, 566 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_1] = GAUDI2_EVENT_TPC16_QM, 567 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_2] = GAUDI2_EVENT_TPC16_QM, 568 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] = GAUDI2_EVENT_TPC16_QM, 569 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_0] = GAUDI2_EVENT_TPC17_QM, 570 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_1] = GAUDI2_EVENT_TPC17_QM, 571 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_2] = GAUDI2_EVENT_TPC17_QM, 572 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] = GAUDI2_EVENT_TPC17_QM, 573 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0] = GAUDI2_EVENT_HDMA6_QM, 574 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_1] = GAUDI2_EVENT_HDMA6_QM, 575 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_2] = GAUDI2_EVENT_HDMA6_QM, 576 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] = GAUDI2_EVENT_HDMA6_QM, 577 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0] = GAUDI2_EVENT_HDMA7_QM, 578 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1] = GAUDI2_EVENT_HDMA7_QM, 579 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2] = GAUDI2_EVENT_HDMA7_QM, 580 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] = GAUDI2_EVENT_HDMA7_QM, 581 [GAUDI2_QUEUE_ID_DCORE3_MME_0_0] = GAUDI2_EVENT_MME3_QM, 582 [GAUDI2_QUEUE_ID_DCORE3_MME_0_1] = GAUDI2_EVENT_MME3_QM, 583 [GAUDI2_QUEUE_ID_DCORE3_MME_0_2] = GAUDI2_EVENT_MME3_QM, 584 [GAUDI2_QUEUE_ID_DCORE3_MME_0_3] = GAUDI2_EVENT_MME3_QM, 585 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_0] = GAUDI2_EVENT_TPC18_QM, 586 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_1] = GAUDI2_EVENT_TPC18_QM, 587 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_2] = GAUDI2_EVENT_TPC18_QM, 588 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] = GAUDI2_EVENT_TPC18_QM, 589 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_0] = GAUDI2_EVENT_TPC19_QM, 590 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_1] = GAUDI2_EVENT_TPC19_QM, 591 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_2] = GAUDI2_EVENT_TPC19_QM, 592 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] = GAUDI2_EVENT_TPC19_QM, 593 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_0] = GAUDI2_EVENT_TPC20_QM, 594 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_1] = GAUDI2_EVENT_TPC20_QM, 595 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_2] = GAUDI2_EVENT_TPC20_QM, 596 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] = GAUDI2_EVENT_TPC20_QM, 597 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_0] = GAUDI2_EVENT_TPC21_QM, 598 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_1] = GAUDI2_EVENT_TPC21_QM, 599 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_2] = GAUDI2_EVENT_TPC21_QM, 600 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] = GAUDI2_EVENT_TPC21_QM, 601 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_0] = GAUDI2_EVENT_TPC22_QM, 602 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_1] = GAUDI2_EVENT_TPC22_QM, 603 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_2] = GAUDI2_EVENT_TPC22_QM, 604 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] = GAUDI2_EVENT_TPC22_QM, 605 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_0] = GAUDI2_EVENT_TPC23_QM, 606 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_1] = GAUDI2_EVENT_TPC23_QM, 607 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_2] = GAUDI2_EVENT_TPC23_QM, 608 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] = GAUDI2_EVENT_TPC23_QM, 609 [GAUDI2_QUEUE_ID_NIC_0_0] = GAUDI2_EVENT_NIC0_QM0, 610 [GAUDI2_QUEUE_ID_NIC_0_1] = GAUDI2_EVENT_NIC0_QM0, 611 [GAUDI2_QUEUE_ID_NIC_0_2] = GAUDI2_EVENT_NIC0_QM0, 612 [GAUDI2_QUEUE_ID_NIC_0_3] = GAUDI2_EVENT_NIC0_QM0, 613 [GAUDI2_QUEUE_ID_NIC_1_0] = GAUDI2_EVENT_NIC0_QM1, 614 [GAUDI2_QUEUE_ID_NIC_1_1] = GAUDI2_EVENT_NIC0_QM1, 615 [GAUDI2_QUEUE_ID_NIC_1_2] = GAUDI2_EVENT_NIC0_QM1, 616 [GAUDI2_QUEUE_ID_NIC_1_3] = GAUDI2_EVENT_NIC0_QM1, 617 [GAUDI2_QUEUE_ID_NIC_2_0] = GAUDI2_EVENT_NIC1_QM0, 618 [GAUDI2_QUEUE_ID_NIC_2_1] = GAUDI2_EVENT_NIC1_QM0, 619 [GAUDI2_QUEUE_ID_NIC_2_2] = GAUDI2_EVENT_NIC1_QM0, 620 [GAUDI2_QUEUE_ID_NIC_2_3] = GAUDI2_EVENT_NIC1_QM0, 621 [GAUDI2_QUEUE_ID_NIC_3_0] = GAUDI2_EVENT_NIC1_QM1, 622 [GAUDI2_QUEUE_ID_NIC_3_1] = GAUDI2_EVENT_NIC1_QM1, 623 [GAUDI2_QUEUE_ID_NIC_3_2] = GAUDI2_EVENT_NIC1_QM1, 624 [GAUDI2_QUEUE_ID_NIC_3_3] = GAUDI2_EVENT_NIC1_QM1, 625 [GAUDI2_QUEUE_ID_NIC_4_0] = GAUDI2_EVENT_NIC2_QM0, 626 [GAUDI2_QUEUE_ID_NIC_4_1] = GAUDI2_EVENT_NIC2_QM0, 627 [GAUDI2_QUEUE_ID_NIC_4_2] = GAUDI2_EVENT_NIC2_QM0, 628 [GAUDI2_QUEUE_ID_NIC_4_3] = GAUDI2_EVENT_NIC2_QM0, 629 [GAUDI2_QUEUE_ID_NIC_5_0] = GAUDI2_EVENT_NIC2_QM1, 630 [GAUDI2_QUEUE_ID_NIC_5_1] = GAUDI2_EVENT_NIC2_QM1, 631 [GAUDI2_QUEUE_ID_NIC_5_2] = GAUDI2_EVENT_NIC2_QM1, 632 [GAUDI2_QUEUE_ID_NIC_5_3] = GAUDI2_EVENT_NIC2_QM1, 633 [GAUDI2_QUEUE_ID_NIC_6_0] = GAUDI2_EVENT_NIC3_QM0, 634 [GAUDI2_QUEUE_ID_NIC_6_1] = GAUDI2_EVENT_NIC3_QM0, 635 [GAUDI2_QUEUE_ID_NIC_6_2] = GAUDI2_EVENT_NIC3_QM0, 636 [GAUDI2_QUEUE_ID_NIC_6_3] = GAUDI2_EVENT_NIC3_QM0, 637 [GAUDI2_QUEUE_ID_NIC_7_0] = GAUDI2_EVENT_NIC3_QM1, 638 [GAUDI2_QUEUE_ID_NIC_7_1] = GAUDI2_EVENT_NIC3_QM1, 639 [GAUDI2_QUEUE_ID_NIC_7_2] = GAUDI2_EVENT_NIC3_QM1, 640 [GAUDI2_QUEUE_ID_NIC_7_3] = GAUDI2_EVENT_NIC3_QM1, 641 [GAUDI2_QUEUE_ID_NIC_8_0] = GAUDI2_EVENT_NIC4_QM0, 642 [GAUDI2_QUEUE_ID_NIC_8_1] = GAUDI2_EVENT_NIC4_QM0, 643 [GAUDI2_QUEUE_ID_NIC_8_2] = GAUDI2_EVENT_NIC4_QM0, 644 [GAUDI2_QUEUE_ID_NIC_8_3] = GAUDI2_EVENT_NIC4_QM0, 645 [GAUDI2_QUEUE_ID_NIC_9_0] = GAUDI2_EVENT_NIC4_QM1, 646 [GAUDI2_QUEUE_ID_NIC_9_1] = GAUDI2_EVENT_NIC4_QM1, 647 [GAUDI2_QUEUE_ID_NIC_9_2] = GAUDI2_EVENT_NIC4_QM1, 648 [GAUDI2_QUEUE_ID_NIC_9_3] = GAUDI2_EVENT_NIC4_QM1, 649 [GAUDI2_QUEUE_ID_NIC_10_0] = GAUDI2_EVENT_NIC5_QM0, 650 [GAUDI2_QUEUE_ID_NIC_10_1] = GAUDI2_EVENT_NIC5_QM0, 651 [GAUDI2_QUEUE_ID_NIC_10_2] = GAUDI2_EVENT_NIC5_QM0, 652 [GAUDI2_QUEUE_ID_NIC_10_3] = GAUDI2_EVENT_NIC5_QM0, 653 [GAUDI2_QUEUE_ID_NIC_11_0] = GAUDI2_EVENT_NIC5_QM1, 654 [GAUDI2_QUEUE_ID_NIC_11_1] = GAUDI2_EVENT_NIC5_QM1, 655 [GAUDI2_QUEUE_ID_NIC_11_2] = GAUDI2_EVENT_NIC5_QM1, 656 [GAUDI2_QUEUE_ID_NIC_11_3] = GAUDI2_EVENT_NIC5_QM1, 657 [GAUDI2_QUEUE_ID_NIC_12_0] = GAUDI2_EVENT_NIC6_QM0, 658 [GAUDI2_QUEUE_ID_NIC_12_1] = GAUDI2_EVENT_NIC6_QM0, 659 [GAUDI2_QUEUE_ID_NIC_12_2] = GAUDI2_EVENT_NIC6_QM0, 660 [GAUDI2_QUEUE_ID_NIC_12_3] = GAUDI2_EVENT_NIC6_QM0, 661 [GAUDI2_QUEUE_ID_NIC_13_0] = GAUDI2_EVENT_NIC6_QM1, 662 [GAUDI2_QUEUE_ID_NIC_13_1] = GAUDI2_EVENT_NIC6_QM1, 663 [GAUDI2_QUEUE_ID_NIC_13_2] = GAUDI2_EVENT_NIC6_QM1, 664 [GAUDI2_QUEUE_ID_NIC_13_3] = GAUDI2_EVENT_NIC6_QM1, 665 [GAUDI2_QUEUE_ID_NIC_14_0] = GAUDI2_EVENT_NIC7_QM0, 666 [GAUDI2_QUEUE_ID_NIC_14_1] = GAUDI2_EVENT_NIC7_QM0, 667 [GAUDI2_QUEUE_ID_NIC_14_2] = GAUDI2_EVENT_NIC7_QM0, 668 [GAUDI2_QUEUE_ID_NIC_14_3] = GAUDI2_EVENT_NIC7_QM0, 669 [GAUDI2_QUEUE_ID_NIC_15_0] = GAUDI2_EVENT_NIC7_QM1, 670 [GAUDI2_QUEUE_ID_NIC_15_1] = GAUDI2_EVENT_NIC7_QM1, 671 [GAUDI2_QUEUE_ID_NIC_15_2] = GAUDI2_EVENT_NIC7_QM1, 672 [GAUDI2_QUEUE_ID_NIC_15_3] = GAUDI2_EVENT_NIC7_QM1, 673 [GAUDI2_QUEUE_ID_NIC_16_0] = GAUDI2_EVENT_NIC8_QM0, 674 [GAUDI2_QUEUE_ID_NIC_16_1] = GAUDI2_EVENT_NIC8_QM0, 675 [GAUDI2_QUEUE_ID_NIC_16_2] = GAUDI2_EVENT_NIC8_QM0, 676 [GAUDI2_QUEUE_ID_NIC_16_3] = GAUDI2_EVENT_NIC8_QM0, 677 [GAUDI2_QUEUE_ID_NIC_17_0] = GAUDI2_EVENT_NIC8_QM1, 678 [GAUDI2_QUEUE_ID_NIC_17_1] = GAUDI2_EVENT_NIC8_QM1, 679 [GAUDI2_QUEUE_ID_NIC_17_2] = GAUDI2_EVENT_NIC8_QM1, 680 [GAUDI2_QUEUE_ID_NIC_17_3] = GAUDI2_EVENT_NIC8_QM1, 681 [GAUDI2_QUEUE_ID_NIC_18_0] = GAUDI2_EVENT_NIC9_QM0, 682 [GAUDI2_QUEUE_ID_NIC_18_1] = GAUDI2_EVENT_NIC9_QM0, 683 [GAUDI2_QUEUE_ID_NIC_18_2] = GAUDI2_EVENT_NIC9_QM0, 684 [GAUDI2_QUEUE_ID_NIC_18_3] = GAUDI2_EVENT_NIC9_QM0, 685 [GAUDI2_QUEUE_ID_NIC_19_0] = GAUDI2_EVENT_NIC9_QM1, 686 [GAUDI2_QUEUE_ID_NIC_19_1] = GAUDI2_EVENT_NIC9_QM1, 687 [GAUDI2_QUEUE_ID_NIC_19_2] = GAUDI2_EVENT_NIC9_QM1, 688 [GAUDI2_QUEUE_ID_NIC_19_3] = GAUDI2_EVENT_NIC9_QM1, 689 [GAUDI2_QUEUE_ID_NIC_20_0] = GAUDI2_EVENT_NIC10_QM0, 690 [GAUDI2_QUEUE_ID_NIC_20_1] = GAUDI2_EVENT_NIC10_QM0, 691 [GAUDI2_QUEUE_ID_NIC_20_2] = GAUDI2_EVENT_NIC10_QM0, 692 [GAUDI2_QUEUE_ID_NIC_20_3] = GAUDI2_EVENT_NIC10_QM0, 693 [GAUDI2_QUEUE_ID_NIC_21_0] = GAUDI2_EVENT_NIC10_QM1, 694 [GAUDI2_QUEUE_ID_NIC_21_1] = GAUDI2_EVENT_NIC10_QM1, 695 [GAUDI2_QUEUE_ID_NIC_21_2] = GAUDI2_EVENT_NIC10_QM1, 696 [GAUDI2_QUEUE_ID_NIC_21_3] = GAUDI2_EVENT_NIC10_QM1, 697 [GAUDI2_QUEUE_ID_NIC_22_0] = GAUDI2_EVENT_NIC11_QM0, 698 [GAUDI2_QUEUE_ID_NIC_22_1] = GAUDI2_EVENT_NIC11_QM0, 699 [GAUDI2_QUEUE_ID_NIC_22_2] = GAUDI2_EVENT_NIC11_QM0, 700 [GAUDI2_QUEUE_ID_NIC_22_3] = GAUDI2_EVENT_NIC11_QM0, 701 [GAUDI2_QUEUE_ID_NIC_23_0] = GAUDI2_EVENT_NIC11_QM1, 702 [GAUDI2_QUEUE_ID_NIC_23_1] = GAUDI2_EVENT_NIC11_QM1, 703 [GAUDI2_QUEUE_ID_NIC_23_2] = GAUDI2_EVENT_NIC11_QM1, 704 [GAUDI2_QUEUE_ID_NIC_23_3] = GAUDI2_EVENT_NIC11_QM1, 705 [GAUDI2_QUEUE_ID_ROT_0_0] = GAUDI2_EVENT_ROTATOR0_ROT0_QM, 706 [GAUDI2_QUEUE_ID_ROT_0_1] = GAUDI2_EVENT_ROTATOR0_ROT0_QM, 707 [GAUDI2_QUEUE_ID_ROT_0_2] = GAUDI2_EVENT_ROTATOR0_ROT0_QM, 708 [GAUDI2_QUEUE_ID_ROT_0_3] = GAUDI2_EVENT_ROTATOR0_ROT0_QM, 709 [GAUDI2_QUEUE_ID_ROT_1_0] = GAUDI2_EVENT_ROTATOR1_ROT1_QM, 710 [GAUDI2_QUEUE_ID_ROT_1_1] = GAUDI2_EVENT_ROTATOR1_ROT1_QM, 711 [GAUDI2_QUEUE_ID_ROT_1_2] = GAUDI2_EVENT_ROTATOR1_ROT1_QM, 712 [GAUDI2_QUEUE_ID_ROT_1_3] = GAUDI2_EVENT_ROTATOR1_ROT1_QM 713 }; 714 715 static const int gaudi2_dma_core_async_event_id[] = { 716 [DMA_CORE_ID_EDMA0] = GAUDI2_EVENT_HDMA0_CORE, 717 [DMA_CORE_ID_EDMA1] = GAUDI2_EVENT_HDMA1_CORE, 718 [DMA_CORE_ID_EDMA2] = GAUDI2_EVENT_HDMA2_CORE, 719 [DMA_CORE_ID_EDMA3] = GAUDI2_EVENT_HDMA3_CORE, 720 [DMA_CORE_ID_EDMA4] = GAUDI2_EVENT_HDMA4_CORE, 721 [DMA_CORE_ID_EDMA5] = GAUDI2_EVENT_HDMA5_CORE, 722 [DMA_CORE_ID_EDMA6] = GAUDI2_EVENT_HDMA6_CORE, 723 [DMA_CORE_ID_EDMA7] = GAUDI2_EVENT_HDMA7_CORE, 724 [DMA_CORE_ID_PDMA0] = GAUDI2_EVENT_PDMA0_CORE, 725 [DMA_CORE_ID_PDMA1] = GAUDI2_EVENT_PDMA1_CORE, 726 [DMA_CORE_ID_KDMA] = GAUDI2_EVENT_KDMA0_CORE, 727 }; 728 729 static const char * const gaudi2_qm_sei_error_cause[GAUDI2_NUM_OF_QM_SEI_ERR_CAUSE] = { 730 "qman sei intr", 731 "arc sei intr" 732 }; 733 734 static const char * const gaudi2_cpu_sei_error_cause[GAUDI2_NUM_OF_CPU_SEI_ERR_CAUSE] = { 735 "AXI_TERMINATOR WR", 736 "AXI_TERMINATOR RD", 737 "AXI SPLIT SEI Status" 738 }; 739 740 static const char * const gaudi2_arc_sei_error_cause[GAUDI2_NUM_OF_ARC_SEI_ERR_CAUSE] = { 741 "cbu_bresp_sei_intr_cause", 742 "cbu_rresp_sei_intr_cause", 743 "lbu_bresp_sei_intr_cause", 744 "lbu_rresp_sei_intr_cause", 745 "cbu_axi_split_intr_cause", 746 "lbu_axi_split_intr_cause", 747 "arc_ip_excptn_sei_intr_cause", 748 "dmi_bresp_sei_intr_cause", 749 "aux2apb_err_sei_intr_cause", 750 "cfg_lbw_wr_terminated_intr_cause", 751 "cfg_lbw_rd_terminated_intr_cause", 752 "cfg_dccm_wr_terminated_intr_cause", 753 "cfg_dccm_rd_terminated_intr_cause", 754 "cfg_hbw_rd_terminated_intr_cause" 755 }; 756 757 static const char * const gaudi2_dec_error_cause[GAUDI2_NUM_OF_DEC_ERR_CAUSE] = { 758 "msix_vcd_hbw_sei", 759 "msix_l2c_hbw_sei", 760 "msix_nrm_hbw_sei", 761 "msix_abnrm_hbw_sei", 762 "msix_vcd_lbw_sei", 763 "msix_l2c_lbw_sei", 764 "msix_nrm_lbw_sei", 765 "msix_abnrm_lbw_sei", 766 "apb_vcd_lbw_sei", 767 "apb_l2c_lbw_sei", 768 "apb_nrm_lbw_sei", 769 "apb_abnrm_lbw_sei", 770 "dec_sei", 771 "dec_apb_sei", 772 "trc_apb_sei", 773 "lbw_mstr_if_sei", 774 "axi_split_bresp_err_sei", 775 "hbw_axi_wr_viol_sei", 776 "hbw_axi_rd_viol_sei", 777 "lbw_axi_wr_viol_sei", 778 "lbw_axi_rd_viol_sei", 779 "vcd_spi", 780 "l2c_spi", 781 "nrm_spi", 782 "abnrm_spi", 783 }; 784 785 static const char * const gaudi2_qman_error_cause[GAUDI2_NUM_OF_QM_ERR_CAUSE] = { 786 "PQ AXI HBW error", 787 "CQ AXI HBW error", 788 "CP AXI HBW error", 789 "CP error due to undefined OPCODE", 790 "CP encountered STOP OPCODE", 791 "CP AXI LBW error", 792 "CP WRREG32 or WRBULK returned error", 793 "N/A", 794 "FENCE 0 inc over max value and clipped", 795 "FENCE 1 inc over max value and clipped", 796 "FENCE 2 inc over max value and clipped", 797 "FENCE 3 inc over max value and clipped", 798 "FENCE 0 dec under min value and clipped", 799 "FENCE 1 dec under min value and clipped", 800 "FENCE 2 dec under min value and clipped", 801 "FENCE 3 dec under min value and clipped", 802 "CPDMA Up overflow", 803 "PQC L2H error" 804 }; 805 806 static const char * const gaudi2_lower_qman_error_cause[GAUDI2_NUM_OF_LOWER_QM_ERR_CAUSE] = { 807 "RSVD0", 808 "CQ AXI HBW error", 809 "CP AXI HBW error", 810 "CP error due to undefined OPCODE", 811 "CP encountered STOP OPCODE", 812 "CP AXI LBW error", 813 "CP WRREG32 or WRBULK returned error", 814 "N/A", 815 "FENCE 0 inc over max value and clipped", 816 "FENCE 1 inc over max value and clipped", 817 "FENCE 2 inc over max value and clipped", 818 "FENCE 3 inc over max value and clipped", 819 "FENCE 0 dec under min value and clipped", 820 "FENCE 1 dec under min value and clipped", 821 "FENCE 2 dec under min value and clipped", 822 "FENCE 3 dec under min value and clipped", 823 "CPDMA Up overflow", 824 "RSVD17", 825 "CQ_WR_IFIFO_CI_ERR", 826 "CQ_WR_CTL_CI_ERR", 827 "ARC_CQF_RD_ERR", 828 "ARC_CQ_WR_IFIFO_CI_ERR", 829 "ARC_CQ_WR_CTL_CI_ERR", 830 "ARC_AXI_ERR", 831 "CP_SWITCH_WDT_ERR" 832 }; 833 834 static const char * const gaudi2_qman_arb_error_cause[GAUDI2_NUM_OF_QM_ARB_ERR_CAUSE] = { 835 "Choice push while full error", 836 "Choice Q watchdog error", 837 "MSG AXI LBW returned with error" 838 }; 839 840 static const char * const guadi2_rot_error_cause[GAUDI2_NUM_OF_ROT_ERR_CAUSE] = { 841 "qm_axi_err", 842 "qm_trace_fence_events", 843 "qm_sw_err", 844 "qm_cp_sw_stop", 845 "lbw_mstr_rresp_err", 846 "lbw_mstr_bresp_err", 847 "lbw_msg_slverr", 848 "hbw_msg_slverr", 849 "wbc_slverr", 850 "hbw_mstr_rresp_err", 851 "hbw_mstr_bresp_err", 852 "sb_resp_intr", 853 "mrsb_resp_intr", 854 "core_dw_status_0", 855 "core_dw_status_1", 856 "core_dw_status_2", 857 "core_dw_status_3", 858 "core_dw_status_4", 859 "core_dw_status_5", 860 "core_dw_status_6", 861 "core_dw_status_7", 862 "async_arc2cpu_sei_intr", 863 }; 864 865 static const char * const gaudi2_tpc_interrupts_cause[GAUDI2_NUM_OF_TPC_INTR_CAUSE] = { 866 "tpc_address_exceed_slm", 867 "tpc_div_by_0", 868 "tpc_spu_mac_overflow", 869 "tpc_spu_addsub_overflow", 870 "tpc_spu_abs_overflow", 871 "tpc_spu_fma_fp_dst_nan", 872 "tpc_spu_fma_fp_dst_inf", 873 "tpc_spu_convert_fp_dst_nan", 874 "tpc_spu_convert_fp_dst_inf", 875 "tpc_spu_fp_dst_denorm", 876 "tpc_vpu_mac_overflow", 877 "tpc_vpu_addsub_overflow", 878 "tpc_vpu_abs_overflow", 879 "tpc_vpu_convert_fp_dst_nan", 880 "tpc_vpu_convert_fp_dst_inf", 881 "tpc_vpu_fma_fp_dst_nan", 882 "tpc_vpu_fma_fp_dst_inf", 883 "tpc_vpu_fp_dst_denorm", 884 "tpc_assertions", 885 "tpc_illegal_instruction", 886 "tpc_pc_wrap_around", 887 "tpc_qm_sw_err", 888 "tpc_hbw_rresp_err", 889 "tpc_hbw_bresp_err", 890 "tpc_lbw_rresp_err", 891 "tpc_lbw_bresp_err", 892 "st_unlock_already_locked", 893 "invalid_lock_access", 894 "LD_L protection violation", 895 "ST_L protection violation", 896 "D$ L0CS mismatch", 897 }; 898 899 static const char * const guadi2_mme_error_cause[GAUDI2_NUM_OF_MME_ERR_CAUSE] = { 900 "agu_resp_intr", 901 "qman_axi_err", 902 "wap sei (wbc axi err)", 903 "arc sei", 904 "cfg access error", 905 "qm_sw_err", 906 "sbte_dbg_intr_0", 907 "sbte_dbg_intr_1", 908 "sbte_dbg_intr_2", 909 "sbte_dbg_intr_3", 910 "sbte_dbg_intr_4", 911 "sbte_prtn_intr_0", 912 "sbte_prtn_intr_1", 913 "sbte_prtn_intr_2", 914 "sbte_prtn_intr_3", 915 "sbte_prtn_intr_4", 916 }; 917 918 static const char * const guadi2_mme_wap_error_cause[GAUDI2_NUM_OF_MME_WAP_ERR_CAUSE] = { 919 "WBC ERR RESP_0", 920 "WBC ERR RESP_1", 921 "AP SOURCE POS INF", 922 "AP SOURCE NEG INF", 923 "AP SOURCE NAN", 924 "AP RESULT POS INF", 925 "AP RESULT NEG INF", 926 }; 927 928 static const char * const gaudi2_dma_core_interrupts_cause[GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE] = { 929 "HBW Read returned with error RRESP", 930 "HBW write returned with error BRESP", 931 "LBW write returned with error BRESP", 932 "descriptor_fifo_overflow", 933 "KDMA SB LBW Read returned with error", 934 "KDMA WBC LBW Write returned with error", 935 "TRANSPOSE ENGINE DESC FIFO OVERFLOW", 936 "WRONG CFG FOR COMMIT IN LIN DMA" 937 }; 938 939 static const char * const gaudi2_kdma_core_interrupts_cause[GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE] = { 940 "HBW/LBW Read returned with error RRESP", 941 "HBW/LBW write returned with error BRESP", 942 "LBW write returned with error BRESP", 943 "descriptor_fifo_overflow", 944 "KDMA SB LBW Read returned with error", 945 "KDMA WBC LBW Write returned with error", 946 "TRANSPOSE ENGINE DESC FIFO OVERFLOW", 947 "WRONG CFG FOR COMMIT IN LIN DMA" 948 }; 949 950 struct gaudi2_sm_sei_cause_data { 951 const char *cause_name; 952 const char *log_name; 953 }; 954 955 static const struct gaudi2_sm_sei_cause_data 956 gaudi2_sm_sei_cause[GAUDI2_NUM_OF_SM_SEI_ERR_CAUSE] = { 957 {"calculated SO value overflow/underflow", "SOB ID"}, 958 {"payload address of monitor is not aligned to 4B", "monitor addr"}, 959 {"armed monitor write got BRESP (SLVERR or DECERR)", "AXI id"}, 960 }; 961 962 static const char * const 963 gaudi2_pmmu_fatal_interrupts_cause[GAUDI2_NUM_OF_PMMU_FATAL_ERR_CAUSE] = { 964 "LATENCY_RD_OUT_FIFO_OVERRUN", 965 "LATENCY_WR_OUT_FIFO_OVERRUN", 966 }; 967 968 static const char * const 969 gaudi2_hif_fatal_interrupts_cause[GAUDI2_NUM_OF_HIF_FATAL_ERR_CAUSE] = { 970 "LATENCY_RD_OUT_FIFO_OVERRUN", 971 "LATENCY_WR_OUT_FIFO_OVERRUN", 972 }; 973 974 static const char * const 975 gaudi2_psoc_axi_drain_interrupts_cause[GAUDI2_NUM_OF_AXI_DRAIN_ERR_CAUSE] = { 976 "AXI drain HBW", 977 "AXI drain LBW", 978 }; 979 980 static const char * const 981 gaudi2_pcie_addr_dec_error_cause[GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE] = { 982 "HBW error response", 983 "LBW error response", 984 "TLP is blocked by RR" 985 }; 986 987 static const int gaudi2_queue_id_to_engine_id[] = { 988 [GAUDI2_QUEUE_ID_PDMA_0_0...GAUDI2_QUEUE_ID_PDMA_0_3] = GAUDI2_ENGINE_ID_PDMA_0, 989 [GAUDI2_QUEUE_ID_PDMA_1_0...GAUDI2_QUEUE_ID_PDMA_1_3] = GAUDI2_ENGINE_ID_PDMA_1, 990 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] = 991 GAUDI2_DCORE0_ENGINE_ID_EDMA_0, 992 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] = 993 GAUDI2_DCORE0_ENGINE_ID_EDMA_1, 994 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] = 995 GAUDI2_DCORE1_ENGINE_ID_EDMA_0, 996 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0...GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] = 997 GAUDI2_DCORE1_ENGINE_ID_EDMA_1, 998 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] = 999 GAUDI2_DCORE2_ENGINE_ID_EDMA_0, 1000 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0...GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] = 1001 GAUDI2_DCORE2_ENGINE_ID_EDMA_1, 1002 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] = 1003 GAUDI2_DCORE3_ENGINE_ID_EDMA_0, 1004 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0...GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] = 1005 GAUDI2_DCORE3_ENGINE_ID_EDMA_1, 1006 [GAUDI2_QUEUE_ID_DCORE0_MME_0_0...GAUDI2_QUEUE_ID_DCORE0_MME_0_3] = 1007 GAUDI2_DCORE0_ENGINE_ID_MME, 1008 [GAUDI2_QUEUE_ID_DCORE1_MME_0_0...GAUDI2_QUEUE_ID_DCORE1_MME_0_3] = 1009 GAUDI2_DCORE1_ENGINE_ID_MME, 1010 [GAUDI2_QUEUE_ID_DCORE2_MME_0_0...GAUDI2_QUEUE_ID_DCORE2_MME_0_3] = 1011 GAUDI2_DCORE2_ENGINE_ID_MME, 1012 [GAUDI2_QUEUE_ID_DCORE3_MME_0_0...GAUDI2_QUEUE_ID_DCORE3_MME_0_3] = 1013 GAUDI2_DCORE3_ENGINE_ID_MME, 1014 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_0...GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] = 1015 GAUDI2_DCORE0_ENGINE_ID_TPC_0, 1016 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_0...GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] = 1017 GAUDI2_DCORE0_ENGINE_ID_TPC_1, 1018 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_0...GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] = 1019 GAUDI2_DCORE0_ENGINE_ID_TPC_2, 1020 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_0...GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] = 1021 GAUDI2_DCORE0_ENGINE_ID_TPC_3, 1022 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_0...GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] = 1023 GAUDI2_DCORE0_ENGINE_ID_TPC_4, 1024 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_0...GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] = 1025 GAUDI2_DCORE0_ENGINE_ID_TPC_5, 1026 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_0...GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] = 1027 GAUDI2_DCORE0_ENGINE_ID_TPC_6, 1028 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_0...GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] = 1029 GAUDI2_DCORE1_ENGINE_ID_TPC_0, 1030 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_0...GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] = 1031 GAUDI2_DCORE1_ENGINE_ID_TPC_1, 1032 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_0...GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] = 1033 GAUDI2_DCORE1_ENGINE_ID_TPC_2, 1034 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_0...GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] = 1035 GAUDI2_DCORE1_ENGINE_ID_TPC_3, 1036 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_0...GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] = 1037 GAUDI2_DCORE1_ENGINE_ID_TPC_4, 1038 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_0...GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] = 1039 GAUDI2_DCORE1_ENGINE_ID_TPC_5, 1040 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_0...GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] = 1041 GAUDI2_DCORE2_ENGINE_ID_TPC_0, 1042 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_0...GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] = 1043 GAUDI2_DCORE2_ENGINE_ID_TPC_1, 1044 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_0...GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] = 1045 GAUDI2_DCORE2_ENGINE_ID_TPC_2, 1046 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_0...GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] = 1047 GAUDI2_DCORE2_ENGINE_ID_TPC_3, 1048 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_0...GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] = 1049 GAUDI2_DCORE2_ENGINE_ID_TPC_4, 1050 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_0...GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] = 1051 GAUDI2_DCORE2_ENGINE_ID_TPC_5, 1052 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_0...GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] = 1053 GAUDI2_DCORE3_ENGINE_ID_TPC_0, 1054 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_0...GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] = 1055 GAUDI2_DCORE3_ENGINE_ID_TPC_1, 1056 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_0...GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] = 1057 GAUDI2_DCORE3_ENGINE_ID_TPC_2, 1058 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_0...GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] = 1059 GAUDI2_DCORE3_ENGINE_ID_TPC_3, 1060 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_0...GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] = 1061 GAUDI2_DCORE3_ENGINE_ID_TPC_4, 1062 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_0...GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] = 1063 GAUDI2_DCORE3_ENGINE_ID_TPC_5, 1064 [GAUDI2_QUEUE_ID_NIC_0_0...GAUDI2_QUEUE_ID_NIC_0_3] = GAUDI2_ENGINE_ID_NIC0_0, 1065 [GAUDI2_QUEUE_ID_NIC_1_0...GAUDI2_QUEUE_ID_NIC_1_3] = GAUDI2_ENGINE_ID_NIC0_1, 1066 [GAUDI2_QUEUE_ID_NIC_2_0...GAUDI2_QUEUE_ID_NIC_2_3] = GAUDI2_ENGINE_ID_NIC1_0, 1067 [GAUDI2_QUEUE_ID_NIC_3_0...GAUDI2_QUEUE_ID_NIC_3_3] = GAUDI2_ENGINE_ID_NIC1_1, 1068 [GAUDI2_QUEUE_ID_NIC_4_0...GAUDI2_QUEUE_ID_NIC_4_3] = GAUDI2_ENGINE_ID_NIC2_0, 1069 [GAUDI2_QUEUE_ID_NIC_5_0...GAUDI2_QUEUE_ID_NIC_5_3] = GAUDI2_ENGINE_ID_NIC2_1, 1070 [GAUDI2_QUEUE_ID_NIC_6_0...GAUDI2_QUEUE_ID_NIC_6_3] = GAUDI2_ENGINE_ID_NIC3_0, 1071 [GAUDI2_QUEUE_ID_NIC_7_0...GAUDI2_QUEUE_ID_NIC_7_3] = GAUDI2_ENGINE_ID_NIC3_1, 1072 [GAUDI2_QUEUE_ID_NIC_8_0...GAUDI2_QUEUE_ID_NIC_8_3] = GAUDI2_ENGINE_ID_NIC4_0, 1073 [GAUDI2_QUEUE_ID_NIC_9_0...GAUDI2_QUEUE_ID_NIC_9_3] = GAUDI2_ENGINE_ID_NIC4_1, 1074 [GAUDI2_QUEUE_ID_NIC_10_0...GAUDI2_QUEUE_ID_NIC_10_3] = GAUDI2_ENGINE_ID_NIC5_0, 1075 [GAUDI2_QUEUE_ID_NIC_11_0...GAUDI2_QUEUE_ID_NIC_11_3] = GAUDI2_ENGINE_ID_NIC5_1, 1076 [GAUDI2_QUEUE_ID_NIC_12_0...GAUDI2_QUEUE_ID_NIC_12_3] = GAUDI2_ENGINE_ID_NIC6_0, 1077 [GAUDI2_QUEUE_ID_NIC_13_0...GAUDI2_QUEUE_ID_NIC_13_3] = GAUDI2_ENGINE_ID_NIC6_1, 1078 [GAUDI2_QUEUE_ID_NIC_14_0...GAUDI2_QUEUE_ID_NIC_14_3] = GAUDI2_ENGINE_ID_NIC7_0, 1079 [GAUDI2_QUEUE_ID_NIC_15_0...GAUDI2_QUEUE_ID_NIC_15_3] = GAUDI2_ENGINE_ID_NIC7_1, 1080 [GAUDI2_QUEUE_ID_NIC_16_0...GAUDI2_QUEUE_ID_NIC_16_3] = GAUDI2_ENGINE_ID_NIC8_0, 1081 [GAUDI2_QUEUE_ID_NIC_17_0...GAUDI2_QUEUE_ID_NIC_17_3] = GAUDI2_ENGINE_ID_NIC8_1, 1082 [GAUDI2_QUEUE_ID_NIC_18_0...GAUDI2_QUEUE_ID_NIC_18_3] = GAUDI2_ENGINE_ID_NIC9_0, 1083 [GAUDI2_QUEUE_ID_NIC_19_0...GAUDI2_QUEUE_ID_NIC_19_3] = GAUDI2_ENGINE_ID_NIC9_1, 1084 [GAUDI2_QUEUE_ID_NIC_20_0...GAUDI2_QUEUE_ID_NIC_20_3] = GAUDI2_ENGINE_ID_NIC10_0, 1085 [GAUDI2_QUEUE_ID_NIC_21_0...GAUDI2_QUEUE_ID_NIC_21_3] = GAUDI2_ENGINE_ID_NIC10_1, 1086 [GAUDI2_QUEUE_ID_NIC_22_0...GAUDI2_QUEUE_ID_NIC_22_3] = GAUDI2_ENGINE_ID_NIC11_0, 1087 [GAUDI2_QUEUE_ID_NIC_23_0...GAUDI2_QUEUE_ID_NIC_23_3] = GAUDI2_ENGINE_ID_NIC11_1, 1088 [GAUDI2_QUEUE_ID_ROT_0_0...GAUDI2_QUEUE_ID_ROT_0_3] = GAUDI2_ENGINE_ID_ROT_0, 1089 [GAUDI2_QUEUE_ID_ROT_1_0...GAUDI2_QUEUE_ID_ROT_1_3] = GAUDI2_ENGINE_ID_ROT_1, 1090 }; 1091 1092 const u32 gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_SIZE] = { 1093 [GAUDI2_QUEUE_ID_PDMA_0_0] = mmPDMA0_QM_BASE, 1094 [GAUDI2_QUEUE_ID_PDMA_0_1] = mmPDMA0_QM_BASE, 1095 [GAUDI2_QUEUE_ID_PDMA_0_2] = mmPDMA0_QM_BASE, 1096 [GAUDI2_QUEUE_ID_PDMA_0_3] = mmPDMA0_QM_BASE, 1097 [GAUDI2_QUEUE_ID_PDMA_1_0] = mmPDMA1_QM_BASE, 1098 [GAUDI2_QUEUE_ID_PDMA_1_1] = mmPDMA1_QM_BASE, 1099 [GAUDI2_QUEUE_ID_PDMA_1_2] = mmPDMA1_QM_BASE, 1100 [GAUDI2_QUEUE_ID_PDMA_1_3] = mmPDMA1_QM_BASE, 1101 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0] = mmDCORE0_EDMA0_QM_BASE, 1102 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_1] = mmDCORE0_EDMA0_QM_BASE, 1103 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_2] = mmDCORE0_EDMA0_QM_BASE, 1104 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] = mmDCORE0_EDMA0_QM_BASE, 1105 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0] = mmDCORE0_EDMA1_QM_BASE, 1106 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_1] = mmDCORE0_EDMA1_QM_BASE, 1107 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_2] = mmDCORE0_EDMA1_QM_BASE, 1108 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] = mmDCORE0_EDMA1_QM_BASE, 1109 [GAUDI2_QUEUE_ID_DCORE0_MME_0_0] = mmDCORE0_MME_QM_BASE, 1110 [GAUDI2_QUEUE_ID_DCORE0_MME_0_1] = mmDCORE0_MME_QM_BASE, 1111 [GAUDI2_QUEUE_ID_DCORE0_MME_0_2] = mmDCORE0_MME_QM_BASE, 1112 [GAUDI2_QUEUE_ID_DCORE0_MME_0_3] = mmDCORE0_MME_QM_BASE, 1113 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_0] = mmDCORE0_TPC0_QM_BASE, 1114 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_1] = mmDCORE0_TPC0_QM_BASE, 1115 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_2] = mmDCORE0_TPC0_QM_BASE, 1116 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] = mmDCORE0_TPC0_QM_BASE, 1117 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_0] = mmDCORE0_TPC1_QM_BASE, 1118 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_1] = mmDCORE0_TPC1_QM_BASE, 1119 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_2] = mmDCORE0_TPC1_QM_BASE, 1120 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] = mmDCORE0_TPC1_QM_BASE, 1121 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_0] = mmDCORE0_TPC2_QM_BASE, 1122 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_1] = mmDCORE0_TPC2_QM_BASE, 1123 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_2] = mmDCORE0_TPC2_QM_BASE, 1124 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] = mmDCORE0_TPC2_QM_BASE, 1125 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_0] = mmDCORE0_TPC3_QM_BASE, 1126 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_1] = mmDCORE0_TPC3_QM_BASE, 1127 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_2] = mmDCORE0_TPC3_QM_BASE, 1128 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] = mmDCORE0_TPC3_QM_BASE, 1129 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_0] = mmDCORE0_TPC4_QM_BASE, 1130 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_1] = mmDCORE0_TPC4_QM_BASE, 1131 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_2] = mmDCORE0_TPC4_QM_BASE, 1132 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] = mmDCORE0_TPC4_QM_BASE, 1133 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_0] = mmDCORE0_TPC5_QM_BASE, 1134 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_1] = mmDCORE0_TPC5_QM_BASE, 1135 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_2] = mmDCORE0_TPC5_QM_BASE, 1136 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] = mmDCORE0_TPC5_QM_BASE, 1137 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_0] = mmDCORE0_TPC6_QM_BASE, 1138 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_1] = mmDCORE0_TPC6_QM_BASE, 1139 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_2] = mmDCORE0_TPC6_QM_BASE, 1140 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] = mmDCORE0_TPC6_QM_BASE, 1141 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0] = mmDCORE1_EDMA0_QM_BASE, 1142 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_1] = mmDCORE1_EDMA0_QM_BASE, 1143 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_2] = mmDCORE1_EDMA0_QM_BASE, 1144 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] = mmDCORE1_EDMA0_QM_BASE, 1145 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0] = mmDCORE1_EDMA1_QM_BASE, 1146 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_1] = mmDCORE1_EDMA1_QM_BASE, 1147 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_2] = mmDCORE1_EDMA1_QM_BASE, 1148 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] = mmDCORE1_EDMA1_QM_BASE, 1149 [GAUDI2_QUEUE_ID_DCORE1_MME_0_0] = mmDCORE1_MME_QM_BASE, 1150 [GAUDI2_QUEUE_ID_DCORE1_MME_0_1] = mmDCORE1_MME_QM_BASE, 1151 [GAUDI2_QUEUE_ID_DCORE1_MME_0_2] = mmDCORE1_MME_QM_BASE, 1152 [GAUDI2_QUEUE_ID_DCORE1_MME_0_3] = mmDCORE1_MME_QM_BASE, 1153 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_0] = mmDCORE1_TPC0_QM_BASE, 1154 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_1] = mmDCORE1_TPC0_QM_BASE, 1155 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_2] = mmDCORE1_TPC0_QM_BASE, 1156 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] = mmDCORE1_TPC0_QM_BASE, 1157 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_0] = mmDCORE1_TPC1_QM_BASE, 1158 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_1] = mmDCORE1_TPC1_QM_BASE, 1159 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_2] = mmDCORE1_TPC1_QM_BASE, 1160 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] = mmDCORE1_TPC1_QM_BASE, 1161 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_0] = mmDCORE1_TPC2_QM_BASE, 1162 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_1] = mmDCORE1_TPC2_QM_BASE, 1163 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_2] = mmDCORE1_TPC2_QM_BASE, 1164 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] = mmDCORE1_TPC2_QM_BASE, 1165 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_0] = mmDCORE1_TPC3_QM_BASE, 1166 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_1] = mmDCORE1_TPC3_QM_BASE, 1167 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_2] = mmDCORE1_TPC3_QM_BASE, 1168 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] = mmDCORE1_TPC3_QM_BASE, 1169 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_0] = mmDCORE1_TPC4_QM_BASE, 1170 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_1] = mmDCORE1_TPC4_QM_BASE, 1171 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_2] = mmDCORE1_TPC4_QM_BASE, 1172 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] = mmDCORE1_TPC4_QM_BASE, 1173 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_0] = mmDCORE1_TPC5_QM_BASE, 1174 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_1] = mmDCORE1_TPC5_QM_BASE, 1175 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_2] = mmDCORE1_TPC5_QM_BASE, 1176 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] = mmDCORE1_TPC5_QM_BASE, 1177 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0] = mmDCORE2_EDMA0_QM_BASE, 1178 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_1] = mmDCORE2_EDMA0_QM_BASE, 1179 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_2] = mmDCORE2_EDMA0_QM_BASE, 1180 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] = mmDCORE2_EDMA0_QM_BASE, 1181 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0] = mmDCORE2_EDMA1_QM_BASE, 1182 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_1] = mmDCORE2_EDMA1_QM_BASE, 1183 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_2] = mmDCORE2_EDMA1_QM_BASE, 1184 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] = mmDCORE2_EDMA1_QM_BASE, 1185 [GAUDI2_QUEUE_ID_DCORE2_MME_0_0] = mmDCORE2_MME_QM_BASE, 1186 [GAUDI2_QUEUE_ID_DCORE2_MME_0_1] = mmDCORE2_MME_QM_BASE, 1187 [GAUDI2_QUEUE_ID_DCORE2_MME_0_2] = mmDCORE2_MME_QM_BASE, 1188 [GAUDI2_QUEUE_ID_DCORE2_MME_0_3] = mmDCORE2_MME_QM_BASE, 1189 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_0] = mmDCORE2_TPC0_QM_BASE, 1190 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_1] = mmDCORE2_TPC0_QM_BASE, 1191 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_2] = mmDCORE2_TPC0_QM_BASE, 1192 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] = mmDCORE2_TPC0_QM_BASE, 1193 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_0] = mmDCORE2_TPC1_QM_BASE, 1194 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_1] = mmDCORE2_TPC1_QM_BASE, 1195 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_2] = mmDCORE2_TPC1_QM_BASE, 1196 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] = mmDCORE2_TPC1_QM_BASE, 1197 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_0] = mmDCORE2_TPC2_QM_BASE, 1198 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_1] = mmDCORE2_TPC2_QM_BASE, 1199 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_2] = mmDCORE2_TPC2_QM_BASE, 1200 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] = mmDCORE2_TPC2_QM_BASE, 1201 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_0] = mmDCORE2_TPC3_QM_BASE, 1202 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_1] = mmDCORE2_TPC3_QM_BASE, 1203 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_2] = mmDCORE2_TPC3_QM_BASE, 1204 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] = mmDCORE2_TPC3_QM_BASE, 1205 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_0] = mmDCORE2_TPC4_QM_BASE, 1206 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_1] = mmDCORE2_TPC4_QM_BASE, 1207 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_2] = mmDCORE2_TPC4_QM_BASE, 1208 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] = mmDCORE2_TPC4_QM_BASE, 1209 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_0] = mmDCORE2_TPC5_QM_BASE, 1210 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_1] = mmDCORE2_TPC5_QM_BASE, 1211 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_2] = mmDCORE2_TPC5_QM_BASE, 1212 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] = mmDCORE2_TPC5_QM_BASE, 1213 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0] = mmDCORE3_EDMA0_QM_BASE, 1214 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_1] = mmDCORE3_EDMA0_QM_BASE, 1215 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_2] = mmDCORE3_EDMA0_QM_BASE, 1216 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] = mmDCORE3_EDMA0_QM_BASE, 1217 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0] = mmDCORE3_EDMA1_QM_BASE, 1218 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1] = mmDCORE3_EDMA1_QM_BASE, 1219 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2] = mmDCORE3_EDMA1_QM_BASE, 1220 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] = mmDCORE3_EDMA1_QM_BASE, 1221 [GAUDI2_QUEUE_ID_DCORE3_MME_0_0] = mmDCORE3_MME_QM_BASE, 1222 [GAUDI2_QUEUE_ID_DCORE3_MME_0_1] = mmDCORE3_MME_QM_BASE, 1223 [GAUDI2_QUEUE_ID_DCORE3_MME_0_2] = mmDCORE3_MME_QM_BASE, 1224 [GAUDI2_QUEUE_ID_DCORE3_MME_0_3] = mmDCORE3_MME_QM_BASE, 1225 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_0] = mmDCORE3_TPC0_QM_BASE, 1226 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_1] = mmDCORE3_TPC0_QM_BASE, 1227 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_2] = mmDCORE3_TPC0_QM_BASE, 1228 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] = mmDCORE3_TPC0_QM_BASE, 1229 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_0] = mmDCORE3_TPC1_QM_BASE, 1230 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_1] = mmDCORE3_TPC1_QM_BASE, 1231 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_2] = mmDCORE3_TPC1_QM_BASE, 1232 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] = mmDCORE3_TPC1_QM_BASE, 1233 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_0] = mmDCORE3_TPC2_QM_BASE, 1234 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_1] = mmDCORE3_TPC2_QM_BASE, 1235 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_2] = mmDCORE3_TPC2_QM_BASE, 1236 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] = mmDCORE3_TPC2_QM_BASE, 1237 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_0] = mmDCORE3_TPC3_QM_BASE, 1238 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_1] = mmDCORE3_TPC3_QM_BASE, 1239 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_2] = mmDCORE3_TPC3_QM_BASE, 1240 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] = mmDCORE3_TPC3_QM_BASE, 1241 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_0] = mmDCORE3_TPC4_QM_BASE, 1242 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_1] = mmDCORE3_TPC4_QM_BASE, 1243 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_2] = mmDCORE3_TPC4_QM_BASE, 1244 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] = mmDCORE3_TPC4_QM_BASE, 1245 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_0] = mmDCORE3_TPC5_QM_BASE, 1246 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_1] = mmDCORE3_TPC5_QM_BASE, 1247 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_2] = mmDCORE3_TPC5_QM_BASE, 1248 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] = mmDCORE3_TPC5_QM_BASE, 1249 [GAUDI2_QUEUE_ID_NIC_0_0] = mmNIC0_QM0_BASE, 1250 [GAUDI2_QUEUE_ID_NIC_0_1] = mmNIC0_QM0_BASE, 1251 [GAUDI2_QUEUE_ID_NIC_0_2] = mmNIC0_QM0_BASE, 1252 [GAUDI2_QUEUE_ID_NIC_0_3] = mmNIC0_QM0_BASE, 1253 [GAUDI2_QUEUE_ID_NIC_1_0] = mmNIC0_QM1_BASE, 1254 [GAUDI2_QUEUE_ID_NIC_1_1] = mmNIC0_QM1_BASE, 1255 [GAUDI2_QUEUE_ID_NIC_1_2] = mmNIC0_QM1_BASE, 1256 [GAUDI2_QUEUE_ID_NIC_1_3] = mmNIC0_QM1_BASE, 1257 [GAUDI2_QUEUE_ID_NIC_2_0] = mmNIC1_QM0_BASE, 1258 [GAUDI2_QUEUE_ID_NIC_2_1] = mmNIC1_QM0_BASE, 1259 [GAUDI2_QUEUE_ID_NIC_2_2] = mmNIC1_QM0_BASE, 1260 [GAUDI2_QUEUE_ID_NIC_2_3] = mmNIC1_QM0_BASE, 1261 [GAUDI2_QUEUE_ID_NIC_3_0] = mmNIC1_QM1_BASE, 1262 [GAUDI2_QUEUE_ID_NIC_3_1] = mmNIC1_QM1_BASE, 1263 [GAUDI2_QUEUE_ID_NIC_3_2] = mmNIC1_QM1_BASE, 1264 [GAUDI2_QUEUE_ID_NIC_3_3] = mmNIC1_QM1_BASE, 1265 [GAUDI2_QUEUE_ID_NIC_4_0] = mmNIC2_QM0_BASE, 1266 [GAUDI2_QUEUE_ID_NIC_4_1] = mmNIC2_QM0_BASE, 1267 [GAUDI2_QUEUE_ID_NIC_4_2] = mmNIC2_QM0_BASE, 1268 [GAUDI2_QUEUE_ID_NIC_4_3] = mmNIC2_QM0_BASE, 1269 [GAUDI2_QUEUE_ID_NIC_5_0] = mmNIC2_QM1_BASE, 1270 [GAUDI2_QUEUE_ID_NIC_5_1] = mmNIC2_QM1_BASE, 1271 [GAUDI2_QUEUE_ID_NIC_5_2] = mmNIC2_QM1_BASE, 1272 [GAUDI2_QUEUE_ID_NIC_5_3] = mmNIC2_QM1_BASE, 1273 [GAUDI2_QUEUE_ID_NIC_6_0] = mmNIC3_QM0_BASE, 1274 [GAUDI2_QUEUE_ID_NIC_6_1] = mmNIC3_QM0_BASE, 1275 [GAUDI2_QUEUE_ID_NIC_6_2] = mmNIC3_QM0_BASE, 1276 [GAUDI2_QUEUE_ID_NIC_6_3] = mmNIC3_QM0_BASE, 1277 [GAUDI2_QUEUE_ID_NIC_7_0] = mmNIC3_QM1_BASE, 1278 [GAUDI2_QUEUE_ID_NIC_7_1] = mmNIC3_QM1_BASE, 1279 [GAUDI2_QUEUE_ID_NIC_7_2] = mmNIC3_QM1_BASE, 1280 [GAUDI2_QUEUE_ID_NIC_7_3] = mmNIC3_QM1_BASE, 1281 [GAUDI2_QUEUE_ID_NIC_8_0] = mmNIC4_QM0_BASE, 1282 [GAUDI2_QUEUE_ID_NIC_8_1] = mmNIC4_QM0_BASE, 1283 [GAUDI2_QUEUE_ID_NIC_8_2] = mmNIC4_QM0_BASE, 1284 [GAUDI2_QUEUE_ID_NIC_8_3] = mmNIC4_QM0_BASE, 1285 [GAUDI2_QUEUE_ID_NIC_9_0] = mmNIC4_QM1_BASE, 1286 [GAUDI2_QUEUE_ID_NIC_9_1] = mmNIC4_QM1_BASE, 1287 [GAUDI2_QUEUE_ID_NIC_9_2] = mmNIC4_QM1_BASE, 1288 [GAUDI2_QUEUE_ID_NIC_9_3] = mmNIC4_QM1_BASE, 1289 [GAUDI2_QUEUE_ID_NIC_10_0] = mmNIC5_QM0_BASE, 1290 [GAUDI2_QUEUE_ID_NIC_10_1] = mmNIC5_QM0_BASE, 1291 [GAUDI2_QUEUE_ID_NIC_10_2] = mmNIC5_QM0_BASE, 1292 [GAUDI2_QUEUE_ID_NIC_10_3] = mmNIC5_QM0_BASE, 1293 [GAUDI2_QUEUE_ID_NIC_11_0] = mmNIC5_QM1_BASE, 1294 [GAUDI2_QUEUE_ID_NIC_11_1] = mmNIC5_QM1_BASE, 1295 [GAUDI2_QUEUE_ID_NIC_11_2] = mmNIC5_QM1_BASE, 1296 [GAUDI2_QUEUE_ID_NIC_11_3] = mmNIC5_QM1_BASE, 1297 [GAUDI2_QUEUE_ID_NIC_12_0] = mmNIC6_QM0_BASE, 1298 [GAUDI2_QUEUE_ID_NIC_12_1] = mmNIC6_QM0_BASE, 1299 [GAUDI2_QUEUE_ID_NIC_12_2] = mmNIC6_QM0_BASE, 1300 [GAUDI2_QUEUE_ID_NIC_12_3] = mmNIC6_QM0_BASE, 1301 [GAUDI2_QUEUE_ID_NIC_13_0] = mmNIC6_QM1_BASE, 1302 [GAUDI2_QUEUE_ID_NIC_13_1] = mmNIC6_QM1_BASE, 1303 [GAUDI2_QUEUE_ID_NIC_13_2] = mmNIC6_QM1_BASE, 1304 [GAUDI2_QUEUE_ID_NIC_13_3] = mmNIC6_QM1_BASE, 1305 [GAUDI2_QUEUE_ID_NIC_14_0] = mmNIC7_QM0_BASE, 1306 [GAUDI2_QUEUE_ID_NIC_14_1] = mmNIC7_QM0_BASE, 1307 [GAUDI2_QUEUE_ID_NIC_14_2] = mmNIC7_QM0_BASE, 1308 [GAUDI2_QUEUE_ID_NIC_14_3] = mmNIC7_QM0_BASE, 1309 [GAUDI2_QUEUE_ID_NIC_15_0] = mmNIC7_QM1_BASE, 1310 [GAUDI2_QUEUE_ID_NIC_15_1] = mmNIC7_QM1_BASE, 1311 [GAUDI2_QUEUE_ID_NIC_15_2] = mmNIC7_QM1_BASE, 1312 [GAUDI2_QUEUE_ID_NIC_15_3] = mmNIC7_QM1_BASE, 1313 [GAUDI2_QUEUE_ID_NIC_16_0] = mmNIC8_QM0_BASE, 1314 [GAUDI2_QUEUE_ID_NIC_16_1] = mmNIC8_QM0_BASE, 1315 [GAUDI2_QUEUE_ID_NIC_16_2] = mmNIC8_QM0_BASE, 1316 [GAUDI2_QUEUE_ID_NIC_16_3] = mmNIC8_QM0_BASE, 1317 [GAUDI2_QUEUE_ID_NIC_17_0] = mmNIC8_QM1_BASE, 1318 [GAUDI2_QUEUE_ID_NIC_17_1] = mmNIC8_QM1_BASE, 1319 [GAUDI2_QUEUE_ID_NIC_17_2] = mmNIC8_QM1_BASE, 1320 [GAUDI2_QUEUE_ID_NIC_17_3] = mmNIC8_QM1_BASE, 1321 [GAUDI2_QUEUE_ID_NIC_18_0] = mmNIC9_QM0_BASE, 1322 [GAUDI2_QUEUE_ID_NIC_18_1] = mmNIC9_QM0_BASE, 1323 [GAUDI2_QUEUE_ID_NIC_18_2] = mmNIC9_QM0_BASE, 1324 [GAUDI2_QUEUE_ID_NIC_18_3] = mmNIC9_QM0_BASE, 1325 [GAUDI2_QUEUE_ID_NIC_19_0] = mmNIC9_QM1_BASE, 1326 [GAUDI2_QUEUE_ID_NIC_19_1] = mmNIC9_QM1_BASE, 1327 [GAUDI2_QUEUE_ID_NIC_19_2] = mmNIC9_QM1_BASE, 1328 [GAUDI2_QUEUE_ID_NIC_19_3] = mmNIC9_QM1_BASE, 1329 [GAUDI2_QUEUE_ID_NIC_20_0] = mmNIC10_QM0_BASE, 1330 [GAUDI2_QUEUE_ID_NIC_20_1] = mmNIC10_QM0_BASE, 1331 [GAUDI2_QUEUE_ID_NIC_20_2] = mmNIC10_QM0_BASE, 1332 [GAUDI2_QUEUE_ID_NIC_20_3] = mmNIC10_QM0_BASE, 1333 [GAUDI2_QUEUE_ID_NIC_21_0] = mmNIC10_QM1_BASE, 1334 [GAUDI2_QUEUE_ID_NIC_21_1] = mmNIC10_QM1_BASE, 1335 [GAUDI2_QUEUE_ID_NIC_21_2] = mmNIC10_QM1_BASE, 1336 [GAUDI2_QUEUE_ID_NIC_21_3] = mmNIC10_QM1_BASE, 1337 [GAUDI2_QUEUE_ID_NIC_22_0] = mmNIC11_QM0_BASE, 1338 [GAUDI2_QUEUE_ID_NIC_22_1] = mmNIC11_QM0_BASE, 1339 [GAUDI2_QUEUE_ID_NIC_22_2] = mmNIC11_QM0_BASE, 1340 [GAUDI2_QUEUE_ID_NIC_22_3] = mmNIC11_QM0_BASE, 1341 [GAUDI2_QUEUE_ID_NIC_23_0] = mmNIC11_QM1_BASE, 1342 [GAUDI2_QUEUE_ID_NIC_23_1] = mmNIC11_QM1_BASE, 1343 [GAUDI2_QUEUE_ID_NIC_23_2] = mmNIC11_QM1_BASE, 1344 [GAUDI2_QUEUE_ID_NIC_23_3] = mmNIC11_QM1_BASE, 1345 [GAUDI2_QUEUE_ID_ROT_0_0] = mmROT0_QM_BASE, 1346 [GAUDI2_QUEUE_ID_ROT_0_1] = mmROT0_QM_BASE, 1347 [GAUDI2_QUEUE_ID_ROT_0_2] = mmROT0_QM_BASE, 1348 [GAUDI2_QUEUE_ID_ROT_0_3] = mmROT0_QM_BASE, 1349 [GAUDI2_QUEUE_ID_ROT_1_0] = mmROT1_QM_BASE, 1350 [GAUDI2_QUEUE_ID_ROT_1_1] = mmROT1_QM_BASE, 1351 [GAUDI2_QUEUE_ID_ROT_1_2] = mmROT1_QM_BASE, 1352 [GAUDI2_QUEUE_ID_ROT_1_3] = mmROT1_QM_BASE 1353 }; 1354 1355 static const u32 gaudi2_arc_blocks_bases[NUM_ARC_CPUS] = { 1356 [CPU_ID_SCHED_ARC0] = mmARC_FARM_ARC0_AUX_BASE, 1357 [CPU_ID_SCHED_ARC1] = mmARC_FARM_ARC1_AUX_BASE, 1358 [CPU_ID_SCHED_ARC2] = mmARC_FARM_ARC2_AUX_BASE, 1359 [CPU_ID_SCHED_ARC3] = mmARC_FARM_ARC3_AUX_BASE, 1360 [CPU_ID_SCHED_ARC4] = mmDCORE1_MME_QM_ARC_AUX_BASE, 1361 [CPU_ID_SCHED_ARC5] = mmDCORE3_MME_QM_ARC_AUX_BASE, 1362 [CPU_ID_TPC_QMAN_ARC0] = mmDCORE0_TPC0_QM_ARC_AUX_BASE, 1363 [CPU_ID_TPC_QMAN_ARC1] = mmDCORE0_TPC1_QM_ARC_AUX_BASE, 1364 [CPU_ID_TPC_QMAN_ARC2] = mmDCORE0_TPC2_QM_ARC_AUX_BASE, 1365 [CPU_ID_TPC_QMAN_ARC3] = mmDCORE0_TPC3_QM_ARC_AUX_BASE, 1366 [CPU_ID_TPC_QMAN_ARC4] = mmDCORE0_TPC4_QM_ARC_AUX_BASE, 1367 [CPU_ID_TPC_QMAN_ARC5] = mmDCORE0_TPC5_QM_ARC_AUX_BASE, 1368 [CPU_ID_TPC_QMAN_ARC6] = mmDCORE1_TPC0_QM_ARC_AUX_BASE, 1369 [CPU_ID_TPC_QMAN_ARC7] = mmDCORE1_TPC1_QM_ARC_AUX_BASE, 1370 [CPU_ID_TPC_QMAN_ARC8] = mmDCORE1_TPC2_QM_ARC_AUX_BASE, 1371 [CPU_ID_TPC_QMAN_ARC9] = mmDCORE1_TPC3_QM_ARC_AUX_BASE, 1372 [CPU_ID_TPC_QMAN_ARC10] = mmDCORE1_TPC4_QM_ARC_AUX_BASE, 1373 [CPU_ID_TPC_QMAN_ARC11] = mmDCORE1_TPC5_QM_ARC_AUX_BASE, 1374 [CPU_ID_TPC_QMAN_ARC12] = mmDCORE2_TPC0_QM_ARC_AUX_BASE, 1375 [CPU_ID_TPC_QMAN_ARC13] = mmDCORE2_TPC1_QM_ARC_AUX_BASE, 1376 [CPU_ID_TPC_QMAN_ARC14] = mmDCORE2_TPC2_QM_ARC_AUX_BASE, 1377 [CPU_ID_TPC_QMAN_ARC15] = mmDCORE2_TPC3_QM_ARC_AUX_BASE, 1378 [CPU_ID_TPC_QMAN_ARC16] = mmDCORE2_TPC4_QM_ARC_AUX_BASE, 1379 [CPU_ID_TPC_QMAN_ARC17] = mmDCORE2_TPC5_QM_ARC_AUX_BASE, 1380 [CPU_ID_TPC_QMAN_ARC18] = mmDCORE3_TPC0_QM_ARC_AUX_BASE, 1381 [CPU_ID_TPC_QMAN_ARC19] = mmDCORE3_TPC1_QM_ARC_AUX_BASE, 1382 [CPU_ID_TPC_QMAN_ARC20] = mmDCORE3_TPC2_QM_ARC_AUX_BASE, 1383 [CPU_ID_TPC_QMAN_ARC21] = mmDCORE3_TPC3_QM_ARC_AUX_BASE, 1384 [CPU_ID_TPC_QMAN_ARC22] = mmDCORE3_TPC4_QM_ARC_AUX_BASE, 1385 [CPU_ID_TPC_QMAN_ARC23] = mmDCORE3_TPC5_QM_ARC_AUX_BASE, 1386 [CPU_ID_TPC_QMAN_ARC24] = mmDCORE0_TPC6_QM_ARC_AUX_BASE, 1387 [CPU_ID_MME_QMAN_ARC0] = mmDCORE0_MME_QM_ARC_AUX_BASE, 1388 [CPU_ID_MME_QMAN_ARC1] = mmDCORE2_MME_QM_ARC_AUX_BASE, 1389 [CPU_ID_EDMA_QMAN_ARC0] = mmDCORE0_EDMA0_QM_ARC_AUX_BASE, 1390 [CPU_ID_EDMA_QMAN_ARC1] = mmDCORE0_EDMA1_QM_ARC_AUX_BASE, 1391 [CPU_ID_EDMA_QMAN_ARC2] = mmDCORE1_EDMA0_QM_ARC_AUX_BASE, 1392 [CPU_ID_EDMA_QMAN_ARC3] = mmDCORE1_EDMA1_QM_ARC_AUX_BASE, 1393 [CPU_ID_EDMA_QMAN_ARC4] = mmDCORE2_EDMA0_QM_ARC_AUX_BASE, 1394 [CPU_ID_EDMA_QMAN_ARC5] = mmDCORE2_EDMA1_QM_ARC_AUX_BASE, 1395 [CPU_ID_EDMA_QMAN_ARC6] = mmDCORE3_EDMA0_QM_ARC_AUX_BASE, 1396 [CPU_ID_EDMA_QMAN_ARC7] = mmDCORE3_EDMA1_QM_ARC_AUX_BASE, 1397 [CPU_ID_PDMA_QMAN_ARC0] = mmPDMA0_QM_ARC_AUX_BASE, 1398 [CPU_ID_PDMA_QMAN_ARC1] = mmPDMA1_QM_ARC_AUX_BASE, 1399 [CPU_ID_ROT_QMAN_ARC0] = mmROT0_QM_ARC_AUX_BASE, 1400 [CPU_ID_ROT_QMAN_ARC1] = mmROT1_QM_ARC_AUX_BASE, 1401 [CPU_ID_NIC_QMAN_ARC0] = mmNIC0_QM_ARC_AUX0_BASE, 1402 [CPU_ID_NIC_QMAN_ARC1] = mmNIC0_QM_ARC_AUX1_BASE, 1403 [CPU_ID_NIC_QMAN_ARC2] = mmNIC1_QM_ARC_AUX0_BASE, 1404 [CPU_ID_NIC_QMAN_ARC3] = mmNIC1_QM_ARC_AUX1_BASE, 1405 [CPU_ID_NIC_QMAN_ARC4] = mmNIC2_QM_ARC_AUX0_BASE, 1406 [CPU_ID_NIC_QMAN_ARC5] = mmNIC2_QM_ARC_AUX1_BASE, 1407 [CPU_ID_NIC_QMAN_ARC6] = mmNIC3_QM_ARC_AUX0_BASE, 1408 [CPU_ID_NIC_QMAN_ARC7] = mmNIC3_QM_ARC_AUX1_BASE, 1409 [CPU_ID_NIC_QMAN_ARC8] = mmNIC4_QM_ARC_AUX0_BASE, 1410 [CPU_ID_NIC_QMAN_ARC9] = mmNIC4_QM_ARC_AUX1_BASE, 1411 [CPU_ID_NIC_QMAN_ARC10] = mmNIC5_QM_ARC_AUX0_BASE, 1412 [CPU_ID_NIC_QMAN_ARC11] = mmNIC5_QM_ARC_AUX1_BASE, 1413 [CPU_ID_NIC_QMAN_ARC12] = mmNIC6_QM_ARC_AUX0_BASE, 1414 [CPU_ID_NIC_QMAN_ARC13] = mmNIC6_QM_ARC_AUX1_BASE, 1415 [CPU_ID_NIC_QMAN_ARC14] = mmNIC7_QM_ARC_AUX0_BASE, 1416 [CPU_ID_NIC_QMAN_ARC15] = mmNIC7_QM_ARC_AUX1_BASE, 1417 [CPU_ID_NIC_QMAN_ARC16] = mmNIC8_QM_ARC_AUX0_BASE, 1418 [CPU_ID_NIC_QMAN_ARC17] = mmNIC8_QM_ARC_AUX1_BASE, 1419 [CPU_ID_NIC_QMAN_ARC18] = mmNIC9_QM_ARC_AUX0_BASE, 1420 [CPU_ID_NIC_QMAN_ARC19] = mmNIC9_QM_ARC_AUX1_BASE, 1421 [CPU_ID_NIC_QMAN_ARC20] = mmNIC10_QM_ARC_AUX0_BASE, 1422 [CPU_ID_NIC_QMAN_ARC21] = mmNIC10_QM_ARC_AUX1_BASE, 1423 [CPU_ID_NIC_QMAN_ARC22] = mmNIC11_QM_ARC_AUX0_BASE, 1424 [CPU_ID_NIC_QMAN_ARC23] = mmNIC11_QM_ARC_AUX1_BASE, 1425 }; 1426 1427 static const u32 gaudi2_arc_dccm_bases[NUM_ARC_CPUS] = { 1428 [CPU_ID_SCHED_ARC0] = mmARC_FARM_ARC0_DCCM0_BASE, 1429 [CPU_ID_SCHED_ARC1] = mmARC_FARM_ARC1_DCCM0_BASE, 1430 [CPU_ID_SCHED_ARC2] = mmARC_FARM_ARC2_DCCM0_BASE, 1431 [CPU_ID_SCHED_ARC3] = mmARC_FARM_ARC3_DCCM0_BASE, 1432 [CPU_ID_SCHED_ARC4] = mmDCORE1_MME_QM_ARC_DCCM_BASE, 1433 [CPU_ID_SCHED_ARC5] = mmDCORE3_MME_QM_ARC_DCCM_BASE, 1434 [CPU_ID_TPC_QMAN_ARC0] = mmDCORE0_TPC0_QM_DCCM_BASE, 1435 [CPU_ID_TPC_QMAN_ARC1] = mmDCORE0_TPC1_QM_DCCM_BASE, 1436 [CPU_ID_TPC_QMAN_ARC2] = mmDCORE0_TPC2_QM_DCCM_BASE, 1437 [CPU_ID_TPC_QMAN_ARC3] = mmDCORE0_TPC3_QM_DCCM_BASE, 1438 [CPU_ID_TPC_QMAN_ARC4] = mmDCORE0_TPC4_QM_DCCM_BASE, 1439 [CPU_ID_TPC_QMAN_ARC5] = mmDCORE0_TPC5_QM_DCCM_BASE, 1440 [CPU_ID_TPC_QMAN_ARC6] = mmDCORE1_TPC0_QM_DCCM_BASE, 1441 [CPU_ID_TPC_QMAN_ARC7] = mmDCORE1_TPC1_QM_DCCM_BASE, 1442 [CPU_ID_TPC_QMAN_ARC8] = mmDCORE1_TPC2_QM_DCCM_BASE, 1443 [CPU_ID_TPC_QMAN_ARC9] = mmDCORE1_TPC3_QM_DCCM_BASE, 1444 [CPU_ID_TPC_QMAN_ARC10] = mmDCORE1_TPC4_QM_DCCM_BASE, 1445 [CPU_ID_TPC_QMAN_ARC11] = mmDCORE1_TPC5_QM_DCCM_BASE, 1446 [CPU_ID_TPC_QMAN_ARC12] = mmDCORE2_TPC0_QM_DCCM_BASE, 1447 [CPU_ID_TPC_QMAN_ARC13] = mmDCORE2_TPC1_QM_DCCM_BASE, 1448 [CPU_ID_TPC_QMAN_ARC14] = mmDCORE2_TPC2_QM_DCCM_BASE, 1449 [CPU_ID_TPC_QMAN_ARC15] = mmDCORE2_TPC3_QM_DCCM_BASE, 1450 [CPU_ID_TPC_QMAN_ARC16] = mmDCORE2_TPC4_QM_DCCM_BASE, 1451 [CPU_ID_TPC_QMAN_ARC17] = mmDCORE2_TPC5_QM_DCCM_BASE, 1452 [CPU_ID_TPC_QMAN_ARC18] = mmDCORE3_TPC0_QM_DCCM_BASE, 1453 [CPU_ID_TPC_QMAN_ARC19] = mmDCORE3_TPC1_QM_DCCM_BASE, 1454 [CPU_ID_TPC_QMAN_ARC20] = mmDCORE3_TPC2_QM_DCCM_BASE, 1455 [CPU_ID_TPC_QMAN_ARC21] = mmDCORE3_TPC3_QM_DCCM_BASE, 1456 [CPU_ID_TPC_QMAN_ARC22] = mmDCORE3_TPC4_QM_DCCM_BASE, 1457 [CPU_ID_TPC_QMAN_ARC23] = mmDCORE3_TPC5_QM_DCCM_BASE, 1458 [CPU_ID_TPC_QMAN_ARC24] = mmDCORE0_TPC6_QM_DCCM_BASE, 1459 [CPU_ID_MME_QMAN_ARC0] = mmDCORE0_MME_QM_ARC_DCCM_BASE, 1460 [CPU_ID_MME_QMAN_ARC1] = mmDCORE2_MME_QM_ARC_DCCM_BASE, 1461 [CPU_ID_EDMA_QMAN_ARC0] = mmDCORE0_EDMA0_QM_DCCM_BASE, 1462 [CPU_ID_EDMA_QMAN_ARC1] = mmDCORE0_EDMA1_QM_DCCM_BASE, 1463 [CPU_ID_EDMA_QMAN_ARC2] = mmDCORE1_EDMA0_QM_DCCM_BASE, 1464 [CPU_ID_EDMA_QMAN_ARC3] = mmDCORE1_EDMA1_QM_DCCM_BASE, 1465 [CPU_ID_EDMA_QMAN_ARC4] = mmDCORE2_EDMA0_QM_DCCM_BASE, 1466 [CPU_ID_EDMA_QMAN_ARC5] = mmDCORE2_EDMA1_QM_DCCM_BASE, 1467 [CPU_ID_EDMA_QMAN_ARC6] = mmDCORE3_EDMA0_QM_DCCM_BASE, 1468 [CPU_ID_EDMA_QMAN_ARC7] = mmDCORE3_EDMA1_QM_DCCM_BASE, 1469 [CPU_ID_PDMA_QMAN_ARC0] = mmPDMA0_QM_ARC_DCCM_BASE, 1470 [CPU_ID_PDMA_QMAN_ARC1] = mmPDMA1_QM_ARC_DCCM_BASE, 1471 [CPU_ID_ROT_QMAN_ARC0] = mmROT0_QM_ARC_DCCM_BASE, 1472 [CPU_ID_ROT_QMAN_ARC1] = mmROT1_QM_ARC_DCCM_BASE, 1473 [CPU_ID_NIC_QMAN_ARC0] = mmNIC0_QM_DCCM0_BASE, 1474 [CPU_ID_NIC_QMAN_ARC1] = mmNIC0_QM_DCCM1_BASE, 1475 [CPU_ID_NIC_QMAN_ARC2] = mmNIC1_QM_DCCM0_BASE, 1476 [CPU_ID_NIC_QMAN_ARC3] = mmNIC1_QM_DCCM1_BASE, 1477 [CPU_ID_NIC_QMAN_ARC4] = mmNIC2_QM_DCCM0_BASE, 1478 [CPU_ID_NIC_QMAN_ARC5] = mmNIC2_QM_DCCM1_BASE, 1479 [CPU_ID_NIC_QMAN_ARC6] = mmNIC3_QM_DCCM0_BASE, 1480 [CPU_ID_NIC_QMAN_ARC7] = mmNIC3_QM_DCCM1_BASE, 1481 [CPU_ID_NIC_QMAN_ARC8] = mmNIC4_QM_DCCM0_BASE, 1482 [CPU_ID_NIC_QMAN_ARC9] = mmNIC4_QM_DCCM1_BASE, 1483 [CPU_ID_NIC_QMAN_ARC10] = mmNIC5_QM_DCCM0_BASE, 1484 [CPU_ID_NIC_QMAN_ARC11] = mmNIC5_QM_DCCM1_BASE, 1485 [CPU_ID_NIC_QMAN_ARC12] = mmNIC6_QM_DCCM0_BASE, 1486 [CPU_ID_NIC_QMAN_ARC13] = mmNIC6_QM_DCCM1_BASE, 1487 [CPU_ID_NIC_QMAN_ARC14] = mmNIC7_QM_DCCM0_BASE, 1488 [CPU_ID_NIC_QMAN_ARC15] = mmNIC7_QM_DCCM1_BASE, 1489 [CPU_ID_NIC_QMAN_ARC16] = mmNIC8_QM_DCCM0_BASE, 1490 [CPU_ID_NIC_QMAN_ARC17] = mmNIC8_QM_DCCM1_BASE, 1491 [CPU_ID_NIC_QMAN_ARC18] = mmNIC9_QM_DCCM0_BASE, 1492 [CPU_ID_NIC_QMAN_ARC19] = mmNIC9_QM_DCCM1_BASE, 1493 [CPU_ID_NIC_QMAN_ARC20] = mmNIC10_QM_DCCM0_BASE, 1494 [CPU_ID_NIC_QMAN_ARC21] = mmNIC10_QM_DCCM1_BASE, 1495 [CPU_ID_NIC_QMAN_ARC22] = mmNIC11_QM_DCCM0_BASE, 1496 [CPU_ID_NIC_QMAN_ARC23] = mmNIC11_QM_DCCM1_BASE, 1497 }; 1498 1499 const u32 gaudi2_mme_ctrl_lo_blocks_bases[MME_ID_SIZE] = { 1500 [MME_ID_DCORE0] = mmDCORE0_MME_CTRL_LO_BASE, 1501 [MME_ID_DCORE1] = mmDCORE1_MME_CTRL_LO_BASE, 1502 [MME_ID_DCORE2] = mmDCORE2_MME_CTRL_LO_BASE, 1503 [MME_ID_DCORE3] = mmDCORE3_MME_CTRL_LO_BASE, 1504 }; 1505 1506 static const u32 gaudi2_queue_id_to_arc_id[GAUDI2_QUEUE_ID_SIZE] = { 1507 [GAUDI2_QUEUE_ID_PDMA_0_0] = CPU_ID_PDMA_QMAN_ARC0, 1508 [GAUDI2_QUEUE_ID_PDMA_0_1] = CPU_ID_PDMA_QMAN_ARC0, 1509 [GAUDI2_QUEUE_ID_PDMA_0_2] = CPU_ID_PDMA_QMAN_ARC0, 1510 [GAUDI2_QUEUE_ID_PDMA_0_3] = CPU_ID_PDMA_QMAN_ARC0, 1511 [GAUDI2_QUEUE_ID_PDMA_1_0] = CPU_ID_PDMA_QMAN_ARC1, 1512 [GAUDI2_QUEUE_ID_PDMA_1_1] = CPU_ID_PDMA_QMAN_ARC1, 1513 [GAUDI2_QUEUE_ID_PDMA_1_2] = CPU_ID_PDMA_QMAN_ARC1, 1514 [GAUDI2_QUEUE_ID_PDMA_1_3] = CPU_ID_PDMA_QMAN_ARC1, 1515 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC0, 1516 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC0, 1517 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC0, 1518 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC0, 1519 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC1, 1520 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC1, 1521 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC1, 1522 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC1, 1523 [GAUDI2_QUEUE_ID_DCORE0_MME_0_0] = CPU_ID_MME_QMAN_ARC0, 1524 [GAUDI2_QUEUE_ID_DCORE0_MME_0_1] = CPU_ID_MME_QMAN_ARC0, 1525 [GAUDI2_QUEUE_ID_DCORE0_MME_0_2] = CPU_ID_MME_QMAN_ARC0, 1526 [GAUDI2_QUEUE_ID_DCORE0_MME_0_3] = CPU_ID_MME_QMAN_ARC0, 1527 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_0] = CPU_ID_TPC_QMAN_ARC0, 1528 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_1] = CPU_ID_TPC_QMAN_ARC0, 1529 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_2] = CPU_ID_TPC_QMAN_ARC0, 1530 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] = CPU_ID_TPC_QMAN_ARC0, 1531 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_0] = CPU_ID_TPC_QMAN_ARC1, 1532 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_1] = CPU_ID_TPC_QMAN_ARC1, 1533 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_2] = CPU_ID_TPC_QMAN_ARC1, 1534 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] = CPU_ID_TPC_QMAN_ARC1, 1535 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_0] = CPU_ID_TPC_QMAN_ARC2, 1536 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_1] = CPU_ID_TPC_QMAN_ARC2, 1537 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_2] = CPU_ID_TPC_QMAN_ARC2, 1538 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] = CPU_ID_TPC_QMAN_ARC2, 1539 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_0] = CPU_ID_TPC_QMAN_ARC3, 1540 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_1] = CPU_ID_TPC_QMAN_ARC3, 1541 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_2] = CPU_ID_TPC_QMAN_ARC3, 1542 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] = CPU_ID_TPC_QMAN_ARC3, 1543 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_0] = CPU_ID_TPC_QMAN_ARC4, 1544 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_1] = CPU_ID_TPC_QMAN_ARC4, 1545 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_2] = CPU_ID_TPC_QMAN_ARC4, 1546 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] = CPU_ID_TPC_QMAN_ARC4, 1547 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_0] = CPU_ID_TPC_QMAN_ARC5, 1548 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_1] = CPU_ID_TPC_QMAN_ARC5, 1549 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_2] = CPU_ID_TPC_QMAN_ARC5, 1550 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] = CPU_ID_TPC_QMAN_ARC5, 1551 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_0] = CPU_ID_TPC_QMAN_ARC24, 1552 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_1] = CPU_ID_TPC_QMAN_ARC24, 1553 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_2] = CPU_ID_TPC_QMAN_ARC24, 1554 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] = CPU_ID_TPC_QMAN_ARC24, 1555 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC2, 1556 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC2, 1557 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC2, 1558 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC2, 1559 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC3, 1560 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC3, 1561 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC3, 1562 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC3, 1563 [GAUDI2_QUEUE_ID_DCORE1_MME_0_0] = CPU_ID_SCHED_ARC4, 1564 [GAUDI2_QUEUE_ID_DCORE1_MME_0_1] = CPU_ID_SCHED_ARC4, 1565 [GAUDI2_QUEUE_ID_DCORE1_MME_0_2] = CPU_ID_SCHED_ARC4, 1566 [GAUDI2_QUEUE_ID_DCORE1_MME_0_3] = CPU_ID_SCHED_ARC4, 1567 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_0] = CPU_ID_TPC_QMAN_ARC6, 1568 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_1] = CPU_ID_TPC_QMAN_ARC6, 1569 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_2] = CPU_ID_TPC_QMAN_ARC6, 1570 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] = CPU_ID_TPC_QMAN_ARC6, 1571 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_0] = CPU_ID_TPC_QMAN_ARC7, 1572 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_1] = CPU_ID_TPC_QMAN_ARC7, 1573 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_2] = CPU_ID_TPC_QMAN_ARC7, 1574 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] = CPU_ID_TPC_QMAN_ARC7, 1575 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_0] = CPU_ID_TPC_QMAN_ARC8, 1576 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_1] = CPU_ID_TPC_QMAN_ARC8, 1577 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_2] = CPU_ID_TPC_QMAN_ARC8, 1578 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] = CPU_ID_TPC_QMAN_ARC8, 1579 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_0] = CPU_ID_TPC_QMAN_ARC9, 1580 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_1] = CPU_ID_TPC_QMAN_ARC9, 1581 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_2] = CPU_ID_TPC_QMAN_ARC9, 1582 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] = CPU_ID_TPC_QMAN_ARC9, 1583 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_0] = CPU_ID_TPC_QMAN_ARC10, 1584 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_1] = CPU_ID_TPC_QMAN_ARC10, 1585 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_2] = CPU_ID_TPC_QMAN_ARC10, 1586 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] = CPU_ID_TPC_QMAN_ARC10, 1587 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_0] = CPU_ID_TPC_QMAN_ARC11, 1588 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_1] = CPU_ID_TPC_QMAN_ARC11, 1589 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_2] = CPU_ID_TPC_QMAN_ARC11, 1590 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] = CPU_ID_TPC_QMAN_ARC11, 1591 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC4, 1592 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC4, 1593 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC4, 1594 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC4, 1595 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC5, 1596 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC5, 1597 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC5, 1598 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC5, 1599 [GAUDI2_QUEUE_ID_DCORE2_MME_0_0] = CPU_ID_MME_QMAN_ARC1, 1600 [GAUDI2_QUEUE_ID_DCORE2_MME_0_1] = CPU_ID_MME_QMAN_ARC1, 1601 [GAUDI2_QUEUE_ID_DCORE2_MME_0_2] = CPU_ID_MME_QMAN_ARC1, 1602 [GAUDI2_QUEUE_ID_DCORE2_MME_0_3] = CPU_ID_MME_QMAN_ARC1, 1603 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_0] = CPU_ID_TPC_QMAN_ARC12, 1604 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_1] = CPU_ID_TPC_QMAN_ARC12, 1605 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_2] = CPU_ID_TPC_QMAN_ARC12, 1606 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] = CPU_ID_TPC_QMAN_ARC12, 1607 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_0] = CPU_ID_TPC_QMAN_ARC13, 1608 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_1] = CPU_ID_TPC_QMAN_ARC13, 1609 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_2] = CPU_ID_TPC_QMAN_ARC13, 1610 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] = CPU_ID_TPC_QMAN_ARC13, 1611 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_0] = CPU_ID_TPC_QMAN_ARC14, 1612 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_1] = CPU_ID_TPC_QMAN_ARC14, 1613 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_2] = CPU_ID_TPC_QMAN_ARC14, 1614 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] = CPU_ID_TPC_QMAN_ARC14, 1615 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_0] = CPU_ID_TPC_QMAN_ARC15, 1616 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_1] = CPU_ID_TPC_QMAN_ARC15, 1617 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_2] = CPU_ID_TPC_QMAN_ARC15, 1618 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] = CPU_ID_TPC_QMAN_ARC15, 1619 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_0] = CPU_ID_TPC_QMAN_ARC16, 1620 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_1] = CPU_ID_TPC_QMAN_ARC16, 1621 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_2] = CPU_ID_TPC_QMAN_ARC16, 1622 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] = CPU_ID_TPC_QMAN_ARC16, 1623 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_0] = CPU_ID_TPC_QMAN_ARC17, 1624 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_1] = CPU_ID_TPC_QMAN_ARC17, 1625 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_2] = CPU_ID_TPC_QMAN_ARC17, 1626 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] = CPU_ID_TPC_QMAN_ARC17, 1627 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC6, 1628 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC6, 1629 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC6, 1630 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC6, 1631 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC7, 1632 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC7, 1633 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC7, 1634 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC7, 1635 [GAUDI2_QUEUE_ID_DCORE3_MME_0_0] = CPU_ID_SCHED_ARC5, 1636 [GAUDI2_QUEUE_ID_DCORE3_MME_0_1] = CPU_ID_SCHED_ARC5, 1637 [GAUDI2_QUEUE_ID_DCORE3_MME_0_2] = CPU_ID_SCHED_ARC5, 1638 [GAUDI2_QUEUE_ID_DCORE3_MME_0_3] = CPU_ID_SCHED_ARC5, 1639 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_0] = CPU_ID_TPC_QMAN_ARC18, 1640 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_1] = CPU_ID_TPC_QMAN_ARC18, 1641 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_2] = CPU_ID_TPC_QMAN_ARC18, 1642 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] = CPU_ID_TPC_QMAN_ARC18, 1643 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_0] = CPU_ID_TPC_QMAN_ARC19, 1644 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_1] = CPU_ID_TPC_QMAN_ARC19, 1645 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_2] = CPU_ID_TPC_QMAN_ARC19, 1646 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] = CPU_ID_TPC_QMAN_ARC19, 1647 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_0] = CPU_ID_TPC_QMAN_ARC20, 1648 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_1] = CPU_ID_TPC_QMAN_ARC20, 1649 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_2] = CPU_ID_TPC_QMAN_ARC20, 1650 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] = CPU_ID_TPC_QMAN_ARC20, 1651 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_0] = CPU_ID_TPC_QMAN_ARC21, 1652 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_1] = CPU_ID_TPC_QMAN_ARC21, 1653 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_2] = CPU_ID_TPC_QMAN_ARC21, 1654 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] = CPU_ID_TPC_QMAN_ARC21, 1655 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_0] = CPU_ID_TPC_QMAN_ARC22, 1656 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_1] = CPU_ID_TPC_QMAN_ARC22, 1657 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_2] = CPU_ID_TPC_QMAN_ARC22, 1658 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] = CPU_ID_TPC_QMAN_ARC22, 1659 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_0] = CPU_ID_TPC_QMAN_ARC23, 1660 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_1] = CPU_ID_TPC_QMAN_ARC23, 1661 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_2] = CPU_ID_TPC_QMAN_ARC23, 1662 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] = CPU_ID_TPC_QMAN_ARC23, 1663 [GAUDI2_QUEUE_ID_NIC_0_0] = CPU_ID_NIC_QMAN_ARC0, 1664 [GAUDI2_QUEUE_ID_NIC_0_1] = CPU_ID_NIC_QMAN_ARC0, 1665 [GAUDI2_QUEUE_ID_NIC_0_2] = CPU_ID_NIC_QMAN_ARC0, 1666 [GAUDI2_QUEUE_ID_NIC_0_3] = CPU_ID_NIC_QMAN_ARC0, 1667 [GAUDI2_QUEUE_ID_NIC_1_0] = CPU_ID_NIC_QMAN_ARC1, 1668 [GAUDI2_QUEUE_ID_NIC_1_1] = CPU_ID_NIC_QMAN_ARC1, 1669 [GAUDI2_QUEUE_ID_NIC_1_2] = CPU_ID_NIC_QMAN_ARC1, 1670 [GAUDI2_QUEUE_ID_NIC_1_3] = CPU_ID_NIC_QMAN_ARC1, 1671 [GAUDI2_QUEUE_ID_NIC_2_0] = CPU_ID_NIC_QMAN_ARC2, 1672 [GAUDI2_QUEUE_ID_NIC_2_1] = CPU_ID_NIC_QMAN_ARC2, 1673 [GAUDI2_QUEUE_ID_NIC_2_2] = CPU_ID_NIC_QMAN_ARC2, 1674 [GAUDI2_QUEUE_ID_NIC_2_3] = CPU_ID_NIC_QMAN_ARC2, 1675 [GAUDI2_QUEUE_ID_NIC_3_0] = CPU_ID_NIC_QMAN_ARC3, 1676 [GAUDI2_QUEUE_ID_NIC_3_1] = CPU_ID_NIC_QMAN_ARC3, 1677 [GAUDI2_QUEUE_ID_NIC_3_2] = CPU_ID_NIC_QMAN_ARC3, 1678 [GAUDI2_QUEUE_ID_NIC_3_3] = CPU_ID_NIC_QMAN_ARC3, 1679 [GAUDI2_QUEUE_ID_NIC_4_0] = CPU_ID_NIC_QMAN_ARC4, 1680 [GAUDI2_QUEUE_ID_NIC_4_1] = CPU_ID_NIC_QMAN_ARC4, 1681 [GAUDI2_QUEUE_ID_NIC_4_2] = CPU_ID_NIC_QMAN_ARC4, 1682 [GAUDI2_QUEUE_ID_NIC_4_3] = CPU_ID_NIC_QMAN_ARC4, 1683 [GAUDI2_QUEUE_ID_NIC_5_0] = CPU_ID_NIC_QMAN_ARC5, 1684 [GAUDI2_QUEUE_ID_NIC_5_1] = CPU_ID_NIC_QMAN_ARC5, 1685 [GAUDI2_QUEUE_ID_NIC_5_2] = CPU_ID_NIC_QMAN_ARC5, 1686 [GAUDI2_QUEUE_ID_NIC_5_3] = CPU_ID_NIC_QMAN_ARC5, 1687 [GAUDI2_QUEUE_ID_NIC_6_0] = CPU_ID_NIC_QMAN_ARC6, 1688 [GAUDI2_QUEUE_ID_NIC_6_1] = CPU_ID_NIC_QMAN_ARC6, 1689 [GAUDI2_QUEUE_ID_NIC_6_2] = CPU_ID_NIC_QMAN_ARC6, 1690 [GAUDI2_QUEUE_ID_NIC_6_3] = CPU_ID_NIC_QMAN_ARC6, 1691 [GAUDI2_QUEUE_ID_NIC_7_0] = CPU_ID_NIC_QMAN_ARC7, 1692 [GAUDI2_QUEUE_ID_NIC_7_1] = CPU_ID_NIC_QMAN_ARC7, 1693 [GAUDI2_QUEUE_ID_NIC_7_2] = CPU_ID_NIC_QMAN_ARC7, 1694 [GAUDI2_QUEUE_ID_NIC_7_3] = CPU_ID_NIC_QMAN_ARC7, 1695 [GAUDI2_QUEUE_ID_NIC_8_0] = CPU_ID_NIC_QMAN_ARC8, 1696 [GAUDI2_QUEUE_ID_NIC_8_1] = CPU_ID_NIC_QMAN_ARC8, 1697 [GAUDI2_QUEUE_ID_NIC_8_2] = CPU_ID_NIC_QMAN_ARC8, 1698 [GAUDI2_QUEUE_ID_NIC_8_3] = CPU_ID_NIC_QMAN_ARC8, 1699 [GAUDI2_QUEUE_ID_NIC_9_0] = CPU_ID_NIC_QMAN_ARC9, 1700 [GAUDI2_QUEUE_ID_NIC_9_1] = CPU_ID_NIC_QMAN_ARC9, 1701 [GAUDI2_QUEUE_ID_NIC_9_2] = CPU_ID_NIC_QMAN_ARC9, 1702 [GAUDI2_QUEUE_ID_NIC_9_3] = CPU_ID_NIC_QMAN_ARC9, 1703 [GAUDI2_QUEUE_ID_NIC_10_0] = CPU_ID_NIC_QMAN_ARC10, 1704 [GAUDI2_QUEUE_ID_NIC_10_1] = CPU_ID_NIC_QMAN_ARC10, 1705 [GAUDI2_QUEUE_ID_NIC_10_2] = CPU_ID_NIC_QMAN_ARC10, 1706 [GAUDI2_QUEUE_ID_NIC_10_3] = CPU_ID_NIC_QMAN_ARC10, 1707 [GAUDI2_QUEUE_ID_NIC_11_0] = CPU_ID_NIC_QMAN_ARC11, 1708 [GAUDI2_QUEUE_ID_NIC_11_1] = CPU_ID_NIC_QMAN_ARC11, 1709 [GAUDI2_QUEUE_ID_NIC_11_2] = CPU_ID_NIC_QMAN_ARC11, 1710 [GAUDI2_QUEUE_ID_NIC_11_3] = CPU_ID_NIC_QMAN_ARC11, 1711 [GAUDI2_QUEUE_ID_NIC_12_0] = CPU_ID_NIC_QMAN_ARC12, 1712 [GAUDI2_QUEUE_ID_NIC_12_1] = CPU_ID_NIC_QMAN_ARC12, 1713 [GAUDI2_QUEUE_ID_NIC_12_2] = CPU_ID_NIC_QMAN_ARC12, 1714 [GAUDI2_QUEUE_ID_NIC_12_3] = CPU_ID_NIC_QMAN_ARC12, 1715 [GAUDI2_QUEUE_ID_NIC_13_0] = CPU_ID_NIC_QMAN_ARC13, 1716 [GAUDI2_QUEUE_ID_NIC_13_1] = CPU_ID_NIC_QMAN_ARC13, 1717 [GAUDI2_QUEUE_ID_NIC_13_2] = CPU_ID_NIC_QMAN_ARC13, 1718 [GAUDI2_QUEUE_ID_NIC_13_3] = CPU_ID_NIC_QMAN_ARC13, 1719 [GAUDI2_QUEUE_ID_NIC_14_0] = CPU_ID_NIC_QMAN_ARC14, 1720 [GAUDI2_QUEUE_ID_NIC_14_1] = CPU_ID_NIC_QMAN_ARC14, 1721 [GAUDI2_QUEUE_ID_NIC_14_2] = CPU_ID_NIC_QMAN_ARC14, 1722 [GAUDI2_QUEUE_ID_NIC_14_3] = CPU_ID_NIC_QMAN_ARC14, 1723 [GAUDI2_QUEUE_ID_NIC_15_0] = CPU_ID_NIC_QMAN_ARC15, 1724 [GAUDI2_QUEUE_ID_NIC_15_1] = CPU_ID_NIC_QMAN_ARC15, 1725 [GAUDI2_QUEUE_ID_NIC_15_2] = CPU_ID_NIC_QMAN_ARC15, 1726 [GAUDI2_QUEUE_ID_NIC_15_3] = CPU_ID_NIC_QMAN_ARC15, 1727 [GAUDI2_QUEUE_ID_NIC_16_0] = CPU_ID_NIC_QMAN_ARC16, 1728 [GAUDI2_QUEUE_ID_NIC_16_1] = CPU_ID_NIC_QMAN_ARC16, 1729 [GAUDI2_QUEUE_ID_NIC_16_2] = CPU_ID_NIC_QMAN_ARC16, 1730 [GAUDI2_QUEUE_ID_NIC_16_3] = CPU_ID_NIC_QMAN_ARC16, 1731 [GAUDI2_QUEUE_ID_NIC_17_0] = CPU_ID_NIC_QMAN_ARC17, 1732 [GAUDI2_QUEUE_ID_NIC_17_1] = CPU_ID_NIC_QMAN_ARC17, 1733 [GAUDI2_QUEUE_ID_NIC_17_2] = CPU_ID_NIC_QMAN_ARC17, 1734 [GAUDI2_QUEUE_ID_NIC_17_3] = CPU_ID_NIC_QMAN_ARC17, 1735 [GAUDI2_QUEUE_ID_NIC_18_0] = CPU_ID_NIC_QMAN_ARC18, 1736 [GAUDI2_QUEUE_ID_NIC_18_1] = CPU_ID_NIC_QMAN_ARC18, 1737 [GAUDI2_QUEUE_ID_NIC_18_2] = CPU_ID_NIC_QMAN_ARC18, 1738 [GAUDI2_QUEUE_ID_NIC_18_3] = CPU_ID_NIC_QMAN_ARC18, 1739 [GAUDI2_QUEUE_ID_NIC_19_0] = CPU_ID_NIC_QMAN_ARC19, 1740 [GAUDI2_QUEUE_ID_NIC_19_1] = CPU_ID_NIC_QMAN_ARC19, 1741 [GAUDI2_QUEUE_ID_NIC_19_2] = CPU_ID_NIC_QMAN_ARC19, 1742 [GAUDI2_QUEUE_ID_NIC_19_3] = CPU_ID_NIC_QMAN_ARC19, 1743 [GAUDI2_QUEUE_ID_NIC_20_0] = CPU_ID_NIC_QMAN_ARC20, 1744 [GAUDI2_QUEUE_ID_NIC_20_1] = CPU_ID_NIC_QMAN_ARC20, 1745 [GAUDI2_QUEUE_ID_NIC_20_2] = CPU_ID_NIC_QMAN_ARC20, 1746 [GAUDI2_QUEUE_ID_NIC_20_3] = CPU_ID_NIC_QMAN_ARC20, 1747 [GAUDI2_QUEUE_ID_NIC_21_0] = CPU_ID_NIC_QMAN_ARC21, 1748 [GAUDI2_QUEUE_ID_NIC_21_1] = CPU_ID_NIC_QMAN_ARC21, 1749 [GAUDI2_QUEUE_ID_NIC_21_2] = CPU_ID_NIC_QMAN_ARC21, 1750 [GAUDI2_QUEUE_ID_NIC_21_3] = CPU_ID_NIC_QMAN_ARC21, 1751 [GAUDI2_QUEUE_ID_NIC_22_0] = CPU_ID_NIC_QMAN_ARC22, 1752 [GAUDI2_QUEUE_ID_NIC_22_1] = CPU_ID_NIC_QMAN_ARC22, 1753 [GAUDI2_QUEUE_ID_NIC_22_2] = CPU_ID_NIC_QMAN_ARC22, 1754 [GAUDI2_QUEUE_ID_NIC_22_3] = CPU_ID_NIC_QMAN_ARC22, 1755 [GAUDI2_QUEUE_ID_NIC_23_0] = CPU_ID_NIC_QMAN_ARC23, 1756 [GAUDI2_QUEUE_ID_NIC_23_1] = CPU_ID_NIC_QMAN_ARC23, 1757 [GAUDI2_QUEUE_ID_NIC_23_2] = CPU_ID_NIC_QMAN_ARC23, 1758 [GAUDI2_QUEUE_ID_NIC_23_3] = CPU_ID_NIC_QMAN_ARC23, 1759 [GAUDI2_QUEUE_ID_ROT_0_0] = CPU_ID_ROT_QMAN_ARC0, 1760 [GAUDI2_QUEUE_ID_ROT_0_1] = CPU_ID_ROT_QMAN_ARC0, 1761 [GAUDI2_QUEUE_ID_ROT_0_2] = CPU_ID_ROT_QMAN_ARC0, 1762 [GAUDI2_QUEUE_ID_ROT_0_3] = CPU_ID_ROT_QMAN_ARC0, 1763 [GAUDI2_QUEUE_ID_ROT_1_0] = CPU_ID_ROT_QMAN_ARC1, 1764 [GAUDI2_QUEUE_ID_ROT_1_1] = CPU_ID_ROT_QMAN_ARC1, 1765 [GAUDI2_QUEUE_ID_ROT_1_2] = CPU_ID_ROT_QMAN_ARC1, 1766 [GAUDI2_QUEUE_ID_ROT_1_3] = CPU_ID_ROT_QMAN_ARC1 1767 }; 1768 1769 const u32 gaudi2_dma_core_blocks_bases[DMA_CORE_ID_SIZE] = { 1770 [DMA_CORE_ID_PDMA0] = mmPDMA0_CORE_BASE, 1771 [DMA_CORE_ID_PDMA1] = mmPDMA1_CORE_BASE, 1772 [DMA_CORE_ID_EDMA0] = mmDCORE0_EDMA0_CORE_BASE, 1773 [DMA_CORE_ID_EDMA1] = mmDCORE0_EDMA1_CORE_BASE, 1774 [DMA_CORE_ID_EDMA2] = mmDCORE1_EDMA0_CORE_BASE, 1775 [DMA_CORE_ID_EDMA3] = mmDCORE1_EDMA1_CORE_BASE, 1776 [DMA_CORE_ID_EDMA4] = mmDCORE2_EDMA0_CORE_BASE, 1777 [DMA_CORE_ID_EDMA5] = mmDCORE2_EDMA1_CORE_BASE, 1778 [DMA_CORE_ID_EDMA6] = mmDCORE3_EDMA0_CORE_BASE, 1779 [DMA_CORE_ID_EDMA7] = mmDCORE3_EDMA1_CORE_BASE, 1780 [DMA_CORE_ID_KDMA] = mmARC_FARM_KDMA_BASE 1781 }; 1782 1783 const u32 gaudi2_mme_acc_blocks_bases[MME_ID_SIZE] = { 1784 [MME_ID_DCORE0] = mmDCORE0_MME_ACC_BASE, 1785 [MME_ID_DCORE1] = mmDCORE1_MME_ACC_BASE, 1786 [MME_ID_DCORE2] = mmDCORE2_MME_ACC_BASE, 1787 [MME_ID_DCORE3] = mmDCORE3_MME_ACC_BASE 1788 }; 1789 1790 static const u32 gaudi2_tpc_cfg_blocks_bases[TPC_ID_SIZE] = { 1791 [TPC_ID_DCORE0_TPC0] = mmDCORE0_TPC0_CFG_BASE, 1792 [TPC_ID_DCORE0_TPC1] = mmDCORE0_TPC1_CFG_BASE, 1793 [TPC_ID_DCORE0_TPC2] = mmDCORE0_TPC2_CFG_BASE, 1794 [TPC_ID_DCORE0_TPC3] = mmDCORE0_TPC3_CFG_BASE, 1795 [TPC_ID_DCORE0_TPC4] = mmDCORE0_TPC4_CFG_BASE, 1796 [TPC_ID_DCORE0_TPC5] = mmDCORE0_TPC5_CFG_BASE, 1797 [TPC_ID_DCORE1_TPC0] = mmDCORE1_TPC0_CFG_BASE, 1798 [TPC_ID_DCORE1_TPC1] = mmDCORE1_TPC1_CFG_BASE, 1799 [TPC_ID_DCORE1_TPC2] = mmDCORE1_TPC2_CFG_BASE, 1800 [TPC_ID_DCORE1_TPC3] = mmDCORE1_TPC3_CFG_BASE, 1801 [TPC_ID_DCORE1_TPC4] = mmDCORE1_TPC4_CFG_BASE, 1802 [TPC_ID_DCORE1_TPC5] = mmDCORE1_TPC5_CFG_BASE, 1803 [TPC_ID_DCORE2_TPC0] = mmDCORE2_TPC0_CFG_BASE, 1804 [TPC_ID_DCORE2_TPC1] = mmDCORE2_TPC1_CFG_BASE, 1805 [TPC_ID_DCORE2_TPC2] = mmDCORE2_TPC2_CFG_BASE, 1806 [TPC_ID_DCORE2_TPC3] = mmDCORE2_TPC3_CFG_BASE, 1807 [TPC_ID_DCORE2_TPC4] = mmDCORE2_TPC4_CFG_BASE, 1808 [TPC_ID_DCORE2_TPC5] = mmDCORE2_TPC5_CFG_BASE, 1809 [TPC_ID_DCORE3_TPC0] = mmDCORE3_TPC0_CFG_BASE, 1810 [TPC_ID_DCORE3_TPC1] = mmDCORE3_TPC1_CFG_BASE, 1811 [TPC_ID_DCORE3_TPC2] = mmDCORE3_TPC2_CFG_BASE, 1812 [TPC_ID_DCORE3_TPC3] = mmDCORE3_TPC3_CFG_BASE, 1813 [TPC_ID_DCORE3_TPC4] = mmDCORE3_TPC4_CFG_BASE, 1814 [TPC_ID_DCORE3_TPC5] = mmDCORE3_TPC5_CFG_BASE, 1815 [TPC_ID_DCORE0_TPC6] = mmDCORE0_TPC6_CFG_BASE, 1816 }; 1817 1818 static const u32 gaudi2_tpc_eml_cfg_blocks_bases[TPC_ID_SIZE] = { 1819 [TPC_ID_DCORE0_TPC0] = mmDCORE0_TPC0_EML_CFG_BASE, 1820 [TPC_ID_DCORE0_TPC1] = mmDCORE0_TPC1_EML_CFG_BASE, 1821 [TPC_ID_DCORE0_TPC2] = mmDCORE0_TPC2_EML_CFG_BASE, 1822 [TPC_ID_DCORE0_TPC3] = mmDCORE0_TPC3_EML_CFG_BASE, 1823 [TPC_ID_DCORE0_TPC4] = mmDCORE0_TPC4_EML_CFG_BASE, 1824 [TPC_ID_DCORE0_TPC5] = mmDCORE0_TPC5_EML_CFG_BASE, 1825 [TPC_ID_DCORE1_TPC0] = mmDCORE1_TPC0_EML_CFG_BASE, 1826 [TPC_ID_DCORE1_TPC1] = mmDCORE1_TPC1_EML_CFG_BASE, 1827 [TPC_ID_DCORE1_TPC2] = mmDCORE1_TPC2_EML_CFG_BASE, 1828 [TPC_ID_DCORE1_TPC3] = mmDCORE1_TPC3_EML_CFG_BASE, 1829 [TPC_ID_DCORE1_TPC4] = mmDCORE1_TPC4_EML_CFG_BASE, 1830 [TPC_ID_DCORE1_TPC5] = mmDCORE1_TPC5_EML_CFG_BASE, 1831 [TPC_ID_DCORE2_TPC0] = mmDCORE2_TPC0_EML_CFG_BASE, 1832 [TPC_ID_DCORE2_TPC1] = mmDCORE2_TPC1_EML_CFG_BASE, 1833 [TPC_ID_DCORE2_TPC2] = mmDCORE2_TPC2_EML_CFG_BASE, 1834 [TPC_ID_DCORE2_TPC3] = mmDCORE2_TPC3_EML_CFG_BASE, 1835 [TPC_ID_DCORE2_TPC4] = mmDCORE2_TPC4_EML_CFG_BASE, 1836 [TPC_ID_DCORE2_TPC5] = mmDCORE2_TPC5_EML_CFG_BASE, 1837 [TPC_ID_DCORE3_TPC0] = mmDCORE3_TPC0_EML_CFG_BASE, 1838 [TPC_ID_DCORE3_TPC1] = mmDCORE3_TPC1_EML_CFG_BASE, 1839 [TPC_ID_DCORE3_TPC2] = mmDCORE3_TPC2_EML_CFG_BASE, 1840 [TPC_ID_DCORE3_TPC3] = mmDCORE3_TPC3_EML_CFG_BASE, 1841 [TPC_ID_DCORE3_TPC4] = mmDCORE3_TPC4_EML_CFG_BASE, 1842 [TPC_ID_DCORE3_TPC5] = mmDCORE3_TPC5_EML_CFG_BASE, 1843 [TPC_ID_DCORE0_TPC6] = mmDCORE0_TPC6_EML_CFG_BASE, 1844 }; 1845 1846 const u32 gaudi2_rot_blocks_bases[ROTATOR_ID_SIZE] = { 1847 [ROTATOR_ID_0] = mmROT0_BASE, 1848 [ROTATOR_ID_1] = mmROT1_BASE 1849 }; 1850 1851 static const u32 gaudi2_tpc_id_to_queue_id[TPC_ID_SIZE] = { 1852 [TPC_ID_DCORE0_TPC0] = GAUDI2_QUEUE_ID_DCORE0_TPC_0_0, 1853 [TPC_ID_DCORE0_TPC1] = GAUDI2_QUEUE_ID_DCORE0_TPC_1_0, 1854 [TPC_ID_DCORE0_TPC2] = GAUDI2_QUEUE_ID_DCORE0_TPC_2_0, 1855 [TPC_ID_DCORE0_TPC3] = GAUDI2_QUEUE_ID_DCORE0_TPC_3_0, 1856 [TPC_ID_DCORE0_TPC4] = GAUDI2_QUEUE_ID_DCORE0_TPC_4_0, 1857 [TPC_ID_DCORE0_TPC5] = GAUDI2_QUEUE_ID_DCORE0_TPC_5_0, 1858 [TPC_ID_DCORE1_TPC0] = GAUDI2_QUEUE_ID_DCORE1_TPC_0_0, 1859 [TPC_ID_DCORE1_TPC1] = GAUDI2_QUEUE_ID_DCORE1_TPC_1_0, 1860 [TPC_ID_DCORE1_TPC2] = GAUDI2_QUEUE_ID_DCORE1_TPC_2_0, 1861 [TPC_ID_DCORE1_TPC3] = GAUDI2_QUEUE_ID_DCORE1_TPC_3_0, 1862 [TPC_ID_DCORE1_TPC4] = GAUDI2_QUEUE_ID_DCORE1_TPC_4_0, 1863 [TPC_ID_DCORE1_TPC5] = GAUDI2_QUEUE_ID_DCORE1_TPC_5_0, 1864 [TPC_ID_DCORE2_TPC0] = GAUDI2_QUEUE_ID_DCORE2_TPC_0_0, 1865 [TPC_ID_DCORE2_TPC1] = GAUDI2_QUEUE_ID_DCORE2_TPC_1_0, 1866 [TPC_ID_DCORE2_TPC2] = GAUDI2_QUEUE_ID_DCORE2_TPC_2_0, 1867 [TPC_ID_DCORE2_TPC3] = GAUDI2_QUEUE_ID_DCORE2_TPC_3_0, 1868 [TPC_ID_DCORE2_TPC4] = GAUDI2_QUEUE_ID_DCORE2_TPC_4_0, 1869 [TPC_ID_DCORE2_TPC5] = GAUDI2_QUEUE_ID_DCORE2_TPC_5_0, 1870 [TPC_ID_DCORE3_TPC0] = GAUDI2_QUEUE_ID_DCORE3_TPC_0_0, 1871 [TPC_ID_DCORE3_TPC1] = GAUDI2_QUEUE_ID_DCORE3_TPC_1_0, 1872 [TPC_ID_DCORE3_TPC2] = GAUDI2_QUEUE_ID_DCORE3_TPC_2_0, 1873 [TPC_ID_DCORE3_TPC3] = GAUDI2_QUEUE_ID_DCORE3_TPC_3_0, 1874 [TPC_ID_DCORE3_TPC4] = GAUDI2_QUEUE_ID_DCORE3_TPC_4_0, 1875 [TPC_ID_DCORE3_TPC5] = GAUDI2_QUEUE_ID_DCORE3_TPC_5_0, 1876 [TPC_ID_DCORE0_TPC6] = GAUDI2_QUEUE_ID_DCORE0_TPC_6_0, 1877 }; 1878 1879 static const u32 gaudi2_rot_id_to_queue_id[ROTATOR_ID_SIZE] = { 1880 [ROTATOR_ID_0] = GAUDI2_QUEUE_ID_ROT_0_0, 1881 [ROTATOR_ID_1] = GAUDI2_QUEUE_ID_ROT_1_0, 1882 }; 1883 1884 static const u32 gaudi2_tpc_engine_id_to_tpc_id[] = { 1885 [GAUDI2_DCORE0_ENGINE_ID_TPC_0] = TPC_ID_DCORE0_TPC0, 1886 [GAUDI2_DCORE0_ENGINE_ID_TPC_1] = TPC_ID_DCORE0_TPC1, 1887 [GAUDI2_DCORE0_ENGINE_ID_TPC_2] = TPC_ID_DCORE0_TPC2, 1888 [GAUDI2_DCORE0_ENGINE_ID_TPC_3] = TPC_ID_DCORE0_TPC3, 1889 [GAUDI2_DCORE0_ENGINE_ID_TPC_4] = TPC_ID_DCORE0_TPC4, 1890 [GAUDI2_DCORE0_ENGINE_ID_TPC_5] = TPC_ID_DCORE0_TPC5, 1891 [GAUDI2_DCORE1_ENGINE_ID_TPC_0] = TPC_ID_DCORE1_TPC0, 1892 [GAUDI2_DCORE1_ENGINE_ID_TPC_1] = TPC_ID_DCORE1_TPC1, 1893 [GAUDI2_DCORE1_ENGINE_ID_TPC_2] = TPC_ID_DCORE1_TPC2, 1894 [GAUDI2_DCORE1_ENGINE_ID_TPC_3] = TPC_ID_DCORE1_TPC3, 1895 [GAUDI2_DCORE1_ENGINE_ID_TPC_4] = TPC_ID_DCORE1_TPC4, 1896 [GAUDI2_DCORE1_ENGINE_ID_TPC_5] = TPC_ID_DCORE1_TPC5, 1897 [GAUDI2_DCORE2_ENGINE_ID_TPC_0] = TPC_ID_DCORE2_TPC0, 1898 [GAUDI2_DCORE2_ENGINE_ID_TPC_1] = TPC_ID_DCORE2_TPC1, 1899 [GAUDI2_DCORE2_ENGINE_ID_TPC_2] = TPC_ID_DCORE2_TPC2, 1900 [GAUDI2_DCORE2_ENGINE_ID_TPC_3] = TPC_ID_DCORE2_TPC3, 1901 [GAUDI2_DCORE2_ENGINE_ID_TPC_4] = TPC_ID_DCORE2_TPC4, 1902 [GAUDI2_DCORE2_ENGINE_ID_TPC_5] = TPC_ID_DCORE2_TPC5, 1903 [GAUDI2_DCORE3_ENGINE_ID_TPC_0] = TPC_ID_DCORE3_TPC0, 1904 [GAUDI2_DCORE3_ENGINE_ID_TPC_1] = TPC_ID_DCORE3_TPC1, 1905 [GAUDI2_DCORE3_ENGINE_ID_TPC_2] = TPC_ID_DCORE3_TPC2, 1906 [GAUDI2_DCORE3_ENGINE_ID_TPC_3] = TPC_ID_DCORE3_TPC3, 1907 [GAUDI2_DCORE3_ENGINE_ID_TPC_4] = TPC_ID_DCORE3_TPC4, 1908 [GAUDI2_DCORE3_ENGINE_ID_TPC_5] = TPC_ID_DCORE3_TPC5, 1909 /* the PCI TPC is placed last (mapped liked HW) */ 1910 [GAUDI2_DCORE0_ENGINE_ID_TPC_6] = TPC_ID_DCORE0_TPC6, 1911 }; 1912 1913 static const u32 gaudi2_mme_engine_id_to_mme_id[] = { 1914 [GAUDI2_DCORE0_ENGINE_ID_MME] = MME_ID_DCORE0, 1915 [GAUDI2_DCORE1_ENGINE_ID_MME] = MME_ID_DCORE1, 1916 [GAUDI2_DCORE2_ENGINE_ID_MME] = MME_ID_DCORE2, 1917 [GAUDI2_DCORE3_ENGINE_ID_MME] = MME_ID_DCORE3, 1918 }; 1919 1920 static const u32 gaudi2_edma_engine_id_to_edma_id[] = { 1921 [GAUDI2_ENGINE_ID_PDMA_0] = DMA_CORE_ID_PDMA0, 1922 [GAUDI2_ENGINE_ID_PDMA_1] = DMA_CORE_ID_PDMA1, 1923 [GAUDI2_DCORE0_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA0, 1924 [GAUDI2_DCORE0_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA1, 1925 [GAUDI2_DCORE1_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA2, 1926 [GAUDI2_DCORE1_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA3, 1927 [GAUDI2_DCORE2_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA4, 1928 [GAUDI2_DCORE2_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA5, 1929 [GAUDI2_DCORE3_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA6, 1930 [GAUDI2_DCORE3_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA7, 1931 [GAUDI2_ENGINE_ID_KDMA] = DMA_CORE_ID_KDMA, 1932 }; 1933 1934 const u32 edma_stream_base[NUM_OF_EDMA_PER_DCORE * NUM_OF_DCORES] = { 1935 GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0, 1936 GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0, 1937 GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0, 1938 GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0, 1939 GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0, 1940 GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0, 1941 GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0, 1942 GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0, 1943 }; 1944 1945 static const char gaudi2_vdec_irq_name[GAUDI2_VDEC_MSIX_ENTRIES][GAUDI2_MAX_STRING_LEN] = { 1946 "gaudi2 vdec 0_0", "gaudi2 vdec 0_0 abnormal", 1947 "gaudi2 vdec 0_1", "gaudi2 vdec 0_1 abnormal", 1948 "gaudi2 vdec 1_0", "gaudi2 vdec 1_0 abnormal", 1949 "gaudi2 vdec 1_1", "gaudi2 vdec 1_1 abnormal", 1950 "gaudi2 vdec 2_0", "gaudi2 vdec 2_0 abnormal", 1951 "gaudi2 vdec 2_1", "gaudi2 vdec 2_1 abnormal", 1952 "gaudi2 vdec 3_0", "gaudi2 vdec 3_0 abnormal", 1953 "gaudi2 vdec 3_1", "gaudi2 vdec 3_1 abnormal", 1954 "gaudi2 vdec s_0", "gaudi2 vdec s_0 abnormal", 1955 "gaudi2 vdec s_1", "gaudi2 vdec s_1 abnormal" 1956 }; 1957 1958 enum rtr_id { 1959 DCORE0_RTR0, 1960 DCORE0_RTR1, 1961 DCORE0_RTR2, 1962 DCORE0_RTR3, 1963 DCORE0_RTR4, 1964 DCORE0_RTR5, 1965 DCORE0_RTR6, 1966 DCORE0_RTR7, 1967 DCORE1_RTR0, 1968 DCORE1_RTR1, 1969 DCORE1_RTR2, 1970 DCORE1_RTR3, 1971 DCORE1_RTR4, 1972 DCORE1_RTR5, 1973 DCORE1_RTR6, 1974 DCORE1_RTR7, 1975 DCORE2_RTR0, 1976 DCORE2_RTR1, 1977 DCORE2_RTR2, 1978 DCORE2_RTR3, 1979 DCORE2_RTR4, 1980 DCORE2_RTR5, 1981 DCORE2_RTR6, 1982 DCORE2_RTR7, 1983 DCORE3_RTR0, 1984 DCORE3_RTR1, 1985 DCORE3_RTR2, 1986 DCORE3_RTR3, 1987 DCORE3_RTR4, 1988 DCORE3_RTR5, 1989 DCORE3_RTR6, 1990 DCORE3_RTR7, 1991 }; 1992 1993 static const u32 gaudi2_tpc_initiator_hbw_rtr_id[NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1] = { 1994 DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR2, DCORE0_RTR2, DCORE0_RTR3, DCORE0_RTR3, 1995 DCORE1_RTR6, DCORE1_RTR6, DCORE1_RTR5, DCORE1_RTR5, DCORE1_RTR4, DCORE1_RTR4, 1996 DCORE2_RTR3, DCORE2_RTR3, DCORE2_RTR2, DCORE2_RTR2, DCORE2_RTR1, DCORE2_RTR1, 1997 DCORE3_RTR4, DCORE3_RTR4, DCORE3_RTR5, DCORE3_RTR5, DCORE3_RTR6, DCORE3_RTR6, 1998 DCORE0_RTR0 1999 }; 2000 2001 static const u32 gaudi2_tpc_initiator_lbw_rtr_id[NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1] = { 2002 DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR2, DCORE0_RTR2, 2003 DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR6, DCORE1_RTR6, DCORE1_RTR5, DCORE1_RTR5, 2004 DCORE2_RTR2, DCORE2_RTR2, DCORE2_RTR1, DCORE2_RTR1, DCORE2_RTR0, DCORE2_RTR0, 2005 DCORE3_RTR5, DCORE3_RTR5, DCORE3_RTR6, DCORE3_RTR6, DCORE3_RTR7, DCORE3_RTR7, 2006 DCORE0_RTR0 2007 }; 2008 2009 static const u32 gaudi2_dec_initiator_hbw_rtr_id[NUMBER_OF_DEC] = { 2010 DCORE0_RTR0, DCORE0_RTR0, DCORE1_RTR7, DCORE1_RTR7, DCORE2_RTR0, DCORE2_RTR0, 2011 DCORE3_RTR7, DCORE3_RTR7, DCORE0_RTR0, DCORE0_RTR0 2012 }; 2013 2014 static const u32 gaudi2_dec_initiator_lbw_rtr_id[NUMBER_OF_DEC] = { 2015 DCORE0_RTR1, DCORE0_RTR1, DCORE1_RTR6, DCORE1_RTR6, DCORE2_RTR1, DCORE2_RTR1, 2016 DCORE3_RTR6, DCORE3_RTR6, DCORE0_RTR0, DCORE0_RTR0 2017 }; 2018 2019 static const u32 gaudi2_nic_initiator_hbw_rtr_id[NIC_NUMBER_OF_MACROS] = { 2020 DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE2_RTR0, 2021 DCORE2_RTR0, DCORE2_RTR0, DCORE2_RTR0, DCORE3_RTR7, DCORE3_RTR7, DCORE3_RTR7 2022 }; 2023 2024 static const u32 gaudi2_nic_initiator_lbw_rtr_id[NIC_NUMBER_OF_MACROS] = { 2025 DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE2_RTR0, 2026 DCORE2_RTR0, DCORE2_RTR0, DCORE2_RTR0, DCORE3_RTR7, DCORE3_RTR7, DCORE3_RTR7 2027 }; 2028 2029 static const u32 gaudi2_edma_initiator_hbw_sft[NUM_OF_EDMA_PER_DCORE * NUM_OF_DCORES] = { 2030 mmSFT0_HBW_RTR_IF1_MSTR_IF_RR_SHRD_HBW_BASE, 2031 mmSFT0_HBW_RTR_IF0_MSTR_IF_RR_SHRD_HBW_BASE, 2032 mmSFT1_HBW_RTR_IF1_MSTR_IF_RR_SHRD_HBW_BASE, 2033 mmSFT1_HBW_RTR_IF0_MSTR_IF_RR_SHRD_HBW_BASE, 2034 mmSFT2_HBW_RTR_IF0_MSTR_IF_RR_SHRD_HBW_BASE, 2035 mmSFT2_HBW_RTR_IF1_MSTR_IF_RR_SHRD_HBW_BASE, 2036 mmSFT3_HBW_RTR_IF0_MSTR_IF_RR_SHRD_HBW_BASE, 2037 mmSFT3_HBW_RTR_IF1_MSTR_IF_RR_SHRD_HBW_BASE 2038 }; 2039 2040 static const u32 gaudi2_pdma_initiator_hbw_rtr_id[NUM_OF_PDMA] = { 2041 DCORE0_RTR0, DCORE0_RTR0 2042 }; 2043 2044 static const u32 gaudi2_pdma_initiator_lbw_rtr_id[NUM_OF_PDMA] = { 2045 DCORE0_RTR2, DCORE0_RTR2 2046 }; 2047 2048 static const u32 gaudi2_rot_initiator_hbw_rtr_id[NUM_OF_ROT] = { 2049 DCORE2_RTR0, DCORE3_RTR7 2050 }; 2051 2052 static const u32 gaudi2_rot_initiator_lbw_rtr_id[NUM_OF_ROT] = { 2053 DCORE2_RTR2, DCORE3_RTR5 2054 }; 2055 2056 struct mme_initiators_rtr_id { 2057 u32 wap0; 2058 u32 wap1; 2059 u32 write; 2060 u32 read; 2061 u32 sbte0; 2062 u32 sbte1; 2063 u32 sbte2; 2064 u32 sbte3; 2065 u32 sbte4; 2066 }; 2067 2068 enum mme_initiators { 2069 MME_WAP0 = 0, 2070 MME_WAP1, 2071 MME_WRITE, 2072 MME_READ, 2073 MME_SBTE0, 2074 MME_SBTE1, 2075 MME_SBTE2, 2076 MME_SBTE3, 2077 MME_SBTE4, 2078 MME_INITIATORS_MAX 2079 }; 2080 2081 static const struct mme_initiators_rtr_id 2082 gaudi2_mme_initiator_rtr_id[NUM_OF_MME_PER_DCORE * NUM_OF_DCORES] = { 2083 { .wap0 = 5, .wap1 = 7, .write = 6, .read = 7, 2084 .sbte0 = 7, .sbte1 = 4, .sbte2 = 4, .sbte3 = 5, .sbte4 = 6}, 2085 { .wap0 = 10, .wap1 = 8, .write = 9, .read = 8, 2086 .sbte0 = 11, .sbte1 = 11, .sbte2 = 10, .sbte3 = 9, .sbte4 = 8}, 2087 { .wap0 = 21, .wap1 = 23, .write = 22, .read = 23, 2088 .sbte0 = 20, .sbte1 = 20, .sbte2 = 21, .sbte3 = 22, .sbte4 = 23}, 2089 { .wap0 = 30, .wap1 = 28, .write = 29, .read = 30, 2090 .sbte0 = 31, .sbte1 = 31, .sbte2 = 30, .sbte3 = 29, .sbte4 = 28}, 2091 }; 2092 2093 enum razwi_event_sources { 2094 RAZWI_TPC, 2095 RAZWI_MME, 2096 RAZWI_EDMA, 2097 RAZWI_PDMA, 2098 RAZWI_NIC, 2099 RAZWI_DEC, 2100 RAZWI_ROT, 2101 RAZWI_ARC_FARM 2102 }; 2103 2104 struct hbm_mc_error_causes { 2105 u32 mask; 2106 char cause[50]; 2107 }; 2108 2109 static struct hl_special_block_info gaudi2_special_blocks[] = GAUDI2_SPECIAL_BLOCKS; 2110 2111 /* Special blocks iterator is currently used to configure security protection bits, 2112 * and read global errors. Most HW blocks are addressable and those who aren't (N/A)- 2113 * must be skipped. Following configurations are commonly used for both PB config 2114 * and global error reading, since currently they both share the same settings. 2115 * Once it changes, we must remember to use separate configurations for either one. 2116 */ 2117 static int gaudi2_iterator_skip_block_types[] = { 2118 GAUDI2_BLOCK_TYPE_PLL, 2119 GAUDI2_BLOCK_TYPE_EU_BIST, 2120 GAUDI2_BLOCK_TYPE_HBM, 2121 GAUDI2_BLOCK_TYPE_XFT 2122 }; 2123 2124 static struct range gaudi2_iterator_skip_block_ranges[] = { 2125 /* Skip all PSOC blocks except for PSOC_GLOBAL_CONF */ 2126 {mmPSOC_I2C_M0_BASE, mmPSOC_EFUSE_BASE}, 2127 {mmPSOC_BTL_BASE, mmPSOC_MSTR_IF_RR_SHRD_HBW_BASE}, 2128 /* Skip all CPU blocks except for CPU_IF */ 2129 {mmCPU_CA53_CFG_BASE, mmCPU_CA53_CFG_BASE}, 2130 {mmCPU_TIMESTAMP_BASE, mmCPU_MSTR_IF_RR_SHRD_HBW_BASE} 2131 }; 2132 2133 static struct hbm_mc_error_causes hbm_mc_spi[GAUDI2_NUM_OF_HBM_MC_SPI_CAUSE] = { 2134 {HBM_MC_SPI_TEMP_PIN_CHG_MASK, "temperature pins changed"}, 2135 {HBM_MC_SPI_THR_ENG_MASK, "temperature-based throttling engaged"}, 2136 {HBM_MC_SPI_THR_DIS_ENG_MASK, "temperature-based throttling disengaged"}, 2137 {HBM_MC_SPI_IEEE1500_COMP_MASK, "IEEE1500 op comp"}, 2138 {HBM_MC_SPI_IEEE1500_PAUSED_MASK, "IEEE1500 op paused"}, 2139 }; 2140 2141 static const char * const hbm_mc_sei_cause[GAUDI2_NUM_OF_HBM_SEI_CAUSE] = { 2142 [HBM_SEI_CMD_PARITY_EVEN] = "SEI C/A parity even", 2143 [HBM_SEI_CMD_PARITY_ODD] = "SEI C/A parity odd", 2144 [HBM_SEI_READ_ERR] = "SEI read data error", 2145 [HBM_SEI_WRITE_DATA_PARITY_ERR] = "SEI write data parity error", 2146 [HBM_SEI_CATTRIP] = "SEI CATTRIP asserted", 2147 [HBM_SEI_MEM_BIST_FAIL] = "SEI memory BIST fail", 2148 [HBM_SEI_DFI] = "SEI DFI error", 2149 [HBM_SEI_INV_TEMP_READ_OUT] = "SEI invalid temp read", 2150 [HBM_SEI_BIST_FAIL] = "SEI BIST fail" 2151 }; 2152 2153 struct mmu_spi_sei_cause { 2154 char cause[50]; 2155 int clear_bit; 2156 }; 2157 2158 static const struct mmu_spi_sei_cause gaudi2_mmu_spi_sei[GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE] = { 2159 {"page fault", 1}, /* INTERRUPT_CLR[1] */ 2160 {"page access", 1}, /* INTERRUPT_CLR[1] */ 2161 {"bypass ddr", 2}, /* INTERRUPT_CLR[2] */ 2162 {"multi hit", 2}, /* INTERRUPT_CLR[2] */ 2163 {"mmu rei0", -1}, /* no clear register bit */ 2164 {"mmu rei1", -1}, /* no clear register bit */ 2165 {"stlb rei0", -1}, /* no clear register bit */ 2166 {"stlb rei1", -1}, /* no clear register bit */ 2167 {"rr privileged write hit", 2}, /* INTERRUPT_CLR[2] */ 2168 {"rr privileged read hit", 2}, /* INTERRUPT_CLR[2] */ 2169 {"rr secure write hit", 2}, /* INTERRUPT_CLR[2] */ 2170 {"rr secure read hit", 2}, /* INTERRUPT_CLR[2] */ 2171 {"bist_fail no use", 2}, /* INTERRUPT_CLR[2] */ 2172 {"bist_fail no use", 2}, /* INTERRUPT_CLR[2] */ 2173 {"bist_fail no use", 2}, /* INTERRUPT_CLR[2] */ 2174 {"bist_fail no use", 2}, /* INTERRUPT_CLR[2] */ 2175 {"slave error", 16}, /* INTERRUPT_CLR[16] */ 2176 {"dec error", 17}, /* INTERRUPT_CLR[17] */ 2177 {"burst fifo full", 2} /* INTERRUPT_CLR[2] */ 2178 }; 2179 2180 struct gaudi2_cache_invld_params { 2181 u64 start_va; 2182 u64 end_va; 2183 u32 inv_start_val; 2184 u32 flags; 2185 bool range_invalidation; 2186 }; 2187 2188 struct gaudi2_tpc_idle_data { 2189 struct engines_data *e; 2190 unsigned long *mask; 2191 bool *is_idle; 2192 const char *tpc_fmt; 2193 }; 2194 2195 struct gaudi2_tpc_mmu_data { 2196 u32 rw_asid; 2197 }; 2198 2199 static s64 gaudi2_state_dump_specs_props[SP_MAX] = {0}; 2200 2201 static int gaudi2_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, u64 val); 2202 static bool gaudi2_is_queue_enabled(struct hl_device *hdev, u32 hw_queue_id); 2203 static bool gaudi2_is_arc_enabled(struct hl_device *hdev, u64 arc_id); 2204 static void gaudi2_clr_arc_id_cap(struct hl_device *hdev, u64 arc_id); 2205 static void gaudi2_set_arc_id_cap(struct hl_device *hdev, u64 arc_id); 2206 static void gaudi2_memset_device_lbw(struct hl_device *hdev, u32 addr, u32 size, u32 val); 2207 static int gaudi2_send_job_to_kdma(struct hl_device *hdev, u64 src_addr, u64 dst_addr, u32 size, 2208 bool is_memset); 2209 static bool gaudi2_get_tpc_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 2210 struct engines_data *e); 2211 static bool gaudi2_get_mme_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 2212 struct engines_data *e); 2213 static bool gaudi2_get_edma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 2214 struct engines_data *e); 2215 static u64 gaudi2_mmu_scramble_addr(struct hl_device *hdev, u64 raw_addr); 2216 static u64 gaudi2_mmu_descramble_addr(struct hl_device *hdev, u64 scrambled_addr); 2217 2218 static void gaudi2_init_scrambler_hbm(struct hl_device *hdev) 2219 { 2220 2221 } 2222 2223 static u32 gaudi2_get_signal_cb_size(struct hl_device *hdev) 2224 { 2225 return sizeof(struct packet_msg_short); 2226 } 2227 2228 static u32 gaudi2_get_wait_cb_size(struct hl_device *hdev) 2229 { 2230 return sizeof(struct packet_msg_short) * 4 + sizeof(struct packet_fence); 2231 } 2232 2233 void gaudi2_iterate_tpcs(struct hl_device *hdev, struct iterate_module_ctx *ctx) 2234 { 2235 struct asic_fixed_properties *prop = &hdev->asic_prop; 2236 int dcore, inst, tpc_seq; 2237 u32 offset; 2238 2239 /* init the return code */ 2240 ctx->rc = 0; 2241 2242 for (dcore = 0; dcore < NUM_OF_DCORES; dcore++) { 2243 for (inst = 0; inst < NUM_OF_TPC_PER_DCORE; inst++) { 2244 tpc_seq = dcore * NUM_OF_TPC_PER_DCORE + inst; 2245 2246 if (!(prop->tpc_enabled_mask & BIT(tpc_seq))) 2247 continue; 2248 2249 offset = (DCORE_OFFSET * dcore) + (DCORE_TPC_OFFSET * inst); 2250 2251 ctx->fn(hdev, dcore, inst, offset, ctx); 2252 if (ctx->rc) { 2253 dev_err(hdev->dev, "TPC iterator failed for DCORE%d TPC%d\n", 2254 dcore, inst); 2255 return; 2256 } 2257 } 2258 } 2259 2260 if (!(prop->tpc_enabled_mask & BIT(TPC_ID_DCORE0_TPC6))) 2261 return; 2262 2263 /* special check for PCI TPC (DCORE0_TPC6) */ 2264 offset = DCORE_TPC_OFFSET * (NUM_DCORE0_TPC - 1); 2265 ctx->fn(hdev, 0, NUM_DCORE0_TPC - 1, offset, ctx); 2266 if (ctx->rc) 2267 dev_err(hdev->dev, "TPC iterator failed for DCORE0 TPC6\n"); 2268 } 2269 2270 static bool gaudi2_host_phys_addr_valid(u64 addr) 2271 { 2272 if ((addr < HOST_PHYS_BASE_0 + HOST_PHYS_SIZE_0) || (addr >= HOST_PHYS_BASE_1)) 2273 return true; 2274 2275 return false; 2276 } 2277 2278 static int set_number_of_functional_hbms(struct hl_device *hdev) 2279 { 2280 struct asic_fixed_properties *prop = &hdev->asic_prop; 2281 u8 faulty_hbms = hweight64(hdev->dram_binning); 2282 2283 /* check if all HBMs should be used */ 2284 if (!faulty_hbms) { 2285 dev_dbg(hdev->dev, "All HBM are in use (no binning)\n"); 2286 prop->num_functional_hbms = GAUDI2_HBM_NUM; 2287 return 0; 2288 } 2289 2290 /* 2291 * check for error condition in which number of binning 2292 * candidates is higher than the maximum supported by the 2293 * driver (in which case binning mask shall be ignored and driver will 2294 * set the default) 2295 */ 2296 if (faulty_hbms > MAX_FAULTY_HBMS) { 2297 dev_err(hdev->dev, 2298 "HBM binning supports max of %d faulty HBMs, supplied mask 0x%llx.\n", 2299 MAX_FAULTY_HBMS, hdev->dram_binning); 2300 return -EINVAL; 2301 } 2302 2303 /* 2304 * by default, number of functional HBMs in Gaudi2 is always 2305 * GAUDI2_HBM_NUM - 1. 2306 */ 2307 prop->num_functional_hbms = GAUDI2_HBM_NUM - faulty_hbms; 2308 return 0; 2309 } 2310 2311 static bool gaudi2_is_edma_queue_id(u32 queue_id) 2312 { 2313 2314 switch (queue_id) { 2315 case GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3: 2316 case GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3: 2317 case GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3: 2318 case GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3: 2319 return true; 2320 default: 2321 return false; 2322 } 2323 } 2324 2325 static int gaudi2_set_dram_properties(struct hl_device *hdev) 2326 { 2327 struct asic_fixed_properties *prop = &hdev->asic_prop; 2328 u64 hbm_drv_base_offset = 0, edma_pq_base_addr; 2329 u32 basic_hbm_page_size, edma_idx = 0; 2330 int rc, i; 2331 2332 rc = set_number_of_functional_hbms(hdev); 2333 if (rc) 2334 return -EINVAL; 2335 2336 /* 2337 * Due to HW bug in which TLB size is x16 smaller than expected we use a workaround 2338 * in which we are using x16 bigger page size to be able to populate the entire 2339 * HBM mappings in the TLB 2340 */ 2341 basic_hbm_page_size = prop->num_functional_hbms * SZ_8M; 2342 prop->dram_page_size = GAUDI2_COMPENSATE_TLB_PAGE_SIZE_FACTOR * basic_hbm_page_size; 2343 prop->device_mem_alloc_default_page_size = prop->dram_page_size; 2344 prop->dram_size = prop->num_functional_hbms * SZ_16G; 2345 prop->dram_base_address = DRAM_PHYS_BASE; 2346 prop->dram_end_address = prop->dram_base_address + prop->dram_size; 2347 prop->dram_supports_virtual_memory = true; 2348 2349 prop->dram_user_base_address = DRAM_PHYS_BASE + prop->dram_page_size; 2350 prop->dram_hints_align_mask = ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK; 2351 prop->hints_dram_reserved_va_range.start_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HBM_START; 2352 prop->hints_dram_reserved_va_range.end_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HBM_END; 2353 2354 /* since DRAM page size differs from DMMU page size we need to allocate 2355 * DRAM memory in units of dram_page size and mapping this memory in 2356 * units of DMMU page size. we overcome this size mismatch using a 2357 * scrambling routine which takes a DRAM page and converts it to a DMMU 2358 * page. 2359 * We therefore: 2360 * 1. partition the virtual address space to DRAM-page (whole) pages. 2361 * (suppose we get n such pages) 2362 * 2. limit the amount of virtual address space we got from 1 above to 2363 * a multiple of 64M as we don't want the scrambled address to cross 2364 * the DRAM virtual address space. 2365 * ( m = (n * DRAM_page_size) / DMMU_page_size). 2366 * 3. determine the and address accordingly 2367 * end_addr = start_addr + m * 48M 2368 * 2369 * the DRAM address MSBs (63:48) are not part of the roundup calculation 2370 */ 2371 prop->dmmu.start_addr = prop->dram_base_address + 2372 (prop->dram_page_size * 2373 DIV_ROUND_UP_SECTOR_T(prop->dram_size, prop->dram_page_size)); 2374 prop->dmmu.end_addr = prop->dmmu.start_addr + prop->dram_page_size * 2375 div_u64((VA_HBM_SPACE_END - prop->dmmu.start_addr), prop->dmmu.page_size); 2376 /* 2377 * Driver can't share an (48MB) HBM page with the F/W in order to prevent FW to block 2378 * the driver part by range register, so it must start at the next (48MB) page 2379 */ 2380 hbm_drv_base_offset = roundup(CPU_FW_IMAGE_SIZE, prop->num_functional_hbms * SZ_8M); 2381 2382 /* 2383 * The NIC driver section size and the HMMU page tables section in the HBM needs 2384 * to be the remaining size in the first dram page after taking into 2385 * account the F/W image size 2386 */ 2387 2388 /* Reserve region in HBM for HMMU page tables */ 2389 prop->mmu_pgt_addr = DRAM_PHYS_BASE + hbm_drv_base_offset + 2390 ((prop->dram_page_size - hbm_drv_base_offset) - 2391 (HMMU_PAGE_TABLES_SIZE + EDMA_PQS_SIZE + EDMA_SCRATCHPAD_SIZE)); 2392 2393 /* Set EDMA PQs HBM addresses */ 2394 edma_pq_base_addr = prop->mmu_pgt_addr + HMMU_PAGE_TABLES_SIZE; 2395 2396 for (i = 0 ; i < GAUDI2_QUEUE_ID_CPU_PQ ; i++) { 2397 if (gaudi2_is_edma_queue_id(i)) { 2398 prop->hw_queues_props[i].q_dram_bd_address = edma_pq_base_addr + 2399 (edma_idx * HL_QUEUE_SIZE_IN_BYTES); 2400 edma_idx++; 2401 } 2402 } 2403 2404 return 0; 2405 } 2406 2407 static int gaudi2_set_fixed_properties(struct hl_device *hdev) 2408 { 2409 struct asic_fixed_properties *prop = &hdev->asic_prop; 2410 struct hw_queue_properties *q_props; 2411 u32 num_sync_stream_queues = 0; 2412 int i, rc; 2413 2414 prop->max_queues = GAUDI2_QUEUE_ID_SIZE; 2415 prop->hw_queues_props = kcalloc(prop->max_queues, sizeof(struct hw_queue_properties), 2416 GFP_KERNEL); 2417 2418 if (!prop->hw_queues_props) 2419 return -ENOMEM; 2420 2421 q_props = prop->hw_queues_props; 2422 2423 for (i = 0 ; i < GAUDI2_QUEUE_ID_CPU_PQ ; i++) { 2424 q_props[i].type = QUEUE_TYPE_HW; 2425 q_props[i].driver_only = 0; 2426 2427 if (i >= GAUDI2_QUEUE_ID_NIC_0_0 && i <= GAUDI2_QUEUE_ID_NIC_23_3) { 2428 q_props[i].supports_sync_stream = 0; 2429 } else { 2430 q_props[i].supports_sync_stream = 1; 2431 num_sync_stream_queues++; 2432 } 2433 2434 q_props[i].cb_alloc_flags = CB_ALLOC_USER; 2435 2436 if (gaudi2_is_edma_queue_id(i)) 2437 q_props[i].dram_bd = 1; 2438 } 2439 2440 q_props[GAUDI2_QUEUE_ID_CPU_PQ].type = QUEUE_TYPE_CPU; 2441 q_props[GAUDI2_QUEUE_ID_CPU_PQ].driver_only = 1; 2442 q_props[GAUDI2_QUEUE_ID_CPU_PQ].cb_alloc_flags = CB_ALLOC_KERNEL; 2443 2444 prop->cache_line_size = DEVICE_CACHE_LINE_SIZE; 2445 prop->cfg_base_address = CFG_BASE; 2446 prop->device_dma_offset_for_host_access = HOST_PHYS_BASE_0; 2447 prop->host_base_address = HOST_PHYS_BASE_0; 2448 prop->host_end_address = prop->host_base_address + HOST_PHYS_SIZE_0; 2449 prop->max_pending_cs = GAUDI2_MAX_PENDING_CS; 2450 prop->completion_queues_count = GAUDI2_RESERVED_CQ_NUMBER; 2451 prop->user_dec_intr_count = NUMBER_OF_DEC; 2452 prop->user_interrupt_count = GAUDI2_IRQ_NUM_USER_LAST - GAUDI2_IRQ_NUM_USER_FIRST + 1; 2453 prop->completion_mode = HL_COMPLETION_MODE_CS; 2454 prop->sync_stream_first_sob = GAUDI2_RESERVED_SOB_NUMBER; 2455 prop->sync_stream_first_mon = GAUDI2_RESERVED_MON_NUMBER; 2456 2457 prop->sram_base_address = SRAM_BASE_ADDR; 2458 prop->sram_size = SRAM_SIZE; 2459 prop->sram_end_address = prop->sram_base_address + prop->sram_size; 2460 prop->sram_user_base_address = prop->sram_base_address + SRAM_USER_BASE_OFFSET; 2461 2462 prop->hints_range_reservation = true; 2463 2464 prop->rotator_enabled_mask = BIT(NUM_OF_ROT) - 1; 2465 2466 prop->max_asid = 2; 2467 2468 prop->dmmu.pgt_size = HMMU_PAGE_TABLES_SIZE; 2469 prop->mmu_pte_size = HL_PTE_SIZE; 2470 prop->mmu_hop_table_size = HOP_TABLE_SIZE_512_PTE; 2471 prop->mmu_hop0_tables_total_size = HOP_TABLE_SIZE_512_PTE * prop->max_asid; 2472 2473 prop->dmmu.hop_shifts[MMU_HOP0] = DHOP0_SHIFT; 2474 prop->dmmu.hop_shifts[MMU_HOP1] = DHOP1_SHIFT; 2475 prop->dmmu.hop_shifts[MMU_HOP2] = DHOP2_SHIFT; 2476 prop->dmmu.hop_shifts[MMU_HOP3] = DHOP3_SHIFT; 2477 prop->dmmu.hop_masks[MMU_HOP0] = DHOP0_MASK; 2478 prop->dmmu.hop_masks[MMU_HOP1] = DHOP1_MASK; 2479 prop->dmmu.hop_masks[MMU_HOP2] = DHOP2_MASK; 2480 prop->dmmu.hop_masks[MMU_HOP3] = DHOP3_MASK; 2481 prop->dmmu.page_size = PAGE_SIZE_1GB; 2482 prop->dmmu.num_hops = MMU_ARCH_4_HOPS; 2483 prop->dmmu.last_mask = LAST_MASK; 2484 prop->dmmu.host_resident = 0; 2485 prop->dmmu.hop_table_size = prop->mmu_hop_table_size; 2486 prop->dmmu.hop0_tables_total_size = prop->mmu_hop0_tables_total_size; 2487 2488 /* As we need to set the pgt address in dram for HMMU init so we cannot 2489 * wait to the fw cpucp info to set the dram props as mmu init comes before 2490 * hw init 2491 */ 2492 rc = hdev->asic_funcs->set_dram_properties(hdev); 2493 if (rc) 2494 goto free_qprops; 2495 2496 prop->mmu_pgt_size = PMMU_PAGE_TABLES_SIZE; 2497 2498 prop->pmmu.pgt_size = prop->mmu_pgt_size; 2499 hdev->pmmu_huge_range = true; 2500 prop->pmmu.host_resident = 1; 2501 prop->pmmu.num_hops = MMU_ARCH_6_HOPS; 2502 prop->pmmu.last_mask = LAST_MASK; 2503 prop->pmmu.hop_table_size = prop->mmu_hop_table_size; 2504 prop->pmmu.hop0_tables_total_size = prop->mmu_hop0_tables_total_size; 2505 2506 prop->hints_host_reserved_va_range.start_addr = RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START; 2507 prop->hints_host_reserved_va_range.end_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HOST_END; 2508 prop->hints_host_hpage_reserved_va_range.start_addr = 2509 RESERVED_VA_RANGE_FOR_ARC_ON_HOST_HPAGE_START; 2510 prop->hints_host_hpage_reserved_va_range.end_addr = 2511 RESERVED_VA_RANGE_FOR_ARC_ON_HOST_HPAGE_END; 2512 2513 if (PAGE_SIZE == SZ_64K) { 2514 prop->pmmu.hop_shifts[MMU_HOP0] = HOP0_SHIFT_64K; 2515 prop->pmmu.hop_shifts[MMU_HOP1] = HOP1_SHIFT_64K; 2516 prop->pmmu.hop_shifts[MMU_HOP2] = HOP2_SHIFT_64K; 2517 prop->pmmu.hop_shifts[MMU_HOP3] = HOP3_SHIFT_64K; 2518 prop->pmmu.hop_shifts[MMU_HOP4] = HOP4_SHIFT_64K; 2519 prop->pmmu.hop_shifts[MMU_HOP5] = HOP5_SHIFT_64K; 2520 prop->pmmu.hop_masks[MMU_HOP0] = HOP0_MASK_64K; 2521 prop->pmmu.hop_masks[MMU_HOP1] = HOP1_MASK_64K; 2522 prop->pmmu.hop_masks[MMU_HOP2] = HOP2_MASK_64K; 2523 prop->pmmu.hop_masks[MMU_HOP3] = HOP3_MASK_64K; 2524 prop->pmmu.hop_masks[MMU_HOP4] = HOP4_MASK_64K; 2525 prop->pmmu.hop_masks[MMU_HOP5] = HOP5_MASK_64K; 2526 prop->pmmu.start_addr = VA_HOST_SPACE_PAGE_START; 2527 prop->pmmu.end_addr = VA_HOST_SPACE_PAGE_END; 2528 prop->pmmu.page_size = PAGE_SIZE_64KB; 2529 2530 /* shifts and masks are the same in PMMU and HPMMU */ 2531 memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu)); 2532 prop->pmmu_huge.page_size = PAGE_SIZE_16MB; 2533 prop->pmmu_huge.start_addr = VA_HOST_SPACE_HPAGE_START; 2534 prop->pmmu_huge.end_addr = VA_HOST_SPACE_HPAGE_END; 2535 } else { 2536 prop->pmmu.hop_shifts[MMU_HOP0] = HOP0_SHIFT_4K; 2537 prop->pmmu.hop_shifts[MMU_HOP1] = HOP1_SHIFT_4K; 2538 prop->pmmu.hop_shifts[MMU_HOP2] = HOP2_SHIFT_4K; 2539 prop->pmmu.hop_shifts[MMU_HOP3] = HOP3_SHIFT_4K; 2540 prop->pmmu.hop_shifts[MMU_HOP4] = HOP4_SHIFT_4K; 2541 prop->pmmu.hop_shifts[MMU_HOP5] = HOP5_SHIFT_4K; 2542 prop->pmmu.hop_masks[MMU_HOP0] = HOP0_MASK_4K; 2543 prop->pmmu.hop_masks[MMU_HOP1] = HOP1_MASK_4K; 2544 prop->pmmu.hop_masks[MMU_HOP2] = HOP2_MASK_4K; 2545 prop->pmmu.hop_masks[MMU_HOP3] = HOP3_MASK_4K; 2546 prop->pmmu.hop_masks[MMU_HOP4] = HOP4_MASK_4K; 2547 prop->pmmu.hop_masks[MMU_HOP5] = HOP5_MASK_4K; 2548 prop->pmmu.start_addr = VA_HOST_SPACE_PAGE_START; 2549 prop->pmmu.end_addr = VA_HOST_SPACE_PAGE_END; 2550 prop->pmmu.page_size = PAGE_SIZE_4KB; 2551 2552 /* shifts and masks are the same in PMMU and HPMMU */ 2553 memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu)); 2554 prop->pmmu_huge.page_size = PAGE_SIZE_2MB; 2555 prop->pmmu_huge.start_addr = VA_HOST_SPACE_HPAGE_START; 2556 prop->pmmu_huge.end_addr = VA_HOST_SPACE_HPAGE_END; 2557 } 2558 2559 prop->max_num_of_engines = GAUDI2_ENGINE_ID_SIZE; 2560 prop->num_engine_cores = CPU_ID_MAX; 2561 prop->cfg_size = CFG_SIZE; 2562 prop->num_of_events = GAUDI2_EVENT_SIZE; 2563 2564 prop->supports_engine_modes = true; 2565 2566 prop->dc_power_default = DC_POWER_DEFAULT; 2567 2568 prop->cb_pool_cb_cnt = GAUDI2_CB_POOL_CB_CNT; 2569 prop->cb_pool_cb_size = GAUDI2_CB_POOL_CB_SIZE; 2570 prop->pcie_dbi_base_address = CFG_BASE + mmPCIE_DBI_BASE; 2571 prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI; 2572 2573 strscpy_pad(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME, CARD_NAME_MAX_LEN); 2574 2575 prop->mme_master_slave_mode = 1; 2576 2577 prop->first_available_user_sob[0] = GAUDI2_RESERVED_SOB_NUMBER + 2578 (num_sync_stream_queues * HL_RSVD_SOBS); 2579 2580 prop->first_available_user_mon[0] = GAUDI2_RESERVED_MON_NUMBER + 2581 (num_sync_stream_queues * HL_RSVD_MONS); 2582 2583 prop->first_available_user_interrupt = GAUDI2_IRQ_NUM_USER_FIRST; 2584 prop->tpc_interrupt_id = GAUDI2_IRQ_NUM_TPC_ASSERT; 2585 prop->eq_interrupt_id = GAUDI2_IRQ_NUM_EVENT_QUEUE; 2586 2587 prop->first_available_cq[0] = GAUDI2_RESERVED_CQ_NUMBER; 2588 2589 prop->fw_cpu_boot_dev_sts0_valid = false; 2590 prop->fw_cpu_boot_dev_sts1_valid = false; 2591 prop->hard_reset_done_by_fw = false; 2592 prop->gic_interrupts_enable = true; 2593 2594 prop->server_type = HL_SERVER_TYPE_UNKNOWN; 2595 2596 prop->max_dec = NUMBER_OF_DEC; 2597 2598 prop->clk_pll_index = HL_GAUDI2_MME_PLL; 2599 2600 prop->dma_mask = 64; 2601 2602 prop->hbw_flush_reg = mmPCIE_WRAP_SPECIAL_GLBL_SPARE_0; 2603 2604 return 0; 2605 2606 free_qprops: 2607 kfree(prop->hw_queues_props); 2608 return rc; 2609 } 2610 2611 static int gaudi2_pci_bars_map(struct hl_device *hdev) 2612 { 2613 static const char * const name[] = {"CFG_SRAM", "MSIX", "DRAM"}; 2614 bool is_wc[3] = {false, false, true}; 2615 int rc; 2616 2617 rc = hl_pci_bars_map(hdev, name, is_wc); 2618 if (rc) 2619 return rc; 2620 2621 hdev->rmmio = hdev->pcie_bar[SRAM_CFG_BAR_ID] + (CFG_BASE - STM_FLASH_BASE_ADDR); 2622 2623 return 0; 2624 } 2625 2626 static u64 gaudi2_set_hbm_bar_base(struct hl_device *hdev, u64 addr) 2627 { 2628 struct gaudi2_device *gaudi2 = hdev->asic_specific; 2629 struct hl_inbound_pci_region pci_region; 2630 u64 old_addr = addr; 2631 int rc; 2632 2633 if ((gaudi2) && (gaudi2->dram_bar_cur_addr == addr)) 2634 return old_addr; 2635 2636 if (hdev->asic_prop.iatu_done_by_fw) 2637 return U64_MAX; 2638 2639 /* Inbound Region 2 - Bar 4 - Point to DRAM */ 2640 pci_region.mode = PCI_BAR_MATCH_MODE; 2641 pci_region.bar = DRAM_BAR_ID; 2642 pci_region.addr = addr; 2643 rc = hl_pci_set_inbound_region(hdev, 2, &pci_region); 2644 if (rc) 2645 return U64_MAX; 2646 2647 if (gaudi2) { 2648 old_addr = gaudi2->dram_bar_cur_addr; 2649 gaudi2->dram_bar_cur_addr = addr; 2650 } 2651 2652 return old_addr; 2653 } 2654 2655 static int gaudi2_init_iatu(struct hl_device *hdev) 2656 { 2657 struct hl_inbound_pci_region inbound_region; 2658 struct hl_outbound_pci_region outbound_region; 2659 u32 bar_addr_low, bar_addr_high; 2660 int rc; 2661 2662 if (hdev->asic_prop.iatu_done_by_fw) 2663 return 0; 2664 2665 /* Temporary inbound Region 0 - Bar 0 - Point to CFG 2666 * We must map this region in BAR match mode in order to 2667 * fetch BAR physical base address 2668 */ 2669 inbound_region.mode = PCI_BAR_MATCH_MODE; 2670 inbound_region.bar = SRAM_CFG_BAR_ID; 2671 /* Base address must be aligned to Bar size which is 256 MB */ 2672 inbound_region.addr = STM_FLASH_BASE_ADDR - STM_FLASH_ALIGNED_OFF; 2673 rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region); 2674 if (rc) 2675 return rc; 2676 2677 /* Fetch physical BAR address */ 2678 bar_addr_high = RREG32(mmPCIE_DBI_BAR1_REG + STM_FLASH_ALIGNED_OFF); 2679 bar_addr_low = RREG32(mmPCIE_DBI_BAR0_REG + STM_FLASH_ALIGNED_OFF) & ~0xF; 2680 2681 hdev->pcie_bar_phys[SRAM_CFG_BAR_ID] = (u64)bar_addr_high << 32 | bar_addr_low; 2682 2683 /* Inbound Region 0 - Bar 0 - Point to CFG */ 2684 inbound_region.mode = PCI_ADDRESS_MATCH_MODE; 2685 inbound_region.bar = SRAM_CFG_BAR_ID; 2686 inbound_region.offset_in_bar = 0; 2687 inbound_region.addr = STM_FLASH_BASE_ADDR; 2688 inbound_region.size = CFG_REGION_SIZE; 2689 rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region); 2690 if (rc) 2691 return rc; 2692 2693 /* Inbound Region 1 - Bar 0 - Point to BAR0_RESERVED + SRAM */ 2694 inbound_region.mode = PCI_ADDRESS_MATCH_MODE; 2695 inbound_region.bar = SRAM_CFG_BAR_ID; 2696 inbound_region.offset_in_bar = CFG_REGION_SIZE; 2697 inbound_region.addr = BAR0_RSRVD_BASE_ADDR; 2698 inbound_region.size = BAR0_RSRVD_SIZE + SRAM_SIZE; 2699 rc = hl_pci_set_inbound_region(hdev, 1, &inbound_region); 2700 if (rc) 2701 return rc; 2702 2703 /* Inbound Region 2 - Bar 4 - Point to DRAM */ 2704 inbound_region.mode = PCI_BAR_MATCH_MODE; 2705 inbound_region.bar = DRAM_BAR_ID; 2706 inbound_region.addr = DRAM_PHYS_BASE; 2707 rc = hl_pci_set_inbound_region(hdev, 2, &inbound_region); 2708 if (rc) 2709 return rc; 2710 2711 /* Outbound Region 0 - Point to Host */ 2712 outbound_region.addr = HOST_PHYS_BASE_0; 2713 outbound_region.size = HOST_PHYS_SIZE_0; 2714 rc = hl_pci_set_outbound_region(hdev, &outbound_region); 2715 2716 return rc; 2717 } 2718 2719 static enum hl_device_hw_state gaudi2_get_hw_state(struct hl_device *hdev) 2720 { 2721 return RREG32(mmHW_STATE); 2722 } 2723 2724 static int gaudi2_tpc_binning_init_prop(struct hl_device *hdev) 2725 { 2726 struct asic_fixed_properties *prop = &hdev->asic_prop; 2727 2728 /* 2729 * check for error condition in which number of binning candidates 2730 * is higher than the maximum supported by the driver 2731 */ 2732 if (hweight64(hdev->tpc_binning) > MAX_CLUSTER_BINNING_FAULTY_TPCS) { 2733 dev_err(hdev->dev, "TPC binning is supported for max of %d faulty TPCs, provided mask 0x%llx\n", 2734 MAX_CLUSTER_BINNING_FAULTY_TPCS, 2735 hdev->tpc_binning); 2736 return -EINVAL; 2737 } 2738 2739 prop->tpc_binning_mask = hdev->tpc_binning; 2740 prop->tpc_enabled_mask = GAUDI2_TPC_FULL_MASK; 2741 2742 return 0; 2743 } 2744 2745 static int gaudi2_set_tpc_binning_masks(struct hl_device *hdev) 2746 { 2747 struct asic_fixed_properties *prop = &hdev->asic_prop; 2748 struct hw_queue_properties *q_props = prop->hw_queues_props; 2749 u64 tpc_binning_mask; 2750 u8 subst_idx = 0; 2751 int i, rc; 2752 2753 rc = gaudi2_tpc_binning_init_prop(hdev); 2754 if (rc) 2755 return rc; 2756 2757 tpc_binning_mask = prop->tpc_binning_mask; 2758 2759 for (i = 0 ; i < MAX_FAULTY_TPCS ; i++) { 2760 u8 subst_seq, binned, qid_base; 2761 2762 if (tpc_binning_mask == 0) 2763 break; 2764 2765 if (subst_idx == 0) { 2766 subst_seq = TPC_ID_DCORE0_TPC6; 2767 qid_base = GAUDI2_QUEUE_ID_DCORE0_TPC_6_0; 2768 } else { 2769 subst_seq = TPC_ID_DCORE3_TPC5; 2770 qid_base = GAUDI2_QUEUE_ID_DCORE3_TPC_5_0; 2771 } 2772 2773 2774 /* clear bit from mask */ 2775 binned = __ffs(tpc_binning_mask); 2776 /* 2777 * Coverity complains about possible out-of-bound access in 2778 * clear_bit 2779 */ 2780 if (binned >= TPC_ID_SIZE) { 2781 dev_err(hdev->dev, 2782 "Invalid binned TPC (binning mask: %llx)\n", 2783 tpc_binning_mask); 2784 return -EINVAL; 2785 } 2786 clear_bit(binned, (unsigned long *)&tpc_binning_mask); 2787 2788 /* also clear replacing TPC bit from enabled mask */ 2789 clear_bit(subst_seq, (unsigned long *)&prop->tpc_enabled_mask); 2790 2791 /* bin substite TPC's Qs */ 2792 q_props[qid_base].binned = 1; 2793 q_props[qid_base + 1].binned = 1; 2794 q_props[qid_base + 2].binned = 1; 2795 q_props[qid_base + 3].binned = 1; 2796 2797 subst_idx++; 2798 } 2799 2800 return 0; 2801 } 2802 2803 static int gaudi2_set_dec_binning_masks(struct hl_device *hdev) 2804 { 2805 struct asic_fixed_properties *prop = &hdev->asic_prop; 2806 u8 num_faulty; 2807 2808 num_faulty = hweight32(hdev->decoder_binning); 2809 2810 /* 2811 * check for error condition in which number of binning candidates 2812 * is higher than the maximum supported by the driver 2813 */ 2814 if (num_faulty > MAX_FAULTY_DECODERS) { 2815 dev_err(hdev->dev, "decoder binning is supported for max of single faulty decoder, provided mask 0x%x\n", 2816 hdev->decoder_binning); 2817 return -EINVAL; 2818 } 2819 2820 prop->decoder_binning_mask = (hdev->decoder_binning & GAUDI2_DECODER_FULL_MASK); 2821 2822 if (prop->decoder_binning_mask) 2823 prop->decoder_enabled_mask = (GAUDI2_DECODER_FULL_MASK & ~BIT(DEC_ID_PCIE_VDEC1)); 2824 else 2825 prop->decoder_enabled_mask = GAUDI2_DECODER_FULL_MASK; 2826 2827 return 0; 2828 } 2829 2830 static void gaudi2_set_dram_binning_masks(struct hl_device *hdev) 2831 { 2832 struct asic_fixed_properties *prop = &hdev->asic_prop; 2833 2834 /* check if we should override default binning */ 2835 if (!hdev->dram_binning) { 2836 prop->dram_binning_mask = 0; 2837 prop->dram_enabled_mask = GAUDI2_DRAM_FULL_MASK; 2838 return; 2839 } 2840 2841 /* set DRAM binning constraints */ 2842 prop->faulty_dram_cluster_map |= hdev->dram_binning; 2843 prop->dram_binning_mask = hdev->dram_binning; 2844 prop->dram_enabled_mask = GAUDI2_DRAM_FULL_MASK & ~BIT(HBM_ID5); 2845 } 2846 2847 static int gaudi2_set_edma_binning_masks(struct hl_device *hdev) 2848 { 2849 struct asic_fixed_properties *prop = &hdev->asic_prop; 2850 struct hw_queue_properties *q_props; 2851 u8 seq, num_faulty; 2852 2853 num_faulty = hweight32(hdev->edma_binning); 2854 2855 /* 2856 * check for error condition in which number of binning candidates 2857 * is higher than the maximum supported by the driver 2858 */ 2859 if (num_faulty > MAX_FAULTY_EDMAS) { 2860 dev_err(hdev->dev, 2861 "EDMA binning is supported for max of single faulty EDMA, provided mask 0x%x\n", 2862 hdev->edma_binning); 2863 return -EINVAL; 2864 } 2865 2866 if (!hdev->edma_binning) { 2867 prop->edma_binning_mask = 0; 2868 prop->edma_enabled_mask = GAUDI2_EDMA_FULL_MASK; 2869 return 0; 2870 } 2871 2872 seq = __ffs((unsigned long)hdev->edma_binning); 2873 2874 /* set binning constraints */ 2875 prop->faulty_dram_cluster_map |= BIT(edma_to_hbm_cluster[seq]); 2876 prop->edma_binning_mask = hdev->edma_binning; 2877 prop->edma_enabled_mask = GAUDI2_EDMA_FULL_MASK & ~BIT(EDMA_ID_DCORE3_INSTANCE1); 2878 2879 /* bin substitute EDMA's queue */ 2880 q_props = prop->hw_queues_props; 2881 q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0].binned = 1; 2882 q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1].binned = 1; 2883 q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2].binned = 1; 2884 q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3].binned = 1; 2885 2886 return 0; 2887 } 2888 2889 static int gaudi2_set_xbar_edge_enable_mask(struct hl_device *hdev, u32 xbar_edge_iso_mask) 2890 { 2891 struct asic_fixed_properties *prop = &hdev->asic_prop; 2892 u8 num_faulty, seq; 2893 2894 /* check if we should override default binning */ 2895 if (!xbar_edge_iso_mask) { 2896 prop->xbar_edge_enabled_mask = GAUDI2_XBAR_EDGE_FULL_MASK; 2897 return 0; 2898 } 2899 2900 /* 2901 * note that it can be set to value other than 0 only after cpucp packet (i.e. 2902 * only the FW can set a redundancy value). for user it'll always be 0. 2903 */ 2904 num_faulty = hweight32(xbar_edge_iso_mask); 2905 2906 /* 2907 * check for error condition in which number of binning candidates 2908 * is higher than the maximum supported by the driver 2909 */ 2910 if (num_faulty > MAX_FAULTY_XBARS) { 2911 dev_err(hdev->dev, "we cannot have more than %d faulty XBAR EDGE\n", 2912 MAX_FAULTY_XBARS); 2913 return -EINVAL; 2914 } 2915 2916 seq = __ffs((unsigned long)xbar_edge_iso_mask); 2917 2918 /* set binning constraints */ 2919 prop->faulty_dram_cluster_map |= BIT(xbar_edge_to_hbm_cluster[seq]); 2920 prop->xbar_edge_enabled_mask = (~xbar_edge_iso_mask) & GAUDI2_XBAR_EDGE_FULL_MASK; 2921 2922 return 0; 2923 } 2924 2925 static int gaudi2_set_cluster_binning_masks_common(struct hl_device *hdev, u8 xbar_edge_iso_mask) 2926 { 2927 int rc; 2928 2929 /* 2930 * mark all clusters as good, each component will "fail" cluster 2931 * based on eFuse/user values. 2932 * If more than single cluster is faulty- the chip is unusable 2933 */ 2934 hdev->asic_prop.faulty_dram_cluster_map = 0; 2935 2936 gaudi2_set_dram_binning_masks(hdev); 2937 2938 rc = gaudi2_set_edma_binning_masks(hdev); 2939 if (rc) 2940 return rc; 2941 2942 rc = gaudi2_set_xbar_edge_enable_mask(hdev, xbar_edge_iso_mask); 2943 if (rc) 2944 return rc; 2945 2946 2947 /* always initially set to full mask */ 2948 hdev->asic_prop.hmmu_hif_enabled_mask = GAUDI2_HIF_HMMU_FULL_MASK; 2949 2950 return 0; 2951 } 2952 2953 static int gaudi2_set_cluster_binning_masks(struct hl_device *hdev) 2954 { 2955 struct asic_fixed_properties *prop = &hdev->asic_prop; 2956 int rc; 2957 2958 rc = gaudi2_set_cluster_binning_masks_common(hdev, prop->cpucp_info.xbar_binning_mask); 2959 if (rc) 2960 return rc; 2961 2962 /* if we have DRAM binning reported by FW we should perform cluster config */ 2963 if (prop->faulty_dram_cluster_map) { 2964 u8 cluster_seq = __ffs((unsigned long)prop->faulty_dram_cluster_map); 2965 2966 prop->hmmu_hif_enabled_mask = cluster_hmmu_hif_enabled_mask[cluster_seq]; 2967 } 2968 2969 return 0; 2970 } 2971 2972 static int gaudi2_set_binning_masks(struct hl_device *hdev) 2973 { 2974 int rc; 2975 2976 rc = gaudi2_set_cluster_binning_masks(hdev); 2977 if (rc) 2978 return rc; 2979 2980 rc = gaudi2_set_tpc_binning_masks(hdev); 2981 if (rc) 2982 return rc; 2983 2984 rc = gaudi2_set_dec_binning_masks(hdev); 2985 if (rc) 2986 return rc; 2987 2988 return 0; 2989 } 2990 2991 static int gaudi2_cpucp_info_get(struct hl_device *hdev) 2992 { 2993 struct gaudi2_device *gaudi2 = hdev->asic_specific; 2994 struct asic_fixed_properties *prop = &hdev->asic_prop; 2995 long max_power; 2996 u64 dram_size; 2997 int rc; 2998 2999 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 3000 return 0; 3001 3002 /* No point of asking this information again when not doing hard reset, as the device 3003 * CPU hasn't been reset 3004 */ 3005 if (hdev->reset_info.in_compute_reset) 3006 return 0; 3007 3008 rc = hl_fw_cpucp_handshake(hdev, mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_DEV_STS1, mmCPU_BOOT_ERR0, 3009 mmCPU_BOOT_ERR1); 3010 if (rc) 3011 return rc; 3012 3013 dram_size = le64_to_cpu(prop->cpucp_info.dram_size); 3014 if (dram_size) { 3015 /* we can have wither 5 or 6 HBMs. other values are invalid */ 3016 3017 if ((dram_size != ((GAUDI2_HBM_NUM - 1) * SZ_16G)) && 3018 (dram_size != (GAUDI2_HBM_NUM * SZ_16G))) { 3019 dev_err(hdev->dev, 3020 "F/W reported invalid DRAM size %llu. Trying to use default size %llu\n", 3021 dram_size, prop->dram_size); 3022 dram_size = prop->dram_size; 3023 } 3024 3025 prop->dram_size = dram_size; 3026 prop->dram_end_address = prop->dram_base_address + dram_size; 3027 } 3028 3029 if (!strlen(prop->cpucp_info.card_name)) 3030 strscpy_pad(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME, 3031 CARD_NAME_MAX_LEN); 3032 3033 /* Overwrite binning masks with the actual binning values from F/W */ 3034 hdev->dram_binning = prop->cpucp_info.dram_binning_mask; 3035 hdev->edma_binning = prop->cpucp_info.edma_binning_mask; 3036 hdev->tpc_binning = le64_to_cpu(prop->cpucp_info.tpc_binning_mask); 3037 hdev->decoder_binning = lower_32_bits(le64_to_cpu(prop->cpucp_info.decoder_binning_mask)); 3038 3039 dev_dbg(hdev->dev, "Read binning masks: tpc: 0x%llx, dram: 0x%llx, edma: 0x%x, dec: 0x%x\n", 3040 hdev->tpc_binning, hdev->dram_binning, hdev->edma_binning, 3041 hdev->decoder_binning); 3042 3043 /* 3044 * at this point the DRAM parameters need to be updated according to data obtained 3045 * from the FW 3046 */ 3047 rc = hdev->asic_funcs->set_dram_properties(hdev); 3048 if (rc) 3049 return rc; 3050 3051 rc = hdev->asic_funcs->set_binning_masks(hdev); 3052 if (rc) 3053 return rc; 3054 3055 max_power = hl_fw_get_max_power(hdev); 3056 if (max_power < 0) 3057 return max_power; 3058 3059 prop->max_power_default = (u64) max_power; 3060 3061 return 0; 3062 } 3063 3064 static int gaudi2_fetch_psoc_frequency(struct hl_device *hdev) 3065 { 3066 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3067 u16 pll_freq_arr[HL_PLL_NUM_OUTPUTS]; 3068 int rc; 3069 3070 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 3071 return 0; 3072 3073 rc = hl_fw_cpucp_pll_info_get(hdev, HL_GAUDI2_CPU_PLL, pll_freq_arr); 3074 if (rc) 3075 return rc; 3076 3077 hdev->asic_prop.psoc_timestamp_frequency = pll_freq_arr[3]; 3078 3079 return 0; 3080 } 3081 3082 static int gaudi2_mmu_clear_pgt_range(struct hl_device *hdev) 3083 { 3084 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3085 struct asic_fixed_properties *prop = &hdev->asic_prop; 3086 int rc; 3087 3088 if (!(gaudi2->hw_cap_initialized & HW_CAP_MMU_MASK)) 3089 return 0; 3090 3091 if (prop->dmmu.host_resident) 3092 return 0; 3093 3094 rc = gaudi2_memset_device_memory(hdev, prop->mmu_pgt_addr, prop->dmmu.pgt_size, 0); 3095 if (rc) 3096 dev_err(hdev->dev, "Failed to clear mmu pgt"); 3097 3098 return rc; 3099 } 3100 3101 static int gaudi2_early_init(struct hl_device *hdev) 3102 { 3103 struct asic_fixed_properties *prop = &hdev->asic_prop; 3104 struct pci_dev *pdev = hdev->pdev; 3105 resource_size_t pci_bar_size; 3106 int rc; 3107 3108 rc = gaudi2_set_fixed_properties(hdev); 3109 if (rc) 3110 return rc; 3111 3112 /* Check BAR sizes */ 3113 pci_bar_size = pci_resource_len(pdev, SRAM_CFG_BAR_ID); 3114 3115 if (pci_bar_size != CFG_BAR_SIZE) { 3116 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n", 3117 SRAM_CFG_BAR_ID, &pci_bar_size, CFG_BAR_SIZE); 3118 rc = -ENODEV; 3119 goto free_queue_props; 3120 } 3121 3122 pci_bar_size = pci_resource_len(pdev, MSIX_BAR_ID); 3123 if (pci_bar_size != MSIX_BAR_SIZE) { 3124 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n", 3125 MSIX_BAR_ID, &pci_bar_size, MSIX_BAR_SIZE); 3126 rc = -ENODEV; 3127 goto free_queue_props; 3128 } 3129 3130 prop->dram_pci_bar_size = pci_resource_len(pdev, DRAM_BAR_ID); 3131 hdev->dram_pci_bar_start = pci_resource_start(pdev, DRAM_BAR_ID); 3132 3133 /* 3134 * Only in pldm driver config iATU 3135 */ 3136 if (hdev->pldm) 3137 hdev->asic_prop.iatu_done_by_fw = false; 3138 else 3139 hdev->asic_prop.iatu_done_by_fw = true; 3140 3141 rc = hl_pci_init(hdev); 3142 if (rc) 3143 goto free_queue_props; 3144 3145 /* Before continuing in the initialization, we need to read the preboot 3146 * version to determine whether we run with a security-enabled firmware 3147 */ 3148 rc = hl_fw_read_preboot_status(hdev); 3149 if (rc) { 3150 if (hdev->reset_on_preboot_fail) 3151 /* we are already on failure flow, so don't check if hw_fini fails. */ 3152 hdev->asic_funcs->hw_fini(hdev, true, false); 3153 goto pci_fini; 3154 } 3155 3156 if (gaudi2_get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) { 3157 dev_dbg(hdev->dev, "H/W state is dirty, must reset before initializing\n"); 3158 rc = hdev->asic_funcs->hw_fini(hdev, true, false); 3159 if (rc) { 3160 dev_err(hdev->dev, "failed to reset HW in dirty state (%d)\n", rc); 3161 goto pci_fini; 3162 } 3163 } 3164 3165 return 0; 3166 3167 pci_fini: 3168 hl_pci_fini(hdev); 3169 free_queue_props: 3170 kfree(hdev->asic_prop.hw_queues_props); 3171 return rc; 3172 } 3173 3174 static int gaudi2_early_fini(struct hl_device *hdev) 3175 { 3176 kfree(hdev->asic_prop.hw_queues_props); 3177 hl_pci_fini(hdev); 3178 3179 return 0; 3180 } 3181 3182 static bool gaudi2_is_arc_nic_owned(u64 arc_id) 3183 { 3184 switch (arc_id) { 3185 case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23: 3186 return true; 3187 default: 3188 return false; 3189 } 3190 } 3191 3192 static bool gaudi2_is_arc_tpc_owned(u64 arc_id) 3193 { 3194 switch (arc_id) { 3195 case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24: 3196 return true; 3197 default: 3198 return false; 3199 } 3200 } 3201 3202 static void gaudi2_init_arcs(struct hl_device *hdev) 3203 { 3204 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 3205 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3206 u64 arc_id; 3207 u32 i; 3208 3209 for (i = CPU_ID_SCHED_ARC0 ; i <= CPU_ID_SCHED_ARC3 ; i++) { 3210 if (gaudi2_is_arc_enabled(hdev, i)) 3211 continue; 3212 3213 gaudi2_set_arc_id_cap(hdev, i); 3214 } 3215 3216 for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ ; i += 4) { 3217 if (!gaudi2_is_queue_enabled(hdev, i)) 3218 continue; 3219 3220 arc_id = gaudi2_queue_id_to_arc_id[i]; 3221 if (gaudi2_is_arc_enabled(hdev, arc_id)) 3222 continue; 3223 3224 if (gaudi2_is_arc_nic_owned(arc_id) && 3225 !(hdev->nic_ports_mask & BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0))) 3226 continue; 3227 3228 if (gaudi2_is_arc_tpc_owned(arc_id) && !(gaudi2->tpc_hw_cap_initialized & 3229 BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0))) 3230 continue; 3231 3232 gaudi2_set_arc_id_cap(hdev, arc_id); 3233 } 3234 3235 /* Fetch ARC scratchpad address */ 3236 hdev->asic_prop.engine_core_interrupt_reg_addr = 3237 CFG_BASE + le32_to_cpu(dyn_regs->eng_arc_irq_ctrl); 3238 } 3239 3240 static int gaudi2_scrub_arc_dccm(struct hl_device *hdev, u32 cpu_id) 3241 { 3242 u32 reg_base, reg_val; 3243 int rc; 3244 3245 switch (cpu_id) { 3246 case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC3: 3247 /* Each ARC scheduler has 2 consecutive DCCM blocks */ 3248 rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id], 3249 ARC_DCCM_BLOCK_SIZE * 2, true); 3250 if (rc) 3251 return rc; 3252 break; 3253 case CPU_ID_SCHED_ARC4: 3254 case CPU_ID_SCHED_ARC5: 3255 case CPU_ID_MME_QMAN_ARC0: 3256 case CPU_ID_MME_QMAN_ARC1: 3257 reg_base = gaudi2_arc_blocks_bases[cpu_id]; 3258 3259 /* Scrub lower DCCM block */ 3260 rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id], 3261 ARC_DCCM_BLOCK_SIZE, true); 3262 if (rc) 3263 return rc; 3264 3265 /* Switch to upper DCCM block */ 3266 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_MME_ARC_UPPER_DCCM_EN_VAL_MASK, 1); 3267 WREG32(reg_base + ARC_DCCM_UPPER_EN_OFFSET, reg_val); 3268 3269 /* Scrub upper DCCM block */ 3270 rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id], 3271 ARC_DCCM_BLOCK_SIZE, true); 3272 if (rc) 3273 return rc; 3274 3275 /* Switch to lower DCCM block */ 3276 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_MME_ARC_UPPER_DCCM_EN_VAL_MASK, 0); 3277 WREG32(reg_base + ARC_DCCM_UPPER_EN_OFFSET, reg_val); 3278 break; 3279 default: 3280 rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id], 3281 ARC_DCCM_BLOCK_SIZE, true); 3282 if (rc) 3283 return rc; 3284 } 3285 3286 return 0; 3287 } 3288 3289 static int gaudi2_scrub_arcs_dccm(struct hl_device *hdev) 3290 { 3291 u16 arc_id; 3292 int rc; 3293 3294 for (arc_id = CPU_ID_SCHED_ARC0 ; arc_id < CPU_ID_MAX ; arc_id++) { 3295 if (!gaudi2_is_arc_enabled(hdev, arc_id)) 3296 continue; 3297 3298 rc = gaudi2_scrub_arc_dccm(hdev, arc_id); 3299 if (rc) 3300 return rc; 3301 } 3302 3303 return 0; 3304 } 3305 3306 static int gaudi2_late_init(struct hl_device *hdev) 3307 { 3308 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3309 int rc; 3310 3311 hdev->asic_prop.supports_advanced_cpucp_rc = true; 3312 3313 rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_ENABLE_PCI_ACCESS, 3314 gaudi2->virt_msix_db_dma_addr); 3315 if (rc) { 3316 dev_err(hdev->dev, "Failed to enable PCI access from CPU\n"); 3317 return rc; 3318 } 3319 3320 rc = gaudi2_fetch_psoc_frequency(hdev); 3321 if (rc) { 3322 dev_err(hdev->dev, "Failed to fetch psoc frequency\n"); 3323 goto disable_pci_access; 3324 } 3325 3326 rc = gaudi2_mmu_clear_pgt_range(hdev); 3327 if (rc) { 3328 dev_err(hdev->dev, "Failed to clear MMU page tables range\n"); 3329 goto disable_pci_access; 3330 } 3331 3332 gaudi2_init_arcs(hdev); 3333 3334 rc = gaudi2_scrub_arcs_dccm(hdev); 3335 if (rc) { 3336 dev_err(hdev->dev, "Failed to scrub arcs DCCM\n"); 3337 goto disable_pci_access; 3338 } 3339 3340 gaudi2_init_security(hdev); 3341 3342 return 0; 3343 3344 disable_pci_access: 3345 hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0); 3346 3347 return rc; 3348 } 3349 3350 static void gaudi2_late_fini(struct hl_device *hdev) 3351 { 3352 hl_hwmon_release_resources(hdev); 3353 } 3354 3355 static void gaudi2_user_mapped_dec_init(struct gaudi2_device *gaudi2, u32 start_idx) 3356 { 3357 struct user_mapped_block *blocks = gaudi2->mapped_blocks; 3358 3359 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE0_DEC0_CMD_BASE, HL_BLOCK_SIZE); 3360 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE0_DEC1_CMD_BASE, HL_BLOCK_SIZE); 3361 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE1_DEC0_CMD_BASE, HL_BLOCK_SIZE); 3362 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE1_DEC1_CMD_BASE, HL_BLOCK_SIZE); 3363 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE2_DEC0_CMD_BASE, HL_BLOCK_SIZE); 3364 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE2_DEC1_CMD_BASE, HL_BLOCK_SIZE); 3365 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE3_DEC0_CMD_BASE, HL_BLOCK_SIZE); 3366 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE3_DEC1_CMD_BASE, HL_BLOCK_SIZE); 3367 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmPCIE_DEC0_CMD_BASE, HL_BLOCK_SIZE); 3368 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx], mmPCIE_DEC1_CMD_BASE, HL_BLOCK_SIZE); 3369 } 3370 3371 static void gaudi2_user_mapped_blocks_init(struct hl_device *hdev) 3372 { 3373 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3374 struct user_mapped_block *blocks = gaudi2->mapped_blocks; 3375 u32 block_size, umr_start_idx, num_umr_blocks; 3376 int i; 3377 3378 for (i = 0 ; i < NUM_ARC_CPUS ; i++) { 3379 if (i >= CPU_ID_SCHED_ARC0 && i <= CPU_ID_SCHED_ARC3) 3380 block_size = ARC_DCCM_BLOCK_SIZE * 2; 3381 else 3382 block_size = ARC_DCCM_BLOCK_SIZE; 3383 3384 blocks[i].address = gaudi2_arc_dccm_bases[i]; 3385 blocks[i].size = block_size; 3386 } 3387 3388 blocks[NUM_ARC_CPUS].address = mmARC_FARM_ARC0_ACP_ENG_BASE; 3389 blocks[NUM_ARC_CPUS].size = HL_BLOCK_SIZE; 3390 3391 blocks[NUM_ARC_CPUS + 1].address = mmARC_FARM_ARC1_ACP_ENG_BASE; 3392 blocks[NUM_ARC_CPUS + 1].size = HL_BLOCK_SIZE; 3393 3394 blocks[NUM_ARC_CPUS + 2].address = mmARC_FARM_ARC2_ACP_ENG_BASE; 3395 blocks[NUM_ARC_CPUS + 2].size = HL_BLOCK_SIZE; 3396 3397 blocks[NUM_ARC_CPUS + 3].address = mmARC_FARM_ARC3_ACP_ENG_BASE; 3398 blocks[NUM_ARC_CPUS + 3].size = HL_BLOCK_SIZE; 3399 3400 blocks[NUM_ARC_CPUS + 4].address = mmDCORE0_MME_QM_ARC_ACP_ENG_BASE; 3401 blocks[NUM_ARC_CPUS + 4].size = HL_BLOCK_SIZE; 3402 3403 blocks[NUM_ARC_CPUS + 5].address = mmDCORE1_MME_QM_ARC_ACP_ENG_BASE; 3404 blocks[NUM_ARC_CPUS + 5].size = HL_BLOCK_SIZE; 3405 3406 blocks[NUM_ARC_CPUS + 6].address = mmDCORE2_MME_QM_ARC_ACP_ENG_BASE; 3407 blocks[NUM_ARC_CPUS + 6].size = HL_BLOCK_SIZE; 3408 3409 blocks[NUM_ARC_CPUS + 7].address = mmDCORE3_MME_QM_ARC_ACP_ENG_BASE; 3410 blocks[NUM_ARC_CPUS + 7].size = HL_BLOCK_SIZE; 3411 3412 umr_start_idx = NUM_ARC_CPUS + NUM_OF_USER_ACP_BLOCKS; 3413 num_umr_blocks = NIC_NUMBER_OF_ENGINES * NUM_OF_USER_NIC_UMR_BLOCKS; 3414 for (i = 0 ; i < num_umr_blocks ; i++) { 3415 u8 nic_id, umr_block_id; 3416 3417 nic_id = i / NUM_OF_USER_NIC_UMR_BLOCKS; 3418 umr_block_id = i % NUM_OF_USER_NIC_UMR_BLOCKS; 3419 3420 blocks[umr_start_idx + i].address = 3421 mmNIC0_UMR0_0_UNSECURE_DOORBELL0_BASE + 3422 (nic_id / NIC_NUMBER_OF_QM_PER_MACRO) * NIC_OFFSET + 3423 (nic_id % NIC_NUMBER_OF_QM_PER_MACRO) * NIC_QM_OFFSET + 3424 umr_block_id * NIC_UMR_OFFSET; 3425 blocks[umr_start_idx + i].size = HL_BLOCK_SIZE; 3426 } 3427 3428 /* Expose decoder HW configuration block to user */ 3429 gaudi2_user_mapped_dec_init(gaudi2, USR_MAPPED_BLK_DEC_START_IDX); 3430 3431 for (i = 1; i < NUM_OF_DCORES; ++i) { 3432 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1)].size = SM_OBJS_BLOCK_SIZE; 3433 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1) + 1].size = HL_BLOCK_SIZE; 3434 3435 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1)].address = 3436 mmDCORE0_SYNC_MNGR_OBJS_BASE + i * DCORE_OFFSET; 3437 3438 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1) + 1].address = 3439 mmDCORE0_SYNC_MNGR_GLBL_BASE + i * DCORE_OFFSET; 3440 } 3441 } 3442 3443 static int gaudi2_alloc_cpu_accessible_dma_mem(struct hl_device *hdev) 3444 { 3445 dma_addr_t dma_addr_arr[GAUDI2_ALLOC_CPU_MEM_RETRY_CNT] = {}, end_addr; 3446 void *virt_addr_arr[GAUDI2_ALLOC_CPU_MEM_RETRY_CNT] = {}; 3447 int i, j, rc = 0; 3448 3449 /* The device ARC works with 32-bits addresses, and because there is a single HW register 3450 * that holds the extension bits (49..28), these bits must be identical in all the allocated 3451 * range. 3452 */ 3453 3454 for (i = 0 ; i < GAUDI2_ALLOC_CPU_MEM_RETRY_CNT ; i++) { 3455 virt_addr_arr[i] = hl_asic_dma_alloc_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, 3456 &dma_addr_arr[i], GFP_KERNEL | __GFP_ZERO); 3457 if (!virt_addr_arr[i]) { 3458 rc = -ENOMEM; 3459 goto free_dma_mem_arr; 3460 } 3461 3462 end_addr = dma_addr_arr[i] + HL_CPU_ACCESSIBLE_MEM_SIZE - 1; 3463 if (GAUDI2_ARC_PCI_MSB_ADDR(dma_addr_arr[i]) == GAUDI2_ARC_PCI_MSB_ADDR(end_addr)) 3464 break; 3465 } 3466 3467 if (i == GAUDI2_ALLOC_CPU_MEM_RETRY_CNT) { 3468 dev_err(hdev->dev, 3469 "MSB of ARC accessible DMA memory are not identical in all range\n"); 3470 rc = -EFAULT; 3471 goto free_dma_mem_arr; 3472 } 3473 3474 hdev->cpu_accessible_dma_mem = virt_addr_arr[i]; 3475 hdev->cpu_accessible_dma_address = dma_addr_arr[i]; 3476 3477 free_dma_mem_arr: 3478 for (j = 0 ; j < i ; j++) 3479 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, virt_addr_arr[j], 3480 dma_addr_arr[j]); 3481 3482 return rc; 3483 } 3484 3485 static void gaudi2_set_pci_memory_regions(struct hl_device *hdev) 3486 { 3487 struct asic_fixed_properties *prop = &hdev->asic_prop; 3488 struct pci_mem_region *region; 3489 3490 /* CFG */ 3491 region = &hdev->pci_mem_region[PCI_REGION_CFG]; 3492 region->region_base = CFG_BASE; 3493 region->region_size = CFG_SIZE; 3494 region->offset_in_bar = CFG_BASE - STM_FLASH_BASE_ADDR; 3495 region->bar_size = CFG_BAR_SIZE; 3496 region->bar_id = SRAM_CFG_BAR_ID; 3497 region->used = 1; 3498 3499 /* SRAM */ 3500 region = &hdev->pci_mem_region[PCI_REGION_SRAM]; 3501 region->region_base = SRAM_BASE_ADDR; 3502 region->region_size = SRAM_SIZE; 3503 region->offset_in_bar = CFG_REGION_SIZE + BAR0_RSRVD_SIZE; 3504 region->bar_size = CFG_BAR_SIZE; 3505 region->bar_id = SRAM_CFG_BAR_ID; 3506 region->used = 1; 3507 3508 /* DRAM */ 3509 region = &hdev->pci_mem_region[PCI_REGION_DRAM]; 3510 region->region_base = DRAM_PHYS_BASE; 3511 region->region_size = hdev->asic_prop.dram_size; 3512 region->offset_in_bar = 0; 3513 region->bar_size = prop->dram_pci_bar_size; 3514 region->bar_id = DRAM_BAR_ID; 3515 region->used = 1; 3516 } 3517 3518 static void gaudi2_user_interrupt_setup(struct hl_device *hdev) 3519 { 3520 struct asic_fixed_properties *prop = &hdev->asic_prop; 3521 int i, j, k; 3522 3523 /* Initialize TPC interrupt */ 3524 HL_USR_INTR_STRUCT_INIT(hdev->tpc_interrupt, hdev, 0, HL_USR_INTERRUPT_TPC); 3525 3526 /* Initialize unexpected error interrupt */ 3527 HL_USR_INTR_STRUCT_INIT(hdev->unexpected_error_interrupt, hdev, 0, 3528 HL_USR_INTERRUPT_UNEXPECTED); 3529 3530 /* Initialize common user CQ interrupt */ 3531 HL_USR_INTR_STRUCT_INIT(hdev->common_user_cq_interrupt, hdev, 3532 HL_COMMON_USER_CQ_INTERRUPT_ID, HL_USR_INTERRUPT_CQ); 3533 3534 /* Initialize common decoder interrupt */ 3535 HL_USR_INTR_STRUCT_INIT(hdev->common_decoder_interrupt, hdev, 3536 HL_COMMON_DEC_INTERRUPT_ID, HL_USR_INTERRUPT_DECODER); 3537 3538 /* User interrupts structure holds both decoder and user interrupts from various engines. 3539 * We first initialize the decoder interrupts and then we add the user interrupts. 3540 * The only limitation is that the last decoder interrupt id must be smaller 3541 * then GAUDI2_IRQ_NUM_USER_FIRST. This is checked at compilation time. 3542 */ 3543 3544 /* Initialize decoder interrupts, expose only normal interrupts, 3545 * error interrupts to be handled by driver 3546 */ 3547 for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM, j = 0 ; i <= GAUDI2_IRQ_NUM_SHARED_DEC1_NRM; 3548 i += 2, j++) 3549 HL_USR_INTR_STRUCT_INIT(hdev->user_interrupt[j], hdev, i, 3550 HL_USR_INTERRUPT_DECODER); 3551 3552 for (i = GAUDI2_IRQ_NUM_USER_FIRST, k = 0 ; k < prop->user_interrupt_count; i++, j++, k++) 3553 HL_USR_INTR_STRUCT_INIT(hdev->user_interrupt[j], hdev, i, HL_USR_INTERRUPT_CQ); 3554 } 3555 3556 static inline int gaudi2_get_non_zero_random_int(void) 3557 { 3558 int rand = get_random_u32(); 3559 3560 return rand ? rand : 1; 3561 } 3562 3563 static void gaudi2_special_blocks_free(struct hl_device *hdev) 3564 { 3565 struct asic_fixed_properties *prop = &hdev->asic_prop; 3566 struct hl_skip_blocks_cfg *skip_special_blocks_cfg = 3567 &prop->skip_special_blocks_cfg; 3568 3569 kfree(prop->special_blocks); 3570 kfree(skip_special_blocks_cfg->block_types); 3571 kfree(skip_special_blocks_cfg->block_ranges); 3572 } 3573 3574 static void gaudi2_special_blocks_iterator_free(struct hl_device *hdev) 3575 { 3576 gaudi2_special_blocks_free(hdev); 3577 } 3578 3579 static bool gaudi2_special_block_skip(struct hl_device *hdev, 3580 struct hl_special_blocks_cfg *special_blocks_cfg, 3581 u32 blk_idx, u32 major, u32 minor, u32 sub_minor) 3582 { 3583 return false; 3584 } 3585 3586 static int gaudi2_special_blocks_config(struct hl_device *hdev) 3587 { 3588 struct asic_fixed_properties *prop = &hdev->asic_prop; 3589 int i, rc; 3590 3591 /* Configure Special blocks */ 3592 prop->glbl_err_cause_num = GAUDI2_NUM_OF_GLBL_ERR_CAUSE; 3593 prop->num_of_special_blocks = ARRAY_SIZE(gaudi2_special_blocks); 3594 prop->special_blocks = kmalloc_array(prop->num_of_special_blocks, 3595 sizeof(*prop->special_blocks), GFP_KERNEL); 3596 if (!prop->special_blocks) 3597 return -ENOMEM; 3598 3599 for (i = 0 ; i < prop->num_of_special_blocks ; i++) 3600 memcpy(&prop->special_blocks[i], &gaudi2_special_blocks[i], 3601 sizeof(*prop->special_blocks)); 3602 3603 /* Configure when to skip Special blocks */ 3604 memset(&prop->skip_special_blocks_cfg, 0, sizeof(prop->skip_special_blocks_cfg)); 3605 prop->skip_special_blocks_cfg.skip_block_hook = gaudi2_special_block_skip; 3606 3607 if (ARRAY_SIZE(gaudi2_iterator_skip_block_types)) { 3608 prop->skip_special_blocks_cfg.block_types = 3609 kmalloc_array(ARRAY_SIZE(gaudi2_iterator_skip_block_types), 3610 sizeof(gaudi2_iterator_skip_block_types[0]), GFP_KERNEL); 3611 if (!prop->skip_special_blocks_cfg.block_types) { 3612 rc = -ENOMEM; 3613 goto free_special_blocks; 3614 } 3615 3616 memcpy(prop->skip_special_blocks_cfg.block_types, gaudi2_iterator_skip_block_types, 3617 sizeof(gaudi2_iterator_skip_block_types)); 3618 3619 prop->skip_special_blocks_cfg.block_types_len = 3620 ARRAY_SIZE(gaudi2_iterator_skip_block_types); 3621 } 3622 3623 if (ARRAY_SIZE(gaudi2_iterator_skip_block_ranges)) { 3624 prop->skip_special_blocks_cfg.block_ranges = 3625 kmalloc_array(ARRAY_SIZE(gaudi2_iterator_skip_block_ranges), 3626 sizeof(gaudi2_iterator_skip_block_ranges[0]), GFP_KERNEL); 3627 if (!prop->skip_special_blocks_cfg.block_ranges) { 3628 rc = -ENOMEM; 3629 goto free_skip_special_blocks_types; 3630 } 3631 3632 for (i = 0 ; i < ARRAY_SIZE(gaudi2_iterator_skip_block_ranges) ; i++) 3633 memcpy(&prop->skip_special_blocks_cfg.block_ranges[i], 3634 &gaudi2_iterator_skip_block_ranges[i], 3635 sizeof(struct range)); 3636 3637 prop->skip_special_blocks_cfg.block_ranges_len = 3638 ARRAY_SIZE(gaudi2_iterator_skip_block_ranges); 3639 } 3640 3641 return 0; 3642 3643 free_skip_special_blocks_types: 3644 kfree(prop->skip_special_blocks_cfg.block_types); 3645 free_special_blocks: 3646 kfree(prop->special_blocks); 3647 3648 return rc; 3649 } 3650 3651 static int gaudi2_special_blocks_iterator_config(struct hl_device *hdev) 3652 { 3653 return gaudi2_special_blocks_config(hdev); 3654 } 3655 3656 static void gaudi2_test_queues_msgs_free(struct hl_device *hdev) 3657 { 3658 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3659 struct gaudi2_queues_test_info *msg_info = gaudi2->queues_test_info; 3660 int i; 3661 3662 for (i = 0 ; i < GAUDI2_NUM_TESTED_QS ; i++) { 3663 /* bail-out if this is an allocation failure point */ 3664 if (!msg_info[i].kern_addr) 3665 break; 3666 3667 hl_asic_dma_pool_free(hdev, msg_info[i].kern_addr, msg_info[i].dma_addr); 3668 msg_info[i].kern_addr = NULL; 3669 } 3670 } 3671 3672 static int gaudi2_test_queues_msgs_alloc(struct hl_device *hdev) 3673 { 3674 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3675 struct gaudi2_queues_test_info *msg_info = gaudi2->queues_test_info; 3676 int i, rc; 3677 3678 /* allocate a message-short buf for each Q we intend to test */ 3679 for (i = 0 ; i < GAUDI2_NUM_TESTED_QS ; i++) { 3680 msg_info[i].kern_addr = 3681 (void *)hl_asic_dma_pool_zalloc(hdev, sizeof(struct packet_msg_short), 3682 GFP_KERNEL, &msg_info[i].dma_addr); 3683 if (!msg_info[i].kern_addr) { 3684 dev_err(hdev->dev, 3685 "Failed to allocate dma memory for H/W queue %d testing\n", i); 3686 rc = -ENOMEM; 3687 goto err_exit; 3688 } 3689 } 3690 3691 return 0; 3692 3693 err_exit: 3694 gaudi2_test_queues_msgs_free(hdev); 3695 return rc; 3696 } 3697 3698 static int gaudi2_sw_init(struct hl_device *hdev) 3699 { 3700 struct asic_fixed_properties *prop = &hdev->asic_prop; 3701 struct gaudi2_device *gaudi2; 3702 int i, rc; 3703 3704 /* Allocate device structure */ 3705 gaudi2 = kzalloc(sizeof(*gaudi2), GFP_KERNEL); 3706 if (!gaudi2) 3707 return -ENOMEM; 3708 3709 for (i = 0 ; i < ARRAY_SIZE(gaudi2_irq_map_table) ; i++) { 3710 if (gaudi2_irq_map_table[i].msg || !gaudi2_irq_map_table[i].valid) 3711 continue; 3712 3713 if (gaudi2->num_of_valid_hw_events == GAUDI2_EVENT_SIZE) { 3714 dev_err(hdev->dev, "H/W events array exceeds the limit of %u events\n", 3715 GAUDI2_EVENT_SIZE); 3716 rc = -EINVAL; 3717 goto free_gaudi2_device; 3718 } 3719 3720 gaudi2->hw_events[gaudi2->num_of_valid_hw_events++] = gaudi2_irq_map_table[i].fc_id; 3721 } 3722 3723 for (i = 0 ; i < MME_NUM_OF_LFSR_SEEDS ; i++) 3724 gaudi2->lfsr_rand_seeds[i] = gaudi2_get_non_zero_random_int(); 3725 3726 gaudi2->cpucp_info_get = gaudi2_cpucp_info_get; 3727 3728 hdev->asic_specific = gaudi2; 3729 3730 /* Create DMA pool for small allocations. 3731 * Use DEVICE_CACHE_LINE_SIZE for alignment since the NIC memory-mapped 3732 * PI/CI registers allocated from this pool have this restriction 3733 */ 3734 hdev->dma_pool = dma_pool_create(dev_name(hdev->dev), &hdev->pdev->dev, 3735 GAUDI2_DMA_POOL_BLK_SIZE, DEVICE_CACHE_LINE_SIZE, 0); 3736 if (!hdev->dma_pool) { 3737 dev_err(hdev->dev, "failed to create DMA pool\n"); 3738 rc = -ENOMEM; 3739 goto free_gaudi2_device; 3740 } 3741 3742 rc = gaudi2_alloc_cpu_accessible_dma_mem(hdev); 3743 if (rc) 3744 goto free_dma_pool; 3745 3746 hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1); 3747 if (!hdev->cpu_accessible_dma_pool) { 3748 dev_err(hdev->dev, "Failed to create CPU accessible DMA pool\n"); 3749 rc = -ENOMEM; 3750 goto free_cpu_dma_mem; 3751 } 3752 3753 rc = gen_pool_add(hdev->cpu_accessible_dma_pool, (uintptr_t) hdev->cpu_accessible_dma_mem, 3754 HL_CPU_ACCESSIBLE_MEM_SIZE, -1); 3755 if (rc) { 3756 dev_err(hdev->dev, "Failed to add memory to CPU accessible DMA pool\n"); 3757 rc = -EFAULT; 3758 goto free_cpu_accessible_dma_pool; 3759 } 3760 3761 gaudi2->virt_msix_db_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, prop->pmmu.page_size, 3762 &gaudi2->virt_msix_db_dma_addr); 3763 if (!gaudi2->virt_msix_db_cpu_addr) { 3764 dev_err(hdev->dev, "Failed to allocate DMA memory for virtual MSI-X doorbell\n"); 3765 rc = -ENOMEM; 3766 goto free_cpu_accessible_dma_pool; 3767 } 3768 3769 spin_lock_init(&gaudi2->hw_queues_lock); 3770 3771 gaudi2->scratchpad_bus_address = prop->mmu_pgt_addr + HMMU_PAGE_TABLES_SIZE + EDMA_PQS_SIZE; 3772 3773 gaudi2_user_mapped_blocks_init(hdev); 3774 3775 /* Initialize user interrupts */ 3776 gaudi2_user_interrupt_setup(hdev); 3777 3778 hdev->supports_coresight = true; 3779 hdev->supports_sync_stream = true; 3780 hdev->supports_cb_mapping = true; 3781 hdev->supports_wait_for_multi_cs = false; 3782 3783 prop->supports_compute_reset = true; 3784 3785 /* Event queue sanity check added in FW version 1.11 */ 3786 if (hl_is_fw_sw_ver_below(hdev, 1, 11)) 3787 hdev->event_queue.check_eqe_index = false; 3788 else 3789 hdev->event_queue.check_eqe_index = true; 3790 3791 hdev->asic_funcs->set_pci_memory_regions(hdev); 3792 3793 rc = gaudi2_special_blocks_iterator_config(hdev); 3794 if (rc) 3795 goto free_virt_msix_db_mem; 3796 3797 rc = gaudi2_test_queues_msgs_alloc(hdev); 3798 if (rc) 3799 goto special_blocks_free; 3800 3801 return 0; 3802 3803 special_blocks_free: 3804 gaudi2_special_blocks_iterator_free(hdev); 3805 free_virt_msix_db_mem: 3806 hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr); 3807 free_cpu_accessible_dma_pool: 3808 gen_pool_destroy(hdev->cpu_accessible_dma_pool); 3809 free_cpu_dma_mem: 3810 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem, 3811 hdev->cpu_accessible_dma_address); 3812 free_dma_pool: 3813 dma_pool_destroy(hdev->dma_pool); 3814 free_gaudi2_device: 3815 kfree(gaudi2); 3816 return rc; 3817 } 3818 3819 static int gaudi2_sw_fini(struct hl_device *hdev) 3820 { 3821 struct asic_fixed_properties *prop = &hdev->asic_prop; 3822 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3823 3824 gaudi2_test_queues_msgs_free(hdev); 3825 3826 gaudi2_special_blocks_iterator_free(hdev); 3827 3828 hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr); 3829 3830 gen_pool_destroy(hdev->cpu_accessible_dma_pool); 3831 3832 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem, 3833 hdev->cpu_accessible_dma_address); 3834 3835 dma_pool_destroy(hdev->dma_pool); 3836 3837 kfree(gaudi2); 3838 3839 return 0; 3840 } 3841 3842 static void gaudi2_stop_qman_common(struct hl_device *hdev, u32 reg_base) 3843 { 3844 WREG32(reg_base + QM_GLBL_CFG1_OFFSET, QM_GLBL_CFG1_PQF_STOP | 3845 QM_GLBL_CFG1_CQF_STOP | 3846 QM_GLBL_CFG1_CP_STOP); 3847 3848 /* stop also the ARC */ 3849 WREG32(reg_base + QM_GLBL_CFG2_OFFSET, QM_GLBL_CFG2_ARC_CQF_STOP); 3850 } 3851 3852 static void gaudi2_flush_qman_common(struct hl_device *hdev, u32 reg_base) 3853 { 3854 WREG32(reg_base + QM_GLBL_CFG1_OFFSET, QM_GLBL_CFG1_PQF_FLUSH | 3855 QM_GLBL_CFG1_CQF_FLUSH | 3856 QM_GLBL_CFG1_CP_FLUSH); 3857 } 3858 3859 static void gaudi2_flush_qman_arc_common(struct hl_device *hdev, u32 reg_base) 3860 { 3861 WREG32(reg_base + QM_GLBL_CFG2_OFFSET, QM_GLBL_CFG2_ARC_CQF_FLUSH); 3862 } 3863 3864 /** 3865 * gaudi2_clear_qm_fence_counters_common - clear QM's fence counters 3866 * 3867 * @hdev: pointer to the habanalabs device structure 3868 * @queue_id: queue to clear fence counters to 3869 * @skip_fence: if true set maximum fence value to all fence counters to avoid 3870 * getting stuck on any fence value. otherwise set all fence 3871 * counters to 0 (standard clear of fence counters) 3872 */ 3873 static void gaudi2_clear_qm_fence_counters_common(struct hl_device *hdev, u32 queue_id, 3874 bool skip_fence) 3875 { 3876 u32 size, reg_base; 3877 u32 addr, val; 3878 3879 reg_base = gaudi2_qm_blocks_bases[queue_id]; 3880 3881 addr = reg_base + QM_CP_FENCE0_CNT_0_OFFSET; 3882 size = mmPDMA0_QM_CP_BARRIER_CFG - mmPDMA0_QM_CP_FENCE0_CNT_0; 3883 3884 /* 3885 * in case we want to make sure that QM that is stuck on a fence will 3886 * be released we should set the fence counter to a higher value that 3887 * the value the QM waiting for. to comply with any fence counter of 3888 * any value we set maximum fence value to all counters 3889 */ 3890 val = skip_fence ? U32_MAX : 0; 3891 gaudi2_memset_device_lbw(hdev, addr, size, val); 3892 } 3893 3894 static void gaudi2_qman_manual_flush_common(struct hl_device *hdev, u32 queue_id) 3895 { 3896 u32 reg_base = gaudi2_qm_blocks_bases[queue_id]; 3897 3898 gaudi2_clear_qm_fence_counters_common(hdev, queue_id, true); 3899 gaudi2_flush_qman_common(hdev, reg_base); 3900 gaudi2_flush_qman_arc_common(hdev, reg_base); 3901 } 3902 3903 static void gaudi2_stop_dma_qmans(struct hl_device *hdev) 3904 { 3905 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3906 int dcore, inst; 3907 3908 if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK)) 3909 goto stop_edma_qmans; 3910 3911 /* Stop CPs of PDMA QMANs */ 3912 gaudi2_stop_qman_common(hdev, mmPDMA0_QM_BASE); 3913 gaudi2_stop_qman_common(hdev, mmPDMA1_QM_BASE); 3914 3915 stop_edma_qmans: 3916 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) 3917 return; 3918 3919 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 3920 for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) { 3921 u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst; 3922 u32 qm_base; 3923 3924 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq))) 3925 continue; 3926 3927 qm_base = mmDCORE0_EDMA0_QM_BASE + dcore * DCORE_OFFSET + 3928 inst * DCORE_EDMA_OFFSET; 3929 3930 /* Stop CPs of EDMA QMANs */ 3931 gaudi2_stop_qman_common(hdev, qm_base); 3932 } 3933 } 3934 } 3935 3936 static void gaudi2_stop_mme_qmans(struct hl_device *hdev) 3937 { 3938 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3939 u32 offset, i; 3940 3941 offset = mmDCORE1_MME_QM_BASE - mmDCORE0_MME_QM_BASE; 3942 3943 for (i = 0 ; i < NUM_OF_DCORES ; i++) { 3944 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i))) 3945 continue; 3946 3947 gaudi2_stop_qman_common(hdev, mmDCORE0_MME_QM_BASE + (i * offset)); 3948 } 3949 } 3950 3951 static void gaudi2_stop_tpc_qmans(struct hl_device *hdev) 3952 { 3953 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3954 u32 reg_base; 3955 int i; 3956 3957 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) 3958 return; 3959 3960 for (i = 0 ; i < TPC_ID_SIZE ; i++) { 3961 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i))) 3962 continue; 3963 3964 reg_base = gaudi2_qm_blocks_bases[gaudi2_tpc_id_to_queue_id[i]]; 3965 gaudi2_stop_qman_common(hdev, reg_base); 3966 } 3967 } 3968 3969 static void gaudi2_stop_rot_qmans(struct hl_device *hdev) 3970 { 3971 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3972 u32 reg_base; 3973 int i; 3974 3975 if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK)) 3976 return; 3977 3978 for (i = 0 ; i < ROTATOR_ID_SIZE ; i++) { 3979 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i))) 3980 continue; 3981 3982 reg_base = gaudi2_qm_blocks_bases[gaudi2_rot_id_to_queue_id[i]]; 3983 gaudi2_stop_qman_common(hdev, reg_base); 3984 } 3985 } 3986 3987 static void gaudi2_stop_nic_qmans(struct hl_device *hdev) 3988 { 3989 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3990 u32 reg_base, queue_id; 3991 int i; 3992 3993 if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK)) 3994 return; 3995 3996 queue_id = GAUDI2_QUEUE_ID_NIC_0_0; 3997 3998 for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++, queue_id += NUM_OF_PQ_PER_QMAN) { 3999 if (!(hdev->nic_ports_mask & BIT(i))) 4000 continue; 4001 4002 reg_base = gaudi2_qm_blocks_bases[queue_id]; 4003 gaudi2_stop_qman_common(hdev, reg_base); 4004 } 4005 } 4006 4007 static void gaudi2_stall_dma_common(struct hl_device *hdev, u32 reg_base) 4008 { 4009 u32 reg_val; 4010 4011 reg_val = FIELD_PREP(PDMA0_CORE_CFG_1_HALT_MASK, 0x1); 4012 WREG32(reg_base + DMA_CORE_CFG_1_OFFSET, reg_val); 4013 } 4014 4015 static void gaudi2_dma_stall(struct hl_device *hdev) 4016 { 4017 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4018 int dcore, inst; 4019 4020 if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK)) 4021 goto stall_edma; 4022 4023 gaudi2_stall_dma_common(hdev, mmPDMA0_CORE_BASE); 4024 gaudi2_stall_dma_common(hdev, mmPDMA1_CORE_BASE); 4025 4026 stall_edma: 4027 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) 4028 return; 4029 4030 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 4031 for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) { 4032 u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst; 4033 u32 core_base; 4034 4035 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq))) 4036 continue; 4037 4038 core_base = mmDCORE0_EDMA0_CORE_BASE + dcore * DCORE_OFFSET + 4039 inst * DCORE_EDMA_OFFSET; 4040 4041 /* Stall CPs of EDMA QMANs */ 4042 gaudi2_stall_dma_common(hdev, core_base); 4043 } 4044 } 4045 } 4046 4047 static void gaudi2_mme_stall(struct hl_device *hdev) 4048 { 4049 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4050 u32 offset, i; 4051 4052 offset = mmDCORE1_MME_CTRL_LO_QM_STALL - mmDCORE0_MME_CTRL_LO_QM_STALL; 4053 4054 for (i = 0 ; i < NUM_OF_DCORES ; i++) 4055 if (gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i)) 4056 WREG32(mmDCORE0_MME_CTRL_LO_QM_STALL + (i * offset), 1); 4057 } 4058 4059 static void gaudi2_tpc_stall(struct hl_device *hdev) 4060 { 4061 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4062 u32 reg_base; 4063 int i; 4064 4065 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) 4066 return; 4067 4068 for (i = 0 ; i < TPC_ID_SIZE ; i++) { 4069 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i))) 4070 continue; 4071 4072 reg_base = gaudi2_tpc_cfg_blocks_bases[i]; 4073 WREG32(reg_base + TPC_CFG_STALL_OFFSET, 1); 4074 } 4075 } 4076 4077 static void gaudi2_rotator_stall(struct hl_device *hdev) 4078 { 4079 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4080 u32 reg_val; 4081 int i; 4082 4083 if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK)) 4084 return; 4085 4086 reg_val = FIELD_PREP(ROT_MSS_HALT_WBC_MASK, 0x1) | 4087 FIELD_PREP(ROT_MSS_HALT_RSB_MASK, 0x1) | 4088 FIELD_PREP(ROT_MSS_HALT_MRSB_MASK, 0x1); 4089 4090 for (i = 0 ; i < ROTATOR_ID_SIZE ; i++) { 4091 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i))) 4092 continue; 4093 4094 WREG32(mmROT0_MSS_HALT + i * ROT_OFFSET, reg_val); 4095 } 4096 } 4097 4098 static void gaudi2_disable_qman_common(struct hl_device *hdev, u32 reg_base) 4099 { 4100 WREG32(reg_base + QM_GLBL_CFG0_OFFSET, 0); 4101 } 4102 4103 static void gaudi2_disable_dma_qmans(struct hl_device *hdev) 4104 { 4105 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4106 int dcore, inst; 4107 4108 if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK)) 4109 goto stop_edma_qmans; 4110 4111 gaudi2_disable_qman_common(hdev, mmPDMA0_QM_BASE); 4112 gaudi2_disable_qman_common(hdev, mmPDMA1_QM_BASE); 4113 4114 stop_edma_qmans: 4115 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) 4116 return; 4117 4118 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 4119 for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) { 4120 u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst; 4121 u32 qm_base; 4122 4123 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq))) 4124 continue; 4125 4126 qm_base = mmDCORE0_EDMA0_QM_BASE + dcore * DCORE_OFFSET + 4127 inst * DCORE_EDMA_OFFSET; 4128 4129 /* Disable CPs of EDMA QMANs */ 4130 gaudi2_disable_qman_common(hdev, qm_base); 4131 } 4132 } 4133 } 4134 4135 static void gaudi2_disable_mme_qmans(struct hl_device *hdev) 4136 { 4137 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4138 u32 offset, i; 4139 4140 offset = mmDCORE1_MME_QM_BASE - mmDCORE0_MME_QM_BASE; 4141 4142 for (i = 0 ; i < NUM_OF_DCORES ; i++) 4143 if (gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i)) 4144 gaudi2_disable_qman_common(hdev, mmDCORE0_MME_QM_BASE + (i * offset)); 4145 } 4146 4147 static void gaudi2_disable_tpc_qmans(struct hl_device *hdev) 4148 { 4149 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4150 u32 reg_base; 4151 int i; 4152 4153 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) 4154 return; 4155 4156 for (i = 0 ; i < TPC_ID_SIZE ; i++) { 4157 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i))) 4158 continue; 4159 4160 reg_base = gaudi2_qm_blocks_bases[gaudi2_tpc_id_to_queue_id[i]]; 4161 gaudi2_disable_qman_common(hdev, reg_base); 4162 } 4163 } 4164 4165 static void gaudi2_disable_rot_qmans(struct hl_device *hdev) 4166 { 4167 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4168 u32 reg_base; 4169 int i; 4170 4171 if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK)) 4172 return; 4173 4174 for (i = 0 ; i < ROTATOR_ID_SIZE ; i++) { 4175 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i))) 4176 continue; 4177 4178 reg_base = gaudi2_qm_blocks_bases[gaudi2_rot_id_to_queue_id[i]]; 4179 gaudi2_disable_qman_common(hdev, reg_base); 4180 } 4181 } 4182 4183 static void gaudi2_disable_nic_qmans(struct hl_device *hdev) 4184 { 4185 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4186 u32 reg_base, queue_id; 4187 int i; 4188 4189 if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK)) 4190 return; 4191 4192 queue_id = GAUDI2_QUEUE_ID_NIC_0_0; 4193 4194 for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++, queue_id += NUM_OF_PQ_PER_QMAN) { 4195 if (!(hdev->nic_ports_mask & BIT(i))) 4196 continue; 4197 4198 reg_base = gaudi2_qm_blocks_bases[queue_id]; 4199 gaudi2_disable_qman_common(hdev, reg_base); 4200 } 4201 } 4202 4203 static void gaudi2_enable_timestamp(struct hl_device *hdev) 4204 { 4205 /* Disable the timestamp counter */ 4206 WREG32(mmPSOC_TIMESTAMP_BASE, 0); 4207 4208 /* Zero the lower/upper parts of the 64-bit counter */ 4209 WREG32(mmPSOC_TIMESTAMP_BASE + 0xC, 0); 4210 WREG32(mmPSOC_TIMESTAMP_BASE + 0x8, 0); 4211 4212 /* Enable the counter */ 4213 WREG32(mmPSOC_TIMESTAMP_BASE, 1); 4214 } 4215 4216 static void gaudi2_disable_timestamp(struct hl_device *hdev) 4217 { 4218 /* Disable the timestamp counter */ 4219 WREG32(mmPSOC_TIMESTAMP_BASE, 0); 4220 } 4221 4222 static const char *gaudi2_irq_name(u16 irq_number) 4223 { 4224 switch (irq_number) { 4225 case GAUDI2_IRQ_NUM_EVENT_QUEUE: 4226 return "gaudi2 cpu eq"; 4227 case GAUDI2_IRQ_NUM_COMPLETION: 4228 return "gaudi2 completion"; 4229 case GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM ... GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM: 4230 return gaudi2_vdec_irq_name[irq_number - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM]; 4231 case GAUDI2_IRQ_NUM_TPC_ASSERT: 4232 return "gaudi2 tpc assert"; 4233 case GAUDI2_IRQ_NUM_UNEXPECTED_ERROR: 4234 return "gaudi2 unexpected error"; 4235 case GAUDI2_IRQ_NUM_USER_FIRST ... GAUDI2_IRQ_NUM_USER_LAST: 4236 return "gaudi2 user completion"; 4237 case GAUDI2_IRQ_NUM_EQ_ERROR: 4238 return "gaudi2 eq error"; 4239 default: 4240 return "invalid"; 4241 } 4242 } 4243 4244 static void gaudi2_dec_disable_msix(struct hl_device *hdev, u32 max_irq_num) 4245 { 4246 int i, irq, relative_idx; 4247 struct hl_dec *dec; 4248 4249 for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM ; i < max_irq_num ; i++) { 4250 irq = pci_irq_vector(hdev->pdev, i); 4251 relative_idx = i - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM; 4252 4253 dec = hdev->dec + relative_idx / 2; 4254 4255 /* We pass different structures depending on the irq handler. For the abnormal 4256 * interrupt we pass hl_dec and for the regular interrupt we pass the relevant 4257 * user_interrupt entry 4258 */ 4259 free_irq(irq, ((relative_idx % 2) ? 4260 (void *) dec : 4261 (void *) &hdev->user_interrupt[dec->core_id])); 4262 } 4263 } 4264 4265 static int gaudi2_dec_enable_msix(struct hl_device *hdev) 4266 { 4267 int rc, i, irq_init_cnt, irq, relative_idx; 4268 struct hl_dec *dec; 4269 4270 for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM, irq_init_cnt = 0; 4271 i <= GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM; 4272 i++, irq_init_cnt++) { 4273 4274 irq = pci_irq_vector(hdev->pdev, i); 4275 relative_idx = i - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM; 4276 4277 /* We pass different structures depending on the irq handler. For the abnormal 4278 * interrupt we pass hl_dec and for the regular interrupt we pass the relevant 4279 * user_interrupt entry 4280 * 4281 * TODO: change the dec abnrm to threaded irq 4282 */ 4283 4284 dec = hdev->dec + relative_idx / 2; 4285 if (relative_idx % 2) { 4286 rc = request_irq(irq, hl_irq_handler_dec_abnrm, 0, 4287 gaudi2_irq_name(i), (void *) dec); 4288 } else { 4289 rc = request_irq(irq, hl_irq_user_interrupt_handler, 0, gaudi2_irq_name(i), 4290 (void *) &hdev->user_interrupt[dec->core_id]); 4291 } 4292 4293 if (rc) { 4294 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4295 goto free_dec_irqs; 4296 } 4297 } 4298 4299 return 0; 4300 4301 free_dec_irqs: 4302 gaudi2_dec_disable_msix(hdev, (GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + irq_init_cnt)); 4303 return rc; 4304 } 4305 4306 static int gaudi2_enable_msix(struct hl_device *hdev) 4307 { 4308 struct asic_fixed_properties *prop = &hdev->asic_prop; 4309 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4310 int rc, irq, i, j, user_irq_init_cnt; 4311 struct hl_cq *cq; 4312 4313 if (gaudi2->hw_cap_initialized & HW_CAP_MSIX) 4314 return 0; 4315 4316 hl_init_cpu_for_irq(hdev); 4317 4318 rc = pci_alloc_irq_vectors(hdev->pdev, GAUDI2_MSIX_ENTRIES, GAUDI2_MSIX_ENTRIES, 4319 PCI_IRQ_MSIX); 4320 if (rc < 0) { 4321 dev_err(hdev->dev, "MSI-X: Failed to enable support -- %d/%d\n", 4322 GAUDI2_MSIX_ENTRIES, rc); 4323 return rc; 4324 } 4325 4326 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION); 4327 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION]; 4328 rc = request_irq(irq, hl_irq_handler_cq, 0, gaudi2_irq_name(GAUDI2_IRQ_NUM_COMPLETION), cq); 4329 if (rc) { 4330 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4331 goto free_irq_vectors; 4332 } 4333 4334 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE); 4335 rc = request_irq(irq, hl_irq_handler_eq, 0, gaudi2_irq_name(GAUDI2_IRQ_NUM_EVENT_QUEUE), 4336 &hdev->event_queue); 4337 if (rc) { 4338 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4339 goto free_completion_irq; 4340 } 4341 4342 rc = gaudi2_dec_enable_msix(hdev); 4343 if (rc) { 4344 dev_err(hdev->dev, "Failed to enable decoder IRQ"); 4345 goto free_event_irq; 4346 } 4347 4348 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT); 4349 rc = request_threaded_irq(irq, NULL, hl_irq_user_interrupt_thread_handler, IRQF_ONESHOT, 4350 gaudi2_irq_name(GAUDI2_IRQ_NUM_TPC_ASSERT), 4351 &hdev->tpc_interrupt); 4352 if (rc) { 4353 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4354 goto free_dec_irq; 4355 } 4356 4357 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR); 4358 rc = request_threaded_irq(irq, NULL, hl_irq_user_interrupt_thread_handler, IRQF_ONESHOT, 4359 gaudi2_irq_name(GAUDI2_IRQ_NUM_UNEXPECTED_ERROR), 4360 &hdev->unexpected_error_interrupt); 4361 if (rc) { 4362 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4363 goto free_tpc_irq; 4364 } 4365 4366 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, user_irq_init_cnt = 0; 4367 user_irq_init_cnt < prop->user_interrupt_count; 4368 i++, j++, user_irq_init_cnt++) { 4369 4370 irq = pci_irq_vector(hdev->pdev, i); 4371 hl_set_irq_affinity(hdev, irq); 4372 rc = request_irq(irq, hl_irq_user_interrupt_handler, 0, gaudi2_irq_name(i), 4373 &hdev->user_interrupt[j]); 4374 if (rc) { 4375 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4376 goto free_user_irq; 4377 } 4378 } 4379 4380 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EQ_ERROR); 4381 rc = request_threaded_irq(irq, NULL, hl_irq_eq_error_interrupt_thread_handler, 4382 IRQF_ONESHOT, gaudi2_irq_name(GAUDI2_IRQ_NUM_EQ_ERROR), 4383 hdev); 4384 if (rc) { 4385 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4386 goto free_user_irq; 4387 } 4388 4389 gaudi2->hw_cap_initialized |= HW_CAP_MSIX; 4390 4391 return 0; 4392 4393 free_user_irq: 4394 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count; 4395 i < GAUDI2_IRQ_NUM_USER_FIRST + user_irq_init_cnt ; i++, j++) { 4396 4397 irq = pci_irq_vector(hdev->pdev, i); 4398 irq_set_affinity_hint(irq, NULL); 4399 free_irq(irq, &hdev->user_interrupt[j]); 4400 } 4401 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR); 4402 free_irq(irq, &hdev->unexpected_error_interrupt); 4403 free_tpc_irq: 4404 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT); 4405 free_irq(irq, &hdev->tpc_interrupt); 4406 free_dec_irq: 4407 gaudi2_dec_disable_msix(hdev, GAUDI2_IRQ_NUM_DEC_LAST + 1); 4408 free_event_irq: 4409 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE); 4410 free_irq(irq, cq); 4411 4412 free_completion_irq: 4413 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION); 4414 free_irq(irq, cq); 4415 4416 free_irq_vectors: 4417 pci_free_irq_vectors(hdev->pdev); 4418 4419 return rc; 4420 } 4421 4422 static void gaudi2_sync_irqs(struct hl_device *hdev) 4423 { 4424 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4425 int i, j; 4426 int irq; 4427 4428 if (!(gaudi2->hw_cap_initialized & HW_CAP_MSIX)) 4429 return; 4430 4431 /* Wait for all pending IRQs to be finished */ 4432 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION)); 4433 4434 for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM ; i <= GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM ; i++) { 4435 irq = pci_irq_vector(hdev->pdev, i); 4436 synchronize_irq(irq); 4437 } 4438 4439 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT)); 4440 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR)); 4441 4442 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = 0 ; j < hdev->asic_prop.user_interrupt_count; 4443 i++, j++) { 4444 irq = pci_irq_vector(hdev->pdev, i); 4445 synchronize_irq(irq); 4446 } 4447 4448 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE)); 4449 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EQ_ERROR)); 4450 } 4451 4452 static void gaudi2_disable_msix(struct hl_device *hdev) 4453 { 4454 struct asic_fixed_properties *prop = &hdev->asic_prop; 4455 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4456 struct hl_cq *cq; 4457 int irq, i, j, k; 4458 4459 if (!(gaudi2->hw_cap_initialized & HW_CAP_MSIX)) 4460 return; 4461 4462 gaudi2_sync_irqs(hdev); 4463 4464 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE); 4465 free_irq(irq, &hdev->event_queue); 4466 4467 gaudi2_dec_disable_msix(hdev, GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM + 1); 4468 4469 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT); 4470 free_irq(irq, &hdev->tpc_interrupt); 4471 4472 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR); 4473 free_irq(irq, &hdev->unexpected_error_interrupt); 4474 4475 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, k = 0; 4476 k < hdev->asic_prop.user_interrupt_count ; i++, j++, k++) { 4477 4478 irq = pci_irq_vector(hdev->pdev, i); 4479 irq_set_affinity_hint(irq, NULL); 4480 free_irq(irq, &hdev->user_interrupt[j]); 4481 } 4482 4483 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION); 4484 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION]; 4485 free_irq(irq, cq); 4486 4487 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EQ_ERROR); 4488 free_irq(irq, hdev); 4489 4490 pci_free_irq_vectors(hdev->pdev); 4491 4492 gaudi2->hw_cap_initialized &= ~HW_CAP_MSIX; 4493 } 4494 4495 static void gaudi2_stop_dcore_dec(struct hl_device *hdev, int dcore_id) 4496 { 4497 u32 reg_val = FIELD_PREP(DCORE0_VDEC0_BRDG_CTRL_GRACEFUL_STOP_MASK, 0x1); 4498 u32 graceful_pend_mask = DCORE0_VDEC0_BRDG_CTRL_GRACEFUL_PEND_MASK; 4499 u32 timeout_usec, dec_id, dec_bit, offset, graceful; 4500 int rc; 4501 4502 if (hdev->pldm) 4503 timeout_usec = GAUDI2_PLDM_VDEC_TIMEOUT_USEC; 4504 else 4505 timeout_usec = GAUDI2_VDEC_TIMEOUT_USEC; 4506 4507 for (dec_id = 0 ; dec_id < NUM_OF_DEC_PER_DCORE ; dec_id++) { 4508 dec_bit = dcore_id * NUM_OF_DEC_PER_DCORE + dec_id; 4509 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) 4510 continue; 4511 4512 offset = dcore_id * DCORE_OFFSET + dec_id * DCORE_VDEC_OFFSET; 4513 4514 WREG32(mmDCORE0_DEC0_CMD_SWREG16 + offset, 0); 4515 4516 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_GRACEFUL + offset, reg_val); 4517 4518 /* Wait till all traffic from decoder stops 4519 * before apply core reset. 4520 */ 4521 rc = hl_poll_timeout( 4522 hdev, 4523 mmDCORE0_VDEC0_BRDG_CTRL_GRACEFUL + offset, 4524 graceful, 4525 (graceful & graceful_pend_mask), 4526 100, 4527 timeout_usec); 4528 if (rc) 4529 dev_err(hdev->dev, 4530 "Failed to stop traffic from DCORE%d Decoder %d\n", 4531 dcore_id, dec_id); 4532 } 4533 } 4534 4535 static void gaudi2_stop_pcie_dec(struct hl_device *hdev) 4536 { 4537 u32 reg_val = FIELD_PREP(DCORE0_VDEC0_BRDG_CTRL_GRACEFUL_STOP_MASK, 0x1); 4538 u32 graceful_pend_mask = PCIE_VDEC0_BRDG_CTRL_GRACEFUL_PEND_MASK; 4539 u32 timeout_usec, dec_id, dec_bit, offset, graceful; 4540 int rc; 4541 4542 if (hdev->pldm) 4543 timeout_usec = GAUDI2_PLDM_VDEC_TIMEOUT_USEC; 4544 else 4545 timeout_usec = GAUDI2_VDEC_TIMEOUT_USEC; 4546 4547 for (dec_id = 0 ; dec_id < NUM_OF_DEC_PER_DCORE ; dec_id++) { 4548 dec_bit = PCIE_DEC_SHIFT + dec_id; 4549 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) 4550 continue; 4551 4552 offset = dec_id * PCIE_VDEC_OFFSET; 4553 4554 WREG32(mmPCIE_DEC0_CMD_SWREG16 + offset, 0); 4555 4556 WREG32(mmPCIE_VDEC0_BRDG_CTRL_GRACEFUL + offset, reg_val); 4557 4558 /* Wait till all traffic from decoder stops 4559 * before apply core reset. 4560 */ 4561 rc = hl_poll_timeout( 4562 hdev, 4563 mmPCIE_VDEC0_BRDG_CTRL_GRACEFUL + offset, 4564 graceful, 4565 (graceful & graceful_pend_mask), 4566 100, 4567 timeout_usec); 4568 if (rc) 4569 dev_err(hdev->dev, 4570 "Failed to stop traffic from PCIe Decoder %d\n", 4571 dec_id); 4572 } 4573 } 4574 4575 static void gaudi2_stop_dec(struct hl_device *hdev) 4576 { 4577 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4578 int dcore_id; 4579 4580 if ((gaudi2->dec_hw_cap_initialized & HW_CAP_DEC_MASK) == 0) 4581 return; 4582 4583 for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) 4584 gaudi2_stop_dcore_dec(hdev, dcore_id); 4585 4586 gaudi2_stop_pcie_dec(hdev); 4587 } 4588 4589 static void gaudi2_set_arc_running_mode(struct hl_device *hdev, u32 cpu_id, u32 run_mode) 4590 { 4591 u32 reg_base, reg_val; 4592 4593 reg_base = gaudi2_arc_blocks_bases[cpu_id]; 4594 if (run_mode == HL_ENGINE_CORE_RUN) 4595 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_RUN_REQ_MASK, 1); 4596 else 4597 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_HALT_REQ_MASK, 1); 4598 4599 WREG32(reg_base + ARC_HALT_REQ_OFFSET, reg_val); 4600 } 4601 4602 static void gaudi2_halt_arcs(struct hl_device *hdev) 4603 { 4604 u16 arc_id; 4605 4606 for (arc_id = CPU_ID_SCHED_ARC0; arc_id < CPU_ID_MAX; arc_id++) { 4607 if (gaudi2_is_arc_enabled(hdev, arc_id)) 4608 gaudi2_set_arc_running_mode(hdev, arc_id, HL_ENGINE_CORE_HALT); 4609 } 4610 } 4611 4612 static int gaudi2_verify_arc_running_mode(struct hl_device *hdev, u32 cpu_id, u32 run_mode) 4613 { 4614 int rc; 4615 u32 reg_base, val, ack_mask, timeout_usec = 100000; 4616 4617 if (hdev->pldm) 4618 timeout_usec *= 100; 4619 4620 reg_base = gaudi2_arc_blocks_bases[cpu_id]; 4621 if (run_mode == HL_ENGINE_CORE_RUN) 4622 ack_mask = ARC_FARM_ARC0_AUX_RUN_HALT_ACK_RUN_ACK_MASK; 4623 else 4624 ack_mask = ARC_FARM_ARC0_AUX_RUN_HALT_ACK_HALT_ACK_MASK; 4625 4626 rc = hl_poll_timeout(hdev, reg_base + ARC_HALT_ACK_OFFSET, 4627 val, ((val & ack_mask) == ack_mask), 4628 1000, timeout_usec); 4629 4630 if (!rc) { 4631 /* Clear */ 4632 val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_RUN_REQ_MASK, 0); 4633 WREG32(reg_base + ARC_HALT_REQ_OFFSET, val); 4634 } 4635 4636 return rc; 4637 } 4638 4639 static void gaudi2_reset_arcs(struct hl_device *hdev) 4640 { 4641 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4642 u16 arc_id; 4643 4644 if (!gaudi2) 4645 return; 4646 4647 for (arc_id = CPU_ID_SCHED_ARC0; arc_id < CPU_ID_MAX; arc_id++) 4648 if (gaudi2_is_arc_enabled(hdev, arc_id)) 4649 gaudi2_clr_arc_id_cap(hdev, arc_id); 4650 } 4651 4652 static void gaudi2_nic_qmans_manual_flush(struct hl_device *hdev) 4653 { 4654 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4655 u32 queue_id; 4656 int i; 4657 4658 if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK)) 4659 return; 4660 4661 queue_id = GAUDI2_QUEUE_ID_NIC_0_0; 4662 4663 for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++, queue_id += NUM_OF_PQ_PER_QMAN) { 4664 if (!(hdev->nic_ports_mask & BIT(i))) 4665 continue; 4666 4667 gaudi2_qman_manual_flush_common(hdev, queue_id); 4668 } 4669 } 4670 4671 static int gaudi2_set_engine_cores(struct hl_device *hdev, u32 *core_ids, 4672 u32 num_cores, u32 core_command) 4673 { 4674 int i, rc; 4675 4676 for (i = 0 ; i < num_cores ; i++) { 4677 if (gaudi2_is_arc_enabled(hdev, core_ids[i])) 4678 gaudi2_set_arc_running_mode(hdev, core_ids[i], core_command); 4679 } 4680 4681 for (i = 0 ; i < num_cores ; i++) { 4682 if (gaudi2_is_arc_enabled(hdev, core_ids[i])) { 4683 rc = gaudi2_verify_arc_running_mode(hdev, core_ids[i], core_command); 4684 4685 if (rc) { 4686 dev_err(hdev->dev, "failed to %s arc: %d\n", 4687 (core_command == HL_ENGINE_CORE_HALT) ? 4688 "HALT" : "RUN", core_ids[i]); 4689 return -1; 4690 } 4691 } 4692 } 4693 4694 return 0; 4695 } 4696 4697 static int gaudi2_set_tpc_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command) 4698 { 4699 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4700 u32 reg_base, reg_addr, reg_val, tpc_id; 4701 4702 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) 4703 return 0; 4704 4705 tpc_id = gaudi2_tpc_engine_id_to_tpc_id[engine_id]; 4706 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + tpc_id))) 4707 return 0; 4708 4709 reg_base = gaudi2_tpc_cfg_blocks_bases[tpc_id]; 4710 reg_addr = reg_base + TPC_CFG_STALL_OFFSET; 4711 reg_val = FIELD_PREP(DCORE0_TPC0_CFG_TPC_STALL_V_MASK, 4712 (engine_command == HL_ENGINE_STALL) ? 1 : 0); 4713 WREG32(reg_addr, reg_val); 4714 4715 if (engine_command == HL_ENGINE_RESUME) { 4716 reg_base = gaudi2_tpc_eml_cfg_blocks_bases[tpc_id]; 4717 reg_addr = reg_base + TPC_EML_CFG_DBG_CNT_OFFSET; 4718 RMWREG32(reg_addr, 0x1, DCORE0_TPC0_EML_CFG_DBG_CNT_DBG_EXIT_MASK); 4719 } 4720 4721 return 0; 4722 } 4723 4724 static int gaudi2_set_mme_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command) 4725 { 4726 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4727 u32 reg_base, reg_addr, reg_val, mme_id; 4728 4729 mme_id = gaudi2_mme_engine_id_to_mme_id[engine_id]; 4730 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + mme_id))) 4731 return 0; 4732 4733 reg_base = gaudi2_mme_ctrl_lo_blocks_bases[mme_id]; 4734 reg_addr = reg_base + MME_CTRL_LO_QM_STALL_OFFSET; 4735 reg_val = FIELD_PREP(DCORE0_MME_CTRL_LO_QM_STALL_V_MASK, 4736 (engine_command == HL_ENGINE_STALL) ? 1 : 0); 4737 WREG32(reg_addr, reg_val); 4738 4739 return 0; 4740 } 4741 4742 static int gaudi2_set_edma_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command) 4743 { 4744 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4745 u32 reg_base, reg_addr, reg_val, edma_id; 4746 4747 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) 4748 return 0; 4749 4750 edma_id = gaudi2_edma_engine_id_to_edma_id[engine_id]; 4751 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + edma_id))) 4752 return 0; 4753 4754 reg_base = gaudi2_dma_core_blocks_bases[edma_id]; 4755 reg_addr = reg_base + EDMA_CORE_CFG_STALL_OFFSET; 4756 reg_val = FIELD_PREP(DCORE0_EDMA0_CORE_CFG_1_HALT_MASK, 4757 (engine_command == HL_ENGINE_STALL) ? 1 : 0); 4758 WREG32(reg_addr, reg_val); 4759 4760 if (engine_command == HL_ENGINE_STALL) { 4761 reg_val = FIELD_PREP(DCORE0_EDMA0_CORE_CFG_1_HALT_MASK, 0x1) | 4762 FIELD_PREP(DCORE0_EDMA0_CORE_CFG_1_FLUSH_MASK, 0x1); 4763 WREG32(reg_addr, reg_val); 4764 } 4765 4766 return 0; 4767 } 4768 4769 static int gaudi2_set_engine_modes(struct hl_device *hdev, 4770 u32 *engine_ids, u32 num_engines, u32 engine_command) 4771 { 4772 int i, rc; 4773 4774 for (i = 0 ; i < num_engines ; ++i) { 4775 switch (engine_ids[i]) { 4776 case GAUDI2_DCORE0_ENGINE_ID_TPC_0 ... GAUDI2_DCORE0_ENGINE_ID_TPC_5: 4777 case GAUDI2_DCORE1_ENGINE_ID_TPC_0 ... GAUDI2_DCORE1_ENGINE_ID_TPC_5: 4778 case GAUDI2_DCORE2_ENGINE_ID_TPC_0 ... GAUDI2_DCORE2_ENGINE_ID_TPC_5: 4779 case GAUDI2_DCORE3_ENGINE_ID_TPC_0 ... GAUDI2_DCORE3_ENGINE_ID_TPC_5: 4780 rc = gaudi2_set_tpc_engine_mode(hdev, engine_ids[i], engine_command); 4781 if (rc) 4782 return rc; 4783 4784 break; 4785 case GAUDI2_DCORE0_ENGINE_ID_MME: 4786 case GAUDI2_DCORE1_ENGINE_ID_MME: 4787 case GAUDI2_DCORE2_ENGINE_ID_MME: 4788 case GAUDI2_DCORE3_ENGINE_ID_MME: 4789 rc = gaudi2_set_mme_engine_mode(hdev, engine_ids[i], engine_command); 4790 if (rc) 4791 return rc; 4792 4793 break; 4794 case GAUDI2_DCORE0_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE0_ENGINE_ID_EDMA_1: 4795 case GAUDI2_DCORE1_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE1_ENGINE_ID_EDMA_1: 4796 case GAUDI2_DCORE2_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE2_ENGINE_ID_EDMA_1: 4797 case GAUDI2_DCORE3_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE3_ENGINE_ID_EDMA_1: 4798 rc = gaudi2_set_edma_engine_mode(hdev, engine_ids[i], engine_command); 4799 if (rc) 4800 return rc; 4801 4802 break; 4803 default: 4804 dev_err(hdev->dev, "Invalid engine ID %u\n", engine_ids[i]); 4805 return -EINVAL; 4806 } 4807 } 4808 4809 return 0; 4810 } 4811 4812 static int gaudi2_set_engines(struct hl_device *hdev, u32 *engine_ids, 4813 u32 num_engines, u32 engine_command) 4814 { 4815 switch (engine_command) { 4816 case HL_ENGINE_CORE_HALT: 4817 case HL_ENGINE_CORE_RUN: 4818 return gaudi2_set_engine_cores(hdev, engine_ids, num_engines, engine_command); 4819 4820 case HL_ENGINE_STALL: 4821 case HL_ENGINE_RESUME: 4822 return gaudi2_set_engine_modes(hdev, engine_ids, num_engines, engine_command); 4823 4824 default: 4825 dev_err(hdev->dev, "failed to execute command id %u\n", engine_command); 4826 return -EINVAL; 4827 } 4828 } 4829 4830 static void gaudi2_halt_engines(struct hl_device *hdev, bool hard_reset, bool fw_reset) 4831 { 4832 u32 wait_timeout_ms; 4833 4834 if (hdev->pldm) 4835 wait_timeout_ms = GAUDI2_PLDM_RESET_WAIT_MSEC; 4836 else 4837 wait_timeout_ms = GAUDI2_RESET_WAIT_MSEC; 4838 4839 if (fw_reset) 4840 goto skip_engines; 4841 4842 gaudi2_stop_dma_qmans(hdev); 4843 gaudi2_stop_mme_qmans(hdev); 4844 gaudi2_stop_tpc_qmans(hdev); 4845 gaudi2_stop_rot_qmans(hdev); 4846 gaudi2_stop_nic_qmans(hdev); 4847 msleep(wait_timeout_ms); 4848 4849 gaudi2_halt_arcs(hdev); 4850 gaudi2_dma_stall(hdev); 4851 gaudi2_mme_stall(hdev); 4852 gaudi2_tpc_stall(hdev); 4853 gaudi2_rotator_stall(hdev); 4854 4855 msleep(wait_timeout_ms); 4856 4857 gaudi2_stop_dec(hdev); 4858 4859 /* 4860 * in case of soft reset do a manual flush for QMANs (currently called 4861 * only for NIC QMANs 4862 */ 4863 if (!hard_reset) 4864 gaudi2_nic_qmans_manual_flush(hdev); 4865 4866 gaudi2_disable_dma_qmans(hdev); 4867 gaudi2_disable_mme_qmans(hdev); 4868 gaudi2_disable_tpc_qmans(hdev); 4869 gaudi2_disable_rot_qmans(hdev); 4870 gaudi2_disable_nic_qmans(hdev); 4871 gaudi2_disable_timestamp(hdev); 4872 4873 skip_engines: 4874 if (hard_reset) { 4875 gaudi2_disable_msix(hdev); 4876 return; 4877 } 4878 4879 gaudi2_sync_irqs(hdev); 4880 } 4881 4882 static void gaudi2_init_firmware_preload_params(struct hl_device *hdev) 4883 { 4884 struct pre_fw_load_props *pre_fw_load = &hdev->fw_loader.pre_fw_load; 4885 4886 pre_fw_load->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS; 4887 pre_fw_load->sts_boot_dev_sts0_reg = mmCPU_BOOT_DEV_STS0; 4888 pre_fw_load->sts_boot_dev_sts1_reg = mmCPU_BOOT_DEV_STS1; 4889 pre_fw_load->boot_err0_reg = mmCPU_BOOT_ERR0; 4890 pre_fw_load->boot_err1_reg = mmCPU_BOOT_ERR1; 4891 pre_fw_load->wait_for_preboot_timeout = GAUDI2_PREBOOT_REQ_TIMEOUT_USEC; 4892 pre_fw_load->wait_for_preboot_extended_timeout = 4893 GAUDI2_PREBOOT_EXTENDED_REQ_TIMEOUT_USEC; 4894 } 4895 4896 static void gaudi2_init_firmware_loader(struct hl_device *hdev) 4897 { 4898 struct fw_load_mgr *fw_loader = &hdev->fw_loader; 4899 struct dynamic_fw_load_mgr *dynamic_loader; 4900 struct cpu_dyn_regs *dyn_regs; 4901 4902 /* fill common fields */ 4903 fw_loader->fw_comp_loaded = FW_TYPE_NONE; 4904 fw_loader->boot_fit_img.image_name = GAUDI2_BOOT_FIT_FILE; 4905 fw_loader->linux_img.image_name = GAUDI2_LINUX_FW_FILE; 4906 fw_loader->boot_fit_timeout = GAUDI2_BOOT_FIT_REQ_TIMEOUT_USEC; 4907 fw_loader->skip_bmc = false; 4908 fw_loader->sram_bar_id = SRAM_CFG_BAR_ID; 4909 fw_loader->dram_bar_id = DRAM_BAR_ID; 4910 fw_loader->cpu_timeout = GAUDI2_CPU_TIMEOUT_USEC; 4911 4912 /* here we update initial values for few specific dynamic regs (as 4913 * before reading the first descriptor from FW those value has to be 4914 * hard-coded). in later stages of the protocol those values will be 4915 * updated automatically by reading the FW descriptor so data there 4916 * will always be up-to-date 4917 */ 4918 dynamic_loader = &hdev->fw_loader.dynamic_loader; 4919 dyn_regs = &dynamic_loader->comm_desc.cpu_dyn_regs; 4920 dyn_regs->kmd_msg_to_cpu = cpu_to_le32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU); 4921 dyn_regs->cpu_cmd_status_to_host = cpu_to_le32(mmCPU_CMD_STATUS_TO_HOST); 4922 dynamic_loader->wait_for_bl_timeout = GAUDI2_WAIT_FOR_BL_TIMEOUT_USEC; 4923 } 4924 4925 static int gaudi2_init_cpu(struct hl_device *hdev) 4926 { 4927 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4928 int rc; 4929 4930 if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU)) 4931 return 0; 4932 4933 if (gaudi2->hw_cap_initialized & HW_CAP_CPU) 4934 return 0; 4935 4936 rc = hl_fw_init_cpu(hdev); 4937 if (rc) 4938 return rc; 4939 4940 gaudi2->hw_cap_initialized |= HW_CAP_CPU; 4941 4942 return 0; 4943 } 4944 4945 static int gaudi2_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout) 4946 { 4947 struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ]; 4948 struct asic_fixed_properties *prop = &hdev->asic_prop; 4949 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4950 struct cpu_dyn_regs *dyn_regs; 4951 struct hl_eq *eq; 4952 u32 status; 4953 int err; 4954 4955 if (!hdev->cpu_queues_enable) 4956 return 0; 4957 4958 if (gaudi2->hw_cap_initialized & HW_CAP_CPU_Q) 4959 return 0; 4960 4961 eq = &hdev->event_queue; 4962 4963 dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 4964 4965 WREG32(mmCPU_IF_PQ_BASE_ADDR_LOW, lower_32_bits(cpu_pq->bus_address)); 4966 WREG32(mmCPU_IF_PQ_BASE_ADDR_HIGH, upper_32_bits(cpu_pq->bus_address)); 4967 4968 WREG32(mmCPU_IF_EQ_BASE_ADDR_LOW, lower_32_bits(eq->bus_address)); 4969 WREG32(mmCPU_IF_EQ_BASE_ADDR_HIGH, upper_32_bits(eq->bus_address)); 4970 4971 WREG32(mmCPU_IF_CQ_BASE_ADDR_LOW, lower_32_bits(hdev->cpu_accessible_dma_address)); 4972 WREG32(mmCPU_IF_CQ_BASE_ADDR_HIGH, upper_32_bits(hdev->cpu_accessible_dma_address)); 4973 4974 WREG32(mmCPU_IF_PQ_LENGTH, HL_QUEUE_SIZE_IN_BYTES); 4975 WREG32(mmCPU_IF_EQ_LENGTH, HL_EQ_SIZE_IN_BYTES); 4976 WREG32(mmCPU_IF_CQ_LENGTH, HL_CPU_ACCESSIBLE_MEM_SIZE); 4977 4978 /* Used for EQ CI */ 4979 WREG32(mmCPU_IF_EQ_RD_OFFS, 0); 4980 4981 WREG32(mmCPU_IF_PF_PQ_PI, 0); 4982 4983 WREG32(mmCPU_IF_QUEUE_INIT, PQ_INIT_STATUS_READY_FOR_CP); 4984 4985 /* Let the ARC know we are ready as it is now handling those queues */ 4986 4987 WREG32(le32_to_cpu(dyn_regs->gic_host_pi_upd_irq), 4988 gaudi2_irq_map_table[GAUDI2_EVENT_CPU_PI_UPDATE].cpu_id); 4989 4990 err = hl_poll_timeout( 4991 hdev, 4992 mmCPU_IF_QUEUE_INIT, 4993 status, 4994 (status == PQ_INIT_STATUS_READY_FOR_HOST), 4995 1000, 4996 cpu_timeout); 4997 4998 if (err) { 4999 dev_err(hdev->dev, "Failed to communicate with device CPU (timeout)\n"); 5000 return -EIO; 5001 } 5002 5003 /* update FW application security bits */ 5004 if (prop->fw_cpu_boot_dev_sts0_valid) 5005 prop->fw_app_cpu_boot_dev_sts0 = RREG32(mmCPU_BOOT_DEV_STS0); 5006 5007 if (prop->fw_cpu_boot_dev_sts1_valid) 5008 prop->fw_app_cpu_boot_dev_sts1 = RREG32(mmCPU_BOOT_DEV_STS1); 5009 5010 gaudi2->hw_cap_initialized |= HW_CAP_CPU_Q; 5011 return 0; 5012 } 5013 5014 static void gaudi2_init_qman_pq(struct hl_device *hdev, u32 reg_base, 5015 u32 queue_id_base) 5016 { 5017 struct hl_hw_queue *q; 5018 u32 pq_id, pq_offset; 5019 5020 for (pq_id = 0 ; pq_id < NUM_OF_PQ_PER_QMAN ; pq_id++) { 5021 q = &hdev->kernel_queues[queue_id_base + pq_id]; 5022 pq_offset = pq_id * 4; 5023 5024 if (q->dram_bd) { 5025 WREG32(reg_base + QM_PQ_BASE_LO_0_OFFSET + pq_offset, 5026 lower_32_bits(q->pq_dram_address)); 5027 WREG32(reg_base + QM_PQ_BASE_HI_0_OFFSET + pq_offset, 5028 upper_32_bits(q->pq_dram_address)); 5029 } else { 5030 WREG32(reg_base + QM_PQ_BASE_LO_0_OFFSET + pq_offset, 5031 lower_32_bits(q->bus_address)); 5032 WREG32(reg_base + QM_PQ_BASE_HI_0_OFFSET + pq_offset, 5033 upper_32_bits(q->bus_address)); 5034 } 5035 WREG32(reg_base + QM_PQ_SIZE_0_OFFSET + pq_offset, ilog2(HL_QUEUE_LENGTH)); 5036 WREG32(reg_base + QM_PQ_PI_0_OFFSET + pq_offset, 0); 5037 WREG32(reg_base + QM_PQ_CI_0_OFFSET + pq_offset, 0); 5038 } 5039 } 5040 5041 static void gaudi2_init_qman_cp(struct hl_device *hdev, u32 reg_base) 5042 { 5043 u32 cp_id, cp_offset, mtr_base_lo, mtr_base_hi, so_base_lo, so_base_hi; 5044 5045 mtr_base_lo = lower_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); 5046 mtr_base_hi = upper_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); 5047 so_base_lo = lower_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0); 5048 so_base_hi = upper_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0); 5049 5050 for (cp_id = 0 ; cp_id < NUM_OF_CP_PER_QMAN; cp_id++) { 5051 cp_offset = cp_id * 4; 5052 5053 WREG32(reg_base + QM_CP_MSG_BASE0_ADDR_LO_0_OFFSET + cp_offset, mtr_base_lo); 5054 WREG32(reg_base + QM_CP_MSG_BASE0_ADDR_HI_0_OFFSET + cp_offset, mtr_base_hi); 5055 WREG32(reg_base + QM_CP_MSG_BASE1_ADDR_LO_0_OFFSET + cp_offset, so_base_lo); 5056 WREG32(reg_base + QM_CP_MSG_BASE1_ADDR_HI_0_OFFSET + cp_offset, so_base_hi); 5057 } 5058 5059 /* allow QMANs to accept work from ARC CQF */ 5060 WREG32(reg_base + QM_CP_CFG_OFFSET, FIELD_PREP(PDMA0_QM_CP_CFG_SWITCH_EN_MASK, 0x1)); 5061 } 5062 5063 static void gaudi2_init_qman_pqc(struct hl_device *hdev, u32 reg_base, 5064 u32 queue_id_base) 5065 { 5066 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5067 u32 pq_id, pq_offset, so_base_lo, so_base_hi; 5068 5069 so_base_lo = lower_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0); 5070 so_base_hi = upper_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0); 5071 5072 for (pq_id = 0 ; pq_id < NUM_OF_PQ_PER_QMAN ; pq_id++) { 5073 pq_offset = pq_id * 4; 5074 5075 /* Configure QMAN HBW to scratchpad as it is not needed */ 5076 WREG32(reg_base + QM_PQC_HBW_BASE_LO_0_OFFSET + pq_offset, 5077 lower_32_bits(gaudi2->scratchpad_bus_address)); 5078 WREG32(reg_base + QM_PQC_HBW_BASE_HI_0_OFFSET + pq_offset, 5079 upper_32_bits(gaudi2->scratchpad_bus_address)); 5080 WREG32(reg_base + QM_PQC_SIZE_0_OFFSET + pq_offset, 5081 ilog2(PAGE_SIZE / sizeof(struct hl_cq_entry))); 5082 5083 WREG32(reg_base + QM_PQC_PI_0_OFFSET + pq_offset, 0); 5084 WREG32(reg_base + QM_PQC_LBW_WDATA_0_OFFSET + pq_offset, QM_PQC_LBW_WDATA); 5085 WREG32(reg_base + QM_PQC_LBW_BASE_LO_0_OFFSET + pq_offset, so_base_lo); 5086 WREG32(reg_base + QM_PQC_LBW_BASE_HI_0_OFFSET + pq_offset, so_base_hi); 5087 } 5088 5089 /* Enable QMAN H/W completion */ 5090 WREG32(reg_base + QM_PQC_CFG_OFFSET, 1 << PDMA0_QM_PQC_CFG_EN_SHIFT); 5091 } 5092 5093 static u32 gaudi2_get_dyn_sp_reg(struct hl_device *hdev, u32 queue_id_base) 5094 { 5095 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 5096 u32 sp_reg_addr; 5097 5098 switch (queue_id_base) { 5099 case GAUDI2_QUEUE_ID_PDMA_0_0...GAUDI2_QUEUE_ID_PDMA_1_3: 5100 fallthrough; 5101 case GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3: 5102 fallthrough; 5103 case GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3: 5104 fallthrough; 5105 case GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3: 5106 fallthrough; 5107 case GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3: 5108 sp_reg_addr = le32_to_cpu(dyn_regs->gic_dma_qm_irq_ctrl); 5109 break; 5110 case GAUDI2_QUEUE_ID_DCORE0_MME_0_0...GAUDI2_QUEUE_ID_DCORE0_MME_0_3: 5111 fallthrough; 5112 case GAUDI2_QUEUE_ID_DCORE1_MME_0_0...GAUDI2_QUEUE_ID_DCORE1_MME_0_3: 5113 fallthrough; 5114 case GAUDI2_QUEUE_ID_DCORE2_MME_0_0...GAUDI2_QUEUE_ID_DCORE2_MME_0_3: 5115 fallthrough; 5116 case GAUDI2_QUEUE_ID_DCORE3_MME_0_0...GAUDI2_QUEUE_ID_DCORE3_MME_0_3: 5117 sp_reg_addr = le32_to_cpu(dyn_regs->gic_mme_qm_irq_ctrl); 5118 break; 5119 case GAUDI2_QUEUE_ID_DCORE0_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE0_TPC_6_3: 5120 fallthrough; 5121 case GAUDI2_QUEUE_ID_DCORE1_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE1_TPC_5_3: 5122 fallthrough; 5123 case GAUDI2_QUEUE_ID_DCORE2_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE2_TPC_5_3: 5124 fallthrough; 5125 case GAUDI2_QUEUE_ID_DCORE3_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE3_TPC_5_3: 5126 sp_reg_addr = le32_to_cpu(dyn_regs->gic_tpc_qm_irq_ctrl); 5127 break; 5128 case GAUDI2_QUEUE_ID_ROT_0_0...GAUDI2_QUEUE_ID_ROT_1_3: 5129 sp_reg_addr = le32_to_cpu(dyn_regs->gic_rot_qm_irq_ctrl); 5130 break; 5131 case GAUDI2_QUEUE_ID_NIC_0_0...GAUDI2_QUEUE_ID_NIC_23_3: 5132 sp_reg_addr = le32_to_cpu(dyn_regs->gic_nic_qm_irq_ctrl); 5133 break; 5134 default: 5135 dev_err(hdev->dev, "Unexpected h/w queue %d\n", queue_id_base); 5136 return 0; 5137 } 5138 5139 return sp_reg_addr; 5140 } 5141 5142 static void gaudi2_init_qman_common(struct hl_device *hdev, u32 reg_base, 5143 u32 queue_id_base) 5144 { 5145 u32 glbl_prot = QMAN_MAKE_TRUSTED, irq_handler_offset; 5146 int map_table_entry; 5147 5148 WREG32(reg_base + QM_GLBL_PROT_OFFSET, glbl_prot); 5149 5150 irq_handler_offset = gaudi2_get_dyn_sp_reg(hdev, queue_id_base); 5151 WREG32(reg_base + QM_GLBL_ERR_ADDR_LO_OFFSET, lower_32_bits(CFG_BASE + irq_handler_offset)); 5152 WREG32(reg_base + QM_GLBL_ERR_ADDR_HI_OFFSET, upper_32_bits(CFG_BASE + irq_handler_offset)); 5153 5154 map_table_entry = gaudi2_qman_async_event_id[queue_id_base]; 5155 WREG32(reg_base + QM_GLBL_ERR_WDATA_OFFSET, 5156 gaudi2_irq_map_table[map_table_entry].cpu_id); 5157 5158 WREG32(reg_base + QM_ARB_ERR_MSG_EN_OFFSET, QM_ARB_ERR_MSG_EN_MASK); 5159 5160 WREG32(reg_base + QM_ARB_SLV_CHOISE_WDT_OFFSET, GAUDI2_ARB_WDT_TIMEOUT); 5161 WREG32(reg_base + QM_GLBL_CFG1_OFFSET, 0); 5162 WREG32(reg_base + QM_GLBL_CFG2_OFFSET, 0); 5163 5164 /* Enable the QMAN channel. 5165 * PDMA QMAN configuration is different, as we do not allow user to 5166 * access some of the CPs. 5167 * PDMA0: CP2/3 are reserved for the ARC usage. 5168 * PDMA1: CP1/2/3 are reserved for the ARC usage. 5169 */ 5170 if (reg_base == gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_1_0]) 5171 WREG32(reg_base + QM_GLBL_CFG0_OFFSET, PDMA1_QMAN_ENABLE); 5172 else if (reg_base == gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_0_0]) 5173 WREG32(reg_base + QM_GLBL_CFG0_OFFSET, PDMA0_QMAN_ENABLE); 5174 else 5175 WREG32(reg_base + QM_GLBL_CFG0_OFFSET, QMAN_ENABLE); 5176 } 5177 5178 static void gaudi2_init_qman(struct hl_device *hdev, u32 reg_base, 5179 u32 queue_id_base) 5180 { 5181 u32 pq_id; 5182 5183 for (pq_id = 0 ; pq_id < NUM_OF_PQ_PER_QMAN ; pq_id++) 5184 hdev->kernel_queues[queue_id_base + pq_id].cq_id = GAUDI2_RESERVED_CQ_CS_COMPLETION; 5185 5186 gaudi2_init_qman_pq(hdev, reg_base, queue_id_base); 5187 gaudi2_init_qman_cp(hdev, reg_base); 5188 gaudi2_init_qman_pqc(hdev, reg_base, queue_id_base); 5189 gaudi2_init_qman_common(hdev, reg_base, queue_id_base); 5190 } 5191 5192 static void gaudi2_init_dma_core(struct hl_device *hdev, u32 reg_base, 5193 u32 dma_core_id, bool is_secure) 5194 { 5195 u32 prot, irq_handler_offset; 5196 struct cpu_dyn_regs *dyn_regs; 5197 int map_table_entry; 5198 5199 prot = 1 << ARC_FARM_KDMA_PROT_ERR_VAL_SHIFT; 5200 if (is_secure) 5201 prot |= 1 << ARC_FARM_KDMA_PROT_VAL_SHIFT; 5202 5203 WREG32(reg_base + DMA_CORE_PROT_OFFSET, prot); 5204 5205 dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 5206 irq_handler_offset = le32_to_cpu(dyn_regs->gic_dma_core_irq_ctrl); 5207 5208 WREG32(reg_base + DMA_CORE_ERRMSG_ADDR_LO_OFFSET, 5209 lower_32_bits(CFG_BASE + irq_handler_offset)); 5210 5211 WREG32(reg_base + DMA_CORE_ERRMSG_ADDR_HI_OFFSET, 5212 upper_32_bits(CFG_BASE + irq_handler_offset)); 5213 5214 map_table_entry = gaudi2_dma_core_async_event_id[dma_core_id]; 5215 WREG32(reg_base + DMA_CORE_ERRMSG_WDATA_OFFSET, 5216 gaudi2_irq_map_table[map_table_entry].cpu_id); 5217 5218 /* Enable the DMA channel */ 5219 WREG32(reg_base + DMA_CORE_CFG_0_OFFSET, 1 << ARC_FARM_KDMA_CFG_0_EN_SHIFT); 5220 } 5221 5222 static void gaudi2_init_kdma(struct hl_device *hdev) 5223 { 5224 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5225 u32 reg_base; 5226 5227 if ((gaudi2->hw_cap_initialized & HW_CAP_KDMA) == HW_CAP_KDMA) 5228 return; 5229 5230 reg_base = gaudi2_dma_core_blocks_bases[DMA_CORE_ID_KDMA]; 5231 5232 gaudi2_init_dma_core(hdev, reg_base, DMA_CORE_ID_KDMA, true); 5233 5234 gaudi2->hw_cap_initialized |= HW_CAP_KDMA; 5235 } 5236 5237 static void gaudi2_init_pdma(struct hl_device *hdev) 5238 { 5239 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5240 u32 reg_base; 5241 5242 if ((gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK) == HW_CAP_PDMA_MASK) 5243 return; 5244 5245 reg_base = gaudi2_dma_core_blocks_bases[DMA_CORE_ID_PDMA0]; 5246 gaudi2_init_dma_core(hdev, reg_base, DMA_CORE_ID_PDMA0, false); 5247 5248 reg_base = gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_0_0]; 5249 gaudi2_init_qman(hdev, reg_base, GAUDI2_QUEUE_ID_PDMA_0_0); 5250 5251 reg_base = gaudi2_dma_core_blocks_bases[DMA_CORE_ID_PDMA1]; 5252 gaudi2_init_dma_core(hdev, reg_base, DMA_CORE_ID_PDMA1, false); 5253 5254 reg_base = gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_1_0]; 5255 gaudi2_init_qman(hdev, reg_base, GAUDI2_QUEUE_ID_PDMA_1_0); 5256 5257 gaudi2->hw_cap_initialized |= HW_CAP_PDMA_MASK; 5258 } 5259 5260 static void gaudi2_init_edma_instance(struct hl_device *hdev, u8 seq) 5261 { 5262 u32 reg_base, base_edma_core_id, base_edma_qman_id; 5263 5264 base_edma_core_id = DMA_CORE_ID_EDMA0 + seq; 5265 base_edma_qman_id = edma_stream_base[seq]; 5266 5267 reg_base = gaudi2_dma_core_blocks_bases[base_edma_core_id]; 5268 gaudi2_init_dma_core(hdev, reg_base, base_edma_core_id, false); 5269 5270 reg_base = gaudi2_qm_blocks_bases[base_edma_qman_id]; 5271 gaudi2_init_qman(hdev, reg_base, base_edma_qman_id); 5272 } 5273 5274 static void gaudi2_init_edma(struct hl_device *hdev) 5275 { 5276 struct asic_fixed_properties *prop = &hdev->asic_prop; 5277 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5278 int dcore, inst; 5279 5280 if ((gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK) == HW_CAP_EDMA_MASK) 5281 return; 5282 5283 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 5284 for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) { 5285 u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst; 5286 5287 if (!(prop->edma_enabled_mask & BIT(seq))) 5288 continue; 5289 5290 gaudi2_init_edma_instance(hdev, seq); 5291 5292 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_EDMA_SHIFT + seq); 5293 } 5294 } 5295 } 5296 5297 /* 5298 * gaudi2_arm_monitors_for_virt_msix_db() - Arm monitors for writing to the virtual MSI-X doorbell. 5299 * @hdev: pointer to habanalabs device structure. 5300 * @sob_id: sync object ID. 5301 * @first_mon_id: ID of first monitor out of 3 consecutive monitors. 5302 * @interrupt_id: interrupt ID. 5303 * 5304 * Some initiators cannot have HBW address in their completion address registers, and thus cannot 5305 * write directly to the HBW host memory of the virtual MSI-X doorbell. 5306 * Instead, they are configured to LBW write to a sync object, and a monitor will do the HBW write. 5307 * 5308 * The mechanism in the sync manager block is composed of a master monitor with 3 messages. 5309 * In addition to the HBW write, the other 2 messages are for preparing the monitor to next 5310 * completion, by decrementing the sync object value and re-arming the monitor. 5311 */ 5312 static void gaudi2_arm_monitors_for_virt_msix_db(struct hl_device *hdev, u32 sob_id, 5313 u32 first_mon_id, u32 interrupt_id) 5314 { 5315 u32 sob_offset, first_mon_offset, mon_offset, payload, sob_group, mode, arm, config; 5316 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5317 u64 addr; 5318 u8 mask; 5319 5320 /* Reset the SOB value */ 5321 sob_offset = sob_id * sizeof(u32); 5322 WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset, 0); 5323 5324 /* Configure 3 monitors: 5325 * 1. Write interrupt ID to the virtual MSI-X doorbell (master monitor) 5326 * 2. Decrement SOB value by 1. 5327 * 3. Re-arm the master monitor. 5328 */ 5329 5330 first_mon_offset = first_mon_id * sizeof(u32); 5331 5332 /* 2nd monitor: Decrement SOB value by 1 */ 5333 mon_offset = first_mon_offset + sizeof(u32); 5334 5335 addr = CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset; 5336 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, lower_32_bits(addr)); 5337 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + mon_offset, upper_32_bits(addr)); 5338 5339 payload = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 0x7FFF) | /* "-1" */ 5340 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_SIGN_MASK, 1) | 5341 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1); 5342 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, payload); 5343 5344 /* 3rd monitor: Re-arm the master monitor */ 5345 mon_offset = first_mon_offset + 2 * sizeof(u32); 5346 5347 addr = CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + first_mon_offset; 5348 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, lower_32_bits(addr)); 5349 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + mon_offset, upper_32_bits(addr)); 5350 5351 sob_group = sob_id / 8; 5352 mask = ~BIT(sob_id & 0x7); 5353 mode = 0; /* comparison mode is "greater than or equal to" */ 5354 arm = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SID_MASK, sob_group) | 5355 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_MASK_MASK, mask) | 5356 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOP_MASK, mode) | 5357 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOD_MASK, 1); 5358 5359 payload = arm; 5360 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, payload); 5361 5362 /* 1st monitor (master): Write interrupt ID to the virtual MSI-X doorbell */ 5363 mon_offset = first_mon_offset; 5364 5365 config = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_WR_NUM_MASK, 2); /* "2": 3 writes */ 5366 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + mon_offset, config); 5367 5368 addr = gaudi2->virt_msix_db_dma_addr; 5369 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, lower_32_bits(addr)); 5370 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + mon_offset, upper_32_bits(addr)); 5371 5372 payload = interrupt_id; 5373 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, payload); 5374 5375 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + mon_offset, arm); 5376 } 5377 5378 static void gaudi2_prepare_sm_for_virt_msix_db(struct hl_device *hdev) 5379 { 5380 u32 decoder_id, sob_id, first_mon_id, interrupt_id; 5381 struct asic_fixed_properties *prop = &hdev->asic_prop; 5382 5383 /* Decoder normal/abnormal interrupts */ 5384 for (decoder_id = 0 ; decoder_id < NUMBER_OF_DEC ; ++decoder_id) { 5385 if (!(prop->decoder_enabled_mask & BIT(decoder_id))) 5386 continue; 5387 5388 sob_id = GAUDI2_RESERVED_SOB_DEC_NRM_FIRST + decoder_id; 5389 first_mon_id = GAUDI2_RESERVED_MON_DEC_NRM_FIRST + 3 * decoder_id; 5390 interrupt_id = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + 2 * decoder_id; 5391 gaudi2_arm_monitors_for_virt_msix_db(hdev, sob_id, first_mon_id, interrupt_id); 5392 5393 sob_id = GAUDI2_RESERVED_SOB_DEC_ABNRM_FIRST + decoder_id; 5394 first_mon_id = GAUDI2_RESERVED_MON_DEC_ABNRM_FIRST + 3 * decoder_id; 5395 interrupt_id += 1; 5396 gaudi2_arm_monitors_for_virt_msix_db(hdev, sob_id, first_mon_id, interrupt_id); 5397 } 5398 } 5399 5400 static void gaudi2_init_sm(struct hl_device *hdev) 5401 { 5402 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5403 u64 cq_address; 5404 u32 reg_val; 5405 int i; 5406 5407 /* Enable HBW/LBW CQ for completion monitors */ 5408 reg_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_CQ_EN_MASK, 1); 5409 reg_val |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_LBW_EN_MASK, 1); 5410 5411 for (i = 0 ; i < GAUDI2_MAX_PENDING_CS ; i++) 5412 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + (4 * i), reg_val); 5413 5414 /* Enable only HBW CQ for KDMA completion monitor */ 5415 reg_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_CQ_EN_MASK, 1); 5416 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + (4 * i), reg_val); 5417 5418 /* Init CQ0 DB - configure the monitor to trigger MSI-X interrupt */ 5419 WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0, lower_32_bits(gaudi2->virt_msix_db_dma_addr)); 5420 WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0, upper_32_bits(gaudi2->virt_msix_db_dma_addr)); 5421 WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_DATA_0, GAUDI2_IRQ_NUM_COMPLETION); 5422 5423 for (i = 0 ; i < GAUDI2_RESERVED_CQ_NUMBER ; i++) { 5424 cq_address = 5425 hdev->completion_queue[i].bus_address; 5426 5427 WREG32(mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_L_0 + (4 * i), 5428 lower_32_bits(cq_address)); 5429 WREG32(mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_H_0 + (4 * i), 5430 upper_32_bits(cq_address)); 5431 WREG32(mmDCORE0_SYNC_MNGR_GLBL_CQ_SIZE_LOG2_0 + (4 * i), 5432 ilog2(HL_CQ_SIZE_IN_BYTES)); 5433 } 5434 5435 /* Configure kernel ASID and MMU BP*/ 5436 WREG32(mmDCORE0_SYNC_MNGR_GLBL_ASID_SEC, 0x10000); 5437 WREG32(mmDCORE0_SYNC_MNGR_GLBL_ASID_NONE_SEC_PRIV, 0); 5438 5439 /* Initialize sync objects and monitors which are used for the virtual MSI-X doorbell */ 5440 gaudi2_prepare_sm_for_virt_msix_db(hdev); 5441 } 5442 5443 static void gaudi2_init_mme_acc(struct hl_device *hdev, u32 reg_base) 5444 { 5445 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5446 u32 reg_val; 5447 int i; 5448 5449 reg_val = FIELD_PREP(MME_ACC_INTR_MASK_WBC_ERR_RESP_MASK, 0); 5450 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_POS_INF_MASK, 1); 5451 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_NEG_INF_MASK, 1); 5452 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_NAN_MASK, 1); 5453 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_RESULT_POS_INF_MASK, 1); 5454 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_RESULT_NEG_INF_MASK, 1); 5455 5456 WREG32(reg_base + MME_ACC_INTR_MASK_OFFSET, reg_val); 5457 WREG32(reg_base + MME_ACC_AP_LFSR_POLY_OFFSET, 0x80DEADAF); 5458 5459 for (i = 0 ; i < MME_NUM_OF_LFSR_SEEDS ; i++) { 5460 WREG32(reg_base + MME_ACC_AP_LFSR_SEED_SEL_OFFSET, i); 5461 WREG32(reg_base + MME_ACC_AP_LFSR_SEED_WDATA_OFFSET, gaudi2->lfsr_rand_seeds[i]); 5462 } 5463 } 5464 5465 static void gaudi2_init_dcore_mme(struct hl_device *hdev, int dcore_id, 5466 bool config_qman_only) 5467 { 5468 u32 queue_id_base, reg_base; 5469 5470 switch (dcore_id) { 5471 case 0: 5472 queue_id_base = GAUDI2_QUEUE_ID_DCORE0_MME_0_0; 5473 break; 5474 case 1: 5475 queue_id_base = GAUDI2_QUEUE_ID_DCORE1_MME_0_0; 5476 break; 5477 case 2: 5478 queue_id_base = GAUDI2_QUEUE_ID_DCORE2_MME_0_0; 5479 break; 5480 case 3: 5481 queue_id_base = GAUDI2_QUEUE_ID_DCORE3_MME_0_0; 5482 break; 5483 default: 5484 dev_err(hdev->dev, "Invalid dcore id %u\n", dcore_id); 5485 return; 5486 } 5487 5488 if (!config_qman_only) { 5489 reg_base = gaudi2_mme_acc_blocks_bases[dcore_id]; 5490 gaudi2_init_mme_acc(hdev, reg_base); 5491 } 5492 5493 reg_base = gaudi2_qm_blocks_bases[queue_id_base]; 5494 gaudi2_init_qman(hdev, reg_base, queue_id_base); 5495 } 5496 5497 static void gaudi2_init_mme(struct hl_device *hdev) 5498 { 5499 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5500 int i; 5501 5502 if ((gaudi2->hw_cap_initialized & HW_CAP_MME_MASK) == HW_CAP_MME_MASK) 5503 return; 5504 5505 for (i = 0 ; i < NUM_OF_DCORES ; i++) { 5506 gaudi2_init_dcore_mme(hdev, i, false); 5507 5508 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_MME_SHIFT + i); 5509 } 5510 } 5511 5512 static void gaudi2_init_tpc_cfg(struct hl_device *hdev, u32 reg_base) 5513 { 5514 /* Mask arithmetic and QM interrupts in TPC */ 5515 WREG32(reg_base + TPC_CFG_TPC_INTR_MASK_OFFSET, 0x23FFFE); 5516 5517 /* Set 16 cache lines */ 5518 WREG32(reg_base + TPC_CFG_MSS_CONFIG_OFFSET, 5519 2 << DCORE0_TPC0_CFG_MSS_CONFIG_ICACHE_FETCH_LINE_NUM_SHIFT); 5520 } 5521 5522 struct gaudi2_tpc_init_cfg_data { 5523 enum gaudi2_queue_id dcore_tpc_qid_base[NUM_OF_DCORES]; 5524 }; 5525 5526 static void gaudi2_init_tpc_config(struct hl_device *hdev, int dcore, int inst, 5527 u32 offset, struct iterate_module_ctx *ctx) 5528 { 5529 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5530 struct gaudi2_tpc_init_cfg_data *cfg_data = ctx->data; 5531 u32 queue_id_base; 5532 u8 seq; 5533 5534 queue_id_base = cfg_data->dcore_tpc_qid_base[dcore] + (inst * NUM_OF_PQ_PER_QMAN); 5535 5536 if (dcore == 0 && inst == (NUM_DCORE0_TPC - 1)) 5537 /* gets last sequence number */ 5538 seq = NUM_OF_DCORES * NUM_OF_TPC_PER_DCORE; 5539 else 5540 seq = dcore * NUM_OF_TPC_PER_DCORE + inst; 5541 5542 gaudi2_init_tpc_cfg(hdev, mmDCORE0_TPC0_CFG_BASE + offset); 5543 gaudi2_init_qman(hdev, mmDCORE0_TPC0_QM_BASE + offset, queue_id_base); 5544 5545 gaudi2->tpc_hw_cap_initialized |= BIT_ULL(HW_CAP_TPC_SHIFT + seq); 5546 } 5547 5548 static void gaudi2_init_tpc(struct hl_device *hdev) 5549 { 5550 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5551 struct gaudi2_tpc_init_cfg_data init_cfg_data; 5552 struct iterate_module_ctx tpc_iter; 5553 5554 if (!hdev->asic_prop.tpc_enabled_mask) 5555 return; 5556 5557 if ((gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK) == HW_CAP_TPC_MASK) 5558 return; 5559 5560 init_cfg_data.dcore_tpc_qid_base[0] = GAUDI2_QUEUE_ID_DCORE0_TPC_0_0; 5561 init_cfg_data.dcore_tpc_qid_base[1] = GAUDI2_QUEUE_ID_DCORE1_TPC_0_0; 5562 init_cfg_data.dcore_tpc_qid_base[2] = GAUDI2_QUEUE_ID_DCORE2_TPC_0_0; 5563 init_cfg_data.dcore_tpc_qid_base[3] = GAUDI2_QUEUE_ID_DCORE3_TPC_0_0; 5564 tpc_iter.fn = &gaudi2_init_tpc_config; 5565 tpc_iter.data = &init_cfg_data; 5566 gaudi2_iterate_tpcs(hdev, &tpc_iter); 5567 } 5568 5569 static void gaudi2_init_rotator(struct hl_device *hdev) 5570 { 5571 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5572 u32 i, reg_base, queue_id; 5573 5574 queue_id = GAUDI2_QUEUE_ID_ROT_0_0; 5575 5576 for (i = 0 ; i < NUM_OF_ROT ; i++, queue_id += NUM_OF_PQ_PER_QMAN) { 5577 reg_base = gaudi2_qm_blocks_bases[queue_id]; 5578 gaudi2_init_qman(hdev, reg_base, queue_id); 5579 5580 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_ROT_SHIFT + i); 5581 } 5582 } 5583 5584 static void gaudi2_init_vdec_brdg_ctrl(struct hl_device *hdev, u64 base_addr, u32 decoder_id) 5585 { 5586 u32 sob_id; 5587 5588 /* VCMD normal interrupt */ 5589 sob_id = GAUDI2_RESERVED_SOB_DEC_NRM_FIRST + decoder_id; 5590 WREG32(base_addr + BRDG_CTRL_NRM_MSIX_LBW_AWADDR, 5591 mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_id * sizeof(u32)); 5592 WREG32(base_addr + BRDG_CTRL_NRM_MSIX_LBW_WDATA, GAUDI2_SOB_INCREMENT_BY_ONE); 5593 5594 /* VCMD abnormal interrupt */ 5595 sob_id = GAUDI2_RESERVED_SOB_DEC_ABNRM_FIRST + decoder_id; 5596 WREG32(base_addr + BRDG_CTRL_ABNRM_MSIX_LBW_AWADDR, 5597 mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_id * sizeof(u32)); 5598 WREG32(base_addr + BRDG_CTRL_ABNRM_MSIX_LBW_WDATA, GAUDI2_SOB_INCREMENT_BY_ONE); 5599 } 5600 5601 static void gaudi2_init_dec(struct hl_device *hdev) 5602 { 5603 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5604 u32 dcore_id, dec_id, dec_bit; 5605 u64 base_addr; 5606 5607 if (!hdev->asic_prop.decoder_enabled_mask) 5608 return; 5609 5610 if ((gaudi2->dec_hw_cap_initialized & HW_CAP_DEC_MASK) == HW_CAP_DEC_MASK) 5611 return; 5612 5613 for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) 5614 for (dec_id = 0 ; dec_id < NUM_OF_DEC_PER_DCORE ; dec_id++) { 5615 dec_bit = dcore_id * NUM_OF_DEC_PER_DCORE + dec_id; 5616 5617 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) 5618 continue; 5619 5620 base_addr = mmDCORE0_DEC0_CMD_BASE + 5621 BRDG_CTRL_BLOCK_OFFSET + 5622 dcore_id * DCORE_OFFSET + 5623 dec_id * DCORE_VDEC_OFFSET; 5624 5625 gaudi2_init_vdec_brdg_ctrl(hdev, base_addr, dec_bit); 5626 5627 gaudi2->dec_hw_cap_initialized |= BIT_ULL(HW_CAP_DEC_SHIFT + dec_bit); 5628 } 5629 5630 for (dec_id = 0 ; dec_id < NUM_OF_PCIE_VDEC ; dec_id++) { 5631 dec_bit = PCIE_DEC_SHIFT + dec_id; 5632 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) 5633 continue; 5634 5635 base_addr = mmPCIE_DEC0_CMD_BASE + BRDG_CTRL_BLOCK_OFFSET + 5636 dec_id * DCORE_VDEC_OFFSET; 5637 5638 gaudi2_init_vdec_brdg_ctrl(hdev, base_addr, dec_bit); 5639 5640 gaudi2->dec_hw_cap_initialized |= BIT_ULL(HW_CAP_DEC_SHIFT + dec_bit); 5641 } 5642 } 5643 5644 static int gaudi2_mmu_update_asid_hop0_addr(struct hl_device *hdev, 5645 u32 stlb_base, u32 asid, u64 phys_addr) 5646 { 5647 u32 status, timeout_usec; 5648 int rc; 5649 5650 if (hdev->pldm || !hdev->pdev) 5651 timeout_usec = GAUDI2_PLDM_MMU_TIMEOUT_USEC; 5652 else 5653 timeout_usec = MMU_CONFIG_TIMEOUT_USEC; 5654 5655 WREG32(stlb_base + STLB_ASID_OFFSET, asid); 5656 WREG32(stlb_base + STLB_HOP0_PA43_12_OFFSET, phys_addr >> MMU_HOP0_PA43_12_SHIFT); 5657 WREG32(stlb_base + STLB_HOP0_PA63_44_OFFSET, phys_addr >> MMU_HOP0_PA63_44_SHIFT); 5658 WREG32(stlb_base + STLB_BUSY_OFFSET, 0x80000000); 5659 5660 rc = hl_poll_timeout( 5661 hdev, 5662 stlb_base + STLB_BUSY_OFFSET, 5663 status, 5664 !(status & 0x80000000), 5665 1000, 5666 timeout_usec); 5667 5668 if (rc) { 5669 dev_err(hdev->dev, "Timeout during MMU hop0 config of asid %d\n", asid); 5670 return rc; 5671 } 5672 5673 return 0; 5674 } 5675 5676 static void gaudi2_mmu_send_invalidate_cache_cmd(struct hl_device *hdev, u32 stlb_base, 5677 u32 start_offset, u32 inv_start_val, 5678 u32 flags) 5679 { 5680 /* clear PMMU mem line cache (only needed in mmu range invalidation) */ 5681 if (flags & MMU_OP_CLEAR_MEMCACHE) 5682 WREG32(mmPMMU_HBW_STLB_MEM_CACHE_INVALIDATION, 0x1); 5683 5684 if (flags & MMU_OP_SKIP_LOW_CACHE_INV) 5685 return; 5686 5687 WREG32(stlb_base + start_offset, inv_start_val); 5688 } 5689 5690 static int gaudi2_mmu_invalidate_cache_status_poll(struct hl_device *hdev, u32 stlb_base, 5691 struct gaudi2_cache_invld_params *inv_params) 5692 { 5693 u32 status, timeout_usec, start_offset; 5694 int rc; 5695 5696 timeout_usec = (hdev->pldm) ? GAUDI2_PLDM_MMU_TIMEOUT_USEC : 5697 GAUDI2_MMU_CACHE_INV_TIMEOUT_USEC; 5698 5699 /* poll PMMU mem line cache (only needed in mmu range invalidation) */ 5700 if (inv_params->flags & MMU_OP_CLEAR_MEMCACHE) { 5701 rc = hl_poll_timeout( 5702 hdev, 5703 mmPMMU_HBW_STLB_MEM_CACHE_INV_STATUS, 5704 status, 5705 status & 0x1, 5706 1000, 5707 timeout_usec); 5708 5709 if (rc) 5710 return rc; 5711 5712 /* Need to manually reset the status to 0 */ 5713 WREG32(mmPMMU_HBW_STLB_MEM_CACHE_INV_STATUS, 0x0); 5714 } 5715 5716 /* Lower cache does not work with cache lines, hence we can skip its 5717 * invalidation upon map and invalidate only upon unmap 5718 */ 5719 if (inv_params->flags & MMU_OP_SKIP_LOW_CACHE_INV) 5720 return 0; 5721 5722 start_offset = inv_params->range_invalidation ? 5723 STLB_RANGE_CACHE_INVALIDATION_OFFSET : STLB_INV_ALL_START_OFFSET; 5724 5725 rc = hl_poll_timeout( 5726 hdev, 5727 stlb_base + start_offset, 5728 status, 5729 !(status & 0x1), 5730 1000, 5731 timeout_usec); 5732 5733 return rc; 5734 } 5735 5736 bool gaudi2_is_hmmu_enabled(struct hl_device *hdev, int dcore_id, int hmmu_id) 5737 { 5738 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5739 u32 hw_cap; 5740 5741 hw_cap = HW_CAP_DCORE0_DMMU0 << (NUM_OF_HMMU_PER_DCORE * dcore_id + hmmu_id); 5742 5743 if (gaudi2->hw_cap_initialized & hw_cap) 5744 return true; 5745 5746 return false; 5747 } 5748 5749 /* this function shall be called only for HMMUs for which capability bit is set */ 5750 static inline u32 get_hmmu_stlb_base(int dcore_id, int hmmu_id) 5751 { 5752 u32 offset; 5753 5754 offset = (u32) (dcore_id * DCORE_OFFSET + hmmu_id * DCORE_HMMU_OFFSET); 5755 return (u32)(mmDCORE0_HMMU0_STLB_BASE + offset); 5756 } 5757 5758 static void gaudi2_mmu_invalidate_cache_trigger(struct hl_device *hdev, u32 stlb_base, 5759 struct gaudi2_cache_invld_params *inv_params) 5760 { 5761 u32 start_offset; 5762 5763 if (inv_params->range_invalidation) { 5764 /* Set the addresses range 5765 * Note: that the start address we set in register, is not included in 5766 * the range of the invalidation, by design. 5767 * that's why we need to set lower address than the one we actually 5768 * want to be included in the range invalidation. 5769 */ 5770 u64 start = inv_params->start_va - 1; 5771 5772 start_offset = STLB_RANGE_CACHE_INVALIDATION_OFFSET; 5773 5774 WREG32(stlb_base + STLB_RANGE_INV_START_LSB_OFFSET, 5775 start >> MMU_RANGE_INV_VA_LSB_SHIFT); 5776 5777 WREG32(stlb_base + STLB_RANGE_INV_START_MSB_OFFSET, 5778 start >> MMU_RANGE_INV_VA_MSB_SHIFT); 5779 5780 WREG32(stlb_base + STLB_RANGE_INV_END_LSB_OFFSET, 5781 inv_params->end_va >> MMU_RANGE_INV_VA_LSB_SHIFT); 5782 5783 WREG32(stlb_base + STLB_RANGE_INV_END_MSB_OFFSET, 5784 inv_params->end_va >> MMU_RANGE_INV_VA_MSB_SHIFT); 5785 } else { 5786 start_offset = STLB_INV_ALL_START_OFFSET; 5787 } 5788 5789 gaudi2_mmu_send_invalidate_cache_cmd(hdev, stlb_base, start_offset, 5790 inv_params->inv_start_val, inv_params->flags); 5791 } 5792 5793 static inline void gaudi2_hmmu_invalidate_cache_trigger(struct hl_device *hdev, 5794 int dcore_id, int hmmu_id, 5795 struct gaudi2_cache_invld_params *inv_params) 5796 { 5797 u32 stlb_base = get_hmmu_stlb_base(dcore_id, hmmu_id); 5798 5799 gaudi2_mmu_invalidate_cache_trigger(hdev, stlb_base, inv_params); 5800 } 5801 5802 static inline int gaudi2_hmmu_invalidate_cache_status_poll(struct hl_device *hdev, 5803 int dcore_id, int hmmu_id, 5804 struct gaudi2_cache_invld_params *inv_params) 5805 { 5806 u32 stlb_base = get_hmmu_stlb_base(dcore_id, hmmu_id); 5807 5808 return gaudi2_mmu_invalidate_cache_status_poll(hdev, stlb_base, inv_params); 5809 } 5810 5811 static int gaudi2_hmmus_invalidate_cache(struct hl_device *hdev, 5812 struct gaudi2_cache_invld_params *inv_params) 5813 { 5814 int dcore_id, hmmu_id; 5815 5816 /* first send all invalidation commands */ 5817 for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) { 5818 for (hmmu_id = 0 ; hmmu_id < NUM_OF_HMMU_PER_DCORE ; hmmu_id++) { 5819 if (!gaudi2_is_hmmu_enabled(hdev, dcore_id, hmmu_id)) 5820 continue; 5821 5822 gaudi2_hmmu_invalidate_cache_trigger(hdev, dcore_id, hmmu_id, inv_params); 5823 } 5824 } 5825 5826 /* next, poll all invalidations status */ 5827 for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) { 5828 for (hmmu_id = 0 ; hmmu_id < NUM_OF_HMMU_PER_DCORE ; hmmu_id++) { 5829 int rc; 5830 5831 if (!gaudi2_is_hmmu_enabled(hdev, dcore_id, hmmu_id)) 5832 continue; 5833 5834 rc = gaudi2_hmmu_invalidate_cache_status_poll(hdev, dcore_id, hmmu_id, 5835 inv_params); 5836 if (rc) 5837 return rc; 5838 } 5839 } 5840 5841 return 0; 5842 } 5843 5844 static int gaudi2_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, u32 flags) 5845 { 5846 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5847 struct gaudi2_cache_invld_params invld_params; 5848 int rc = 0; 5849 5850 if (hdev->reset_info.hard_reset_pending) 5851 return rc; 5852 5853 invld_params.range_invalidation = false; 5854 invld_params.inv_start_val = 1; 5855 5856 if ((flags & MMU_OP_USERPTR) && (gaudi2->hw_cap_initialized & HW_CAP_PMMU)) { 5857 invld_params.flags = flags; 5858 gaudi2_mmu_invalidate_cache_trigger(hdev, mmPMMU_HBW_STLB_BASE, &invld_params); 5859 rc = gaudi2_mmu_invalidate_cache_status_poll(hdev, mmPMMU_HBW_STLB_BASE, 5860 &invld_params); 5861 } else if (flags & MMU_OP_PHYS_PACK) { 5862 invld_params.flags = 0; 5863 rc = gaudi2_hmmus_invalidate_cache(hdev, &invld_params); 5864 } 5865 5866 return rc; 5867 } 5868 5869 static int gaudi2_mmu_invalidate_cache_range(struct hl_device *hdev, bool is_hard, 5870 u32 flags, u32 asid, u64 va, u64 size) 5871 { 5872 struct gaudi2_cache_invld_params invld_params = {0}; 5873 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5874 u64 start_va, end_va; 5875 u32 inv_start_val; 5876 int rc = 0; 5877 5878 if (hdev->reset_info.hard_reset_pending) 5879 return 0; 5880 5881 inv_start_val = (1 << MMU_RANGE_INV_EN_SHIFT | 5882 1 << MMU_RANGE_INV_ASID_EN_SHIFT | 5883 asid << MMU_RANGE_INV_ASID_SHIFT); 5884 start_va = va; 5885 end_va = start_va + size; 5886 5887 if ((flags & MMU_OP_USERPTR) && (gaudi2->hw_cap_initialized & HW_CAP_PMMU)) { 5888 /* As range invalidation does not support zero address we will 5889 * do full invalidation in this case 5890 */ 5891 if (start_va) { 5892 invld_params.range_invalidation = true; 5893 invld_params.start_va = start_va; 5894 invld_params.end_va = end_va; 5895 invld_params.inv_start_val = inv_start_val; 5896 invld_params.flags = flags | MMU_OP_CLEAR_MEMCACHE; 5897 } else { 5898 invld_params.range_invalidation = false; 5899 invld_params.inv_start_val = 1; 5900 invld_params.flags = flags; 5901 } 5902 5903 5904 gaudi2_mmu_invalidate_cache_trigger(hdev, mmPMMU_HBW_STLB_BASE, &invld_params); 5905 rc = gaudi2_mmu_invalidate_cache_status_poll(hdev, mmPMMU_HBW_STLB_BASE, 5906 &invld_params); 5907 if (rc) 5908 return rc; 5909 5910 } else if (flags & MMU_OP_PHYS_PACK) { 5911 invld_params.start_va = gaudi2_mmu_scramble_addr(hdev, start_va); 5912 invld_params.end_va = gaudi2_mmu_scramble_addr(hdev, end_va); 5913 invld_params.inv_start_val = inv_start_val; 5914 invld_params.flags = flags; 5915 rc = gaudi2_hmmus_invalidate_cache(hdev, &invld_params); 5916 } 5917 5918 return rc; 5919 } 5920 5921 static int gaudi2_mmu_update_hop0_addr(struct hl_device *hdev, u32 stlb_base, 5922 bool host_resident_pgt) 5923 { 5924 struct asic_fixed_properties *prop = &hdev->asic_prop; 5925 u64 hop0_addr; 5926 u32 asid, max_asid = prop->max_asid; 5927 int rc; 5928 5929 /* it takes too much time to init all of the ASIDs on palladium */ 5930 if (hdev->pldm) 5931 max_asid = min((u32) 8, max_asid); 5932 5933 for (asid = 0 ; asid < max_asid ; asid++) { 5934 if (host_resident_pgt) 5935 hop0_addr = hdev->mmu_priv.hr.mmu_asid_hop0[asid].phys_addr; 5936 else 5937 hop0_addr = prop->mmu_pgt_addr + (asid * prop->mmu_hop_table_size); 5938 5939 rc = gaudi2_mmu_update_asid_hop0_addr(hdev, stlb_base, asid, hop0_addr); 5940 if (rc) { 5941 dev_err(hdev->dev, "failed to set hop0 addr for asid %d\n", asid); 5942 return rc; 5943 } 5944 } 5945 5946 return 0; 5947 } 5948 5949 static int gaudi2_mmu_init_common(struct hl_device *hdev, u32 mmu_base, u32 stlb_base, 5950 bool host_resident_pgt) 5951 { 5952 u32 status, timeout_usec; 5953 int rc; 5954 5955 if (hdev->pldm || !hdev->pdev) 5956 timeout_usec = GAUDI2_PLDM_MMU_TIMEOUT_USEC; 5957 else 5958 timeout_usec = GAUDI2_MMU_CACHE_INV_TIMEOUT_USEC; 5959 5960 WREG32(stlb_base + STLB_INV_ALL_START_OFFSET, 1); 5961 5962 rc = hl_poll_timeout( 5963 hdev, 5964 stlb_base + STLB_SRAM_INIT_OFFSET, 5965 status, 5966 !status, 5967 1000, 5968 timeout_usec); 5969 5970 if (rc) 5971 dev_notice_ratelimited(hdev->dev, "Timeout when waiting for MMU SRAM init\n"); 5972 5973 rc = gaudi2_mmu_update_hop0_addr(hdev, stlb_base, host_resident_pgt); 5974 if (rc) 5975 return rc; 5976 5977 WREG32(mmu_base + MMU_BYPASS_OFFSET, 0); 5978 5979 rc = hl_poll_timeout( 5980 hdev, 5981 stlb_base + STLB_INV_ALL_START_OFFSET, 5982 status, 5983 !status, 5984 1000, 5985 timeout_usec); 5986 5987 if (rc) 5988 dev_notice_ratelimited(hdev->dev, "Timeout when waiting for MMU invalidate all\n"); 5989 5990 WREG32(mmu_base + MMU_ENABLE_OFFSET, 1); 5991 5992 return rc; 5993 } 5994 5995 static int gaudi2_pci_mmu_init(struct hl_device *hdev) 5996 { 5997 struct asic_fixed_properties *prop = &hdev->asic_prop; 5998 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5999 u32 mmu_base, stlb_base; 6000 int rc; 6001 6002 if (gaudi2->hw_cap_initialized & HW_CAP_PMMU) 6003 return 0; 6004 6005 mmu_base = mmPMMU_HBW_MMU_BASE; 6006 stlb_base = mmPMMU_HBW_STLB_BASE; 6007 6008 RMWREG32_SHIFTED(stlb_base + STLB_HOP_CONFIGURATION_OFFSET, 6009 (0 << PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_HOP_SHIFT) | 6010 (5 << PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_SHIFT) | 6011 (4 << PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_SHIFT) | 6012 (5 << PMMU_HBW_STLB_HOP_CONFIGURATION_LAST_HOP_SHIFT) | 6013 (5 << PMMU_HBW_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_SHIFT), 6014 PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_HOP_MASK | 6015 PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_MASK | 6016 PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_MASK | 6017 PMMU_HBW_STLB_HOP_CONFIGURATION_LAST_HOP_MASK | 6018 PMMU_HBW_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_MASK); 6019 6020 WREG32(stlb_base + STLB_LL_LOOKUP_MASK_63_32_OFFSET, 0); 6021 6022 if (PAGE_SIZE == SZ_64K) { 6023 /* Set page sizes to 64K on hop5 and 16M on hop4 + enable 8 bit hops */ 6024 RMWREG32_SHIFTED(mmu_base + MMU_STATIC_MULTI_PAGE_SIZE_OFFSET, 6025 FIELD_PREP(DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP5_PAGE_SIZE_MASK, 4) | 6026 FIELD_PREP(DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP4_PAGE_SIZE_MASK, 3) | 6027 FIELD_PREP( 6028 DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_CFG_8_BITS_HOP_MODE_EN_MASK, 6029 1), 6030 DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP5_PAGE_SIZE_MASK | 6031 DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP4_PAGE_SIZE_MASK | 6032 DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_CFG_8_BITS_HOP_MODE_EN_MASK); 6033 } 6034 6035 WREG32(mmu_base + MMU_SPI_SEI_MASK_OFFSET, GAUDI2_PMMU_SPI_SEI_ENABLE_MASK); 6036 6037 rc = gaudi2_mmu_init_common(hdev, mmu_base, stlb_base, prop->pmmu.host_resident); 6038 if (rc) 6039 return rc; 6040 6041 gaudi2->hw_cap_initialized |= HW_CAP_PMMU; 6042 6043 return 0; 6044 } 6045 6046 static int gaudi2_dcore_hmmu_init(struct hl_device *hdev, int dcore_id, 6047 int hmmu_id) 6048 { 6049 struct asic_fixed_properties *prop = &hdev->asic_prop; 6050 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6051 u32 offset, mmu_base, stlb_base, hw_cap; 6052 u8 dmmu_seq; 6053 int rc; 6054 6055 dmmu_seq = NUM_OF_HMMU_PER_DCORE * dcore_id + hmmu_id; 6056 hw_cap = HW_CAP_DCORE0_DMMU0 << dmmu_seq; 6057 6058 /* 6059 * return if DMMU is already initialized or if it's not out of 6060 * isolation (due to cluster binning) 6061 */ 6062 if ((gaudi2->hw_cap_initialized & hw_cap) || !(prop->hmmu_hif_enabled_mask & BIT(dmmu_seq))) 6063 return 0; 6064 6065 offset = (u32) (dcore_id * DCORE_OFFSET + hmmu_id * DCORE_HMMU_OFFSET); 6066 mmu_base = mmDCORE0_HMMU0_MMU_BASE + offset; 6067 stlb_base = mmDCORE0_HMMU0_STLB_BASE + offset; 6068 6069 RMWREG32(mmu_base + MMU_STATIC_MULTI_PAGE_SIZE_OFFSET, 5 /* 64MB */, 6070 MMU_STATIC_MULTI_PAGE_SIZE_HOP4_PAGE_SIZE_MASK); 6071 6072 RMWREG32_SHIFTED(stlb_base + STLB_HOP_CONFIGURATION_OFFSET, 6073 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_HOP_MASK, 0) | 6074 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_MASK, 3) | 6075 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_MASK, 3) | 6076 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_LAST_HOP_MASK, 3) | 6077 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_MASK, 3), 6078 DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_HOP_MASK | 6079 DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_MASK | 6080 DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_MASK | 6081 DCORE0_HMMU0_STLB_HOP_CONFIGURATION_LAST_HOP_MASK | 6082 DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_MASK); 6083 6084 RMWREG32(stlb_base + STLB_HOP_CONFIGURATION_OFFSET, 1, 6085 STLB_HOP_CONFIGURATION_ONLY_LARGE_PAGE_MASK); 6086 6087 WREG32(mmu_base + MMU_SPI_SEI_MASK_OFFSET, GAUDI2_HMMU_SPI_SEI_ENABLE_MASK); 6088 6089 rc = gaudi2_mmu_init_common(hdev, mmu_base, stlb_base, prop->dmmu.host_resident); 6090 if (rc) 6091 return rc; 6092 6093 gaudi2->hw_cap_initialized |= hw_cap; 6094 6095 return 0; 6096 } 6097 6098 static int gaudi2_hbm_mmu_init(struct hl_device *hdev) 6099 { 6100 int rc, dcore_id, hmmu_id; 6101 6102 for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) 6103 for (hmmu_id = 0 ; hmmu_id < NUM_OF_HMMU_PER_DCORE; hmmu_id++) { 6104 rc = gaudi2_dcore_hmmu_init(hdev, dcore_id, hmmu_id); 6105 if (rc) 6106 return rc; 6107 } 6108 6109 return 0; 6110 } 6111 6112 static int gaudi2_mmu_init(struct hl_device *hdev) 6113 { 6114 int rc; 6115 6116 rc = gaudi2_pci_mmu_init(hdev); 6117 if (rc) 6118 return rc; 6119 6120 rc = gaudi2_hbm_mmu_init(hdev); 6121 if (rc) 6122 return rc; 6123 6124 return 0; 6125 } 6126 6127 static int gaudi2_hw_init(struct hl_device *hdev) 6128 { 6129 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6130 int rc; 6131 6132 /* Let's mark in the H/W that we have reached this point. We check 6133 * this value in the reset_before_init function to understand whether 6134 * we need to reset the chip before doing H/W init. This register is 6135 * cleared by the H/W upon H/W reset 6136 */ 6137 WREG32(mmHW_STATE, HL_DEVICE_HW_STATE_DIRTY); 6138 6139 /* Perform read from the device to make sure device is up */ 6140 RREG32(mmHW_STATE); 6141 6142 /* If iATU is done by FW, the HBM bar ALWAYS points to DRAM_PHYS_BASE. 6143 * So we set it here and if anyone tries to move it later to 6144 * a different address, there will be an error 6145 */ 6146 if (hdev->asic_prop.iatu_done_by_fw) 6147 gaudi2->dram_bar_cur_addr = DRAM_PHYS_BASE; 6148 6149 /* 6150 * Before pushing u-boot/linux to device, need to set the hbm bar to 6151 * base address of dram 6152 */ 6153 if (gaudi2_set_hbm_bar_base(hdev, DRAM_PHYS_BASE) == U64_MAX) { 6154 dev_err(hdev->dev, "failed to map HBM bar to DRAM base address\n"); 6155 return -EIO; 6156 } 6157 6158 rc = gaudi2_init_cpu(hdev); 6159 if (rc) { 6160 dev_err(hdev->dev, "failed to initialize CPU\n"); 6161 return rc; 6162 } 6163 6164 gaudi2_init_scrambler_hbm(hdev); 6165 gaudi2_init_kdma(hdev); 6166 6167 rc = gaudi2_init_cpu_queues(hdev, GAUDI2_CPU_TIMEOUT_USEC); 6168 if (rc) { 6169 dev_err(hdev->dev, "failed to initialize CPU H/W queues %d\n", rc); 6170 return rc; 6171 } 6172 6173 rc = gaudi2->cpucp_info_get(hdev); 6174 if (rc) { 6175 dev_err(hdev->dev, "Failed to get cpucp info\n"); 6176 return rc; 6177 } 6178 6179 rc = gaudi2_mmu_init(hdev); 6180 if (rc) 6181 return rc; 6182 6183 gaudi2_init_pdma(hdev); 6184 gaudi2_init_edma(hdev); 6185 gaudi2_init_sm(hdev); 6186 gaudi2_init_tpc(hdev); 6187 gaudi2_init_mme(hdev); 6188 gaudi2_init_rotator(hdev); 6189 gaudi2_init_dec(hdev); 6190 gaudi2_enable_timestamp(hdev); 6191 6192 rc = gaudi2_coresight_init(hdev); 6193 if (rc) 6194 goto disable_queues; 6195 6196 rc = gaudi2_enable_msix(hdev); 6197 if (rc) 6198 goto disable_queues; 6199 6200 /* Perform read from the device to flush all configuration */ 6201 RREG32(mmHW_STATE); 6202 6203 return 0; 6204 6205 disable_queues: 6206 gaudi2_disable_dma_qmans(hdev); 6207 gaudi2_disable_mme_qmans(hdev); 6208 gaudi2_disable_tpc_qmans(hdev); 6209 gaudi2_disable_rot_qmans(hdev); 6210 gaudi2_disable_nic_qmans(hdev); 6211 6212 gaudi2_disable_timestamp(hdev); 6213 6214 return rc; 6215 } 6216 6217 /** 6218 * gaudi2_send_hard_reset_cmd - common function to handle reset 6219 * 6220 * @hdev: pointer to the habanalabs device structure 6221 * 6222 * This function handles the various possible scenarios for reset. 6223 * It considers if reset is handled by driver\FW and what FW components are loaded 6224 */ 6225 static void gaudi2_send_hard_reset_cmd(struct hl_device *hdev) 6226 { 6227 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 6228 bool heartbeat_reset, preboot_only, cpu_initialized = false; 6229 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6230 u32 cpu_boot_status; 6231 6232 preboot_only = (hdev->fw_loader.fw_comp_loaded == FW_TYPE_PREBOOT_CPU); 6233 heartbeat_reset = (hdev->reset_info.curr_reset_cause == HL_RESET_CAUSE_HEARTBEAT); 6234 6235 /* 6236 * Handle corner case where failure was at cpu management app load, 6237 * and driver didn't detect any failure while loading the FW, 6238 * then at such scenario driver will send only HALT_MACHINE 6239 * and no one will respond to this request since FW already back to preboot 6240 * and it cannot handle such cmd. 6241 * In this case next time the management app loads it'll check on events register 6242 * which will still have the halt indication, and will reboot the device. 6243 * The solution is to let preboot clear all relevant registers before next boot 6244 * once driver send COMMS_RST_DEV. 6245 */ 6246 cpu_boot_status = RREG32(mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS); 6247 6248 if (gaudi2 && (gaudi2->hw_cap_initialized & HW_CAP_CPU) && 6249 (cpu_boot_status == CPU_BOOT_STATUS_SRAM_AVAIL)) 6250 cpu_initialized = true; 6251 6252 /* 6253 * when Linux/Bootfit exist this write to the SP can be interpreted in 2 ways: 6254 * 1. FW reset: FW initiate the reset sequence 6255 * 2. driver reset: FW will start HALT sequence (the preparations for the 6256 * reset but not the reset itself as it is not implemented 6257 * on their part) and LKD will wait to let FW complete the 6258 * sequence before issuing the reset 6259 */ 6260 if (!preboot_only && cpu_initialized) { 6261 WREG32(le32_to_cpu(dyn_regs->gic_host_halt_irq), 6262 gaudi2_irq_map_table[GAUDI2_EVENT_CPU_HALT_MACHINE].cpu_id); 6263 6264 msleep(GAUDI2_CPU_RESET_WAIT_MSEC); 6265 } 6266 6267 /* 6268 * When working with preboot (without Linux/Boot fit) we can 6269 * communicate only using the COMMS commands to issue halt/reset. 6270 * 6271 * For the case in which we are working with Linux/Bootfit this is a hail-mary 6272 * attempt to revive the card in the small chance that the f/w has 6273 * experienced a watchdog event, which caused it to return back to preboot. 6274 * In that case, triggering reset through GIC won't help. We need to 6275 * trigger the reset as if Linux wasn't loaded. 6276 * 6277 * We do it only if the reset cause was HB, because that would be the 6278 * indication of such an event. 6279 * 6280 * In case watchdog hasn't expired but we still got HB, then this won't 6281 * do any damage. 6282 */ 6283 6284 if (heartbeat_reset || preboot_only || !cpu_initialized) { 6285 if (hdev->asic_prop.hard_reset_done_by_fw) 6286 hl_fw_ask_hard_reset_without_linux(hdev); 6287 else 6288 hl_fw_ask_halt_machine_without_linux(hdev); 6289 } 6290 } 6291 6292 /** 6293 * gaudi2_execute_hard_reset - execute hard reset by driver/FW 6294 * 6295 * @hdev: pointer to the habanalabs device structure 6296 * 6297 * This function executes hard reset based on if driver/FW should do the reset 6298 */ 6299 static void gaudi2_execute_hard_reset(struct hl_device *hdev) 6300 { 6301 if (hdev->asic_prop.hard_reset_done_by_fw) { 6302 gaudi2_send_hard_reset_cmd(hdev); 6303 return; 6304 } 6305 6306 /* Set device to handle FLR by H/W as we will put the device 6307 * CPU to halt mode 6308 */ 6309 WREG32(mmPCIE_AUX_FLR_CTRL, 6310 (PCIE_AUX_FLR_CTRL_HW_CTRL_MASK | PCIE_AUX_FLR_CTRL_INT_MASK_MASK)); 6311 6312 gaudi2_send_hard_reset_cmd(hdev); 6313 6314 WREG32(mmPSOC_RESET_CONF_SW_ALL_RST, 1); 6315 } 6316 6317 static int gaudi2_get_soft_rst_done_indication(struct hl_device *hdev, u32 poll_timeout_us) 6318 { 6319 int i, rc = 0; 6320 u32 reg_val; 6321 6322 for (i = 0 ; i < GAUDI2_RESET_POLL_CNT ; i++) 6323 rc = hl_poll_timeout( 6324 hdev, 6325 mmCPU_RST_STATUS_TO_HOST, 6326 reg_val, 6327 reg_val == CPU_RST_STATUS_SOFT_RST_DONE, 6328 1000, 6329 poll_timeout_us); 6330 6331 if (rc) 6332 dev_err(hdev->dev, "Timeout while waiting for FW to complete soft reset (0x%x)\n", 6333 reg_val); 6334 return rc; 6335 } 6336 6337 /** 6338 * gaudi2_execute_soft_reset - execute soft reset by driver/FW 6339 * 6340 * @hdev: pointer to the habanalabs device structure 6341 * @driver_performs_reset: true if driver should perform reset instead of f/w. 6342 * @poll_timeout_us: time to wait for response from f/w. 6343 * 6344 * This function executes soft reset based on if driver/FW should do the reset 6345 */ 6346 static int gaudi2_execute_soft_reset(struct hl_device *hdev, bool driver_performs_reset, 6347 u32 poll_timeout_us) 6348 { 6349 int rc; 6350 6351 if (!driver_performs_reset) { 6352 if (hl_is_fw_sw_ver_below(hdev, 1, 10)) { 6353 /* set SP to indicate reset request sent to FW */ 6354 WREG32(mmCPU_RST_STATUS_TO_HOST, CPU_RST_STATUS_NA); 6355 6356 WREG32(mmGIC_HOST_SOFT_RST_IRQ_POLL_REG, 6357 gaudi2_irq_map_table[GAUDI2_EVENT_CPU_SOFT_RESET].cpu_id); 6358 6359 /* wait for f/w response */ 6360 rc = gaudi2_get_soft_rst_done_indication(hdev, poll_timeout_us); 6361 } else { 6362 rc = hl_fw_send_soft_reset(hdev); 6363 } 6364 return rc; 6365 } 6366 6367 /* Block access to engines, QMANs and SM during reset, these 6368 * RRs will be reconfigured after soft reset. 6369 * PCIE_MSIX is left unsecured to allow NIC packets processing during the reset. 6370 */ 6371 gaudi2_write_rr_to_all_lbw_rtrs(hdev, RR_TYPE_LONG, NUM_LONG_LBW_RR - 1, 6372 mmDCORE0_TPC0_QM_DCCM_BASE, mmPCIE_MSIX_BASE); 6373 6374 gaudi2_write_rr_to_all_lbw_rtrs(hdev, RR_TYPE_LONG, NUM_LONG_LBW_RR - 2, 6375 mmPCIE_MSIX_BASE + HL_BLOCK_SIZE, 6376 mmPCIE_VDEC1_MSTR_IF_RR_SHRD_HBW_BASE + HL_BLOCK_SIZE); 6377 6378 WREG32(mmPSOC_RESET_CONF_SOFT_RST, 1); 6379 return 0; 6380 } 6381 6382 static void gaudi2_poll_btm_indication(struct hl_device *hdev, u32 poll_timeout_us) 6383 { 6384 int i, rc = 0; 6385 u32 reg_val; 6386 6387 /* We poll the BTM done indication multiple times after reset due to 6388 * a HW errata 'GAUDI2_0300' 6389 */ 6390 for (i = 0 ; i < GAUDI2_RESET_POLL_CNT ; i++) 6391 rc = hl_poll_timeout( 6392 hdev, 6393 mmPSOC_GLOBAL_CONF_BTM_FSM, 6394 reg_val, 6395 reg_val == 0, 6396 1000, 6397 poll_timeout_us); 6398 6399 if (rc) 6400 dev_err(hdev->dev, "Timeout while waiting for device to reset 0x%x\n", reg_val); 6401 } 6402 6403 static int gaudi2_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset) 6404 { 6405 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6406 u32 poll_timeout_us, reset_sleep_ms; 6407 bool driver_performs_reset = false; 6408 int rc; 6409 6410 if (hdev->pldm) { 6411 reset_sleep_ms = hard_reset ? GAUDI2_PLDM_HRESET_TIMEOUT_MSEC : 6412 GAUDI2_PLDM_SRESET_TIMEOUT_MSEC; 6413 poll_timeout_us = GAUDI2_PLDM_RESET_POLL_TIMEOUT_USEC; 6414 } else { 6415 reset_sleep_ms = GAUDI2_RESET_TIMEOUT_MSEC; 6416 poll_timeout_us = GAUDI2_RESET_POLL_TIMEOUT_USEC; 6417 } 6418 6419 if (fw_reset) 6420 goto skip_reset; 6421 6422 gaudi2_reset_arcs(hdev); 6423 6424 if (hard_reset) { 6425 driver_performs_reset = !hdev->asic_prop.hard_reset_done_by_fw; 6426 gaudi2_execute_hard_reset(hdev); 6427 } else { 6428 /* 6429 * As we have to support also work with preboot only (which does not supports 6430 * soft reset) we have to make sure that security is disabled before letting driver 6431 * do the reset. user shall control the BFE flags to avoid asking soft reset in 6432 * secured device with preboot only. 6433 */ 6434 driver_performs_reset = (hdev->fw_components == FW_TYPE_PREBOOT_CPU && 6435 !hdev->asic_prop.fw_security_enabled); 6436 rc = gaudi2_execute_soft_reset(hdev, driver_performs_reset, poll_timeout_us); 6437 if (rc) 6438 return rc; 6439 } 6440 6441 skip_reset: 6442 if (driver_performs_reset || hard_reset) { 6443 /* 6444 * Instead of waiting for BTM indication we should wait for preboot ready: 6445 * Consider the below scenario: 6446 * 1. FW update is being triggered 6447 * - setting the dirty bit 6448 * 2. hard reset will be triggered due to the dirty bit 6449 * 3. FW initiates the reset: 6450 * - dirty bit cleared 6451 * - BTM indication cleared 6452 * - preboot ready indication cleared 6453 * 4. during hard reset: 6454 * - BTM indication will be set 6455 * - BIST test performed and another reset triggered 6456 * 5. only after this reset the preboot will set the preboot ready 6457 * 6458 * when polling on BTM indication alone we can lose sync with FW while trying to 6459 * communicate with FW that is during reset. 6460 * to overcome this we will always wait to preboot ready indication 6461 */ 6462 6463 /* without this sleep reset will not work */ 6464 msleep(reset_sleep_ms); 6465 6466 if (hdev->fw_components & FW_TYPE_PREBOOT_CPU) 6467 hl_fw_wait_preboot_ready(hdev); 6468 else 6469 gaudi2_poll_btm_indication(hdev, poll_timeout_us); 6470 } 6471 6472 if (!gaudi2) 6473 return 0; 6474 6475 gaudi2->dec_hw_cap_initialized &= ~(HW_CAP_DEC_MASK); 6476 gaudi2->tpc_hw_cap_initialized &= ~(HW_CAP_TPC_MASK); 6477 6478 /* 6479 * Clear NIC capability mask in order for driver to re-configure 6480 * NIC QMANs. NIC ports will not be re-configured during soft 6481 * reset as we call gaudi2_nic_init only during hard reset 6482 */ 6483 gaudi2->nic_hw_cap_initialized &= ~(HW_CAP_NIC_MASK); 6484 6485 if (hard_reset) { 6486 gaudi2->hw_cap_initialized &= 6487 ~(HW_CAP_DRAM | HW_CAP_CLK_GATE | HW_CAP_HBM_SCRAMBLER_MASK | 6488 HW_CAP_PMMU | HW_CAP_CPU | HW_CAP_CPU_Q | 6489 HW_CAP_SRAM_SCRAMBLER | HW_CAP_DMMU_MASK | 6490 HW_CAP_PDMA_MASK | HW_CAP_EDMA_MASK | HW_CAP_KDMA | 6491 HW_CAP_MME_MASK | HW_CAP_ROT_MASK); 6492 6493 memset(gaudi2->events_stat, 0, sizeof(gaudi2->events_stat)); 6494 } else { 6495 gaudi2->hw_cap_initialized &= 6496 ~(HW_CAP_CLK_GATE | HW_CAP_HBM_SCRAMBLER_SW_RESET | 6497 HW_CAP_PDMA_MASK | HW_CAP_EDMA_MASK | HW_CAP_MME_MASK | 6498 HW_CAP_ROT_MASK); 6499 } 6500 return 0; 6501 } 6502 6503 static int gaudi2_suspend(struct hl_device *hdev) 6504 { 6505 int rc; 6506 6507 rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0); 6508 if (rc) 6509 dev_err(hdev->dev, "Failed to disable PCI access from CPU\n"); 6510 6511 return rc; 6512 } 6513 6514 static int gaudi2_resume(struct hl_device *hdev) 6515 { 6516 return gaudi2_init_iatu(hdev); 6517 } 6518 6519 static int gaudi2_mmap(struct hl_device *hdev, struct vm_area_struct *vma, 6520 void *cpu_addr, dma_addr_t dma_addr, size_t size) 6521 { 6522 int rc; 6523 6524 vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | 6525 VM_DONTCOPY | VM_NORESERVE); 6526 6527 #ifdef _HAS_DMA_MMAP_COHERENT 6528 6529 rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr, dma_addr, size); 6530 if (rc) 6531 dev_err(hdev->dev, "dma_mmap_coherent error %d", rc); 6532 6533 #else 6534 6535 rc = remap_pfn_range(vma, vma->vm_start, 6536 virt_to_phys(cpu_addr) >> PAGE_SHIFT, 6537 size, vma->vm_page_prot); 6538 if (rc) 6539 dev_err(hdev->dev, "remap_pfn_range error %d", rc); 6540 6541 #endif 6542 6543 return rc; 6544 } 6545 6546 static bool gaudi2_is_queue_enabled(struct hl_device *hdev, u32 hw_queue_id) 6547 { 6548 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6549 u64 hw_cap_mask = 0; 6550 u64 hw_tpc_cap_bit = 0; 6551 u64 hw_nic_cap_bit = 0; 6552 u64 hw_test_cap_bit = 0; 6553 6554 switch (hw_queue_id) { 6555 case GAUDI2_QUEUE_ID_PDMA_0_0: 6556 case GAUDI2_QUEUE_ID_PDMA_0_1: 6557 case GAUDI2_QUEUE_ID_PDMA_1_0: 6558 hw_cap_mask = HW_CAP_PDMA_MASK; 6559 break; 6560 case GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3: 6561 hw_test_cap_bit = HW_CAP_EDMA_SHIFT + 6562 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0) >> 2); 6563 break; 6564 case GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3: 6565 hw_test_cap_bit = HW_CAP_EDMA_SHIFT + NUM_OF_EDMA_PER_DCORE + 6566 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0) >> 2); 6567 break; 6568 case GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3: 6569 hw_test_cap_bit = HW_CAP_EDMA_SHIFT + 2 * NUM_OF_EDMA_PER_DCORE + 6570 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0) >> 2); 6571 break; 6572 case GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3: 6573 hw_test_cap_bit = HW_CAP_EDMA_SHIFT + 3 * NUM_OF_EDMA_PER_DCORE + 6574 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0) >> 2); 6575 break; 6576 6577 case GAUDI2_QUEUE_ID_DCORE0_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE0_MME_0_3: 6578 hw_test_cap_bit = HW_CAP_MME_SHIFT; 6579 break; 6580 6581 case GAUDI2_QUEUE_ID_DCORE1_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE1_MME_0_3: 6582 hw_test_cap_bit = HW_CAP_MME_SHIFT + 1; 6583 break; 6584 6585 case GAUDI2_QUEUE_ID_DCORE2_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE2_MME_0_3: 6586 hw_test_cap_bit = HW_CAP_MME_SHIFT + 2; 6587 break; 6588 6589 case GAUDI2_QUEUE_ID_DCORE3_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE3_MME_0_3: 6590 hw_test_cap_bit = HW_CAP_MME_SHIFT + 3; 6591 break; 6592 6593 case GAUDI2_QUEUE_ID_DCORE0_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE0_TPC_5_3: 6594 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + 6595 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE0_TPC_0_0) >> 2); 6596 6597 /* special case where cap bit refers to the first queue id */ 6598 if (!hw_tpc_cap_bit) 6599 return !!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(0)); 6600 break; 6601 6602 case GAUDI2_QUEUE_ID_DCORE1_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE1_TPC_5_3: 6603 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + NUM_OF_TPC_PER_DCORE + 6604 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE1_TPC_0_0) >> 2); 6605 break; 6606 6607 case GAUDI2_QUEUE_ID_DCORE2_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE2_TPC_5_3: 6608 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + (2 * NUM_OF_TPC_PER_DCORE) + 6609 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE2_TPC_0_0) >> 2); 6610 break; 6611 6612 case GAUDI2_QUEUE_ID_DCORE3_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE3_TPC_5_3: 6613 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + (3 * NUM_OF_TPC_PER_DCORE) + 6614 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE3_TPC_0_0) >> 2); 6615 break; 6616 6617 case GAUDI2_QUEUE_ID_DCORE0_TPC_6_0 ... GAUDI2_QUEUE_ID_DCORE0_TPC_6_3: 6618 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + (4 * NUM_OF_TPC_PER_DCORE); 6619 break; 6620 6621 case GAUDI2_QUEUE_ID_ROT_0_0 ... GAUDI2_QUEUE_ID_ROT_1_3: 6622 hw_test_cap_bit = HW_CAP_ROT_SHIFT + ((hw_queue_id - GAUDI2_QUEUE_ID_ROT_0_0) >> 2); 6623 break; 6624 6625 case GAUDI2_QUEUE_ID_NIC_0_0 ... GAUDI2_QUEUE_ID_NIC_23_3: 6626 hw_nic_cap_bit = HW_CAP_NIC_SHIFT + ((hw_queue_id - GAUDI2_QUEUE_ID_NIC_0_0) >> 2); 6627 6628 /* special case where cap bit refers to the first queue id */ 6629 if (!hw_nic_cap_bit) 6630 return !!(gaudi2->nic_hw_cap_initialized & BIT_ULL(0)); 6631 break; 6632 6633 case GAUDI2_QUEUE_ID_CPU_PQ: 6634 return !!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q); 6635 6636 default: 6637 return false; 6638 } 6639 6640 if (hw_tpc_cap_bit) 6641 return !!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(hw_tpc_cap_bit)); 6642 6643 if (hw_nic_cap_bit) 6644 return !!(gaudi2->nic_hw_cap_initialized & BIT_ULL(hw_nic_cap_bit)); 6645 6646 if (hw_test_cap_bit) 6647 hw_cap_mask = BIT_ULL(hw_test_cap_bit); 6648 6649 return !!(gaudi2->hw_cap_initialized & hw_cap_mask); 6650 } 6651 6652 static bool gaudi2_is_arc_enabled(struct hl_device *hdev, u64 arc_id) 6653 { 6654 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6655 6656 switch (arc_id) { 6657 case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC5: 6658 case CPU_ID_MME_QMAN_ARC0...CPU_ID_ROT_QMAN_ARC1: 6659 return !!(gaudi2->active_hw_arc & BIT_ULL(arc_id)); 6660 6661 case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24: 6662 return !!(gaudi2->active_tpc_arc & BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0)); 6663 6664 case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23: 6665 return !!(gaudi2->active_nic_arc & BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0)); 6666 6667 default: 6668 return false; 6669 } 6670 } 6671 6672 static void gaudi2_clr_arc_id_cap(struct hl_device *hdev, u64 arc_id) 6673 { 6674 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6675 6676 switch (arc_id) { 6677 case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC5: 6678 case CPU_ID_MME_QMAN_ARC0...CPU_ID_ROT_QMAN_ARC1: 6679 gaudi2->active_hw_arc &= ~(BIT_ULL(arc_id)); 6680 break; 6681 6682 case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24: 6683 gaudi2->active_tpc_arc &= ~(BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0)); 6684 break; 6685 6686 case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23: 6687 gaudi2->active_nic_arc &= ~(BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0)); 6688 break; 6689 6690 default: 6691 return; 6692 } 6693 } 6694 6695 static void gaudi2_set_arc_id_cap(struct hl_device *hdev, u64 arc_id) 6696 { 6697 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6698 6699 switch (arc_id) { 6700 case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC5: 6701 case CPU_ID_MME_QMAN_ARC0...CPU_ID_ROT_QMAN_ARC1: 6702 gaudi2->active_hw_arc |= BIT_ULL(arc_id); 6703 break; 6704 6705 case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24: 6706 gaudi2->active_tpc_arc |= BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0); 6707 break; 6708 6709 case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23: 6710 gaudi2->active_nic_arc |= BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0); 6711 break; 6712 6713 default: 6714 return; 6715 } 6716 } 6717 6718 static void gaudi2_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi) 6719 { 6720 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 6721 u32 pq_offset, reg_base, db_reg_offset, db_value; 6722 6723 if (hw_queue_id != GAUDI2_QUEUE_ID_CPU_PQ) { 6724 /* 6725 * QMAN has 4 successive PQ_PI registers, 1 for each of the QMAN PQs. 6726 * Masking the H/W queue ID with 0x3 extracts the QMAN internal PQ 6727 * number. 6728 */ 6729 pq_offset = (hw_queue_id & 0x3) * 4; 6730 reg_base = gaudi2_qm_blocks_bases[hw_queue_id]; 6731 db_reg_offset = reg_base + QM_PQ_PI_0_OFFSET + pq_offset; 6732 } else { 6733 db_reg_offset = mmCPU_IF_PF_PQ_PI; 6734 } 6735 6736 db_value = pi; 6737 6738 /* ring the doorbell */ 6739 WREG32(db_reg_offset, db_value); 6740 6741 if (hw_queue_id == GAUDI2_QUEUE_ID_CPU_PQ) { 6742 /* make sure device CPU will read latest data from host */ 6743 mb(); 6744 WREG32(le32_to_cpu(dyn_regs->gic_host_pi_upd_irq), 6745 gaudi2_irq_map_table[GAUDI2_EVENT_CPU_PI_UPDATE].cpu_id); 6746 } 6747 } 6748 6749 static void gaudi2_pqe_write(struct hl_device *hdev, __le64 *pqe, struct hl_bd *bd) 6750 { 6751 __le64 *pbd = (__le64 *) bd; 6752 6753 /* The QMANs are on the host memory so a simple copy suffice */ 6754 pqe[0] = pbd[0]; 6755 pqe[1] = pbd[1]; 6756 } 6757 6758 static void *gaudi2_dma_alloc_coherent(struct hl_device *hdev, size_t size, 6759 dma_addr_t *dma_handle, gfp_t flags) 6760 { 6761 return dma_alloc_coherent(&hdev->pdev->dev, size, dma_handle, flags); 6762 } 6763 6764 static void gaudi2_dma_free_coherent(struct hl_device *hdev, size_t size, 6765 void *cpu_addr, dma_addr_t dma_handle) 6766 { 6767 dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, dma_handle); 6768 } 6769 6770 static int gaudi2_send_cpu_message(struct hl_device *hdev, u32 *msg, u16 len, 6771 u32 timeout, u64 *result) 6772 { 6773 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6774 6775 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) { 6776 if (result) 6777 *result = 0; 6778 return 0; 6779 } 6780 6781 if (!timeout) 6782 timeout = GAUDI2_MSG_TO_CPU_TIMEOUT_USEC; 6783 6784 return hl_fw_send_cpu_message(hdev, GAUDI2_QUEUE_ID_CPU_PQ, msg, len, timeout, result); 6785 } 6786 6787 static void *gaudi2_dma_pool_zalloc(struct hl_device *hdev, size_t size, 6788 gfp_t mem_flags, dma_addr_t *dma_handle) 6789 { 6790 if (size > GAUDI2_DMA_POOL_BLK_SIZE) 6791 return NULL; 6792 6793 return dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle); 6794 } 6795 6796 static void gaudi2_dma_pool_free(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr) 6797 { 6798 dma_pool_free(hdev->dma_pool, vaddr, dma_addr); 6799 } 6800 6801 static void *gaudi2_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, 6802 dma_addr_t *dma_handle) 6803 { 6804 return hl_fw_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle); 6805 } 6806 6807 static void gaudi2_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr) 6808 { 6809 hl_fw_cpu_accessible_dma_pool_free(hdev, size, vaddr); 6810 } 6811 6812 static int gaudi2_validate_cb_address(struct hl_device *hdev, struct hl_cs_parser *parser) 6813 { 6814 struct asic_fixed_properties *asic_prop = &hdev->asic_prop; 6815 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6816 6817 if (!gaudi2_is_queue_enabled(hdev, parser->hw_queue_id)) { 6818 dev_err(hdev->dev, "h/w queue %d is disabled\n", parser->hw_queue_id); 6819 return -EINVAL; 6820 } 6821 6822 /* Just check if CB address is valid */ 6823 6824 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, 6825 parser->user_cb_size, 6826 asic_prop->sram_user_base_address, 6827 asic_prop->sram_end_address)) 6828 return 0; 6829 6830 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, 6831 parser->user_cb_size, 6832 asic_prop->dram_user_base_address, 6833 asic_prop->dram_end_address)) 6834 return 0; 6835 6836 if ((gaudi2->hw_cap_initialized & HW_CAP_DMMU_MASK) && 6837 hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, 6838 parser->user_cb_size, 6839 asic_prop->dmmu.start_addr, 6840 asic_prop->dmmu.end_addr)) 6841 return 0; 6842 6843 if (gaudi2->hw_cap_initialized & HW_CAP_PMMU) { 6844 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, 6845 parser->user_cb_size, 6846 asic_prop->pmmu.start_addr, 6847 asic_prop->pmmu.end_addr) || 6848 hl_mem_area_inside_range( 6849 (u64) (uintptr_t) parser->user_cb, 6850 parser->user_cb_size, 6851 asic_prop->pmmu_huge.start_addr, 6852 asic_prop->pmmu_huge.end_addr)) 6853 return 0; 6854 6855 } else if (gaudi2_host_phys_addr_valid((u64) (uintptr_t) parser->user_cb)) { 6856 if (!hdev->pdev) 6857 return 0; 6858 6859 if (!device_iommu_mapped(&hdev->pdev->dev)) 6860 return 0; 6861 } 6862 6863 dev_err(hdev->dev, "CB address %p + 0x%x for internal QMAN is not valid\n", 6864 parser->user_cb, parser->user_cb_size); 6865 6866 return -EFAULT; 6867 } 6868 6869 static int gaudi2_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser) 6870 { 6871 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6872 6873 if (!parser->is_kernel_allocated_cb) 6874 return gaudi2_validate_cb_address(hdev, parser); 6875 6876 if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) { 6877 dev_err(hdev->dev, "PMMU not initialized - Unsupported mode in Gaudi2\n"); 6878 return -EINVAL; 6879 } 6880 6881 return 0; 6882 } 6883 6884 static int gaudi2_send_heartbeat(struct hl_device *hdev) 6885 { 6886 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6887 6888 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 6889 return 0; 6890 6891 return hl_fw_send_heartbeat(hdev); 6892 } 6893 6894 /* This is an internal helper function, used to update the KDMA mmu props. 6895 * Should be called with a proper kdma lock. 6896 */ 6897 static void gaudi2_kdma_set_mmbp_asid(struct hl_device *hdev, 6898 bool mmu_bypass, u32 asid) 6899 { 6900 u32 rw_asid, rw_mmu_bp; 6901 6902 rw_asid = (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_SHIFT) | 6903 (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_SHIFT); 6904 6905 rw_mmu_bp = (!!mmu_bypass << ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_RD_SHIFT) | 6906 (!!mmu_bypass << ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_WR_SHIFT); 6907 6908 WREG32(mmARC_FARM_KDMA_CTX_AXUSER_HB_ASID, rw_asid); 6909 WREG32(mmARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP, rw_mmu_bp); 6910 } 6911 6912 static void gaudi2_arm_cq_monitor(struct hl_device *hdev, u32 sob_id, u32 mon_id, u32 cq_id, 6913 u32 mon_payload, u32 sync_value) 6914 { 6915 u32 sob_offset, mon_offset, sync_group_id, mode, mon_arm; 6916 u8 mask; 6917 6918 sob_offset = sob_id * 4; 6919 mon_offset = mon_id * 4; 6920 6921 /* Reset the SOB value */ 6922 WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset, 0); 6923 6924 /* Configure this address with CQ_ID 0 because CQ_EN is set */ 6925 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, cq_id); 6926 6927 /* Configure this address with CS index because CQ_EN is set */ 6928 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, mon_payload); 6929 6930 sync_group_id = sob_id / 8; 6931 mask = ~(1 << (sob_id & 0x7)); 6932 mode = 1; /* comparison mode is "equal to" */ 6933 6934 mon_arm = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOD_MASK, sync_value); 6935 mon_arm |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOP_MASK, mode); 6936 mon_arm |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_MASK_MASK, mask); 6937 mon_arm |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SID_MASK, sync_group_id); 6938 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + mon_offset, mon_arm); 6939 } 6940 6941 /* This is an internal helper function used by gaudi2_send_job_to_kdma only */ 6942 static int gaudi2_send_job_to_kdma(struct hl_device *hdev, 6943 u64 src_addr, u64 dst_addr, 6944 u32 size, bool is_memset) 6945 { 6946 u32 comp_val, commit_mask, *polling_addr, timeout, status = 0; 6947 struct hl_cq_entry *cq_base; 6948 struct hl_cq *cq; 6949 u64 comp_addr; 6950 int rc; 6951 6952 gaudi2_arm_cq_monitor(hdev, GAUDI2_RESERVED_SOB_KDMA_COMPLETION, 6953 GAUDI2_RESERVED_MON_KDMA_COMPLETION, 6954 GAUDI2_RESERVED_CQ_KDMA_COMPLETION, 1, 1); 6955 6956 comp_addr = CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + 6957 (GAUDI2_RESERVED_SOB_KDMA_COMPLETION * sizeof(u32)); 6958 6959 comp_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1) | 6960 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 1); 6961 6962 WREG32(mmARC_FARM_KDMA_CTX_SRC_BASE_LO, lower_32_bits(src_addr)); 6963 WREG32(mmARC_FARM_KDMA_CTX_SRC_BASE_HI, upper_32_bits(src_addr)); 6964 WREG32(mmARC_FARM_KDMA_CTX_DST_BASE_LO, lower_32_bits(dst_addr)); 6965 WREG32(mmARC_FARM_KDMA_CTX_DST_BASE_HI, upper_32_bits(dst_addr)); 6966 WREG32(mmARC_FARM_KDMA_CTX_WR_COMP_ADDR_LO, lower_32_bits(comp_addr)); 6967 WREG32(mmARC_FARM_KDMA_CTX_WR_COMP_ADDR_HI, upper_32_bits(comp_addr)); 6968 WREG32(mmARC_FARM_KDMA_CTX_WR_COMP_WDATA, comp_val); 6969 WREG32(mmARC_FARM_KDMA_CTX_DST_TSIZE_0, size); 6970 6971 commit_mask = FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_LIN_MASK, 1) | 6972 FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_WR_COMP_EN_MASK, 1); 6973 6974 if (is_memset) 6975 commit_mask |= FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_MEM_SET_MASK, 1); 6976 6977 WREG32(mmARC_FARM_KDMA_CTX_COMMIT, commit_mask); 6978 6979 /* Wait for completion */ 6980 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_KDMA_COMPLETION]; 6981 cq_base = cq->kernel_address; 6982 polling_addr = (u32 *)&cq_base[cq->ci]; 6983 6984 if (hdev->pldm) 6985 /* for each 1MB 20 second of timeout */ 6986 timeout = ((size / SZ_1M) + 1) * USEC_PER_SEC * 20; 6987 else 6988 timeout = KDMA_TIMEOUT_USEC; 6989 6990 /* Polling */ 6991 rc = hl_poll_timeout_memory( 6992 hdev, 6993 polling_addr, 6994 status, 6995 (status == 1), 6996 1000, 6997 timeout, 6998 true); 6999 7000 *polling_addr = 0; 7001 7002 if (rc) { 7003 dev_err(hdev->dev, "Timeout while waiting for KDMA to be idle\n"); 7004 WREG32(mmARC_FARM_KDMA_CFG_1, 1 << ARC_FARM_KDMA_CFG_1_HALT_SHIFT); 7005 return rc; 7006 } 7007 7008 cq->ci = hl_cq_inc_ptr(cq->ci); 7009 7010 return 0; 7011 } 7012 7013 static void gaudi2_memset_device_lbw(struct hl_device *hdev, u32 addr, u32 size, u32 val) 7014 { 7015 u32 i; 7016 7017 for (i = 0 ; i < size ; i += sizeof(u32)) 7018 WREG32(addr + i, val); 7019 } 7020 7021 static void gaudi2_qman_set_test_mode(struct hl_device *hdev, u32 hw_queue_id, bool enable) 7022 { 7023 u32 reg_base = gaudi2_qm_blocks_bases[hw_queue_id]; 7024 7025 if (enable) { 7026 WREG32(reg_base + QM_GLBL_PROT_OFFSET, QMAN_MAKE_TRUSTED_TEST_MODE); 7027 WREG32(reg_base + QM_PQC_CFG_OFFSET, 0); 7028 } else { 7029 WREG32(reg_base + QM_GLBL_PROT_OFFSET, QMAN_MAKE_TRUSTED); 7030 WREG32(reg_base + QM_PQC_CFG_OFFSET, 1 << PDMA0_QM_PQC_CFG_EN_SHIFT); 7031 } 7032 } 7033 7034 static inline u32 gaudi2_test_queue_hw_queue_id_to_sob_id(struct hl_device *hdev, u32 hw_queue_id) 7035 { 7036 return hdev->asic_prop.first_available_user_sob[0] + 7037 hw_queue_id - GAUDI2_QUEUE_ID_PDMA_0_0; 7038 } 7039 7040 static void gaudi2_test_queue_clear(struct hl_device *hdev, u32 hw_queue_id) 7041 { 7042 u32 sob_offset = gaudi2_test_queue_hw_queue_id_to_sob_id(hdev, hw_queue_id) * 4; 7043 u32 sob_addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset; 7044 7045 /* Reset the SOB value */ 7046 WREG32(sob_addr, 0); 7047 } 7048 7049 static int gaudi2_test_queue_send_msg_short(struct hl_device *hdev, u32 hw_queue_id, u32 sob_val, 7050 struct gaudi2_queues_test_info *msg_info) 7051 { 7052 u32 sob_offset = gaudi2_test_queue_hw_queue_id_to_sob_id(hdev, hw_queue_id) * 4; 7053 u32 tmp, sob_base = 1; 7054 struct packet_msg_short *msg_short_pkt = msg_info->kern_addr; 7055 size_t pkt_size = sizeof(struct packet_msg_short); 7056 int rc; 7057 7058 tmp = (PACKET_MSG_SHORT << GAUDI2_PKT_CTL_OPCODE_SHIFT) | 7059 (1 << GAUDI2_PKT_CTL_EB_SHIFT) | 7060 (1 << GAUDI2_PKT_CTL_MB_SHIFT) | 7061 (sob_base << GAUDI2_PKT_SHORT_CTL_BASE_SHIFT) | 7062 (sob_offset << GAUDI2_PKT_SHORT_CTL_ADDR_SHIFT); 7063 7064 msg_short_pkt->value = cpu_to_le32(sob_val); 7065 msg_short_pkt->ctl = cpu_to_le32(tmp); 7066 7067 rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, pkt_size, msg_info->dma_addr); 7068 if (rc) 7069 dev_err(hdev->dev, 7070 "Failed to send msg_short packet to H/W queue %d\n", hw_queue_id); 7071 7072 return rc; 7073 } 7074 7075 static int gaudi2_test_queue_wait_completion(struct hl_device *hdev, u32 hw_queue_id, u32 sob_val) 7076 { 7077 u32 sob_offset = gaudi2_test_queue_hw_queue_id_to_sob_id(hdev, hw_queue_id) * 4; 7078 u32 sob_addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset; 7079 u32 timeout_usec, tmp; 7080 int rc; 7081 7082 if (hdev->pldm) 7083 timeout_usec = GAUDI2_PLDM_TEST_QUEUE_WAIT_USEC; 7084 else 7085 timeout_usec = GAUDI2_TEST_QUEUE_WAIT_USEC; 7086 7087 rc = hl_poll_timeout( 7088 hdev, 7089 sob_addr, 7090 tmp, 7091 (tmp == sob_val), 7092 1000, 7093 timeout_usec); 7094 7095 if (rc == -ETIMEDOUT) { 7096 dev_err(hdev->dev, "H/W queue %d test failed (SOB_OBJ_0 == 0x%x)\n", 7097 hw_queue_id, tmp); 7098 rc = -EIO; 7099 } 7100 7101 return rc; 7102 } 7103 7104 static int gaudi2_test_cpu_queue(struct hl_device *hdev) 7105 { 7106 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7107 7108 /* 7109 * check capability here as send_cpu_message() won't update the result 7110 * value if no capability 7111 */ 7112 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 7113 return 0; 7114 7115 return hl_fw_test_cpu_queue(hdev); 7116 } 7117 7118 static int gaudi2_test_queues(struct hl_device *hdev) 7119 { 7120 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7121 struct gaudi2_queues_test_info *msg_info; 7122 u32 sob_val = 0x5a5a; 7123 int i, rc; 7124 7125 /* send test message on all enabled Qs */ 7126 for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ; i++) { 7127 if (!gaudi2_is_queue_enabled(hdev, i) || gaudi2_is_edma_queue_id(i)) 7128 continue; 7129 7130 msg_info = &gaudi2->queues_test_info[i - GAUDI2_QUEUE_ID_PDMA_0_0]; 7131 gaudi2_qman_set_test_mode(hdev, i, true); 7132 gaudi2_test_queue_clear(hdev, i); 7133 rc = gaudi2_test_queue_send_msg_short(hdev, i, sob_val, msg_info); 7134 if (rc) 7135 goto done; 7136 } 7137 7138 rc = gaudi2_test_cpu_queue(hdev); 7139 if (rc) 7140 goto done; 7141 7142 /* verify that all messages were processed */ 7143 for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ; i++) { 7144 if (!gaudi2_is_queue_enabled(hdev, i) || gaudi2_is_edma_queue_id(i)) 7145 continue; 7146 7147 rc = gaudi2_test_queue_wait_completion(hdev, i, sob_val); 7148 if (rc) 7149 /* chip is not usable, no need for cleanups, just bail-out with error */ 7150 goto done; 7151 7152 gaudi2_test_queue_clear(hdev, i); 7153 gaudi2_qman_set_test_mode(hdev, i, false); 7154 } 7155 7156 done: 7157 return rc; 7158 } 7159 7160 static int gaudi2_compute_reset_late_init(struct hl_device *hdev) 7161 { 7162 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7163 size_t irq_arr_size; 7164 int rc; 7165 7166 gaudi2_init_arcs(hdev); 7167 7168 rc = gaudi2_scrub_arcs_dccm(hdev); 7169 if (rc) { 7170 dev_err(hdev->dev, "Failed to scrub arcs DCCM\n"); 7171 return rc; 7172 } 7173 7174 gaudi2_init_security(hdev); 7175 7176 /* Unmask all IRQs since some could have been received during the soft reset */ 7177 irq_arr_size = gaudi2->num_of_valid_hw_events * sizeof(gaudi2->hw_events[0]); 7178 return hl_fw_unmask_irq_arr(hdev, gaudi2->hw_events, irq_arr_size); 7179 } 7180 7181 static bool gaudi2_get_edma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7182 struct engines_data *e) 7183 { 7184 u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts, dma_core_sts0, dma_core_sts1; 7185 struct asic_fixed_properties *prop = &hdev->asic_prop; 7186 unsigned long *mask = (unsigned long *) mask_arr; 7187 const char *edma_fmt = "%-6d%-6d%-9s%#-14x%#-15x%#x\n"; 7188 bool is_idle = true, is_eng_idle; 7189 int engine_idx, i, j; 7190 u64 offset; 7191 7192 if (e) 7193 hl_engine_data_sprintf(e, 7194 "\nCORE EDMA is_idle QM_GLBL_STS0 DMA_CORE_STS0 DMA_CORE_STS1\n" 7195 "---- ---- ------- ------------ ------------- -------------\n"); 7196 7197 for (i = 0; i < NUM_OF_DCORES; i++) { 7198 for (j = 0 ; j < NUM_OF_EDMA_PER_DCORE ; j++) { 7199 int seq = i * NUM_OF_EDMA_PER_DCORE + j; 7200 7201 if (!(prop->edma_enabled_mask & BIT(seq))) 7202 continue; 7203 7204 engine_idx = GAUDI2_DCORE0_ENGINE_ID_EDMA_0 + 7205 i * GAUDI2_ENGINE_ID_DCORE_OFFSET + j; 7206 offset = i * DCORE_OFFSET + j * DCORE_EDMA_OFFSET; 7207 7208 dma_core_sts0 = RREG32(mmDCORE0_EDMA0_CORE_STS0 + offset); 7209 dma_core_sts1 = RREG32(mmDCORE0_EDMA0_CORE_STS1 + offset); 7210 7211 qm_glbl_sts0 = RREG32(mmDCORE0_EDMA0_QM_GLBL_STS0 + offset); 7212 qm_glbl_sts1 = RREG32(mmDCORE0_EDMA0_QM_GLBL_STS1 + offset); 7213 qm_cgm_sts = RREG32(mmDCORE0_EDMA0_QM_CGM_STS + offset); 7214 7215 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) && 7216 IS_DMA_IDLE(dma_core_sts0) && !IS_DMA_HALTED(dma_core_sts1); 7217 is_idle &= is_eng_idle; 7218 7219 if (mask && !is_eng_idle) 7220 set_bit(engine_idx, mask); 7221 7222 if (e) 7223 hl_engine_data_sprintf(e, edma_fmt, i, j, is_eng_idle ? "Y" : "N", 7224 qm_glbl_sts0, dma_core_sts0, dma_core_sts1); 7225 } 7226 } 7227 7228 return is_idle; 7229 } 7230 7231 static bool gaudi2_get_pdma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7232 struct engines_data *e) 7233 { 7234 u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts, dma_core_sts0, dma_core_sts1; 7235 unsigned long *mask = (unsigned long *) mask_arr; 7236 const char *pdma_fmt = "%-6d%-9s%#-14x%#-15x%#x\n"; 7237 bool is_idle = true, is_eng_idle; 7238 int engine_idx, i; 7239 u64 offset; 7240 7241 if (e) 7242 hl_engine_data_sprintf(e, 7243 "\nPDMA is_idle QM_GLBL_STS0 DMA_CORE_STS0 DMA_CORE_STS1\n" 7244 "---- ------- ------------ ------------- -------------\n"); 7245 7246 for (i = 0 ; i < NUM_OF_PDMA ; i++) { 7247 engine_idx = GAUDI2_ENGINE_ID_PDMA_0 + i; 7248 offset = i * PDMA_OFFSET; 7249 dma_core_sts0 = RREG32(mmPDMA0_CORE_STS0 + offset); 7250 dma_core_sts1 = RREG32(mmPDMA0_CORE_STS1 + offset); 7251 7252 qm_glbl_sts0 = RREG32(mmPDMA0_QM_GLBL_STS0 + offset); 7253 qm_glbl_sts1 = RREG32(mmPDMA0_QM_GLBL_STS1 + offset); 7254 qm_cgm_sts = RREG32(mmPDMA0_QM_CGM_STS + offset); 7255 7256 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) && 7257 IS_DMA_IDLE(dma_core_sts0) && !IS_DMA_HALTED(dma_core_sts1); 7258 is_idle &= is_eng_idle; 7259 7260 if (mask && !is_eng_idle) 7261 set_bit(engine_idx, mask); 7262 7263 if (e) 7264 hl_engine_data_sprintf(e, pdma_fmt, i, is_eng_idle ? "Y" : "N", 7265 qm_glbl_sts0, dma_core_sts0, dma_core_sts1); 7266 } 7267 7268 return is_idle; 7269 } 7270 7271 static bool gaudi2_get_nic_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7272 struct engines_data *e) 7273 { 7274 unsigned long *mask = (unsigned long *) mask_arr; 7275 const char *nic_fmt = "%-5d%-9s%#-14x%#-12x\n"; 7276 u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts; 7277 bool is_idle = true, is_eng_idle; 7278 int engine_idx, i; 7279 u64 offset = 0; 7280 7281 /* NIC, twelve macros in Full chip */ 7282 if (e && hdev->nic_ports_mask) 7283 hl_engine_data_sprintf(e, 7284 "\nNIC is_idle QM_GLBL_STS0 QM_CGM_STS\n" 7285 "--- ------- ------------ ----------\n"); 7286 7287 for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++) { 7288 if (!(i & 1)) 7289 offset = i / 2 * NIC_OFFSET; 7290 else 7291 offset += NIC_QM_OFFSET; 7292 7293 if (!(hdev->nic_ports_mask & BIT(i))) 7294 continue; 7295 7296 engine_idx = GAUDI2_ENGINE_ID_NIC0_0 + i; 7297 7298 7299 qm_glbl_sts0 = RREG32(mmNIC0_QM0_GLBL_STS0 + offset); 7300 qm_glbl_sts1 = RREG32(mmNIC0_QM0_GLBL_STS1 + offset); 7301 qm_cgm_sts = RREG32(mmNIC0_QM0_CGM_STS + offset); 7302 7303 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts); 7304 is_idle &= is_eng_idle; 7305 7306 if (mask && !is_eng_idle) 7307 set_bit(engine_idx, mask); 7308 7309 if (e) 7310 hl_engine_data_sprintf(e, nic_fmt, i, is_eng_idle ? "Y" : "N", 7311 qm_glbl_sts0, qm_cgm_sts); 7312 } 7313 7314 return is_idle; 7315 } 7316 7317 static bool gaudi2_get_mme_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7318 struct engines_data *e) 7319 { 7320 u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts, mme_arch_sts; 7321 unsigned long *mask = (unsigned long *) mask_arr; 7322 const char *mme_fmt = "%-5d%-6s%-9s%#-14x%#x\n"; 7323 bool is_idle = true, is_eng_idle; 7324 int engine_idx, i; 7325 u64 offset; 7326 7327 if (e) 7328 hl_engine_data_sprintf(e, 7329 "\nMME Stub is_idle QM_GLBL_STS0 MME_ARCH_STATUS\n" 7330 "--- ---- ------- ------------ ---------------\n"); 7331 /* MME, one per Dcore */ 7332 for (i = 0 ; i < NUM_OF_DCORES ; i++) { 7333 engine_idx = GAUDI2_DCORE0_ENGINE_ID_MME + i * GAUDI2_ENGINE_ID_DCORE_OFFSET; 7334 offset = i * DCORE_OFFSET; 7335 7336 qm_glbl_sts0 = RREG32(mmDCORE0_MME_QM_GLBL_STS0 + offset); 7337 qm_glbl_sts1 = RREG32(mmDCORE0_MME_QM_GLBL_STS1 + offset); 7338 qm_cgm_sts = RREG32(mmDCORE0_MME_QM_CGM_STS + offset); 7339 7340 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts); 7341 is_idle &= is_eng_idle; 7342 7343 mme_arch_sts = RREG32(mmDCORE0_MME_CTRL_LO_ARCH_STATUS + offset); 7344 is_eng_idle &= IS_MME_IDLE(mme_arch_sts); 7345 is_idle &= is_eng_idle; 7346 7347 if (e) 7348 hl_engine_data_sprintf(e, mme_fmt, i, "N", 7349 is_eng_idle ? "Y" : "N", 7350 qm_glbl_sts0, 7351 mme_arch_sts); 7352 7353 if (mask && !is_eng_idle) 7354 set_bit(engine_idx, mask); 7355 } 7356 7357 return is_idle; 7358 } 7359 7360 static void gaudi2_is_tpc_engine_idle(struct hl_device *hdev, int dcore, int inst, u32 offset, 7361 struct iterate_module_ctx *ctx) 7362 { 7363 struct gaudi2_tpc_idle_data *idle_data = ctx->data; 7364 u32 tpc_cfg_sts, qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts; 7365 bool is_eng_idle; 7366 int engine_idx; 7367 7368 if ((dcore == 0) && (inst == (NUM_DCORE0_TPC - 1))) 7369 engine_idx = GAUDI2_DCORE0_ENGINE_ID_TPC_6; 7370 else 7371 engine_idx = GAUDI2_DCORE0_ENGINE_ID_TPC_0 + 7372 dcore * GAUDI2_ENGINE_ID_DCORE_OFFSET + inst; 7373 7374 tpc_cfg_sts = RREG32(mmDCORE0_TPC0_CFG_STATUS + offset); 7375 qm_glbl_sts0 = RREG32(mmDCORE0_TPC0_QM_GLBL_STS0 + offset); 7376 qm_glbl_sts1 = RREG32(mmDCORE0_TPC0_QM_GLBL_STS1 + offset); 7377 qm_cgm_sts = RREG32(mmDCORE0_TPC0_QM_CGM_STS + offset); 7378 7379 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) && 7380 IS_TPC_IDLE(tpc_cfg_sts); 7381 *(idle_data->is_idle) &= is_eng_idle; 7382 7383 if (idle_data->mask && !is_eng_idle) 7384 set_bit(engine_idx, idle_data->mask); 7385 7386 if (idle_data->e) 7387 hl_engine_data_sprintf(idle_data->e, 7388 idle_data->tpc_fmt, dcore, inst, 7389 is_eng_idle ? "Y" : "N", 7390 qm_glbl_sts0, qm_cgm_sts, tpc_cfg_sts); 7391 } 7392 7393 static bool gaudi2_get_tpc_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7394 struct engines_data *e) 7395 { 7396 struct asic_fixed_properties *prop = &hdev->asic_prop; 7397 unsigned long *mask = (unsigned long *) mask_arr; 7398 bool is_idle = true; 7399 7400 struct gaudi2_tpc_idle_data tpc_idle_data = { 7401 .tpc_fmt = "%-6d%-5d%-9s%#-14x%#-12x%#x\n", 7402 .e = e, 7403 .mask = mask, 7404 .is_idle = &is_idle, 7405 }; 7406 struct iterate_module_ctx tpc_iter = { 7407 .fn = &gaudi2_is_tpc_engine_idle, 7408 .data = &tpc_idle_data, 7409 }; 7410 7411 if (e && prop->tpc_enabled_mask) 7412 hl_engine_data_sprintf(e, 7413 "\nCORE TPC is_idle QM_GLBL_STS0 QM_CGM_STS STATUS\n" 7414 "---- --- ------- ------------ ---------- ------\n"); 7415 7416 gaudi2_iterate_tpcs(hdev, &tpc_iter); 7417 7418 return *tpc_idle_data.is_idle; 7419 } 7420 7421 static bool gaudi2_get_decoder_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7422 struct engines_data *e) 7423 { 7424 struct asic_fixed_properties *prop = &hdev->asic_prop; 7425 unsigned long *mask = (unsigned long *) mask_arr; 7426 const char *pcie_dec_fmt = "%-10d%-9s%#x\n"; 7427 const char *dec_fmt = "%-6d%-5d%-9s%#x\n"; 7428 bool is_idle = true, is_eng_idle; 7429 u32 dec_swreg15, dec_enabled_bit; 7430 int engine_idx, i, j; 7431 u64 offset; 7432 7433 /* Decoders, two each Dcore and two shared PCIe decoders */ 7434 if (e && (prop->decoder_enabled_mask & (~PCIE_DEC_EN_MASK))) 7435 hl_engine_data_sprintf(e, 7436 "\nCORE DEC is_idle VSI_CMD_SWREG15\n" 7437 "---- --- ------- ---------------\n"); 7438 7439 for (i = 0 ; i < NUM_OF_DCORES ; i++) { 7440 for (j = 0 ; j < NUM_OF_DEC_PER_DCORE ; j++) { 7441 dec_enabled_bit = 1 << (i * NUM_OF_DEC_PER_DCORE + j); 7442 if (!(prop->decoder_enabled_mask & dec_enabled_bit)) 7443 continue; 7444 7445 engine_idx = GAUDI2_DCORE0_ENGINE_ID_DEC_0 + 7446 i * GAUDI2_ENGINE_ID_DCORE_OFFSET + j; 7447 offset = i * DCORE_OFFSET + j * DCORE_DEC_OFFSET; 7448 7449 dec_swreg15 = RREG32(mmDCORE0_DEC0_CMD_SWREG15 + offset); 7450 is_eng_idle = IS_DEC_IDLE(dec_swreg15); 7451 is_idle &= is_eng_idle; 7452 7453 if (mask && !is_eng_idle) 7454 set_bit(engine_idx, mask); 7455 7456 if (e) 7457 hl_engine_data_sprintf(e, dec_fmt, i, j, 7458 is_eng_idle ? "Y" : "N", dec_swreg15); 7459 } 7460 } 7461 7462 if (e && (prop->decoder_enabled_mask & PCIE_DEC_EN_MASK)) 7463 hl_engine_data_sprintf(e, 7464 "\nPCIe DEC is_idle VSI_CMD_SWREG15\n" 7465 "-------- ------- ---------------\n"); 7466 7467 /* Check shared(PCIe) decoders */ 7468 for (i = 0 ; i < NUM_OF_DEC_PER_DCORE ; i++) { 7469 dec_enabled_bit = PCIE_DEC_SHIFT + i; 7470 if (!(prop->decoder_enabled_mask & BIT(dec_enabled_bit))) 7471 continue; 7472 7473 engine_idx = GAUDI2_PCIE_ENGINE_ID_DEC_0 + i; 7474 offset = i * DCORE_DEC_OFFSET; 7475 dec_swreg15 = RREG32(mmPCIE_DEC0_CMD_SWREG15 + offset); 7476 is_eng_idle = IS_DEC_IDLE(dec_swreg15); 7477 is_idle &= is_eng_idle; 7478 7479 if (mask && !is_eng_idle) 7480 set_bit(engine_idx, mask); 7481 7482 if (e) 7483 hl_engine_data_sprintf(e, pcie_dec_fmt, i, 7484 is_eng_idle ? "Y" : "N", dec_swreg15); 7485 } 7486 7487 return is_idle; 7488 } 7489 7490 static bool gaudi2_get_rotator_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7491 struct engines_data *e) 7492 { 7493 const char *rot_fmt = "%-6d%-5d%-9s%#-14x%#-14x%#x\n"; 7494 unsigned long *mask = (unsigned long *) mask_arr; 7495 u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts; 7496 bool is_idle = true, is_eng_idle; 7497 int engine_idx, i; 7498 u64 offset; 7499 7500 if (e) 7501 hl_engine_data_sprintf(e, 7502 "\nCORE ROT is_idle QM_GLBL_STS0 QM_GLBL_STS1 QM_CGM_STS\n" 7503 "---- --- ------- ------------ ------------ ----------\n"); 7504 7505 for (i = 0 ; i < NUM_OF_ROT ; i++) { 7506 engine_idx = GAUDI2_ENGINE_ID_ROT_0 + i; 7507 7508 offset = i * ROT_OFFSET; 7509 7510 qm_glbl_sts0 = RREG32(mmROT0_QM_GLBL_STS0 + offset); 7511 qm_glbl_sts1 = RREG32(mmROT0_QM_GLBL_STS1 + offset); 7512 qm_cgm_sts = RREG32(mmROT0_QM_CGM_STS + offset); 7513 7514 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts); 7515 is_idle &= is_eng_idle; 7516 7517 if (mask && !is_eng_idle) 7518 set_bit(engine_idx, mask); 7519 7520 if (e) 7521 hl_engine_data_sprintf(e, rot_fmt, i, 0, is_eng_idle ? "Y" : "N", 7522 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts); 7523 } 7524 7525 return is_idle; 7526 } 7527 7528 static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7529 struct engines_data *e) 7530 { 7531 bool is_idle = true; 7532 7533 is_idle &= gaudi2_get_edma_idle_status(hdev, mask_arr, mask_len, e); 7534 is_idle &= gaudi2_get_pdma_idle_status(hdev, mask_arr, mask_len, e); 7535 is_idle &= gaudi2_get_nic_idle_status(hdev, mask_arr, mask_len, e); 7536 is_idle &= gaudi2_get_mme_idle_status(hdev, mask_arr, mask_len, e); 7537 is_idle &= gaudi2_get_tpc_idle_status(hdev, mask_arr, mask_len, e); 7538 is_idle &= gaudi2_get_decoder_idle_status(hdev, mask_arr, mask_len, e); 7539 is_idle &= gaudi2_get_rotator_idle_status(hdev, mask_arr, mask_len, e); 7540 7541 return is_idle; 7542 } 7543 7544 static void gaudi2_hw_queues_lock(struct hl_device *hdev) 7545 __acquires(&gaudi2->hw_queues_lock) 7546 { 7547 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7548 7549 spin_lock(&gaudi2->hw_queues_lock); 7550 } 7551 7552 static void gaudi2_hw_queues_unlock(struct hl_device *hdev) 7553 __releases(&gaudi2->hw_queues_lock) 7554 { 7555 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7556 7557 spin_unlock(&gaudi2->hw_queues_lock); 7558 } 7559 7560 static u32 gaudi2_get_pci_id(struct hl_device *hdev) 7561 { 7562 return hdev->pdev->device; 7563 } 7564 7565 static int gaudi2_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size) 7566 { 7567 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7568 7569 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 7570 return 0; 7571 7572 return hl_fw_get_eeprom_data(hdev, data, max_size); 7573 } 7574 7575 static void gaudi2_update_eq_ci(struct hl_device *hdev, u32 val) 7576 { 7577 WREG32(mmCPU_IF_EQ_RD_OFFS, val); 7578 } 7579 7580 static void *gaudi2_get_events_stat(struct hl_device *hdev, bool aggregate, u32 *size) 7581 { 7582 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7583 7584 if (aggregate) { 7585 *size = (u32) sizeof(gaudi2->events_stat_aggregate); 7586 return gaudi2->events_stat_aggregate; 7587 } 7588 7589 *size = (u32) sizeof(gaudi2->events_stat); 7590 return gaudi2->events_stat; 7591 } 7592 7593 static void gaudi2_mmu_vdec_dcore_prepare(struct hl_device *hdev, int dcore_id, 7594 int dcore_vdec_id, u32 rw_asid, u32 rw_mmu_bp) 7595 { 7596 u32 offset = (mmDCORE0_VDEC1_BRDG_CTRL_BASE - mmDCORE0_VDEC0_BRDG_CTRL_BASE) * 7597 dcore_vdec_id + DCORE_OFFSET * dcore_id; 7598 7599 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_MMU_BP + offset, rw_mmu_bp); 7600 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_ASID + offset, rw_asid); 7601 7602 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_MMU_BP + offset, rw_mmu_bp); 7603 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_ASID + offset, rw_asid); 7604 7605 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_MMU_BP + offset, rw_mmu_bp); 7606 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_ASID + offset, rw_asid); 7607 7608 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_MMU_BP + offset, rw_mmu_bp); 7609 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_ASID + offset, rw_asid); 7610 7611 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_MMU_BP + offset, rw_mmu_bp); 7612 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_ASID + offset, rw_asid); 7613 } 7614 7615 static void gaudi2_mmu_dcore_prepare(struct hl_device *hdev, int dcore_id, u32 asid) 7616 { 7617 u32 rw_asid = (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_SHIFT) | 7618 (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_SHIFT); 7619 struct asic_fixed_properties *prop = &hdev->asic_prop; 7620 u32 dcore_offset = dcore_id * DCORE_OFFSET; 7621 u32 vdec_id, i, ports_offset, reg_val; 7622 u8 edma_seq_base; 7623 7624 /* EDMA */ 7625 edma_seq_base = dcore_id * NUM_OF_EDMA_PER_DCORE; 7626 if (prop->edma_enabled_mask & BIT(edma_seq_base)) { 7627 WREG32(mmDCORE0_EDMA0_QM_AXUSER_NONSECURED_HB_MMU_BP + dcore_offset, 0); 7628 WREG32(mmDCORE0_EDMA0_QM_AXUSER_NONSECURED_HB_ASID + dcore_offset, rw_asid); 7629 WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP + dcore_offset, 0); 7630 WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_ASID + dcore_offset, rw_asid); 7631 } 7632 7633 if (prop->edma_enabled_mask & BIT(edma_seq_base + 1)) { 7634 WREG32(mmDCORE0_EDMA1_QM_AXUSER_NONSECURED_HB_MMU_BP + dcore_offset, 0); 7635 WREG32(mmDCORE0_EDMA1_QM_AXUSER_NONSECURED_HB_ASID + dcore_offset, rw_asid); 7636 WREG32(mmDCORE0_EDMA1_CORE_CTX_AXUSER_HB_ASID + dcore_offset, rw_asid); 7637 WREG32(mmDCORE0_EDMA1_CORE_CTX_AXUSER_HB_MMU_BP + dcore_offset, 0); 7638 } 7639 7640 /* Sync Mngr */ 7641 WREG32(mmDCORE0_SYNC_MNGR_GLBL_ASID_NONE_SEC_PRIV + dcore_offset, asid); 7642 /* 7643 * Sync Mngrs on dcores 1 - 3 are exposed to user, so must use user ASID 7644 * for any access type 7645 */ 7646 if (dcore_id > 0) { 7647 reg_val = (asid << DCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_ASID_RD_SHIFT) | 7648 (asid << DCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_ASID_WR_SHIFT); 7649 WREG32(mmDCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_ASID + dcore_offset, reg_val); 7650 WREG32(mmDCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_MMU_BP + dcore_offset, 0); 7651 } 7652 7653 WREG32(mmDCORE0_MME_CTRL_LO_MME_AXUSER_HB_MMU_BP + dcore_offset, 0); 7654 WREG32(mmDCORE0_MME_CTRL_LO_MME_AXUSER_HB_ASID + dcore_offset, rw_asid); 7655 7656 for (i = 0 ; i < NUM_OF_MME_SBTE_PORTS ; i++) { 7657 ports_offset = i * DCORE_MME_SBTE_OFFSET; 7658 WREG32(mmDCORE0_MME_SBTE0_MSTR_IF_AXUSER_HB_MMU_BP + 7659 dcore_offset + ports_offset, 0); 7660 WREG32(mmDCORE0_MME_SBTE0_MSTR_IF_AXUSER_HB_ASID + 7661 dcore_offset + ports_offset, rw_asid); 7662 } 7663 7664 for (i = 0 ; i < NUM_OF_MME_WB_PORTS ; i++) { 7665 ports_offset = i * DCORE_MME_WB_OFFSET; 7666 WREG32(mmDCORE0_MME_WB0_MSTR_IF_AXUSER_HB_MMU_BP + 7667 dcore_offset + ports_offset, 0); 7668 WREG32(mmDCORE0_MME_WB0_MSTR_IF_AXUSER_HB_ASID + 7669 dcore_offset + ports_offset, rw_asid); 7670 } 7671 7672 WREG32(mmDCORE0_MME_QM_AXUSER_NONSECURED_HB_MMU_BP + dcore_offset, 0); 7673 WREG32(mmDCORE0_MME_QM_AXUSER_NONSECURED_HB_ASID + dcore_offset, rw_asid); 7674 7675 /* 7676 * Decoders 7677 */ 7678 for (vdec_id = 0 ; vdec_id < NUM_OF_DEC_PER_DCORE ; vdec_id++) { 7679 if (prop->decoder_enabled_mask & BIT(dcore_id * NUM_OF_DEC_PER_DCORE + vdec_id)) 7680 gaudi2_mmu_vdec_dcore_prepare(hdev, dcore_id, vdec_id, rw_asid, 0); 7681 } 7682 } 7683 7684 static void gudi2_mmu_vdec_shared_prepare(struct hl_device *hdev, 7685 int shared_vdec_id, u32 rw_asid, u32 rw_mmu_bp) 7686 { 7687 u32 offset = (mmPCIE_VDEC1_BRDG_CTRL_BASE - mmPCIE_VDEC0_BRDG_CTRL_BASE) * shared_vdec_id; 7688 7689 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_MMU_BP + offset, rw_mmu_bp); 7690 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_ASID + offset, rw_asid); 7691 7692 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_MMU_BP + offset, rw_mmu_bp); 7693 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_ASID + offset, rw_asid); 7694 7695 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_MMU_BP + offset, rw_mmu_bp); 7696 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_ASID + offset, rw_asid); 7697 7698 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_MMU_BP + offset, rw_mmu_bp); 7699 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_ASID + offset, rw_asid); 7700 7701 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_MMU_BP + offset, rw_mmu_bp); 7702 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_ASID + offset, rw_asid); 7703 } 7704 7705 static void gudi2_mmu_arc_farm_arc_dup_eng_prepare(struct hl_device *hdev, int arc_farm_id, 7706 u32 rw_asid, u32 rw_mmu_bp) 7707 { 7708 u32 offset = (mmARC_FARM_ARC1_DUP_ENG_BASE - mmARC_FARM_ARC0_DUP_ENG_BASE) * arc_farm_id; 7709 7710 WREG32(mmARC_FARM_ARC0_DUP_ENG_AXUSER_HB_MMU_BP + offset, rw_mmu_bp); 7711 WREG32(mmARC_FARM_ARC0_DUP_ENG_AXUSER_HB_ASID + offset, rw_asid); 7712 } 7713 7714 static void gaudi2_arc_mmu_prepare(struct hl_device *hdev, u32 cpu_id, u32 asid) 7715 { 7716 u32 reg_base, reg_offset, reg_val = 0; 7717 7718 reg_base = gaudi2_arc_blocks_bases[cpu_id]; 7719 7720 /* Enable MMU and configure asid for all relevant ARC regions */ 7721 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_ARC_REGION_CFG_MMU_BP_MASK, 0); 7722 reg_val |= FIELD_PREP(ARC_FARM_ARC0_AUX_ARC_REGION_CFG_0_ASID_MASK, asid); 7723 7724 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION3_GENERAL); 7725 WREG32(reg_base + reg_offset, reg_val); 7726 7727 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION4_HBM0_FW); 7728 WREG32(reg_base + reg_offset, reg_val); 7729 7730 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION5_HBM1_GC_DATA); 7731 WREG32(reg_base + reg_offset, reg_val); 7732 7733 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION6_HBM2_GC_DATA); 7734 WREG32(reg_base + reg_offset, reg_val); 7735 7736 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION7_HBM3_GC_DATA); 7737 WREG32(reg_base + reg_offset, reg_val); 7738 7739 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION9_PCIE); 7740 WREG32(reg_base + reg_offset, reg_val); 7741 7742 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION10_GENERAL); 7743 WREG32(reg_base + reg_offset, reg_val); 7744 7745 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION11_GENERAL); 7746 WREG32(reg_base + reg_offset, reg_val); 7747 7748 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION12_GENERAL); 7749 WREG32(reg_base + reg_offset, reg_val); 7750 7751 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION13_GENERAL); 7752 WREG32(reg_base + reg_offset, reg_val); 7753 7754 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION14_GENERAL); 7755 WREG32(reg_base + reg_offset, reg_val); 7756 } 7757 7758 static int gaudi2_arc_mmu_prepare_all(struct hl_device *hdev, u32 asid) 7759 { 7760 int i; 7761 7762 if (hdev->fw_components & FW_TYPE_BOOT_CPU) 7763 return hl_fw_cpucp_engine_core_asid_set(hdev, asid); 7764 7765 for (i = CPU_ID_SCHED_ARC0 ; i < NUM_OF_ARC_FARMS_ARC ; i++) 7766 gaudi2_arc_mmu_prepare(hdev, i, asid); 7767 7768 for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ ; i += 4) { 7769 if (!gaudi2_is_queue_enabled(hdev, i)) 7770 continue; 7771 7772 gaudi2_arc_mmu_prepare(hdev, gaudi2_queue_id_to_arc_id[i], asid); 7773 } 7774 7775 return 0; 7776 } 7777 7778 static int gaudi2_mmu_shared_prepare(struct hl_device *hdev, u32 asid) 7779 { 7780 struct asic_fixed_properties *prop = &hdev->asic_prop; 7781 u32 rw_asid, offset; 7782 int rc, i; 7783 7784 rw_asid = FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_MASK, asid) | 7785 FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_MASK, asid); 7786 7787 WREG32(mmPDMA0_QM_AXUSER_NONSECURED_HB_ASID, rw_asid); 7788 WREG32(mmPDMA0_QM_AXUSER_NONSECURED_HB_MMU_BP, 0); 7789 WREG32(mmPDMA0_CORE_CTX_AXUSER_HB_ASID, rw_asid); 7790 WREG32(mmPDMA0_CORE_CTX_AXUSER_HB_MMU_BP, 0); 7791 7792 WREG32(mmPDMA1_QM_AXUSER_NONSECURED_HB_ASID, rw_asid); 7793 WREG32(mmPDMA1_QM_AXUSER_NONSECURED_HB_MMU_BP, 0); 7794 WREG32(mmPDMA1_CORE_CTX_AXUSER_HB_ASID, rw_asid); 7795 WREG32(mmPDMA1_CORE_CTX_AXUSER_HB_MMU_BP, 0); 7796 7797 /* ROT */ 7798 for (i = 0 ; i < NUM_OF_ROT ; i++) { 7799 offset = i * ROT_OFFSET; 7800 WREG32(mmROT0_QM_AXUSER_NONSECURED_HB_ASID + offset, rw_asid); 7801 WREG32(mmROT0_QM_AXUSER_NONSECURED_HB_MMU_BP + offset, 0); 7802 RMWREG32(mmROT0_CPL_QUEUE_AWUSER + offset, asid, MMUBP_ASID_MASK); 7803 RMWREG32(mmROT0_DESC_HBW_ARUSER_LO + offset, asid, MMUBP_ASID_MASK); 7804 RMWREG32(mmROT0_DESC_HBW_AWUSER_LO + offset, asid, MMUBP_ASID_MASK); 7805 } 7806 7807 /* Shared Decoders are the last bits in the decoders mask */ 7808 if (prop->decoder_enabled_mask & BIT(NUM_OF_DCORES * NUM_OF_DEC_PER_DCORE + 0)) 7809 gudi2_mmu_vdec_shared_prepare(hdev, 0, rw_asid, 0); 7810 7811 if (prop->decoder_enabled_mask & BIT(NUM_OF_DCORES * NUM_OF_DEC_PER_DCORE + 1)) 7812 gudi2_mmu_vdec_shared_prepare(hdev, 1, rw_asid, 0); 7813 7814 /* arc farm arc dup eng */ 7815 for (i = 0 ; i < NUM_OF_ARC_FARMS_ARC ; i++) 7816 gudi2_mmu_arc_farm_arc_dup_eng_prepare(hdev, i, rw_asid, 0); 7817 7818 rc = gaudi2_arc_mmu_prepare_all(hdev, asid); 7819 if (rc) 7820 return rc; 7821 7822 return 0; 7823 } 7824 7825 static void gaudi2_tpc_mmu_prepare(struct hl_device *hdev, int dcore, int inst, u32 offset, 7826 struct iterate_module_ctx *ctx) 7827 { 7828 struct gaudi2_tpc_mmu_data *mmu_data = ctx->data; 7829 7830 WREG32(mmDCORE0_TPC0_CFG_AXUSER_HB_MMU_BP + offset, 0); 7831 WREG32(mmDCORE0_TPC0_CFG_AXUSER_HB_ASID + offset, mmu_data->rw_asid); 7832 WREG32(mmDCORE0_TPC0_QM_AXUSER_NONSECURED_HB_MMU_BP + offset, 0); 7833 WREG32(mmDCORE0_TPC0_QM_AXUSER_NONSECURED_HB_ASID + offset, mmu_data->rw_asid); 7834 } 7835 7836 /* zero the MMUBP and set the ASID */ 7837 static int gaudi2_mmu_prepare(struct hl_device *hdev, u32 asid) 7838 { 7839 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7840 struct gaudi2_tpc_mmu_data tpc_mmu_data; 7841 struct iterate_module_ctx tpc_iter = { 7842 .fn = &gaudi2_tpc_mmu_prepare, 7843 .data = &tpc_mmu_data, 7844 }; 7845 int rc, i; 7846 7847 if (asid & ~DCORE0_HMMU0_STLB_ASID_ASID_MASK) { 7848 dev_crit(hdev->dev, "asid %u is too big\n", asid); 7849 return -EINVAL; 7850 } 7851 7852 if (!(gaudi2->hw_cap_initialized & HW_CAP_MMU_MASK)) 7853 return 0; 7854 7855 rc = gaudi2_mmu_shared_prepare(hdev, asid); 7856 if (rc) 7857 return rc; 7858 7859 /* configure DCORE MMUs */ 7860 tpc_mmu_data.rw_asid = (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_SHIFT) | 7861 (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_SHIFT); 7862 gaudi2_iterate_tpcs(hdev, &tpc_iter); 7863 for (i = 0 ; i < NUM_OF_DCORES ; i++) 7864 gaudi2_mmu_dcore_prepare(hdev, i, asid); 7865 7866 return 0; 7867 } 7868 7869 static inline bool is_info_event(u32 event) 7870 { 7871 switch (event) { 7872 case GAUDI2_EVENT_CPU_CPLD_SHUTDOWN_CAUSE: 7873 case GAUDI2_EVENT_CPU_FIX_POWER_ENV_S ... GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_E: 7874 case GAUDI2_EVENT_ARC_PWR_BRK_ENTRY ... GAUDI2_EVENT_ARC_PWR_RD_MODE3: 7875 7876 /* return in case of NIC status event - these events are received periodically and not as 7877 * an indication to an error. 7878 */ 7879 case GAUDI2_EVENT_CPU0_STATUS_NIC0_ENG0 ... GAUDI2_EVENT_CPU11_STATUS_NIC11_ENG1: 7880 case GAUDI2_EVENT_ARC_EQ_HEARTBEAT: 7881 return true; 7882 default: 7883 return false; 7884 } 7885 } 7886 7887 static void gaudi2_print_event(struct hl_device *hdev, u16 event_type, 7888 bool ratelimited, const char *fmt, ...) 7889 { 7890 struct va_format vaf; 7891 va_list args; 7892 7893 va_start(args, fmt); 7894 vaf.fmt = fmt; 7895 vaf.va = &args; 7896 7897 if (ratelimited) 7898 dev_err_ratelimited(hdev->dev, "%s: %pV\n", 7899 gaudi2_irq_map_table[event_type].valid ? 7900 gaudi2_irq_map_table[event_type].name : "N/A Event", &vaf); 7901 else 7902 dev_err(hdev->dev, "%s: %pV\n", 7903 gaudi2_irq_map_table[event_type].valid ? 7904 gaudi2_irq_map_table[event_type].name : "N/A Event", &vaf); 7905 7906 va_end(args); 7907 } 7908 7909 static bool gaudi2_handle_ecc_event(struct hl_device *hdev, u16 event_type, 7910 struct hl_eq_ecc_data *ecc_data) 7911 { 7912 u64 ecc_address = 0, ecc_syndrome = 0; 7913 u8 memory_wrapper_idx = 0; 7914 bool has_block_id = false; 7915 u16 block_id; 7916 7917 if (!hl_is_fw_sw_ver_below(hdev, 1, 12)) 7918 has_block_id = true; 7919 7920 ecc_address = le64_to_cpu(ecc_data->ecc_address); 7921 ecc_syndrome = le64_to_cpu(ecc_data->ecc_syndrom); 7922 memory_wrapper_idx = ecc_data->memory_wrapper_idx; 7923 7924 if (has_block_id) { 7925 block_id = le16_to_cpu(ecc_data->block_id); 7926 gaudi2_print_event(hdev, event_type, !ecc_data->is_critical, 7927 "ECC error detected. address: %#llx. Syndrome: %#llx. wrapper id %u. block id %#x. critical %u.", 7928 ecc_address, ecc_syndrome, memory_wrapper_idx, block_id, 7929 ecc_data->is_critical); 7930 } else { 7931 gaudi2_print_event(hdev, event_type, !ecc_data->is_critical, 7932 "ECC error detected. address: %#llx. Syndrome: %#llx. wrapper id %u. critical %u.", 7933 ecc_address, ecc_syndrome, memory_wrapper_idx, ecc_data->is_critical); 7934 } 7935 7936 return !!ecc_data->is_critical; 7937 } 7938 7939 static void handle_lower_qman_data_on_err(struct hl_device *hdev, u64 qman_base, u32 engine_id) 7940 { 7941 struct undefined_opcode_info *undef_opcode = &hdev->captured_err_info.undef_opcode; 7942 u64 cq_ptr, cp_current_inst; 7943 u32 lo, hi, cq_size, cp_sts; 7944 bool is_arc_cq; 7945 7946 cp_sts = RREG32(qman_base + QM_CP_STS_4_OFFSET); 7947 is_arc_cq = FIELD_GET(PDMA0_QM_CP_STS_CUR_CQ_MASK, cp_sts); /* 0 - legacy CQ, 1 - ARC_CQ */ 7948 7949 if (is_arc_cq) { 7950 lo = RREG32(qman_base + QM_ARC_CQ_PTR_LO_STS_OFFSET); 7951 hi = RREG32(qman_base + QM_ARC_CQ_PTR_HI_STS_OFFSET); 7952 cq_ptr = ((u64) hi) << 32 | lo; 7953 cq_size = RREG32(qman_base + QM_ARC_CQ_TSIZE_STS_OFFSET); 7954 } else { 7955 lo = RREG32(qman_base + QM_CQ_PTR_LO_STS_4_OFFSET); 7956 hi = RREG32(qman_base + QM_CQ_PTR_HI_STS_4_OFFSET); 7957 cq_ptr = ((u64) hi) << 32 | lo; 7958 cq_size = RREG32(qman_base + QM_CQ_TSIZE_STS_4_OFFSET); 7959 } 7960 7961 lo = RREG32(qman_base + QM_CP_CURRENT_INST_LO_4_OFFSET); 7962 hi = RREG32(qman_base + QM_CP_CURRENT_INST_HI_4_OFFSET); 7963 cp_current_inst = ((u64) hi) << 32 | lo; 7964 7965 dev_info(hdev->dev, 7966 "LowerQM. %sCQ: {ptr %#llx, size %u}, CP: {instruction %#018llx}\n", 7967 is_arc_cq ? "ARC_" : "", cq_ptr, cq_size, cp_current_inst); 7968 7969 if (undef_opcode->write_enable) { 7970 memset(undef_opcode, 0, sizeof(*undef_opcode)); 7971 undef_opcode->timestamp = ktime_get(); 7972 undef_opcode->cq_addr = cq_ptr; 7973 undef_opcode->cq_size = cq_size; 7974 undef_opcode->engine_id = engine_id; 7975 undef_opcode->stream_id = QMAN_STREAMS; 7976 undef_opcode->write_enable = 0; 7977 } 7978 } 7979 7980 static int gaudi2_handle_qman_err_generic(struct hl_device *hdev, u16 event_type, 7981 u64 qman_base, u32 qid_base, u64 *event_mask) 7982 { 7983 u32 i, j, glbl_sts_val, arb_err_val, num_error_causes, error_count = 0; 7984 u64 glbl_sts_addr, arb_err_addr; 7985 char reg_desc[32]; 7986 7987 glbl_sts_addr = qman_base + (mmDCORE0_TPC0_QM_GLBL_ERR_STS_0 - mmDCORE0_TPC0_QM_BASE); 7988 arb_err_addr = qman_base + (mmDCORE0_TPC0_QM_ARB_ERR_CAUSE - mmDCORE0_TPC0_QM_BASE); 7989 7990 /* Iterate through all stream GLBL_ERR_STS registers + Lower CP */ 7991 for (i = 0 ; i < QMAN_STREAMS + 1 ; i++) { 7992 glbl_sts_val = RREG32(glbl_sts_addr + 4 * i); 7993 7994 if (!glbl_sts_val) 7995 continue; 7996 7997 if (i == QMAN_STREAMS) { 7998 snprintf(reg_desc, ARRAY_SIZE(reg_desc), "LowerQM"); 7999 num_error_causes = GAUDI2_NUM_OF_LOWER_QM_ERR_CAUSE; 8000 } else { 8001 snprintf(reg_desc, ARRAY_SIZE(reg_desc), "stream%u", i); 8002 num_error_causes = GAUDI2_NUM_OF_QM_ERR_CAUSE; 8003 } 8004 8005 for (j = 0 ; j < num_error_causes ; j++) 8006 if (glbl_sts_val & BIT(j)) { 8007 gaudi2_print_event(hdev, event_type, true, 8008 "%s. err cause: %s", reg_desc, 8009 i == QMAN_STREAMS ? 8010 gaudi2_lower_qman_error_cause[j] : 8011 gaudi2_qman_error_cause[j]); 8012 error_count++; 8013 } 8014 8015 /* Check for undefined opcode error in lower QM */ 8016 if ((i == QMAN_STREAMS) && 8017 (glbl_sts_val & PDMA0_QM_GLBL_ERR_STS_CP_UNDEF_CMD_ERR_MASK)) { 8018 handle_lower_qman_data_on_err(hdev, qman_base, 8019 gaudi2_queue_id_to_engine_id[qid_base]); 8020 *event_mask |= HL_NOTIFIER_EVENT_UNDEFINED_OPCODE; 8021 } 8022 } 8023 8024 arb_err_val = RREG32(arb_err_addr); 8025 8026 if (!arb_err_val) 8027 goto out; 8028 8029 for (j = 0 ; j < GAUDI2_NUM_OF_QM_ARB_ERR_CAUSE ; j++) { 8030 if (arb_err_val & BIT(j)) { 8031 gaudi2_print_event(hdev, event_type, true, 8032 "ARB_ERR. err cause: %s", 8033 gaudi2_qman_arb_error_cause[j]); 8034 error_count++; 8035 } 8036 } 8037 8038 out: 8039 return error_count; 8040 } 8041 8042 static void gaudi2_razwi_rr_hbw_shared_printf_info(struct hl_device *hdev, 8043 u64 rtr_mstr_if_base_addr, bool is_write, char *name, 8044 enum gaudi2_engine_id id, u64 *event_mask) 8045 { 8046 u32 razwi_hi, razwi_lo, razwi_xy; 8047 u16 eng_id = id; 8048 u8 rd_wr_flag; 8049 8050 if (is_write) { 8051 razwi_hi = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HI); 8052 razwi_lo = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_LO); 8053 razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_XY); 8054 rd_wr_flag = HL_RAZWI_WRITE; 8055 } else { 8056 razwi_hi = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HI); 8057 razwi_lo = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_LO); 8058 razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_XY); 8059 rd_wr_flag = HL_RAZWI_READ; 8060 } 8061 8062 hl_handle_razwi(hdev, (u64)razwi_hi << 32 | razwi_lo, &eng_id, 1, 8063 rd_wr_flag | HL_RAZWI_HBW, event_mask); 8064 8065 dev_err_ratelimited(hdev->dev, 8066 "%s-RAZWI SHARED RR HBW %s error, address %#llx, Initiator coordinates 0x%x\n", 8067 name, is_write ? "WR" : "RD", (u64)razwi_hi << 32 | razwi_lo, razwi_xy); 8068 } 8069 8070 static void gaudi2_razwi_rr_lbw_shared_printf_info(struct hl_device *hdev, 8071 u64 rtr_mstr_if_base_addr, bool is_write, char *name, 8072 enum gaudi2_engine_id id, u64 *event_mask) 8073 { 8074 u64 razwi_addr = CFG_BASE; 8075 u32 razwi_xy; 8076 u16 eng_id = id; 8077 u8 rd_wr_flag; 8078 8079 if (is_write) { 8080 razwi_addr += RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI); 8081 razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_XY); 8082 rd_wr_flag = HL_RAZWI_WRITE; 8083 } else { 8084 razwi_addr += RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI); 8085 razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_XY); 8086 rd_wr_flag = HL_RAZWI_READ; 8087 } 8088 8089 hl_handle_razwi(hdev, razwi_addr, &eng_id, 1, rd_wr_flag | HL_RAZWI_LBW, event_mask); 8090 dev_err_ratelimited(hdev->dev, 8091 "%s-RAZWI SHARED RR LBW %s error, mstr_if 0x%llx, captured address 0x%llX Initiator coordinates 0x%x\n", 8092 name, is_write ? "WR" : "RD", rtr_mstr_if_base_addr, razwi_addr, 8093 razwi_xy); 8094 } 8095 8096 static enum gaudi2_engine_id gaudi2_razwi_calc_engine_id(struct hl_device *hdev, 8097 enum razwi_event_sources module, u8 module_idx) 8098 { 8099 switch (module) { 8100 case RAZWI_TPC: 8101 if (module_idx == (NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES)) 8102 return GAUDI2_DCORE0_ENGINE_ID_TPC_6; 8103 return (((module_idx / NUM_OF_TPC_PER_DCORE) * ENGINE_ID_DCORE_OFFSET) + 8104 (module_idx % NUM_OF_TPC_PER_DCORE) + 8105 (GAUDI2_DCORE0_ENGINE_ID_TPC_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0)); 8106 8107 case RAZWI_MME: 8108 return ((GAUDI2_DCORE0_ENGINE_ID_MME - GAUDI2_DCORE0_ENGINE_ID_EDMA_0) + 8109 (module_idx * ENGINE_ID_DCORE_OFFSET)); 8110 8111 case RAZWI_EDMA: 8112 return (((module_idx / NUM_OF_EDMA_PER_DCORE) * ENGINE_ID_DCORE_OFFSET) + 8113 (module_idx % NUM_OF_EDMA_PER_DCORE)); 8114 8115 case RAZWI_PDMA: 8116 return (GAUDI2_ENGINE_ID_PDMA_0 + module_idx); 8117 8118 case RAZWI_NIC: 8119 return (GAUDI2_ENGINE_ID_NIC0_0 + (NIC_NUMBER_OF_QM_PER_MACRO * module_idx)); 8120 8121 case RAZWI_DEC: 8122 if (module_idx == 8) 8123 return GAUDI2_PCIE_ENGINE_ID_DEC_0; 8124 8125 if (module_idx == 9) 8126 return GAUDI2_PCIE_ENGINE_ID_DEC_1; 8127 ; 8128 return (((module_idx / NUM_OF_DEC_PER_DCORE) * ENGINE_ID_DCORE_OFFSET) + 8129 (module_idx % NUM_OF_DEC_PER_DCORE) + 8130 (GAUDI2_DCORE0_ENGINE_ID_DEC_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0)); 8131 8132 case RAZWI_ROT: 8133 return GAUDI2_ENGINE_ID_ROT_0 + module_idx; 8134 8135 case RAZWI_ARC_FARM: 8136 return GAUDI2_ENGINE_ID_ARC_FARM; 8137 8138 default: 8139 return GAUDI2_ENGINE_ID_SIZE; 8140 } 8141 } 8142 8143 /* 8144 * This function handles RR(Range register) hit events. 8145 * raised be initiators not PSOC RAZWI. 8146 */ 8147 static void gaudi2_ack_module_razwi_event_handler(struct hl_device *hdev, 8148 enum razwi_event_sources module, u8 module_idx, 8149 u8 module_sub_idx, u64 *event_mask) 8150 { 8151 bool via_sft = false; 8152 u32 hbw_rtr_id, lbw_rtr_id, dcore_id, dcore_rtr_id, eng_id, binned_idx; 8153 u64 hbw_rtr_mstr_if_base_addr, lbw_rtr_mstr_if_base_addr; 8154 u32 hbw_shrd_aw = 0, hbw_shrd_ar = 0; 8155 u32 lbw_shrd_aw = 0, lbw_shrd_ar = 0; 8156 char initiator_name[64]; 8157 8158 switch (module) { 8159 case RAZWI_TPC: 8160 sprintf(initiator_name, "TPC_%u", module_idx); 8161 if (hdev->tpc_binning) { 8162 binned_idx = __ffs(hdev->tpc_binning); 8163 if (binned_idx == module_idx) 8164 module_idx = TPC_ID_DCORE0_TPC6; 8165 } 8166 8167 hbw_rtr_id = gaudi2_tpc_initiator_hbw_rtr_id[module_idx]; 8168 8169 if (hl_is_fw_sw_ver_below(hdev, 1, 9) && 8170 !hdev->asic_prop.fw_security_enabled && 8171 ((module_idx == 0) || (module_idx == 1))) 8172 lbw_rtr_id = DCORE0_RTR0; 8173 else 8174 lbw_rtr_id = gaudi2_tpc_initiator_lbw_rtr_id[module_idx]; 8175 break; 8176 case RAZWI_MME: 8177 sprintf(initiator_name, "MME_%u", module_idx); 8178 switch (module_sub_idx) { 8179 case MME_WAP0: 8180 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].wap0; 8181 break; 8182 case MME_WAP1: 8183 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].wap1; 8184 break; 8185 case MME_WRITE: 8186 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].write; 8187 break; 8188 case MME_READ: 8189 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].read; 8190 break; 8191 case MME_SBTE0: 8192 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte0; 8193 break; 8194 case MME_SBTE1: 8195 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte1; 8196 break; 8197 case MME_SBTE2: 8198 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte2; 8199 break; 8200 case MME_SBTE3: 8201 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte3; 8202 break; 8203 case MME_SBTE4: 8204 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte4; 8205 break; 8206 default: 8207 return; 8208 } 8209 lbw_rtr_id = hbw_rtr_id; 8210 break; 8211 case RAZWI_EDMA: 8212 hbw_rtr_mstr_if_base_addr = gaudi2_edma_initiator_hbw_sft[module_idx]; 8213 dcore_id = module_idx / NUM_OF_EDMA_PER_DCORE; 8214 /* SFT has separate MSTR_IF for LBW, only there we can 8215 * read the LBW razwi related registers 8216 */ 8217 lbw_rtr_mstr_if_base_addr = mmSFT0_LBW_RTR_IF_MSTR_IF_RR_SHRD_HBW_BASE + 8218 dcore_id * SFT_DCORE_OFFSET; 8219 via_sft = true; 8220 sprintf(initiator_name, "EDMA_%u", module_idx); 8221 break; 8222 case RAZWI_PDMA: 8223 hbw_rtr_id = gaudi2_pdma_initiator_hbw_rtr_id[module_idx]; 8224 lbw_rtr_id = gaudi2_pdma_initiator_lbw_rtr_id[module_idx]; 8225 sprintf(initiator_name, "PDMA_%u", module_idx); 8226 break; 8227 case RAZWI_NIC: 8228 hbw_rtr_id = gaudi2_nic_initiator_hbw_rtr_id[module_idx]; 8229 lbw_rtr_id = gaudi2_nic_initiator_lbw_rtr_id[module_idx]; 8230 sprintf(initiator_name, "NIC_%u", module_idx); 8231 break; 8232 case RAZWI_DEC: 8233 sprintf(initiator_name, "DEC_%u", module_idx); 8234 if (hdev->decoder_binning) { 8235 binned_idx = __ffs(hdev->decoder_binning); 8236 if (binned_idx == module_idx) 8237 module_idx = DEC_ID_PCIE_VDEC1; 8238 } 8239 hbw_rtr_id = gaudi2_dec_initiator_hbw_rtr_id[module_idx]; 8240 lbw_rtr_id = gaudi2_dec_initiator_lbw_rtr_id[module_idx]; 8241 break; 8242 case RAZWI_ROT: 8243 hbw_rtr_id = gaudi2_rot_initiator_hbw_rtr_id[module_idx]; 8244 lbw_rtr_id = gaudi2_rot_initiator_lbw_rtr_id[module_idx]; 8245 sprintf(initiator_name, "ROT_%u", module_idx); 8246 break; 8247 case RAZWI_ARC_FARM: 8248 lbw_rtr_id = DCORE1_RTR5; 8249 hbw_rtr_id = DCORE1_RTR7; 8250 sprintf(initiator_name, "ARC_FARM_%u", module_idx); 8251 break; 8252 default: 8253 return; 8254 } 8255 8256 /* Find router mstr_if register base */ 8257 if (!via_sft) { 8258 dcore_id = hbw_rtr_id / NUM_OF_RTR_PER_DCORE; 8259 dcore_rtr_id = hbw_rtr_id % NUM_OF_RTR_PER_DCORE; 8260 hbw_rtr_mstr_if_base_addr = mmDCORE0_RTR0_CTRL_BASE + 8261 dcore_id * DCORE_OFFSET + 8262 dcore_rtr_id * DCORE_RTR_OFFSET + 8263 RTR_MSTR_IF_OFFSET; 8264 lbw_rtr_mstr_if_base_addr = hbw_rtr_mstr_if_base_addr + 8265 (((s32)lbw_rtr_id - hbw_rtr_id) * DCORE_RTR_OFFSET); 8266 } 8267 8268 /* Find out event cause by reading "RAZWI_HAPPENED" registers */ 8269 hbw_shrd_aw = RREG32(hbw_rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HAPPENED); 8270 hbw_shrd_ar = RREG32(hbw_rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HAPPENED); 8271 lbw_shrd_aw = RREG32(lbw_rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_HAPPENED); 8272 lbw_shrd_ar = RREG32(lbw_rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_HAPPENED); 8273 8274 eng_id = gaudi2_razwi_calc_engine_id(hdev, module, module_idx); 8275 if (hbw_shrd_aw) { 8276 gaudi2_razwi_rr_hbw_shared_printf_info(hdev, hbw_rtr_mstr_if_base_addr, true, 8277 initiator_name, eng_id, event_mask); 8278 8279 /* Clear event indication */ 8280 WREG32(hbw_rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HAPPENED, hbw_shrd_aw); 8281 } 8282 8283 if (hbw_shrd_ar) { 8284 gaudi2_razwi_rr_hbw_shared_printf_info(hdev, hbw_rtr_mstr_if_base_addr, false, 8285 initiator_name, eng_id, event_mask); 8286 8287 /* Clear event indication */ 8288 WREG32(hbw_rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HAPPENED, hbw_shrd_ar); 8289 } 8290 8291 if (lbw_shrd_aw) { 8292 gaudi2_razwi_rr_lbw_shared_printf_info(hdev, lbw_rtr_mstr_if_base_addr, true, 8293 initiator_name, eng_id, event_mask); 8294 8295 /* Clear event indication */ 8296 WREG32(lbw_rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_HAPPENED, lbw_shrd_aw); 8297 } 8298 8299 if (lbw_shrd_ar) { 8300 gaudi2_razwi_rr_lbw_shared_printf_info(hdev, lbw_rtr_mstr_if_base_addr, false, 8301 initiator_name, eng_id, event_mask); 8302 8303 /* Clear event indication */ 8304 WREG32(lbw_rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_HAPPENED, lbw_shrd_ar); 8305 } 8306 } 8307 8308 static void gaudi2_check_if_razwi_happened(struct hl_device *hdev) 8309 { 8310 struct asic_fixed_properties *prop = &hdev->asic_prop; 8311 u8 mod_idx, sub_mod; 8312 8313 /* check all TPCs */ 8314 for (mod_idx = 0 ; mod_idx < (NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1) ; mod_idx++) { 8315 if (prop->tpc_enabled_mask & BIT(mod_idx)) 8316 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_TPC, mod_idx, 0, NULL); 8317 } 8318 8319 /* check all MMEs */ 8320 for (mod_idx = 0 ; mod_idx < (NUM_OF_MME_PER_DCORE * NUM_OF_DCORES) ; mod_idx++) 8321 for (sub_mod = MME_WAP0 ; sub_mod < MME_INITIATORS_MAX ; sub_mod++) 8322 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mod_idx, 8323 sub_mod, NULL); 8324 8325 /* check all EDMAs */ 8326 for (mod_idx = 0 ; mod_idx < (NUM_OF_EDMA_PER_DCORE * NUM_OF_DCORES) ; mod_idx++) 8327 if (prop->edma_enabled_mask & BIT(mod_idx)) 8328 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_EDMA, mod_idx, 0, NULL); 8329 8330 /* check all PDMAs */ 8331 for (mod_idx = 0 ; mod_idx < NUM_OF_PDMA ; mod_idx++) 8332 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_PDMA, mod_idx, 0, NULL); 8333 8334 /* check all NICs */ 8335 for (mod_idx = 0 ; mod_idx < NIC_NUMBER_OF_PORTS ; mod_idx++) 8336 if (hdev->nic_ports_mask & BIT(mod_idx)) 8337 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_NIC, mod_idx >> 1, 0, 8338 NULL); 8339 8340 /* check all DECs */ 8341 for (mod_idx = 0 ; mod_idx < NUMBER_OF_DEC ; mod_idx++) 8342 if (prop->decoder_enabled_mask & BIT(mod_idx)) 8343 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_DEC, mod_idx, 0, NULL); 8344 8345 /* check all ROTs */ 8346 for (mod_idx = 0 ; mod_idx < NUM_OF_ROT ; mod_idx++) 8347 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_ROT, mod_idx, 0, NULL); 8348 } 8349 8350 static int gaudi2_psoc_razwi_get_engines(struct gaudi2_razwi_info *razwi_info, u32 array_size, 8351 u32 axuser_xy, u32 *base, u16 *eng_id, 8352 char *eng_name) 8353 { 8354 8355 int i, num_of_eng = 0; 8356 u16 str_size = 0; 8357 8358 for (i = 0 ; i < array_size ; i++) { 8359 if (axuser_xy != razwi_info[i].axuser_xy) 8360 continue; 8361 8362 eng_id[num_of_eng] = razwi_info[i].eng_id; 8363 base[num_of_eng] = razwi_info[i].rtr_ctrl; 8364 if (!num_of_eng) 8365 str_size += scnprintf(eng_name + str_size, 8366 PSOC_RAZWI_ENG_STR_SIZE - str_size, "%s", 8367 razwi_info[i].eng_name); 8368 else 8369 str_size += scnprintf(eng_name + str_size, 8370 PSOC_RAZWI_ENG_STR_SIZE - str_size, " or %s", 8371 razwi_info[i].eng_name); 8372 num_of_eng++; 8373 } 8374 8375 return num_of_eng; 8376 } 8377 8378 static bool gaudi2_handle_psoc_razwi_happened(struct hl_device *hdev, u32 razwi_reg, 8379 u64 *event_mask) 8380 { 8381 u32 axuser_xy = RAZWI_GET_AXUSER_XY(razwi_reg), addr_hi = 0, addr_lo = 0; 8382 u32 base[PSOC_RAZWI_MAX_ENG_PER_RTR]; 8383 u16 num_of_eng, eng_id[PSOC_RAZWI_MAX_ENG_PER_RTR]; 8384 char eng_name_str[PSOC_RAZWI_ENG_STR_SIZE]; 8385 bool razwi_happened = false; 8386 u64 addr; 8387 int i; 8388 8389 num_of_eng = gaudi2_psoc_razwi_get_engines(common_razwi_info, ARRAY_SIZE(common_razwi_info), 8390 axuser_xy, base, eng_id, eng_name_str); 8391 8392 /* If no match for XY coordinates, try to find it in MME razwi table */ 8393 if (!num_of_eng) { 8394 axuser_xy = RAZWI_GET_AXUSER_LOW_XY(razwi_reg); 8395 num_of_eng = gaudi2_psoc_razwi_get_engines(mme_razwi_info, 8396 ARRAY_SIZE(mme_razwi_info), 8397 axuser_xy, base, eng_id, 8398 eng_name_str); 8399 } 8400 8401 for (i = 0 ; i < num_of_eng ; i++) { 8402 if (RREG32(base[i] + DEC_RAZWI_HBW_AW_SET)) { 8403 addr_hi = RREG32(base[i] + DEC_RAZWI_HBW_AW_ADDR_HI); 8404 addr_lo = RREG32(base[i] + DEC_RAZWI_HBW_AW_ADDR_LO); 8405 addr = ((u64)addr_hi << 32) + addr_lo; 8406 if (addr) { 8407 dev_err(hdev->dev, 8408 "PSOC HBW AW RAZWI: %s, address (aligned to 128 byte): 0x%llX\n", 8409 eng_name_str, addr); 8410 hl_handle_razwi(hdev, addr, &eng_id[0], 8411 num_of_eng, HL_RAZWI_HBW | HL_RAZWI_WRITE, event_mask); 8412 razwi_happened = true; 8413 } 8414 } 8415 8416 if (RREG32(base[i] + DEC_RAZWI_HBW_AR_SET)) { 8417 addr_hi = RREG32(base[i] + DEC_RAZWI_HBW_AR_ADDR_HI); 8418 addr_lo = RREG32(base[i] + DEC_RAZWI_HBW_AR_ADDR_LO); 8419 addr = ((u64)addr_hi << 32) + addr_lo; 8420 if (addr) { 8421 dev_err(hdev->dev, 8422 "PSOC HBW AR RAZWI: %s, address (aligned to 128 byte): 0x%llX\n", 8423 eng_name_str, addr); 8424 hl_handle_razwi(hdev, addr, &eng_id[0], 8425 num_of_eng, HL_RAZWI_HBW | HL_RAZWI_READ, event_mask); 8426 razwi_happened = true; 8427 } 8428 } 8429 8430 if (RREG32(base[i] + DEC_RAZWI_LBW_AW_SET)) { 8431 addr_lo = RREG32(base[i] + DEC_RAZWI_LBW_AW_ADDR); 8432 if (addr_lo) { 8433 dev_err(hdev->dev, 8434 "PSOC LBW AW RAZWI: %s, address (aligned to 128 byte): 0x%X\n", 8435 eng_name_str, addr_lo); 8436 hl_handle_razwi(hdev, addr_lo, &eng_id[0], 8437 num_of_eng, HL_RAZWI_LBW | HL_RAZWI_WRITE, event_mask); 8438 razwi_happened = true; 8439 } 8440 } 8441 8442 if (RREG32(base[i] + DEC_RAZWI_LBW_AR_SET)) { 8443 addr_lo = RREG32(base[i] + DEC_RAZWI_LBW_AR_ADDR); 8444 if (addr_lo) { 8445 dev_err(hdev->dev, 8446 "PSOC LBW AR RAZWI: %s, address (aligned to 128 byte): 0x%X\n", 8447 eng_name_str, addr_lo); 8448 hl_handle_razwi(hdev, addr_lo, &eng_id[0], 8449 num_of_eng, HL_RAZWI_LBW | HL_RAZWI_READ, event_mask); 8450 razwi_happened = true; 8451 } 8452 } 8453 /* In common case the loop will break, when there is only one engine id, or 8454 * several engines with the same router. The exceptional case is with psoc razwi 8455 * from EDMA, where it's possible to get axuser id which fits 2 routers (2 8456 * interfaces of sft router). In this case, maybe the first router won't hold info 8457 * and we will need to iterate on the other router. 8458 */ 8459 if (razwi_happened) 8460 break; 8461 } 8462 8463 return razwi_happened; 8464 } 8465 8466 /* PSOC RAZWI interrupt occurs only when trying to access a bad address */ 8467 static int gaudi2_ack_psoc_razwi_event_handler(struct hl_device *hdev, u64 *event_mask) 8468 { 8469 u32 razwi_mask_info, razwi_intr = 0, error_count = 0; 8470 8471 if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX)) { 8472 razwi_intr = RREG32(mmPSOC_GLOBAL_CONF_RAZWI_INTERRUPT); 8473 if (!razwi_intr) 8474 return 0; 8475 } 8476 8477 razwi_mask_info = RREG32(mmPSOC_GLOBAL_CONF_RAZWI_MASK_INFO); 8478 8479 dev_err_ratelimited(hdev->dev, 8480 "PSOC RAZWI interrupt: Mask %d, AR %d, AW %d, AXUSER_L 0x%x AXUSER_H 0x%x\n", 8481 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_MASK_MASK, razwi_mask_info), 8482 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_WAS_AR_MASK, razwi_mask_info), 8483 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_WAS_AW_MASK, razwi_mask_info), 8484 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_AXUSER_L_MASK, razwi_mask_info), 8485 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_AXUSER_H_MASK, razwi_mask_info)); 8486 8487 if (gaudi2_handle_psoc_razwi_happened(hdev, razwi_mask_info, event_mask)) 8488 error_count++; 8489 else 8490 dev_err_ratelimited(hdev->dev, 8491 "PSOC RAZWI interrupt: invalid razwi info (0x%x)\n", 8492 razwi_mask_info); 8493 8494 /* Clear Interrupts only on pldm or if f/w doesn't handle interrupts */ 8495 if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX)) 8496 WREG32(mmPSOC_GLOBAL_CONF_RAZWI_INTERRUPT, razwi_intr); 8497 8498 return error_count; 8499 } 8500 8501 static int _gaudi2_handle_qm_sei_err(struct hl_device *hdev, u64 qman_base, u16 event_type) 8502 { 8503 u32 i, sts_val, sts_clr_val = 0, error_count = 0; 8504 8505 sts_val = RREG32(qman_base + QM_SEI_STATUS_OFFSET); 8506 8507 for (i = 0 ; i < GAUDI2_NUM_OF_QM_SEI_ERR_CAUSE ; i++) { 8508 if (sts_val & BIT(i)) { 8509 gaudi2_print_event(hdev, event_type, true, 8510 "err cause: %s", gaudi2_qm_sei_error_cause[i]); 8511 sts_clr_val |= BIT(i); 8512 error_count++; 8513 } 8514 } 8515 8516 WREG32(qman_base + QM_SEI_STATUS_OFFSET, sts_clr_val); 8517 8518 return error_count; 8519 } 8520 8521 static int gaudi2_handle_qm_sei_err(struct hl_device *hdev, u16 event_type, 8522 bool extended_err_check, u64 *event_mask) 8523 { 8524 enum razwi_event_sources module; 8525 u32 error_count = 0; 8526 u64 qman_base; 8527 u8 index; 8528 8529 switch (event_type) { 8530 case GAUDI2_EVENT_TPC0_AXI_ERR_RSP ... GAUDI2_EVENT_TPC23_AXI_ERR_RSP: 8531 index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP; 8532 qman_base = mmDCORE0_TPC0_QM_BASE + 8533 (index / NUM_OF_TPC_PER_DCORE) * DCORE_OFFSET + 8534 (index % NUM_OF_TPC_PER_DCORE) * DCORE_TPC_OFFSET; 8535 module = RAZWI_TPC; 8536 break; 8537 case GAUDI2_EVENT_TPC24_AXI_ERR_RSP: 8538 qman_base = mmDCORE0_TPC6_QM_BASE; 8539 module = RAZWI_TPC; 8540 break; 8541 case GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE: 8542 case GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE: 8543 case GAUDI2_EVENT_MME2_CTRL_AXI_ERROR_RESPONSE: 8544 case GAUDI2_EVENT_MME3_CTRL_AXI_ERROR_RESPONSE: 8545 index = (event_type - GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE) / 8546 (GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE - 8547 GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE); 8548 qman_base = mmDCORE0_MME_QM_BASE + index * DCORE_OFFSET; 8549 module = RAZWI_MME; 8550 break; 8551 case GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP: 8552 case GAUDI2_EVENT_PDMA_CH1_AXI_ERR_RSP: 8553 index = event_type - GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP; 8554 qman_base = mmPDMA0_QM_BASE + index * PDMA_OFFSET; 8555 module = RAZWI_PDMA; 8556 break; 8557 case GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE: 8558 case GAUDI2_EVENT_ROTATOR1_AXI_ERROR_RESPONSE: 8559 index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE; 8560 qman_base = mmROT0_QM_BASE + index * ROT_OFFSET; 8561 module = RAZWI_ROT; 8562 break; 8563 default: 8564 return 0; 8565 } 8566 8567 error_count = _gaudi2_handle_qm_sei_err(hdev, qman_base, event_type); 8568 8569 /* There is a single event per NIC macro, so should check its both QMAN blocks */ 8570 if (event_type >= GAUDI2_EVENT_NIC0_AXI_ERROR_RESPONSE && 8571 event_type <= GAUDI2_EVENT_NIC11_AXI_ERROR_RESPONSE) 8572 error_count += _gaudi2_handle_qm_sei_err(hdev, 8573 qman_base + NIC_QM_OFFSET, event_type); 8574 8575 if (extended_err_check) { 8576 /* check if RAZWI happened */ 8577 gaudi2_ack_module_razwi_event_handler(hdev, module, 0, 0, event_mask); 8578 hl_check_for_glbl_errors(hdev); 8579 } 8580 8581 return error_count; 8582 } 8583 8584 static int gaudi2_handle_qman_err(struct hl_device *hdev, u16 event_type, u64 *event_mask) 8585 { 8586 u32 qid_base, error_count = 0; 8587 u64 qman_base; 8588 u8 index = 0; 8589 8590 switch (event_type) { 8591 case GAUDI2_EVENT_TPC0_QM ... GAUDI2_EVENT_TPC5_QM: 8592 index = event_type - GAUDI2_EVENT_TPC0_QM; 8593 qid_base = GAUDI2_QUEUE_ID_DCORE0_TPC_0_0 + index * QMAN_STREAMS; 8594 qman_base = mmDCORE0_TPC0_QM_BASE + index * DCORE_TPC_OFFSET; 8595 break; 8596 case GAUDI2_EVENT_TPC6_QM ... GAUDI2_EVENT_TPC11_QM: 8597 index = event_type - GAUDI2_EVENT_TPC6_QM; 8598 qid_base = GAUDI2_QUEUE_ID_DCORE1_TPC_0_0 + index * QMAN_STREAMS; 8599 qman_base = mmDCORE1_TPC0_QM_BASE + index * DCORE_TPC_OFFSET; 8600 break; 8601 case GAUDI2_EVENT_TPC12_QM ... GAUDI2_EVENT_TPC17_QM: 8602 index = event_type - GAUDI2_EVENT_TPC12_QM; 8603 qid_base = GAUDI2_QUEUE_ID_DCORE2_TPC_0_0 + index * QMAN_STREAMS; 8604 qman_base = mmDCORE2_TPC0_QM_BASE + index * DCORE_TPC_OFFSET; 8605 break; 8606 case GAUDI2_EVENT_TPC18_QM ... GAUDI2_EVENT_TPC23_QM: 8607 index = event_type - GAUDI2_EVENT_TPC18_QM; 8608 qid_base = GAUDI2_QUEUE_ID_DCORE3_TPC_0_0 + index * QMAN_STREAMS; 8609 qman_base = mmDCORE3_TPC0_QM_BASE + index * DCORE_TPC_OFFSET; 8610 break; 8611 case GAUDI2_EVENT_TPC24_QM: 8612 qid_base = GAUDI2_QUEUE_ID_DCORE0_TPC_6_0; 8613 qman_base = mmDCORE0_TPC6_QM_BASE; 8614 break; 8615 case GAUDI2_EVENT_MME0_QM: 8616 qid_base = GAUDI2_QUEUE_ID_DCORE0_MME_0_0; 8617 qman_base = mmDCORE0_MME_QM_BASE; 8618 break; 8619 case GAUDI2_EVENT_MME1_QM: 8620 qid_base = GAUDI2_QUEUE_ID_DCORE1_MME_0_0; 8621 qman_base = mmDCORE1_MME_QM_BASE; 8622 break; 8623 case GAUDI2_EVENT_MME2_QM: 8624 qid_base = GAUDI2_QUEUE_ID_DCORE2_MME_0_0; 8625 qman_base = mmDCORE2_MME_QM_BASE; 8626 break; 8627 case GAUDI2_EVENT_MME3_QM: 8628 qid_base = GAUDI2_QUEUE_ID_DCORE3_MME_0_0; 8629 qman_base = mmDCORE3_MME_QM_BASE; 8630 break; 8631 case GAUDI2_EVENT_HDMA0_QM: 8632 index = 0; 8633 qid_base = GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0; 8634 qman_base = mmDCORE0_EDMA0_QM_BASE; 8635 break; 8636 case GAUDI2_EVENT_HDMA1_QM: 8637 index = 1; 8638 qid_base = GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0; 8639 qman_base = mmDCORE0_EDMA1_QM_BASE; 8640 break; 8641 case GAUDI2_EVENT_HDMA2_QM: 8642 index = 2; 8643 qid_base = GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0; 8644 qman_base = mmDCORE1_EDMA0_QM_BASE; 8645 break; 8646 case GAUDI2_EVENT_HDMA3_QM: 8647 index = 3; 8648 qid_base = GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0; 8649 qman_base = mmDCORE1_EDMA1_QM_BASE; 8650 break; 8651 case GAUDI2_EVENT_HDMA4_QM: 8652 index = 4; 8653 qid_base = GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0; 8654 qman_base = mmDCORE2_EDMA0_QM_BASE; 8655 break; 8656 case GAUDI2_EVENT_HDMA5_QM: 8657 index = 5; 8658 qid_base = GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0; 8659 qman_base = mmDCORE2_EDMA1_QM_BASE; 8660 break; 8661 case GAUDI2_EVENT_HDMA6_QM: 8662 index = 6; 8663 qid_base = GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0; 8664 qman_base = mmDCORE3_EDMA0_QM_BASE; 8665 break; 8666 case GAUDI2_EVENT_HDMA7_QM: 8667 index = 7; 8668 qid_base = GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0; 8669 qman_base = mmDCORE3_EDMA1_QM_BASE; 8670 break; 8671 case GAUDI2_EVENT_PDMA0_QM: 8672 qid_base = GAUDI2_QUEUE_ID_PDMA_0_0; 8673 qman_base = mmPDMA0_QM_BASE; 8674 break; 8675 case GAUDI2_EVENT_PDMA1_QM: 8676 qid_base = GAUDI2_QUEUE_ID_PDMA_1_0; 8677 qman_base = mmPDMA1_QM_BASE; 8678 break; 8679 case GAUDI2_EVENT_ROTATOR0_ROT0_QM: 8680 qid_base = GAUDI2_QUEUE_ID_ROT_0_0; 8681 qman_base = mmROT0_QM_BASE; 8682 break; 8683 case GAUDI2_EVENT_ROTATOR1_ROT1_QM: 8684 qid_base = GAUDI2_QUEUE_ID_ROT_1_0; 8685 qman_base = mmROT1_QM_BASE; 8686 break; 8687 default: 8688 return 0; 8689 } 8690 8691 error_count = gaudi2_handle_qman_err_generic(hdev, event_type, qman_base, 8692 qid_base, event_mask); 8693 8694 /* Handle EDMA QM SEI here because there is no AXI error response event for EDMA */ 8695 if (event_type >= GAUDI2_EVENT_HDMA2_QM && event_type <= GAUDI2_EVENT_HDMA5_QM) { 8696 error_count += _gaudi2_handle_qm_sei_err(hdev, qman_base, event_type); 8697 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_EDMA, index, 0, event_mask); 8698 } 8699 8700 hl_check_for_glbl_errors(hdev); 8701 8702 return error_count; 8703 } 8704 8705 static int gaudi2_handle_arc_farm_sei_err(struct hl_device *hdev, u16 event_type, u64 *event_mask) 8706 { 8707 u32 i, sts_val, sts_clr_val, error_count = 0, arc_farm; 8708 8709 for (arc_farm = 0 ; arc_farm < NUM_OF_ARC_FARMS_ARC ; arc_farm++) { 8710 sts_clr_val = 0; 8711 sts_val = RREG32(mmARC_FARM_ARC0_AUX_ARC_SEI_INTR_STS + 8712 (arc_farm * ARC_FARM_OFFSET)); 8713 8714 for (i = 0 ; i < GAUDI2_NUM_OF_ARC_SEI_ERR_CAUSE ; i++) { 8715 if (sts_val & BIT(i)) { 8716 gaudi2_print_event(hdev, event_type, true, 8717 "ARC FARM ARC %u err cause: %s", 8718 arc_farm, gaudi2_arc_sei_error_cause[i]); 8719 sts_clr_val |= BIT(i); 8720 error_count++; 8721 } 8722 } 8723 WREG32(mmARC_FARM_ARC0_AUX_ARC_SEI_INTR_CLR + (arc_farm * ARC_FARM_OFFSET), 8724 sts_clr_val); 8725 } 8726 8727 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_ARC_FARM, 0, 0, event_mask); 8728 hl_check_for_glbl_errors(hdev); 8729 8730 return error_count; 8731 } 8732 8733 static int gaudi2_handle_cpu_sei_err(struct hl_device *hdev, u16 event_type) 8734 { 8735 u32 i, sts_val, sts_clr_val = 0, error_count = 0; 8736 8737 sts_val = RREG32(mmCPU_IF_CPU_SEI_INTR_STS); 8738 8739 for (i = 0 ; i < GAUDI2_NUM_OF_CPU_SEI_ERR_CAUSE ; i++) { 8740 if (sts_val & BIT(i)) { 8741 gaudi2_print_event(hdev, event_type, true, 8742 "err cause: %s", gaudi2_cpu_sei_error_cause[i]); 8743 sts_clr_val |= BIT(i); 8744 error_count++; 8745 } 8746 } 8747 8748 hl_check_for_glbl_errors(hdev); 8749 8750 WREG32(mmCPU_IF_CPU_SEI_INTR_CLR, sts_clr_val); 8751 8752 return error_count; 8753 } 8754 8755 static int gaudi2_handle_rot_err(struct hl_device *hdev, u8 rot_index, u16 event_type, 8756 struct hl_eq_razwi_with_intr_cause *razwi_with_intr_cause, 8757 u64 *event_mask) 8758 { 8759 u64 intr_cause_data = le64_to_cpu(razwi_with_intr_cause->intr_cause.intr_cause_data); 8760 u32 error_count = 0; 8761 int i; 8762 8763 for (i = 0 ; i < GAUDI2_NUM_OF_ROT_ERR_CAUSE ; i++) 8764 if (intr_cause_data & BIT(i)) { 8765 gaudi2_print_event(hdev, event_type, true, 8766 "err cause: %s", guadi2_rot_error_cause[i]); 8767 error_count++; 8768 } 8769 8770 /* check if RAZWI happened */ 8771 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_ROT, rot_index, 0, event_mask); 8772 hl_check_for_glbl_errors(hdev); 8773 8774 return error_count; 8775 } 8776 8777 static int gaudi2_tpc_ack_interrupts(struct hl_device *hdev, u8 tpc_index, u16 event_type, 8778 struct hl_eq_razwi_with_intr_cause *razwi_with_intr_cause, 8779 u64 *event_mask) 8780 { 8781 u64 intr_cause_data = le64_to_cpu(razwi_with_intr_cause->intr_cause.intr_cause_data); 8782 u32 error_count = 0; 8783 int i; 8784 8785 for (i = 0 ; i < GAUDI2_NUM_OF_TPC_INTR_CAUSE ; i++) 8786 if (intr_cause_data & BIT(i)) { 8787 gaudi2_print_event(hdev, event_type, true, 8788 "interrupt cause: %s", gaudi2_tpc_interrupts_cause[i]); 8789 error_count++; 8790 } 8791 8792 /* check if RAZWI happened */ 8793 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_TPC, tpc_index, 0, event_mask); 8794 hl_check_for_glbl_errors(hdev); 8795 8796 return error_count; 8797 } 8798 8799 static int gaudi2_handle_dec_err(struct hl_device *hdev, u8 dec_index, u16 event_type, 8800 u64 *event_mask) 8801 { 8802 u32 sts_addr, sts_val, sts_clr_val = 0, error_count = 0; 8803 int i; 8804 8805 if (dec_index < NUM_OF_VDEC_PER_DCORE * NUM_OF_DCORES) 8806 /* DCORE DEC */ 8807 sts_addr = mmDCORE0_VDEC0_BRDG_CTRL_CAUSE_INTR + 8808 DCORE_OFFSET * (dec_index / NUM_OF_DEC_PER_DCORE) + 8809 DCORE_VDEC_OFFSET * (dec_index % NUM_OF_DEC_PER_DCORE); 8810 else 8811 /* PCIE DEC */ 8812 sts_addr = mmPCIE_VDEC0_BRDG_CTRL_CAUSE_INTR + PCIE_VDEC_OFFSET * 8813 (dec_index - NUM_OF_VDEC_PER_DCORE * NUM_OF_DCORES); 8814 8815 sts_val = RREG32(sts_addr); 8816 8817 for (i = 0 ; i < GAUDI2_NUM_OF_DEC_ERR_CAUSE ; i++) { 8818 if (sts_val & BIT(i)) { 8819 gaudi2_print_event(hdev, event_type, true, 8820 "err cause: %s", gaudi2_dec_error_cause[i]); 8821 sts_clr_val |= BIT(i); 8822 error_count++; 8823 } 8824 } 8825 8826 /* check if RAZWI happened */ 8827 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_DEC, dec_index, 0, event_mask); 8828 hl_check_for_glbl_errors(hdev); 8829 8830 /* Write 1 clear errors */ 8831 WREG32(sts_addr, sts_clr_val); 8832 8833 return error_count; 8834 } 8835 8836 static int gaudi2_handle_mme_err(struct hl_device *hdev, u8 mme_index, u16 event_type, 8837 u64 *event_mask) 8838 { 8839 u32 sts_addr, sts_val, sts_clr_addr, sts_clr_val = 0, error_count = 0; 8840 int i; 8841 8842 sts_addr = mmDCORE0_MME_CTRL_LO_INTR_CAUSE + DCORE_OFFSET * mme_index; 8843 sts_clr_addr = mmDCORE0_MME_CTRL_LO_INTR_CLEAR + DCORE_OFFSET * mme_index; 8844 8845 sts_val = RREG32(sts_addr); 8846 8847 for (i = 0 ; i < GAUDI2_NUM_OF_MME_ERR_CAUSE ; i++) { 8848 if (sts_val & BIT(i)) { 8849 gaudi2_print_event(hdev, event_type, true, 8850 "err cause: %s", guadi2_mme_error_cause[i]); 8851 sts_clr_val |= BIT(i); 8852 error_count++; 8853 } 8854 } 8855 8856 /* check if RAZWI happened */ 8857 for (i = MME_WRITE ; i < MME_INITIATORS_MAX ; i++) 8858 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, i, event_mask); 8859 8860 hl_check_for_glbl_errors(hdev); 8861 8862 WREG32(sts_clr_addr, sts_clr_val); 8863 8864 return error_count; 8865 } 8866 8867 static int gaudi2_handle_mme_sbte_err(struct hl_device *hdev, u16 event_type) 8868 { 8869 /* 8870 * We have a single error cause here but the report mechanism is 8871 * buggy. Hence there is no good reason to fetch the cause so we 8872 * just check for glbl_errors and exit. 8873 */ 8874 hl_check_for_glbl_errors(hdev); 8875 8876 return GAUDI2_NA_EVENT_CAUSE; 8877 } 8878 8879 static int gaudi2_handle_mme_wap_err(struct hl_device *hdev, u8 mme_index, u16 event_type, 8880 u64 *event_mask) 8881 { 8882 u32 sts_addr, sts_val, sts_clr_addr, sts_clr_val = 0, error_count = 0; 8883 int i; 8884 8885 sts_addr = mmDCORE0_MME_ACC_INTR_CAUSE + DCORE_OFFSET * mme_index; 8886 sts_clr_addr = mmDCORE0_MME_ACC_INTR_CLEAR + DCORE_OFFSET * mme_index; 8887 8888 sts_val = RREG32(sts_addr); 8889 8890 for (i = 0 ; i < GAUDI2_NUM_OF_MME_WAP_ERR_CAUSE ; i++) { 8891 if (sts_val & BIT(i)) { 8892 gaudi2_print_event(hdev, event_type, true, 8893 "err cause: %s", guadi2_mme_wap_error_cause[i]); 8894 sts_clr_val |= BIT(i); 8895 error_count++; 8896 } 8897 } 8898 8899 /* check if RAZWI happened on WAP0/1 */ 8900 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, MME_WAP0, event_mask); 8901 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, MME_WAP1, event_mask); 8902 hl_check_for_glbl_errors(hdev); 8903 8904 WREG32(sts_clr_addr, sts_clr_val); 8905 8906 return error_count; 8907 } 8908 8909 static int gaudi2_handle_kdma_core_event(struct hl_device *hdev, u16 event_type, 8910 u64 intr_cause_data) 8911 { 8912 u32 error_count = 0; 8913 int i; 8914 8915 /* If an AXI read or write error is received, an error is reported and 8916 * interrupt message is sent. Due to an HW errata, when reading the cause 8917 * register of the KDMA engine, the reported error is always HBW even if 8918 * the actual error caused by a LBW KDMA transaction. 8919 */ 8920 for (i = 0 ; i < GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE ; i++) 8921 if (intr_cause_data & BIT(i)) { 8922 gaudi2_print_event(hdev, event_type, true, 8923 "err cause: %s", gaudi2_kdma_core_interrupts_cause[i]); 8924 error_count++; 8925 } 8926 8927 hl_check_for_glbl_errors(hdev); 8928 8929 return error_count; 8930 } 8931 8932 static int gaudi2_handle_dma_core_event(struct hl_device *hdev, u16 event_type, u64 intr_cause) 8933 { 8934 u32 error_count = 0; 8935 int i; 8936 8937 for (i = 0 ; i < GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE ; i++) 8938 if (intr_cause & BIT(i)) { 8939 gaudi2_print_event(hdev, event_type, true, 8940 "err cause: %s", gaudi2_dma_core_interrupts_cause[i]); 8941 error_count++; 8942 } 8943 8944 hl_check_for_glbl_errors(hdev); 8945 8946 return error_count; 8947 } 8948 8949 static void gaudi2_print_pcie_mstr_rr_mstr_if_razwi_info(struct hl_device *hdev, u64 *event_mask) 8950 { 8951 u32 mstr_if_base_addr = mmPCIE_MSTR_RR_MSTR_IF_RR_SHRD_HBW_BASE, razwi_happened_addr; 8952 8953 razwi_happened_addr = mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HAPPENED; 8954 if (RREG32(razwi_happened_addr)) { 8955 gaudi2_razwi_rr_hbw_shared_printf_info(hdev, mstr_if_base_addr, true, "PCIE", 8956 GAUDI2_ENGINE_ID_PCIE, event_mask); 8957 WREG32(razwi_happened_addr, 0x1); 8958 } 8959 8960 razwi_happened_addr = mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HAPPENED; 8961 if (RREG32(razwi_happened_addr)) { 8962 gaudi2_razwi_rr_hbw_shared_printf_info(hdev, mstr_if_base_addr, false, "PCIE", 8963 GAUDI2_ENGINE_ID_PCIE, event_mask); 8964 WREG32(razwi_happened_addr, 0x1); 8965 } 8966 8967 razwi_happened_addr = mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_HAPPENED; 8968 if (RREG32(razwi_happened_addr)) { 8969 gaudi2_razwi_rr_lbw_shared_printf_info(hdev, mstr_if_base_addr, true, "PCIE", 8970 GAUDI2_ENGINE_ID_PCIE, event_mask); 8971 WREG32(razwi_happened_addr, 0x1); 8972 } 8973 8974 razwi_happened_addr = mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_HAPPENED; 8975 if (RREG32(razwi_happened_addr)) { 8976 gaudi2_razwi_rr_lbw_shared_printf_info(hdev, mstr_if_base_addr, false, "PCIE", 8977 GAUDI2_ENGINE_ID_PCIE, event_mask); 8978 WREG32(razwi_happened_addr, 0x1); 8979 } 8980 } 8981 8982 static int gaudi2_print_pcie_addr_dec_info(struct hl_device *hdev, u16 event_type, 8983 u64 intr_cause_data, u64 *event_mask) 8984 { 8985 u32 error_count = 0; 8986 int i; 8987 8988 gaudi2_print_event(hdev, event_type, true, 8989 "intr_cause_data: %#llx", intr_cause_data); 8990 8991 for (i = 0 ; i < GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE ; i++) { 8992 if (!(intr_cause_data & BIT_ULL(i))) 8993 continue; 8994 8995 gaudi2_print_event(hdev, event_type, true, 8996 "err cause: %s", gaudi2_pcie_addr_dec_error_cause[i]); 8997 error_count++; 8998 8999 /* 9000 * Always check for LBW and HBW additional info as the indication itself is 9001 * sometimes missing 9002 */ 9003 } 9004 9005 hl_check_for_glbl_errors(hdev); 9006 gaudi2_print_pcie_mstr_rr_mstr_if_razwi_info(hdev, event_mask); 9007 9008 return error_count; 9009 } 9010 9011 static int gaudi2_handle_pif_fatal(struct hl_device *hdev, u16 event_type, 9012 u64 intr_cause_data) 9013 9014 { 9015 u32 error_count = 0; 9016 int i; 9017 9018 for (i = 0 ; i < GAUDI2_NUM_OF_PMMU_FATAL_ERR_CAUSE ; i++) { 9019 if (intr_cause_data & BIT_ULL(i)) { 9020 gaudi2_print_event(hdev, event_type, true, 9021 "err cause: %s", gaudi2_pmmu_fatal_interrupts_cause[i]); 9022 error_count++; 9023 } 9024 } 9025 9026 return error_count; 9027 } 9028 9029 static int gaudi2_handle_hif_fatal(struct hl_device *hdev, u16 event_type, u64 intr_cause_data) 9030 { 9031 u32 error_count = 0; 9032 int i; 9033 9034 for (i = 0 ; i < GAUDI2_NUM_OF_HIF_FATAL_ERR_CAUSE ; i++) { 9035 if (intr_cause_data & BIT_ULL(i)) { 9036 gaudi2_print_event(hdev, event_type, true, 9037 "err cause: %s", gaudi2_hif_fatal_interrupts_cause[i]); 9038 error_count++; 9039 } 9040 } 9041 9042 return error_count; 9043 } 9044 9045 static void gaudi2_handle_page_error(struct hl_device *hdev, u64 mmu_base, bool is_pmmu, 9046 u64 *event_mask) 9047 { 9048 u32 valid, val; 9049 u64 addr; 9050 9051 valid = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID)); 9052 9053 if (!(valid & DCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID_PAGE_ERR_VALID_ENTRY_MASK)) 9054 return; 9055 9056 val = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_PAGE_ERROR_CAPTURE)); 9057 addr = val & DCORE0_HMMU0_MMU_PAGE_ERROR_CAPTURE_VA_63_32_MASK; 9058 addr <<= 32; 9059 addr |= RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_PAGE_ERROR_CAPTURE_VA)); 9060 9061 if (is_pmmu) { 9062 dev_err_ratelimited(hdev->dev, "PMMU page fault on va 0x%llx\n", addr); 9063 } else { 9064 addr = gaudi2_mmu_descramble_addr(hdev, addr); 9065 addr &= HW_UNSCRAMBLED_BITS_MASK; 9066 dev_err_ratelimited(hdev->dev, "HMMU page fault on va range 0x%llx - 0x%llx\n", 9067 addr, addr + ~HW_UNSCRAMBLED_BITS_MASK); 9068 } 9069 9070 hl_handle_page_fault(hdev, addr, 0, is_pmmu, event_mask); 9071 9072 WREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID), 0); 9073 } 9074 9075 static void gaudi2_handle_access_error(struct hl_device *hdev, u64 mmu_base, bool is_pmmu) 9076 { 9077 u32 valid, val; 9078 u64 addr; 9079 9080 valid = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID)); 9081 9082 if (!(valid & DCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID_ACCESS_ERR_VALID_ENTRY_MASK)) 9083 return; 9084 9085 val = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_ERROR_CAPTURE)); 9086 addr = val & DCORE0_HMMU0_MMU_ACCESS_ERROR_CAPTURE_VA_63_32_MASK; 9087 addr <<= 32; 9088 addr |= RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_ERROR_CAPTURE_VA)); 9089 9090 if (!is_pmmu) 9091 addr = gaudi2_mmu_descramble_addr(hdev, addr); 9092 9093 dev_err_ratelimited(hdev->dev, "%s access error on va 0x%llx\n", 9094 is_pmmu ? "PMMU" : "HMMU", addr); 9095 WREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID), 0); 9096 } 9097 9098 static int gaudi2_handle_mmu_spi_sei_generic(struct hl_device *hdev, u16 event_type, 9099 u64 mmu_base, bool is_pmmu, u64 *event_mask) 9100 { 9101 u32 spi_sei_cause, interrupt_clr = 0x0, error_count = 0; 9102 int i; 9103 9104 spi_sei_cause = RREG32(mmu_base + MMU_SPI_SEI_CAUSE_OFFSET); 9105 9106 for (i = 0 ; i < GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE ; i++) { 9107 if (spi_sei_cause & BIT(i)) { 9108 gaudi2_print_event(hdev, event_type, true, 9109 "err cause: %s", gaudi2_mmu_spi_sei[i].cause); 9110 9111 if (i == 0) 9112 gaudi2_handle_page_error(hdev, mmu_base, is_pmmu, event_mask); 9113 else if (i == 1) 9114 gaudi2_handle_access_error(hdev, mmu_base, is_pmmu); 9115 9116 if (gaudi2_mmu_spi_sei[i].clear_bit >= 0) 9117 interrupt_clr |= BIT(gaudi2_mmu_spi_sei[i].clear_bit); 9118 9119 error_count++; 9120 } 9121 } 9122 9123 /* Clear cause */ 9124 WREG32_AND(mmu_base + MMU_SPI_SEI_CAUSE_OFFSET, ~spi_sei_cause); 9125 9126 /* Clear interrupt */ 9127 WREG32(mmu_base + MMU_INTERRUPT_CLR_OFFSET, interrupt_clr); 9128 9129 return error_count; 9130 } 9131 9132 static int gaudi2_handle_sm_err(struct hl_device *hdev, u16 event_type, u8 sm_index) 9133 { 9134 u32 sei_cause_addr, sei_cause_val, sei_cause_cause, sei_cause_log, 9135 cq_intr_addr, cq_intr_val, cq_intr_queue_index, error_count = 0; 9136 int i; 9137 9138 sei_cause_addr = mmDCORE0_SYNC_MNGR_GLBL_SM_SEI_CAUSE + DCORE_OFFSET * sm_index; 9139 cq_intr_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_INTR + DCORE_OFFSET * sm_index; 9140 9141 sei_cause_val = RREG32(sei_cause_addr); 9142 sei_cause_cause = FIELD_GET(DCORE0_SYNC_MNGR_GLBL_SM_SEI_CAUSE_CAUSE_MASK, sei_cause_val); 9143 cq_intr_val = RREG32(cq_intr_addr); 9144 9145 /* SEI interrupt */ 9146 if (sei_cause_cause) { 9147 /* There are corresponding SEI_CAUSE_log bits for every SEI_CAUSE_cause bit */ 9148 sei_cause_log = FIELD_GET(DCORE0_SYNC_MNGR_GLBL_SM_SEI_CAUSE_LOG_MASK, 9149 sei_cause_val); 9150 9151 for (i = 0 ; i < GAUDI2_NUM_OF_SM_SEI_ERR_CAUSE ; i++) { 9152 if (!(sei_cause_cause & BIT(i))) 9153 continue; 9154 9155 gaudi2_print_event(hdev, event_type, true, 9156 "err cause: %s. %s: 0x%X", 9157 gaudi2_sm_sei_cause[i].cause_name, 9158 gaudi2_sm_sei_cause[i].log_name, 9159 sei_cause_log); 9160 error_count++; 9161 break; 9162 } 9163 9164 /* Clear SM_SEI_CAUSE */ 9165 WREG32(sei_cause_addr, 0); 9166 } 9167 9168 /* CQ interrupt */ 9169 if (cq_intr_val & DCORE0_SYNC_MNGR_GLBL_CQ_INTR_CQ_SEC_INTR_MASK) { 9170 cq_intr_queue_index = 9171 FIELD_GET(DCORE0_SYNC_MNGR_GLBL_CQ_INTR_CQ_INTR_QUEUE_INDEX_MASK, 9172 cq_intr_val); 9173 9174 dev_err_ratelimited(hdev->dev, "SM%u err. err cause: CQ_INTR. queue index: %u\n", 9175 sm_index, cq_intr_queue_index); 9176 error_count++; 9177 9178 /* Clear CQ_INTR */ 9179 WREG32(cq_intr_addr, 0); 9180 } 9181 9182 hl_check_for_glbl_errors(hdev); 9183 9184 return error_count; 9185 } 9186 9187 static u64 get_hmmu_base(u16 event_type) 9188 { 9189 u8 dcore, index_in_dcore; 9190 9191 switch (event_type) { 9192 case GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP: 9193 case GAUDI2_EVENT_HMMU0_SPI_BASE ... GAUDI2_EVENT_HMMU0_SECURITY_ERROR: 9194 dcore = 0; 9195 index_in_dcore = 0; 9196 break; 9197 case GAUDI2_EVENT_HMMU_1_AXI_ERR_RSP: 9198 case GAUDI2_EVENT_HMMU1_SPI_BASE ... GAUDI2_EVENT_HMMU1_SECURITY_ERROR: 9199 dcore = 1; 9200 index_in_dcore = 0; 9201 break; 9202 case GAUDI2_EVENT_HMMU_2_AXI_ERR_RSP: 9203 case GAUDI2_EVENT_HMMU2_SPI_BASE ... GAUDI2_EVENT_HMMU2_SECURITY_ERROR: 9204 dcore = 0; 9205 index_in_dcore = 1; 9206 break; 9207 case GAUDI2_EVENT_HMMU_3_AXI_ERR_RSP: 9208 case GAUDI2_EVENT_HMMU3_SPI_BASE ... GAUDI2_EVENT_HMMU3_SECURITY_ERROR: 9209 dcore = 1; 9210 index_in_dcore = 1; 9211 break; 9212 case GAUDI2_EVENT_HMMU_4_AXI_ERR_RSP: 9213 case GAUDI2_EVENT_HMMU4_SPI_BASE ... GAUDI2_EVENT_HMMU4_SECURITY_ERROR: 9214 dcore = 3; 9215 index_in_dcore = 2; 9216 break; 9217 case GAUDI2_EVENT_HMMU_5_AXI_ERR_RSP: 9218 case GAUDI2_EVENT_HMMU5_SPI_BASE ... GAUDI2_EVENT_HMMU5_SECURITY_ERROR: 9219 dcore = 2; 9220 index_in_dcore = 2; 9221 break; 9222 case GAUDI2_EVENT_HMMU_6_AXI_ERR_RSP: 9223 case GAUDI2_EVENT_HMMU6_SPI_BASE ... GAUDI2_EVENT_HMMU6_SECURITY_ERROR: 9224 dcore = 3; 9225 index_in_dcore = 3; 9226 break; 9227 case GAUDI2_EVENT_HMMU_7_AXI_ERR_RSP: 9228 case GAUDI2_EVENT_HMMU7_SPI_BASE ... GAUDI2_EVENT_HMMU7_SECURITY_ERROR: 9229 dcore = 2; 9230 index_in_dcore = 3; 9231 break; 9232 case GAUDI2_EVENT_HMMU_8_AXI_ERR_RSP: 9233 case GAUDI2_EVENT_HMMU8_SPI_BASE ... GAUDI2_EVENT_HMMU8_SECURITY_ERROR: 9234 dcore = 0; 9235 index_in_dcore = 2; 9236 break; 9237 case GAUDI2_EVENT_HMMU_9_AXI_ERR_RSP: 9238 case GAUDI2_EVENT_HMMU9_SPI_BASE ... GAUDI2_EVENT_HMMU9_SECURITY_ERROR: 9239 dcore = 1; 9240 index_in_dcore = 2; 9241 break; 9242 case GAUDI2_EVENT_HMMU_10_AXI_ERR_RSP: 9243 case GAUDI2_EVENT_HMMU10_SPI_BASE ... GAUDI2_EVENT_HMMU10_SECURITY_ERROR: 9244 dcore = 0; 9245 index_in_dcore = 3; 9246 break; 9247 case GAUDI2_EVENT_HMMU_11_AXI_ERR_RSP: 9248 case GAUDI2_EVENT_HMMU11_SPI_BASE ... GAUDI2_EVENT_HMMU11_SECURITY_ERROR: 9249 dcore = 1; 9250 index_in_dcore = 3; 9251 break; 9252 case GAUDI2_EVENT_HMMU_12_AXI_ERR_RSP: 9253 case GAUDI2_EVENT_HMMU12_SPI_BASE ... GAUDI2_EVENT_HMMU12_SECURITY_ERROR: 9254 dcore = 3; 9255 index_in_dcore = 0; 9256 break; 9257 case GAUDI2_EVENT_HMMU_13_AXI_ERR_RSP: 9258 case GAUDI2_EVENT_HMMU13_SPI_BASE ... GAUDI2_EVENT_HMMU13_SECURITY_ERROR: 9259 dcore = 2; 9260 index_in_dcore = 0; 9261 break; 9262 case GAUDI2_EVENT_HMMU_14_AXI_ERR_RSP: 9263 case GAUDI2_EVENT_HMMU14_SPI_BASE ... GAUDI2_EVENT_HMMU14_SECURITY_ERROR: 9264 dcore = 3; 9265 index_in_dcore = 1; 9266 break; 9267 case GAUDI2_EVENT_HMMU_15_AXI_ERR_RSP: 9268 case GAUDI2_EVENT_HMMU15_SPI_BASE ... GAUDI2_EVENT_HMMU15_SECURITY_ERROR: 9269 dcore = 2; 9270 index_in_dcore = 1; 9271 break; 9272 default: 9273 return ULONG_MAX; 9274 } 9275 9276 return mmDCORE0_HMMU0_MMU_BASE + dcore * DCORE_OFFSET + index_in_dcore * DCORE_HMMU_OFFSET; 9277 } 9278 9279 static int gaudi2_handle_mmu_spi_sei_err(struct hl_device *hdev, u16 event_type, u64 *event_mask) 9280 { 9281 bool is_pmmu = false; 9282 u32 error_count = 0; 9283 u64 mmu_base; 9284 9285 switch (event_type) { 9286 case GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP ... GAUDI2_EVENT_HMMU_12_AXI_ERR_RSP: 9287 case GAUDI2_EVENT_HMMU0_SPI_BASE ... GAUDI2_EVENT_HMMU12_SECURITY_ERROR: 9288 mmu_base = get_hmmu_base(event_type); 9289 break; 9290 9291 case GAUDI2_EVENT_PMMU0_PAGE_FAULT_WR_PERM ... GAUDI2_EVENT_PMMU0_SECURITY_ERROR: 9292 case GAUDI2_EVENT_PMMU_AXI_ERR_RSP_0: 9293 is_pmmu = true; 9294 mmu_base = mmPMMU_HBW_MMU_BASE; 9295 break; 9296 default: 9297 return 0; 9298 } 9299 9300 if (mmu_base == ULONG_MAX) 9301 return 0; 9302 9303 error_count = gaudi2_handle_mmu_spi_sei_generic(hdev, event_type, mmu_base, 9304 is_pmmu, event_mask); 9305 hl_check_for_glbl_errors(hdev); 9306 9307 return error_count; 9308 } 9309 9310 9311 /* returns true if hard reset is required (ECC DERR or Read parity), false otherwise (ECC SERR) */ 9312 static bool gaudi2_hbm_sei_handle_read_err(struct hl_device *hdev, 9313 struct hl_eq_hbm_sei_read_err_intr_info *rd_err_data, u32 err_cnt) 9314 { 9315 u32 addr, beat, beat_shift; 9316 bool rc = false; 9317 9318 dev_err_ratelimited(hdev->dev, 9319 "READ ERROR count: ECC SERR: %d, ECC DERR: %d, RD_PARITY: %d\n", 9320 FIELD_GET(HBM_ECC_SERR_CNTR_MASK, err_cnt), 9321 FIELD_GET(HBM_ECC_DERR_CNTR_MASK, err_cnt), 9322 FIELD_GET(HBM_RD_PARITY_CNTR_MASK, err_cnt)); 9323 9324 addr = le32_to_cpu(rd_err_data->dbg_rd_err_addr.rd_addr_val); 9325 dev_err_ratelimited(hdev->dev, 9326 "READ ERROR address: sid(%u), bg(%u), ba(%u), col(%u), row(%u)\n", 9327 FIELD_GET(HBM_RD_ADDR_SID_MASK, addr), 9328 FIELD_GET(HBM_RD_ADDR_BG_MASK, addr), 9329 FIELD_GET(HBM_RD_ADDR_BA_MASK, addr), 9330 FIELD_GET(HBM_RD_ADDR_COL_MASK, addr), 9331 FIELD_GET(HBM_RD_ADDR_ROW_MASK, addr)); 9332 9333 /* For each beat (RDQS edge), look for possible errors and print relevant info */ 9334 for (beat = 0 ; beat < 4 ; beat++) { 9335 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & 9336 (HBM_RD_ERR_SERR_BEAT0_MASK << beat)) 9337 dev_err_ratelimited(hdev->dev, "Beat%d ECC SERR: DM: %#x, Syndrome: %#x\n", 9338 beat, 9339 le32_to_cpu(rd_err_data->dbg_rd_err_dm), 9340 le32_to_cpu(rd_err_data->dbg_rd_err_syndrome)); 9341 9342 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & 9343 (HBM_RD_ERR_DERR_BEAT0_MASK << beat)) { 9344 dev_err_ratelimited(hdev->dev, "Beat%d ECC DERR: DM: %#x, Syndrome: %#x\n", 9345 beat, 9346 le32_to_cpu(rd_err_data->dbg_rd_err_dm), 9347 le32_to_cpu(rd_err_data->dbg_rd_err_syndrome)); 9348 rc |= true; 9349 } 9350 9351 beat_shift = beat * HBM_RD_ERR_BEAT_SHIFT; 9352 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & 9353 (HBM_RD_ERR_PAR_ERR_BEAT0_MASK << beat_shift)) { 9354 dev_err_ratelimited(hdev->dev, 9355 "Beat%d read PARITY: DM: %#x, PAR data: %#x\n", 9356 beat, 9357 le32_to_cpu(rd_err_data->dbg_rd_err_dm), 9358 (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & 9359 (HBM_RD_ERR_PAR_DATA_BEAT0_MASK << beat_shift)) >> 9360 (HBM_RD_ERR_PAR_DATA_BEAT0_SHIFT + beat_shift)); 9361 rc |= true; 9362 } 9363 9364 dev_err_ratelimited(hdev->dev, "Beat%d DQ data:\n", beat); 9365 dev_err_ratelimited(hdev->dev, "\t0x%08x\n", 9366 le32_to_cpu(rd_err_data->dbg_rd_err_data[beat * 2])); 9367 dev_err_ratelimited(hdev->dev, "\t0x%08x\n", 9368 le32_to_cpu(rd_err_data->dbg_rd_err_data[beat * 2 + 1])); 9369 } 9370 9371 return rc; 9372 } 9373 9374 static void gaudi2_hbm_sei_print_wr_par_info(struct hl_device *hdev, 9375 struct hl_eq_hbm_sei_wr_par_intr_info *wr_par_err_data, u32 err_cnt) 9376 { 9377 struct hbm_sei_wr_cmd_address *wr_cmd_addr = wr_par_err_data->dbg_last_wr_cmds; 9378 u32 i, curr_addr, derr = wr_par_err_data->dbg_derr; 9379 9380 dev_err_ratelimited(hdev->dev, "WRITE PARITY ERROR count: %d\n", err_cnt); 9381 9382 dev_err_ratelimited(hdev->dev, "CK-0 DERR: 0x%02x, CK-1 DERR: 0x%02x\n", 9383 derr & 0x3, derr & 0xc); 9384 9385 /* JIRA H6-3286 - the following prints may not be valid */ 9386 dev_err_ratelimited(hdev->dev, "Last latched write commands addresses:\n"); 9387 for (i = 0 ; i < HBM_WR_PAR_CMD_LIFO_LEN ; i++) { 9388 curr_addr = le32_to_cpu(wr_cmd_addr[i].dbg_wr_cmd_addr); 9389 dev_err_ratelimited(hdev->dev, 9390 "\twrite cmd[%u]: Address: SID(%u) BG(%u) BA(%u) COL(%u).\n", 9391 i, 9392 FIELD_GET(WR_PAR_LAST_CMD_SID_MASK, curr_addr), 9393 FIELD_GET(WR_PAR_LAST_CMD_BG_MASK, curr_addr), 9394 FIELD_GET(WR_PAR_LAST_CMD_BA_MASK, curr_addr), 9395 FIELD_GET(WR_PAR_LAST_CMD_COL_MASK, curr_addr)); 9396 } 9397 } 9398 9399 static void gaudi2_hbm_sei_print_ca_par_info(struct hl_device *hdev, 9400 struct hl_eq_hbm_sei_ca_par_intr_info *ca_par_err_data, u32 err_cnt) 9401 { 9402 __le32 *col_cmd = ca_par_err_data->dbg_col; 9403 __le16 *row_cmd = ca_par_err_data->dbg_row; 9404 u32 i; 9405 9406 dev_err_ratelimited(hdev->dev, "CA ERROR count: %d\n", err_cnt); 9407 9408 dev_err_ratelimited(hdev->dev, "Last latched C&R bus commands:\n"); 9409 for (i = 0 ; i < HBM_CA_ERR_CMD_LIFO_LEN ; i++) 9410 dev_err_ratelimited(hdev->dev, "cmd%u: ROW(0x%04x) COL(0x%05x)\n", i, 9411 le16_to_cpu(row_cmd[i]) & (u16)GENMASK(13, 0), 9412 le32_to_cpu(col_cmd[i]) & (u32)GENMASK(17, 0)); 9413 } 9414 9415 /* Returns true if hard reset is needed or false otherwise */ 9416 static bool gaudi2_handle_hbm_mc_sei_err(struct hl_device *hdev, u16 event_type, 9417 struct hl_eq_hbm_sei_data *sei_data) 9418 { 9419 bool require_hard_reset = false; 9420 u32 hbm_id, mc_id, cause_idx; 9421 9422 hbm_id = (event_type - GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE) / 4; 9423 mc_id = ((event_type - GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE) / 2) % 2; 9424 9425 cause_idx = sei_data->hdr.sei_cause; 9426 if (cause_idx > GAUDI2_NUM_OF_HBM_SEI_CAUSE - 1) { 9427 gaudi2_print_event(hdev, event_type, true, 9428 "err cause: %s", 9429 "Invalid HBM SEI event cause (%d) provided by FW", cause_idx); 9430 return true; 9431 } 9432 9433 gaudi2_print_event(hdev, event_type, !sei_data->hdr.is_critical, 9434 "System %s Error Interrupt - HBM(%u) MC(%u) MC_CH(%u) MC_PC(%u). Error cause: %s", 9435 sei_data->hdr.is_critical ? "Critical" : "Non-critical", 9436 hbm_id, mc_id, sei_data->hdr.mc_channel, sei_data->hdr.mc_pseudo_channel, 9437 hbm_mc_sei_cause[cause_idx]); 9438 9439 /* Print error-specific info */ 9440 switch (cause_idx) { 9441 case HBM_SEI_CATTRIP: 9442 require_hard_reset = true; 9443 break; 9444 9445 case HBM_SEI_CMD_PARITY_EVEN: 9446 gaudi2_hbm_sei_print_ca_par_info(hdev, &sei_data->ca_parity_even_info, 9447 le32_to_cpu(sei_data->hdr.cnt)); 9448 require_hard_reset = true; 9449 break; 9450 9451 case HBM_SEI_CMD_PARITY_ODD: 9452 gaudi2_hbm_sei_print_ca_par_info(hdev, &sei_data->ca_parity_odd_info, 9453 le32_to_cpu(sei_data->hdr.cnt)); 9454 require_hard_reset = true; 9455 break; 9456 9457 case HBM_SEI_WRITE_DATA_PARITY_ERR: 9458 gaudi2_hbm_sei_print_wr_par_info(hdev, &sei_data->wr_parity_info, 9459 le32_to_cpu(sei_data->hdr.cnt)); 9460 require_hard_reset = true; 9461 break; 9462 9463 case HBM_SEI_READ_ERR: 9464 /* Unlike other SEI events, read error requires further processing of the 9465 * raw data in order to determine the root cause. 9466 */ 9467 require_hard_reset = gaudi2_hbm_sei_handle_read_err(hdev, 9468 &sei_data->read_err_info, 9469 le32_to_cpu(sei_data->hdr.cnt)); 9470 break; 9471 9472 default: 9473 break; 9474 } 9475 9476 require_hard_reset |= !!sei_data->hdr.is_critical; 9477 9478 return require_hard_reset; 9479 } 9480 9481 static int gaudi2_handle_hbm_cattrip(struct hl_device *hdev, u16 event_type, 9482 u64 intr_cause_data) 9483 { 9484 if (intr_cause_data) { 9485 gaudi2_print_event(hdev, event_type, true, 9486 "temperature error cause: %#llx", intr_cause_data); 9487 return 1; 9488 } 9489 9490 return 0; 9491 } 9492 9493 static int gaudi2_handle_hbm_mc_spi(struct hl_device *hdev, u64 intr_cause_data) 9494 { 9495 u32 i, error_count = 0; 9496 9497 for (i = 0 ; i < GAUDI2_NUM_OF_HBM_MC_SPI_CAUSE ; i++) 9498 if (intr_cause_data & hbm_mc_spi[i].mask) { 9499 dev_dbg(hdev->dev, "HBM spi event: notification cause(%s)\n", 9500 hbm_mc_spi[i].cause); 9501 error_count++; 9502 } 9503 9504 return error_count; 9505 } 9506 9507 static void gaudi2_print_clk_change_info(struct hl_device *hdev, u16 event_type, u64 *event_mask) 9508 { 9509 ktime_t zero_time = ktime_set(0, 0); 9510 9511 mutex_lock(&hdev->clk_throttling.lock); 9512 9513 switch (event_type) { 9514 case GAUDI2_EVENT_CPU_FIX_POWER_ENV_S: 9515 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_POWER; 9516 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_POWER; 9517 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].start = ktime_get(); 9518 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = zero_time; 9519 dev_dbg_ratelimited(hdev->dev, "Clock throttling due to power consumption\n"); 9520 break; 9521 9522 case GAUDI2_EVENT_CPU_FIX_POWER_ENV_E: 9523 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_POWER; 9524 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = ktime_get(); 9525 dev_dbg_ratelimited(hdev->dev, "Power envelop is safe, back to optimal clock\n"); 9526 break; 9527 9528 case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_S: 9529 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_THERMAL; 9530 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_THERMAL; 9531 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].start = ktime_get(); 9532 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = zero_time; 9533 *event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9534 dev_info_ratelimited(hdev->dev, "Clock throttling due to overheating\n"); 9535 break; 9536 9537 case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_E: 9538 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_THERMAL; 9539 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = ktime_get(); 9540 *event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9541 dev_info_ratelimited(hdev->dev, "Thermal envelop is safe, back to optimal clock\n"); 9542 break; 9543 9544 default: 9545 dev_err(hdev->dev, "Received invalid clock change event %d\n", event_type); 9546 break; 9547 } 9548 9549 mutex_unlock(&hdev->clk_throttling.lock); 9550 } 9551 9552 static void gaudi2_print_out_of_sync_info(struct hl_device *hdev, u16 event_type, 9553 struct cpucp_pkt_sync_err *sync_err) 9554 { 9555 struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ]; 9556 9557 gaudi2_print_event(hdev, event_type, false, 9558 "FW: pi=%u, ci=%u, LKD: pi=%u, ci=%d", 9559 le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), 9560 q->pi, atomic_read(&q->ci)); 9561 } 9562 9563 static int gaudi2_handle_pcie_p2p_msix(struct hl_device *hdev, u16 event_type) 9564 { 9565 u32 p2p_intr, msix_gw_intr, error_count = 0; 9566 9567 p2p_intr = RREG32(mmPCIE_WRAP_P2P_INTR); 9568 msix_gw_intr = RREG32(mmPCIE_WRAP_MSIX_GW_INTR); 9569 9570 if (p2p_intr) { 9571 gaudi2_print_event(hdev, event_type, true, 9572 "pcie p2p transaction terminated due to security, req_id(0x%x)", 9573 RREG32(mmPCIE_WRAP_P2P_REQ_ID)); 9574 9575 WREG32(mmPCIE_WRAP_P2P_INTR, 0x1); 9576 error_count++; 9577 } 9578 9579 if (msix_gw_intr) { 9580 gaudi2_print_event(hdev, event_type, true, 9581 "pcie msi-x gen denied due to vector num check failure, vec(0x%X)", 9582 RREG32(mmPCIE_WRAP_MSIX_GW_VEC)); 9583 9584 WREG32(mmPCIE_WRAP_MSIX_GW_INTR, 0x1); 9585 error_count++; 9586 } 9587 9588 return error_count; 9589 } 9590 9591 static int gaudi2_handle_pcie_drain(struct hl_device *hdev, 9592 struct hl_eq_pcie_drain_ind_data *drain_data) 9593 { 9594 u64 lbw_rd, lbw_wr, hbw_rd, hbw_wr, cause, error_count = 0; 9595 9596 cause = le64_to_cpu(drain_data->intr_cause.intr_cause_data); 9597 lbw_rd = le64_to_cpu(drain_data->drain_rd_addr_lbw); 9598 lbw_wr = le64_to_cpu(drain_data->drain_wr_addr_lbw); 9599 hbw_rd = le64_to_cpu(drain_data->drain_rd_addr_hbw); 9600 hbw_wr = le64_to_cpu(drain_data->drain_wr_addr_hbw); 9601 9602 if (cause & BIT_ULL(0)) { 9603 dev_err_ratelimited(hdev->dev, 9604 "PCIE AXI drain LBW completed, read_err %u, write_err %u\n", 9605 !!lbw_rd, !!lbw_wr); 9606 error_count++; 9607 } 9608 9609 if (cause & BIT_ULL(1)) { 9610 dev_err_ratelimited(hdev->dev, 9611 "PCIE AXI drain HBW completed, raddr %#llx, waddr %#llx\n", 9612 hbw_rd, hbw_wr); 9613 error_count++; 9614 } 9615 9616 return error_count; 9617 } 9618 9619 static int gaudi2_handle_psoc_drain(struct hl_device *hdev, u64 intr_cause_data) 9620 { 9621 u32 error_count = 0; 9622 int i; 9623 9624 for (i = 0 ; i < GAUDI2_NUM_OF_AXI_DRAIN_ERR_CAUSE ; i++) { 9625 if (intr_cause_data & BIT_ULL(i)) { 9626 dev_err_ratelimited(hdev->dev, "PSOC %s completed\n", 9627 gaudi2_psoc_axi_drain_interrupts_cause[i]); 9628 error_count++; 9629 } 9630 } 9631 9632 hl_check_for_glbl_errors(hdev); 9633 9634 return error_count; 9635 } 9636 9637 static void gaudi2_print_cpu_pkt_failure_info(struct hl_device *hdev, u16 event_type, 9638 struct cpucp_pkt_sync_err *sync_err) 9639 { 9640 struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ]; 9641 9642 gaudi2_print_event(hdev, event_type, false, 9643 "FW reported sanity check failure, FW: pi=%u, ci=%u, LKD: pi=%u, ci=%d", 9644 le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), q->pi, atomic_read(&q->ci)); 9645 } 9646 9647 static int hl_arc_event_handle(struct hl_device *hdev, u16 event_type, 9648 struct hl_eq_engine_arc_intr_data *data) 9649 { 9650 struct hl_engine_arc_dccm_queue_full_irq *q; 9651 u32 intr_type, engine_id; 9652 u64 payload; 9653 9654 intr_type = le32_to_cpu(data->intr_type); 9655 engine_id = le32_to_cpu(data->engine_id); 9656 payload = le64_to_cpu(data->payload); 9657 9658 switch (intr_type) { 9659 case ENGINE_ARC_DCCM_QUEUE_FULL_IRQ: 9660 q = (struct hl_engine_arc_dccm_queue_full_irq *) &payload; 9661 9662 gaudi2_print_event(hdev, event_type, true, 9663 "ARC DCCM Full event: EngId: %u, Intr_type: %u, Qidx: %u", 9664 engine_id, intr_type, q->queue_index); 9665 return 1; 9666 default: 9667 gaudi2_print_event(hdev, event_type, true, "Unknown ARC event type"); 9668 return 0; 9669 } 9670 } 9671 9672 static u16 event_id_to_engine_id(struct hl_device *hdev, u16 event_type) 9673 { 9674 enum gaudi2_block_types type = GAUDI2_BLOCK_TYPE_MAX; 9675 u16 index; 9676 9677 switch (event_type) { 9678 case GAUDI2_EVENT_TPC0_AXI_ERR_RSP ... GAUDI2_EVENT_TPC24_AXI_ERR_RSP: 9679 index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP; 9680 type = GAUDI2_BLOCK_TYPE_TPC; 9681 break; 9682 case GAUDI2_EVENT_TPC0_QM ... GAUDI2_EVENT_TPC24_QM: 9683 index = event_type - GAUDI2_EVENT_TPC0_QM; 9684 type = GAUDI2_BLOCK_TYPE_TPC; 9685 break; 9686 case GAUDI2_EVENT_MME0_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE: 9687 case GAUDI2_EVENT_MME0_SPI_BASE ... GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID: 9688 case GAUDI2_EVENT_MME0_QM: 9689 index = 0; 9690 type = GAUDI2_BLOCK_TYPE_MME; 9691 break; 9692 case GAUDI2_EVENT_MME1_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE: 9693 case GAUDI2_EVENT_MME1_SPI_BASE ... GAUDI2_EVENT_MME1_WAP_SOURCE_RESULT_INVALID: 9694 case GAUDI2_EVENT_MME1_QM: 9695 index = 1; 9696 type = GAUDI2_BLOCK_TYPE_MME; 9697 break; 9698 case GAUDI2_EVENT_MME2_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME2_CTRL_AXI_ERROR_RESPONSE: 9699 case GAUDI2_EVENT_MME2_SPI_BASE ... GAUDI2_EVENT_MME2_WAP_SOURCE_RESULT_INVALID: 9700 case GAUDI2_EVENT_MME2_QM: 9701 index = 2; 9702 type = GAUDI2_BLOCK_TYPE_MME; 9703 break; 9704 case GAUDI2_EVENT_MME3_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME3_CTRL_AXI_ERROR_RESPONSE: 9705 case GAUDI2_EVENT_MME3_SPI_BASE ... GAUDI2_EVENT_MME3_WAP_SOURCE_RESULT_INVALID: 9706 case GAUDI2_EVENT_MME3_QM: 9707 index = 3; 9708 type = GAUDI2_BLOCK_TYPE_MME; 9709 break; 9710 case GAUDI2_EVENT_KDMA_CH0_AXI_ERR_RSP: 9711 case GAUDI2_EVENT_KDMA_BM_SPMU: 9712 case GAUDI2_EVENT_KDMA0_CORE: 9713 return GAUDI2_ENGINE_ID_KDMA; 9714 case GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP: 9715 case GAUDI2_EVENT_PDMA0_CORE: 9716 case GAUDI2_EVENT_PDMA0_BM_SPMU: 9717 case GAUDI2_EVENT_PDMA0_QM: 9718 return GAUDI2_ENGINE_ID_PDMA_0; 9719 case GAUDI2_EVENT_PDMA_CH1_AXI_ERR_RSP: 9720 case GAUDI2_EVENT_PDMA1_CORE: 9721 case GAUDI2_EVENT_PDMA1_BM_SPMU: 9722 case GAUDI2_EVENT_PDMA1_QM: 9723 return GAUDI2_ENGINE_ID_PDMA_1; 9724 case GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE ... GAUDI2_EVENT_DEC9_AXI_ERR_RSPONSE: 9725 index = event_type - GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE; 9726 type = GAUDI2_BLOCK_TYPE_DEC; 9727 break; 9728 case GAUDI2_EVENT_DEC0_SPI ... GAUDI2_EVENT_DEC9_BMON_SPMU: 9729 index = (event_type - GAUDI2_EVENT_DEC0_SPI) >> 1; 9730 type = GAUDI2_BLOCK_TYPE_DEC; 9731 break; 9732 case GAUDI2_EVENT_NIC0_AXI_ERROR_RESPONSE ... GAUDI2_EVENT_NIC11_AXI_ERROR_RESPONSE: 9733 index = event_type - GAUDI2_EVENT_NIC0_AXI_ERROR_RESPONSE; 9734 return GAUDI2_ENGINE_ID_NIC0_0 + (index * 2); 9735 case GAUDI2_EVENT_NIC0_QM0 ... GAUDI2_EVENT_NIC11_QM1: 9736 index = event_type - GAUDI2_EVENT_NIC0_QM0; 9737 return GAUDI2_ENGINE_ID_NIC0_0 + index; 9738 case GAUDI2_EVENT_NIC0_BMON_SPMU ... GAUDI2_EVENT_NIC11_SW_ERROR: 9739 index = event_type - GAUDI2_EVENT_NIC0_BMON_SPMU; 9740 return GAUDI2_ENGINE_ID_NIC0_0 + (index * 2); 9741 case GAUDI2_EVENT_TPC0_BMON_SPMU ... GAUDI2_EVENT_TPC24_KERNEL_ERR: 9742 index = (event_type - GAUDI2_EVENT_TPC0_BMON_SPMU) >> 1; 9743 type = GAUDI2_BLOCK_TYPE_TPC; 9744 break; 9745 case GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE: 9746 case GAUDI2_EVENT_ROTATOR0_BMON_SPMU: 9747 case GAUDI2_EVENT_ROTATOR0_ROT0_QM: 9748 return GAUDI2_ENGINE_ID_ROT_0; 9749 case GAUDI2_EVENT_ROTATOR1_AXI_ERROR_RESPONSE: 9750 case GAUDI2_EVENT_ROTATOR1_BMON_SPMU: 9751 case GAUDI2_EVENT_ROTATOR1_ROT1_QM: 9752 return GAUDI2_ENGINE_ID_ROT_1; 9753 case GAUDI2_EVENT_HDMA0_BM_SPMU: 9754 case GAUDI2_EVENT_HDMA0_QM: 9755 case GAUDI2_EVENT_HDMA0_CORE: 9756 return GAUDI2_DCORE0_ENGINE_ID_EDMA_0; 9757 case GAUDI2_EVENT_HDMA1_BM_SPMU: 9758 case GAUDI2_EVENT_HDMA1_QM: 9759 case GAUDI2_EVENT_HDMA1_CORE: 9760 return GAUDI2_DCORE0_ENGINE_ID_EDMA_1; 9761 case GAUDI2_EVENT_HDMA2_BM_SPMU: 9762 case GAUDI2_EVENT_HDMA2_QM: 9763 case GAUDI2_EVENT_HDMA2_CORE: 9764 return GAUDI2_DCORE1_ENGINE_ID_EDMA_0; 9765 case GAUDI2_EVENT_HDMA3_BM_SPMU: 9766 case GAUDI2_EVENT_HDMA3_QM: 9767 case GAUDI2_EVENT_HDMA3_CORE: 9768 return GAUDI2_DCORE1_ENGINE_ID_EDMA_1; 9769 case GAUDI2_EVENT_HDMA4_BM_SPMU: 9770 case GAUDI2_EVENT_HDMA4_QM: 9771 case GAUDI2_EVENT_HDMA4_CORE: 9772 return GAUDI2_DCORE2_ENGINE_ID_EDMA_0; 9773 case GAUDI2_EVENT_HDMA5_BM_SPMU: 9774 case GAUDI2_EVENT_HDMA5_QM: 9775 case GAUDI2_EVENT_HDMA5_CORE: 9776 return GAUDI2_DCORE2_ENGINE_ID_EDMA_1; 9777 case GAUDI2_EVENT_HDMA6_BM_SPMU: 9778 case GAUDI2_EVENT_HDMA6_QM: 9779 case GAUDI2_EVENT_HDMA6_CORE: 9780 return GAUDI2_DCORE3_ENGINE_ID_EDMA_0; 9781 case GAUDI2_EVENT_HDMA7_BM_SPMU: 9782 case GAUDI2_EVENT_HDMA7_QM: 9783 case GAUDI2_EVENT_HDMA7_CORE: 9784 return GAUDI2_DCORE3_ENGINE_ID_EDMA_1; 9785 default: 9786 break; 9787 } 9788 9789 switch (type) { 9790 case GAUDI2_BLOCK_TYPE_TPC: 9791 switch (index) { 9792 case TPC_ID_DCORE0_TPC0 ... TPC_ID_DCORE0_TPC5: 9793 return GAUDI2_DCORE0_ENGINE_ID_TPC_0 + index; 9794 case TPC_ID_DCORE1_TPC0 ... TPC_ID_DCORE1_TPC5: 9795 return GAUDI2_DCORE1_ENGINE_ID_TPC_0 + index - TPC_ID_DCORE1_TPC0; 9796 case TPC_ID_DCORE2_TPC0 ... TPC_ID_DCORE2_TPC5: 9797 return GAUDI2_DCORE2_ENGINE_ID_TPC_0 + index - TPC_ID_DCORE2_TPC0; 9798 case TPC_ID_DCORE3_TPC0 ... TPC_ID_DCORE3_TPC5: 9799 return GAUDI2_DCORE3_ENGINE_ID_TPC_0 + index - TPC_ID_DCORE3_TPC0; 9800 default: 9801 break; 9802 } 9803 break; 9804 case GAUDI2_BLOCK_TYPE_MME: 9805 switch (index) { 9806 case MME_ID_DCORE0: return GAUDI2_DCORE0_ENGINE_ID_MME; 9807 case MME_ID_DCORE1: return GAUDI2_DCORE1_ENGINE_ID_MME; 9808 case MME_ID_DCORE2: return GAUDI2_DCORE2_ENGINE_ID_MME; 9809 case MME_ID_DCORE3: return GAUDI2_DCORE3_ENGINE_ID_MME; 9810 default: 9811 break; 9812 } 9813 break; 9814 case GAUDI2_BLOCK_TYPE_DEC: 9815 switch (index) { 9816 case DEC_ID_DCORE0_DEC0: return GAUDI2_DCORE0_ENGINE_ID_DEC_0; 9817 case DEC_ID_DCORE0_DEC1: return GAUDI2_DCORE0_ENGINE_ID_DEC_1; 9818 case DEC_ID_DCORE1_DEC0: return GAUDI2_DCORE1_ENGINE_ID_DEC_0; 9819 case DEC_ID_DCORE1_DEC1: return GAUDI2_DCORE1_ENGINE_ID_DEC_1; 9820 case DEC_ID_DCORE2_DEC0: return GAUDI2_DCORE2_ENGINE_ID_DEC_0; 9821 case DEC_ID_DCORE2_DEC1: return GAUDI2_DCORE2_ENGINE_ID_DEC_1; 9822 case DEC_ID_DCORE3_DEC0: return GAUDI2_DCORE3_ENGINE_ID_DEC_0; 9823 case DEC_ID_DCORE3_DEC1: return GAUDI2_DCORE3_ENGINE_ID_DEC_1; 9824 case DEC_ID_PCIE_VDEC0: return GAUDI2_PCIE_ENGINE_ID_DEC_0; 9825 case DEC_ID_PCIE_VDEC1: return GAUDI2_PCIE_ENGINE_ID_DEC_1; 9826 default: 9827 break; 9828 } 9829 break; 9830 default: 9831 break; 9832 } 9833 9834 return U16_MAX; 9835 } 9836 9837 static void hl_eq_heartbeat_event_handle(struct hl_device *hdev) 9838 { 9839 hdev->eq_heartbeat_received = true; 9840 } 9841 9842 static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry) 9843 { 9844 struct gaudi2_device *gaudi2 = hdev->asic_specific; 9845 bool reset_required = false, is_critical = false; 9846 u32 index, ctl, reset_flags = 0, error_count = 0; 9847 u64 event_mask = 0; 9848 u16 event_type; 9849 9850 ctl = le32_to_cpu(eq_entry->hdr.ctl); 9851 event_type = ((ctl & EQ_CTL_EVENT_TYPE_MASK) >> EQ_CTL_EVENT_TYPE_SHIFT); 9852 9853 if (event_type >= GAUDI2_EVENT_SIZE) { 9854 dev_err(hdev->dev, "Event type %u exceeds maximum of %u", 9855 event_type, GAUDI2_EVENT_SIZE - 1); 9856 return; 9857 } 9858 9859 gaudi2->events_stat[event_type]++; 9860 gaudi2->events_stat_aggregate[event_type]++; 9861 9862 switch (event_type) { 9863 case GAUDI2_EVENT_PCIE_CORE_SERR ... GAUDI2_EVENT_ARC0_ECC_DERR: 9864 fallthrough; 9865 case GAUDI2_EVENT_ROTATOR0_SERR ... GAUDI2_EVENT_ROTATOR1_DERR: 9866 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9867 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9868 reset_required = gaudi2_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data); 9869 is_critical = eq_entry->ecc_data.is_critical; 9870 error_count++; 9871 break; 9872 9873 case GAUDI2_EVENT_TPC0_QM ... GAUDI2_EVENT_PDMA1_QM: 9874 fallthrough; 9875 case GAUDI2_EVENT_ROTATOR0_ROT0_QM ... GAUDI2_EVENT_ROTATOR1_ROT1_QM: 9876 fallthrough; 9877 case GAUDI2_EVENT_NIC0_QM0 ... GAUDI2_EVENT_NIC11_QM1: 9878 error_count = gaudi2_handle_qman_err(hdev, event_type, &event_mask); 9879 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9880 break; 9881 9882 case GAUDI2_EVENT_ARC_AXI_ERROR_RESPONSE_0: 9883 error_count = gaudi2_handle_arc_farm_sei_err(hdev, event_type, &event_mask); 9884 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9885 break; 9886 9887 case GAUDI2_EVENT_CPU_AXI_ERR_RSP: 9888 error_count = gaudi2_handle_cpu_sei_err(hdev, event_type); 9889 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9890 event_mask |= HL_NOTIFIER_EVENT_CRITICL_FW_ERR; 9891 break; 9892 9893 case GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP: 9894 case GAUDI2_EVENT_PDMA_CH1_AXI_ERR_RSP: 9895 error_count = gaudi2_handle_qm_sei_err(hdev, event_type, true, &event_mask); 9896 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9897 break; 9898 9899 case GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE: 9900 case GAUDI2_EVENT_ROTATOR1_AXI_ERROR_RESPONSE: 9901 index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE; 9902 error_count = gaudi2_handle_rot_err(hdev, index, event_type, 9903 &eq_entry->razwi_with_intr_cause, &event_mask); 9904 error_count += gaudi2_handle_qm_sei_err(hdev, event_type, false, &event_mask); 9905 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9906 break; 9907 9908 case GAUDI2_EVENT_TPC0_AXI_ERR_RSP ... GAUDI2_EVENT_TPC24_AXI_ERR_RSP: 9909 index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP; 9910 error_count = gaudi2_tpc_ack_interrupts(hdev, index, event_type, 9911 &eq_entry->razwi_with_intr_cause, &event_mask); 9912 error_count += gaudi2_handle_qm_sei_err(hdev, event_type, false, &event_mask); 9913 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9914 break; 9915 9916 case GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE ... GAUDI2_EVENT_DEC9_AXI_ERR_RSPONSE: 9917 index = event_type - GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE; 9918 error_count = gaudi2_handle_dec_err(hdev, index, event_type, &event_mask); 9919 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9920 break; 9921 9922 case GAUDI2_EVENT_TPC0_KERNEL_ERR: 9923 case GAUDI2_EVENT_TPC1_KERNEL_ERR: 9924 case GAUDI2_EVENT_TPC2_KERNEL_ERR: 9925 case GAUDI2_EVENT_TPC3_KERNEL_ERR: 9926 case GAUDI2_EVENT_TPC4_KERNEL_ERR: 9927 case GAUDI2_EVENT_TPC5_KERNEL_ERR: 9928 case GAUDI2_EVENT_TPC6_KERNEL_ERR: 9929 case GAUDI2_EVENT_TPC7_KERNEL_ERR: 9930 case GAUDI2_EVENT_TPC8_KERNEL_ERR: 9931 case GAUDI2_EVENT_TPC9_KERNEL_ERR: 9932 case GAUDI2_EVENT_TPC10_KERNEL_ERR: 9933 case GAUDI2_EVENT_TPC11_KERNEL_ERR: 9934 case GAUDI2_EVENT_TPC12_KERNEL_ERR: 9935 case GAUDI2_EVENT_TPC13_KERNEL_ERR: 9936 case GAUDI2_EVENT_TPC14_KERNEL_ERR: 9937 case GAUDI2_EVENT_TPC15_KERNEL_ERR: 9938 case GAUDI2_EVENT_TPC16_KERNEL_ERR: 9939 case GAUDI2_EVENT_TPC17_KERNEL_ERR: 9940 case GAUDI2_EVENT_TPC18_KERNEL_ERR: 9941 case GAUDI2_EVENT_TPC19_KERNEL_ERR: 9942 case GAUDI2_EVENT_TPC20_KERNEL_ERR: 9943 case GAUDI2_EVENT_TPC21_KERNEL_ERR: 9944 case GAUDI2_EVENT_TPC22_KERNEL_ERR: 9945 case GAUDI2_EVENT_TPC23_KERNEL_ERR: 9946 case GAUDI2_EVENT_TPC24_KERNEL_ERR: 9947 index = (event_type - GAUDI2_EVENT_TPC0_KERNEL_ERR) / 9948 (GAUDI2_EVENT_TPC1_KERNEL_ERR - GAUDI2_EVENT_TPC0_KERNEL_ERR); 9949 error_count = gaudi2_tpc_ack_interrupts(hdev, index, event_type, 9950 &eq_entry->razwi_with_intr_cause, &event_mask); 9951 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9952 break; 9953 9954 case GAUDI2_EVENT_DEC0_SPI: 9955 case GAUDI2_EVENT_DEC1_SPI: 9956 case GAUDI2_EVENT_DEC2_SPI: 9957 case GAUDI2_EVENT_DEC3_SPI: 9958 case GAUDI2_EVENT_DEC4_SPI: 9959 case GAUDI2_EVENT_DEC5_SPI: 9960 case GAUDI2_EVENT_DEC6_SPI: 9961 case GAUDI2_EVENT_DEC7_SPI: 9962 case GAUDI2_EVENT_DEC8_SPI: 9963 case GAUDI2_EVENT_DEC9_SPI: 9964 index = (event_type - GAUDI2_EVENT_DEC0_SPI) / 9965 (GAUDI2_EVENT_DEC1_SPI - GAUDI2_EVENT_DEC0_SPI); 9966 error_count = gaudi2_handle_dec_err(hdev, index, event_type, &event_mask); 9967 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9968 break; 9969 9970 case GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE: 9971 case GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE: 9972 case GAUDI2_EVENT_MME2_CTRL_AXI_ERROR_RESPONSE: 9973 case GAUDI2_EVENT_MME3_CTRL_AXI_ERROR_RESPONSE: 9974 index = (event_type - GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE) / 9975 (GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE - 9976 GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE); 9977 error_count = gaudi2_handle_mme_err(hdev, index, event_type, &event_mask); 9978 error_count += gaudi2_handle_qm_sei_err(hdev, event_type, false, &event_mask); 9979 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9980 break; 9981 9982 case GAUDI2_EVENT_MME0_QMAN_SW_ERROR: 9983 case GAUDI2_EVENT_MME1_QMAN_SW_ERROR: 9984 case GAUDI2_EVENT_MME2_QMAN_SW_ERROR: 9985 case GAUDI2_EVENT_MME3_QMAN_SW_ERROR: 9986 index = (event_type - GAUDI2_EVENT_MME0_QMAN_SW_ERROR) / 9987 (GAUDI2_EVENT_MME1_QMAN_SW_ERROR - 9988 GAUDI2_EVENT_MME0_QMAN_SW_ERROR); 9989 error_count = gaudi2_handle_mme_err(hdev, index, event_type, &event_mask); 9990 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9991 break; 9992 9993 case GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID: 9994 case GAUDI2_EVENT_MME1_WAP_SOURCE_RESULT_INVALID: 9995 case GAUDI2_EVENT_MME2_WAP_SOURCE_RESULT_INVALID: 9996 case GAUDI2_EVENT_MME3_WAP_SOURCE_RESULT_INVALID: 9997 index = (event_type - GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID) / 9998 (GAUDI2_EVENT_MME1_WAP_SOURCE_RESULT_INVALID - 9999 GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID); 10000 error_count = gaudi2_handle_mme_wap_err(hdev, index, event_type, &event_mask); 10001 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10002 break; 10003 10004 case GAUDI2_EVENT_KDMA_CH0_AXI_ERR_RSP: 10005 case GAUDI2_EVENT_KDMA0_CORE: 10006 error_count = gaudi2_handle_kdma_core_event(hdev, event_type, 10007 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 10008 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10009 break; 10010 10011 case GAUDI2_EVENT_HDMA2_CORE ... GAUDI2_EVENT_HDMA5_CORE: 10012 error_count = gaudi2_handle_dma_core_event(hdev, event_type, 10013 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 10014 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10015 break; 10016 10017 case GAUDI2_EVENT_PDMA0_CORE ... GAUDI2_EVENT_PDMA1_CORE: 10018 error_count = gaudi2_handle_dma_core_event(hdev, event_type, 10019 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 10020 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10021 break; 10022 10023 case GAUDI2_EVENT_PCIE_ADDR_DEC_ERR: 10024 error_count = gaudi2_print_pcie_addr_dec_info(hdev, event_type, 10025 le64_to_cpu(eq_entry->intr_cause.intr_cause_data), &event_mask); 10026 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10027 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10028 break; 10029 10030 case GAUDI2_EVENT_HMMU0_PAGE_FAULT_OR_WR_PERM ... GAUDI2_EVENT_HMMU12_SECURITY_ERROR: 10031 case GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP ... GAUDI2_EVENT_HMMU_12_AXI_ERR_RSP: 10032 case GAUDI2_EVENT_PMMU0_PAGE_FAULT_WR_PERM ... GAUDI2_EVENT_PMMU0_SECURITY_ERROR: 10033 case GAUDI2_EVENT_PMMU_AXI_ERR_RSP_0: 10034 error_count = gaudi2_handle_mmu_spi_sei_err(hdev, event_type, &event_mask); 10035 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10036 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10037 break; 10038 10039 case GAUDI2_EVENT_HIF0_FATAL ... GAUDI2_EVENT_HIF12_FATAL: 10040 error_count = gaudi2_handle_hif_fatal(hdev, event_type, 10041 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 10042 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10043 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10044 break; 10045 10046 case GAUDI2_EVENT_PMMU_FATAL_0: 10047 error_count = gaudi2_handle_pif_fatal(hdev, event_type, 10048 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 10049 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10050 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10051 break; 10052 10053 case GAUDI2_EVENT_PSOC63_RAZWI_OR_PID_MIN_MAX_INTERRUPT: 10054 error_count = gaudi2_ack_psoc_razwi_event_handler(hdev, &event_mask); 10055 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10056 break; 10057 10058 case GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE ... GAUDI2_EVENT_HBM5_MC1_SEI_NON_SEVERE: 10059 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10060 if (gaudi2_handle_hbm_mc_sei_err(hdev, event_type, &eq_entry->sei_data)) { 10061 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10062 reset_required = true; 10063 } 10064 error_count++; 10065 break; 10066 10067 case GAUDI2_EVENT_HBM_CATTRIP_0 ... GAUDI2_EVENT_HBM_CATTRIP_5: 10068 error_count = gaudi2_handle_hbm_cattrip(hdev, event_type, 10069 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 10070 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10071 break; 10072 10073 case GAUDI2_EVENT_HBM0_MC0_SPI ... GAUDI2_EVENT_HBM5_MC1_SPI: 10074 error_count = gaudi2_handle_hbm_mc_spi(hdev, 10075 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 10076 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10077 break; 10078 10079 case GAUDI2_EVENT_PCIE_DRAIN_COMPLETE: 10080 error_count = gaudi2_handle_pcie_drain(hdev, &eq_entry->pcie_drain_ind_data); 10081 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10082 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10083 if (hl_is_fw_sw_ver_equal_or_greater(hdev, 1, 13)) 10084 is_critical = true; 10085 break; 10086 10087 case GAUDI2_EVENT_PSOC59_RPM_ERROR_OR_DRAIN: 10088 error_count = gaudi2_handle_psoc_drain(hdev, 10089 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 10090 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10091 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10092 break; 10093 10094 case GAUDI2_EVENT_CPU_AXI_ECC: 10095 error_count = GAUDI2_NA_EVENT_CAUSE; 10096 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10097 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10098 break; 10099 case GAUDI2_EVENT_CPU_L2_RAM_ECC: 10100 error_count = GAUDI2_NA_EVENT_CAUSE; 10101 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10102 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10103 break; 10104 case GAUDI2_EVENT_MME0_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME0_SBTE4_AXI_ERR_RSP: 10105 case GAUDI2_EVENT_MME1_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME1_SBTE4_AXI_ERR_RSP: 10106 case GAUDI2_EVENT_MME2_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME2_SBTE4_AXI_ERR_RSP: 10107 case GAUDI2_EVENT_MME3_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME3_SBTE4_AXI_ERR_RSP: 10108 error_count = gaudi2_handle_mme_sbte_err(hdev, event_type); 10109 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10110 break; 10111 case GAUDI2_EVENT_VM0_ALARM_A ... GAUDI2_EVENT_VM3_ALARM_B: 10112 error_count = GAUDI2_NA_EVENT_CAUSE; 10113 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10114 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10115 break; 10116 case GAUDI2_EVENT_PSOC_AXI_ERR_RSP: 10117 error_count = GAUDI2_NA_EVENT_CAUSE; 10118 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10119 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10120 break; 10121 case GAUDI2_EVENT_PSOC_PRSTN_FALL: 10122 error_count = GAUDI2_NA_EVENT_CAUSE; 10123 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10124 break; 10125 case GAUDI2_EVENT_PCIE_APB_TIMEOUT: 10126 error_count = GAUDI2_NA_EVENT_CAUSE; 10127 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10128 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10129 break; 10130 case GAUDI2_EVENT_PCIE_FATAL_ERR: 10131 error_count = GAUDI2_NA_EVENT_CAUSE; 10132 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10133 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10134 break; 10135 case GAUDI2_EVENT_TPC0_BMON_SPMU: 10136 case GAUDI2_EVENT_TPC1_BMON_SPMU: 10137 case GAUDI2_EVENT_TPC2_BMON_SPMU: 10138 case GAUDI2_EVENT_TPC3_BMON_SPMU: 10139 case GAUDI2_EVENT_TPC4_BMON_SPMU: 10140 case GAUDI2_EVENT_TPC5_BMON_SPMU: 10141 case GAUDI2_EVENT_TPC6_BMON_SPMU: 10142 case GAUDI2_EVENT_TPC7_BMON_SPMU: 10143 case GAUDI2_EVENT_TPC8_BMON_SPMU: 10144 case GAUDI2_EVENT_TPC9_BMON_SPMU: 10145 case GAUDI2_EVENT_TPC10_BMON_SPMU: 10146 case GAUDI2_EVENT_TPC11_BMON_SPMU: 10147 case GAUDI2_EVENT_TPC12_BMON_SPMU: 10148 case GAUDI2_EVENT_TPC13_BMON_SPMU: 10149 case GAUDI2_EVENT_TPC14_BMON_SPMU: 10150 case GAUDI2_EVENT_TPC15_BMON_SPMU: 10151 case GAUDI2_EVENT_TPC16_BMON_SPMU: 10152 case GAUDI2_EVENT_TPC17_BMON_SPMU: 10153 case GAUDI2_EVENT_TPC18_BMON_SPMU: 10154 case GAUDI2_EVENT_TPC19_BMON_SPMU: 10155 case GAUDI2_EVENT_TPC20_BMON_SPMU: 10156 case GAUDI2_EVENT_TPC21_BMON_SPMU: 10157 case GAUDI2_EVENT_TPC22_BMON_SPMU: 10158 case GAUDI2_EVENT_TPC23_BMON_SPMU: 10159 case GAUDI2_EVENT_TPC24_BMON_SPMU: 10160 case GAUDI2_EVENT_MME0_CTRL_BMON_SPMU: 10161 case GAUDI2_EVENT_MME0_SBTE_BMON_SPMU: 10162 case GAUDI2_EVENT_MME0_WAP_BMON_SPMU: 10163 case GAUDI2_EVENT_MME1_CTRL_BMON_SPMU: 10164 case GAUDI2_EVENT_MME1_SBTE_BMON_SPMU: 10165 case GAUDI2_EVENT_MME1_WAP_BMON_SPMU: 10166 case GAUDI2_EVENT_MME2_CTRL_BMON_SPMU: 10167 case GAUDI2_EVENT_MME2_SBTE_BMON_SPMU: 10168 case GAUDI2_EVENT_MME2_WAP_BMON_SPMU: 10169 case GAUDI2_EVENT_MME3_CTRL_BMON_SPMU: 10170 case GAUDI2_EVENT_MME3_SBTE_BMON_SPMU: 10171 case GAUDI2_EVENT_MME3_WAP_BMON_SPMU: 10172 case GAUDI2_EVENT_HDMA2_BM_SPMU ... GAUDI2_EVENT_PDMA1_BM_SPMU: 10173 fallthrough; 10174 case GAUDI2_EVENT_DEC0_BMON_SPMU: 10175 case GAUDI2_EVENT_DEC1_BMON_SPMU: 10176 case GAUDI2_EVENT_DEC2_BMON_SPMU: 10177 case GAUDI2_EVENT_DEC3_BMON_SPMU: 10178 case GAUDI2_EVENT_DEC4_BMON_SPMU: 10179 case GAUDI2_EVENT_DEC5_BMON_SPMU: 10180 case GAUDI2_EVENT_DEC6_BMON_SPMU: 10181 case GAUDI2_EVENT_DEC7_BMON_SPMU: 10182 case GAUDI2_EVENT_DEC8_BMON_SPMU: 10183 case GAUDI2_EVENT_DEC9_BMON_SPMU: 10184 case GAUDI2_EVENT_ROTATOR0_BMON_SPMU ... GAUDI2_EVENT_SM3_BMON_SPMU: 10185 error_count = GAUDI2_NA_EVENT_CAUSE; 10186 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10187 break; 10188 10189 case GAUDI2_EVENT_CPU_FIX_POWER_ENV_S: 10190 case GAUDI2_EVENT_CPU_FIX_POWER_ENV_E: 10191 case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_S: 10192 case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_E: 10193 gaudi2_print_clk_change_info(hdev, event_type, &event_mask); 10194 error_count = GAUDI2_NA_EVENT_CAUSE; 10195 break; 10196 10197 case GAUDI2_EVENT_CPU_PKT_QUEUE_OUT_SYNC: 10198 gaudi2_print_out_of_sync_info(hdev, event_type, &eq_entry->pkt_sync_err); 10199 error_count = GAUDI2_NA_EVENT_CAUSE; 10200 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10201 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10202 break; 10203 10204 case GAUDI2_EVENT_PCIE_FLR_REQUESTED: 10205 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10206 error_count = GAUDI2_NA_EVENT_CAUSE; 10207 /* Do nothing- FW will handle it */ 10208 break; 10209 10210 case GAUDI2_EVENT_PCIE_P2P_MSIX: 10211 error_count = gaudi2_handle_pcie_p2p_msix(hdev, event_type); 10212 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10213 break; 10214 10215 case GAUDI2_EVENT_SM0_AXI_ERROR_RESPONSE ... GAUDI2_EVENT_SM3_AXI_ERROR_RESPONSE: 10216 index = event_type - GAUDI2_EVENT_SM0_AXI_ERROR_RESPONSE; 10217 error_count = gaudi2_handle_sm_err(hdev, event_type, index); 10218 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10219 break; 10220 10221 case GAUDI2_EVENT_PSOC_MME_PLL_LOCK_ERR ... GAUDI2_EVENT_DCORE2_HBM_PLL_LOCK_ERR: 10222 error_count = GAUDI2_NA_EVENT_CAUSE; 10223 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10224 break; 10225 10226 case GAUDI2_EVENT_CPU_CPLD_SHUTDOWN_CAUSE: 10227 dev_info(hdev->dev, "CPLD shutdown cause, reset reason: 0x%llx\n", 10228 le64_to_cpu(eq_entry->data[0])); 10229 error_count = GAUDI2_NA_EVENT_CAUSE; 10230 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10231 break; 10232 case GAUDI2_EVENT_CPU_CPLD_SHUTDOWN_EVENT: 10233 dev_err(hdev->dev, "CPLD shutdown event, reset reason: 0x%llx\n", 10234 le64_to_cpu(eq_entry->data[0])); 10235 error_count = GAUDI2_NA_EVENT_CAUSE; 10236 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10237 break; 10238 10239 case GAUDI2_EVENT_CPU_PKT_SANITY_FAILED: 10240 gaudi2_print_cpu_pkt_failure_info(hdev, event_type, &eq_entry->pkt_sync_err); 10241 error_count = GAUDI2_NA_EVENT_CAUSE; 10242 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10243 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10244 break; 10245 10246 case GAUDI2_EVENT_ARC_DCCM_FULL: 10247 error_count = hl_arc_event_handle(hdev, event_type, &eq_entry->arc_data); 10248 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10249 break; 10250 10251 case GAUDI2_EVENT_CPU_FP32_NOT_SUPPORTED: 10252 case GAUDI2_EVENT_CPU_DEV_RESET_REQ: 10253 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10254 error_count = GAUDI2_NA_EVENT_CAUSE; 10255 is_critical = true; 10256 break; 10257 10258 case GAUDI2_EVENT_ARC_PWR_BRK_ENTRY: 10259 case GAUDI2_EVENT_ARC_PWR_BRK_EXT: 10260 case GAUDI2_EVENT_ARC_PWR_RD_MODE0: 10261 case GAUDI2_EVENT_ARC_PWR_RD_MODE1: 10262 case GAUDI2_EVENT_ARC_PWR_RD_MODE2: 10263 case GAUDI2_EVENT_ARC_PWR_RD_MODE3: 10264 error_count = GAUDI2_NA_EVENT_CAUSE; 10265 dev_info_ratelimited(hdev->dev, "%s event received\n", 10266 gaudi2_irq_map_table[event_type].name); 10267 break; 10268 10269 case GAUDI2_EVENT_ARC_EQ_HEARTBEAT: 10270 hl_eq_heartbeat_event_handle(hdev); 10271 error_count = GAUDI2_NA_EVENT_CAUSE; 10272 break; 10273 default: 10274 if (gaudi2_irq_map_table[event_type].valid) { 10275 dev_err_ratelimited(hdev->dev, "Cannot find handler for event %d\n", 10276 event_type); 10277 error_count = GAUDI2_NA_EVENT_CAUSE; 10278 } 10279 } 10280 10281 if (event_mask & HL_NOTIFIER_EVENT_USER_ENGINE_ERR) 10282 hl_capture_engine_err(hdev, event_id_to_engine_id(hdev, event_type), error_count); 10283 10284 /* Make sure to dump an error in case no error cause was printed so far. 10285 * Note that although we have counted the errors, we use this number as 10286 * a boolean. 10287 */ 10288 if (error_count == GAUDI2_NA_EVENT_CAUSE && !is_info_event(event_type)) 10289 gaudi2_print_event(hdev, event_type, true, "%d", event_type); 10290 else if (error_count == 0) 10291 gaudi2_print_event(hdev, event_type, true, 10292 "No error cause for H/W event %u", event_type); 10293 10294 if ((gaudi2_irq_map_table[event_type].reset != EVENT_RESET_TYPE_NONE) || 10295 reset_required) { 10296 if (reset_required || 10297 (gaudi2_irq_map_table[event_type].reset == EVENT_RESET_TYPE_HARD)) 10298 reset_flags |= HL_DRV_RESET_HARD; 10299 10300 if (hdev->hard_reset_on_fw_events || 10301 (hdev->asic_prop.fw_security_enabled && is_critical)) 10302 goto reset_device; 10303 } 10304 10305 /* Send unmask irq only for interrupts not classified as MSG */ 10306 if (!gaudi2_irq_map_table[event_type].msg) 10307 hl_fw_unmask_irq(hdev, event_type); 10308 10309 if (event_mask) 10310 hl_notifier_event_send_all(hdev, event_mask); 10311 10312 return; 10313 10314 reset_device: 10315 if (hdev->asic_prop.fw_security_enabled && is_critical) { 10316 reset_flags |= HL_DRV_RESET_BYPASS_REQ_TO_FW; 10317 event_mask |= HL_NOTIFIER_EVENT_DEVICE_UNAVAILABLE; 10318 } else { 10319 reset_flags |= HL_DRV_RESET_DELAY; 10320 } 10321 /* escalate general hw errors to critical/fatal error */ 10322 if (event_mask & HL_NOTIFIER_EVENT_GENERAL_HW_ERR) 10323 hl_handle_critical_hw_err(hdev, event_type, &event_mask); 10324 10325 event_mask |= HL_NOTIFIER_EVENT_DEVICE_RESET; 10326 hl_device_cond_reset(hdev, reset_flags, event_mask); 10327 } 10328 10329 static int gaudi2_memset_memory_chunk_using_edma_qm(struct hl_device *hdev, 10330 struct packet_lin_dma *lin_dma_pkt, 10331 u64 phys_addr, u32 hw_queue_id, u32 size, u64 addr, u32 val) 10332 { 10333 u32 ctl, pkt_size; 10334 int rc = 0, i; 10335 10336 ctl = FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_LIN_DMA); 10337 ctl |= FIELD_PREP(GAUDI2_PKT_LIN_DMA_CTL_MEMSET_MASK, 1); 10338 ctl |= FIELD_PREP(GAUDI2_PKT_LIN_DMA_CTL_WRCOMP_MASK, 1); 10339 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 1); 10340 10341 lin_dma_pkt->ctl = cpu_to_le32(ctl); 10342 lin_dma_pkt->src_addr = cpu_to_le64(val); 10343 lin_dma_pkt->dst_addr = cpu_to_le64(addr); 10344 lin_dma_pkt->tsize = cpu_to_le32(size); 10345 10346 pkt_size = sizeof(struct packet_lin_dma); 10347 10348 for (i = 0; i < 3; i++) 10349 rc = hdev->asic_funcs->access_dev_mem(hdev, PCI_REGION_DRAM, 10350 phys_addr + (i * sizeof(u64)), 10351 ((u64 *)(lin_dma_pkt)) + i, DEBUGFS_WRITE64); 10352 10353 rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, pkt_size, phys_addr); 10354 if (rc) 10355 dev_err(hdev->dev, "Failed to send lin dma packet to H/W queue %d\n", 10356 hw_queue_id); 10357 10358 return rc; 10359 } 10360 10361 static int gaudi2_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, u64 val) 10362 { 10363 u32 edma_queues_id[] = {GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0, 10364 GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0, 10365 GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0, 10366 GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0}; 10367 u32 chunk_size, dcore, edma_idx, sob_offset, sob_addr, comp_val, 10368 old_mmubp, mmubp, num_of_pkts, busy, pkt_size, cb_len; 10369 u64 comp_addr, cur_addr = addr, end_addr = addr + size; 10370 struct asic_fixed_properties *prop = &hdev->asic_prop; 10371 int rc = 0, dma_num = 0, i; 10372 void *lin_dma_pkts_arr; 10373 10374 if (prop->edma_enabled_mask == 0) { 10375 dev_info(hdev->dev, "non of the EDMA engines is enabled - skip dram scrubbing\n"); 10376 return -EIO; 10377 } 10378 10379 sob_offset = hdev->asic_prop.first_available_user_sob[0] * 4; 10380 sob_addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset; 10381 comp_addr = CFG_BASE + sob_addr; 10382 comp_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1) | 10383 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 1); 10384 mmubp = FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_WR_MASK, 1) | 10385 FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_RD_MASK, 1); 10386 10387 /* Calculate how many lin dma pkts we'll need */ 10388 num_of_pkts = div64_u64(round_up(size, SZ_2G), SZ_2G); 10389 pkt_size = sizeof(struct packet_lin_dma); 10390 cb_len = pkt_size * num_of_pkts; 10391 10392 /* 10393 * if we're not scrubing HMMU or NIC reserved sections in hbm, 10394 * then it the scrubing of the user section, as we use the start of the user section 10395 * to store the CB of the EDMA QM, so shift the start address of the scrubbing accordingly 10396 * and scrub the CB section before leaving this function. 10397 */ 10398 if ((addr >= prop->dram_user_base_address) && 10399 (addr < prop->dram_user_base_address + cb_len)) 10400 cur_addr += (prop->dram_user_base_address + cb_len) - addr; 10401 10402 lin_dma_pkts_arr = kvcalloc(num_of_pkts, pkt_size, GFP_KERNEL); 10403 if (!lin_dma_pkts_arr) 10404 return -ENOMEM; 10405 10406 /* 10407 * set mmu bypass for the scrubbing - all ddmas are configured the same so save 10408 * only the first one to restore later 10409 * also set the sob addr for all edma cores for completion. 10410 * set QM as trusted to allow it to access physical address with MMU bp. 10411 */ 10412 old_mmubp = RREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP); 10413 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 10414 for (edma_idx = 0 ; edma_idx < NUM_OF_EDMA_PER_DCORE ; edma_idx++) { 10415 u32 edma_offset = dcore * DCORE_OFFSET + edma_idx * DCORE_EDMA_OFFSET; 10416 u32 edma_bit = dcore * NUM_OF_EDMA_PER_DCORE + edma_idx; 10417 10418 if (!(prop->edma_enabled_mask & BIT(edma_bit))) 10419 continue; 10420 10421 WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP + 10422 edma_offset, mmubp); 10423 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_LO + edma_offset, 10424 lower_32_bits(comp_addr)); 10425 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_HI + edma_offset, 10426 upper_32_bits(comp_addr)); 10427 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_WDATA + edma_offset, 10428 comp_val); 10429 gaudi2_qman_set_test_mode(hdev, 10430 edma_queues_id[dcore] + 4 * edma_idx, true); 10431 } 10432 } 10433 10434 WREG32(sob_addr, 0); 10435 10436 while (cur_addr < end_addr) { 10437 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 10438 for (edma_idx = 0 ; edma_idx < NUM_OF_EDMA_PER_DCORE ; edma_idx++) { 10439 u32 edma_bit = dcore * NUM_OF_EDMA_PER_DCORE + edma_idx; 10440 10441 if (!(prop->edma_enabled_mask & BIT(edma_bit))) 10442 continue; 10443 10444 chunk_size = min_t(u64, SZ_2G, end_addr - cur_addr); 10445 10446 rc = gaudi2_memset_memory_chunk_using_edma_qm(hdev, 10447 (struct packet_lin_dma *)lin_dma_pkts_arr + dma_num, 10448 prop->dram_user_base_address + (dma_num * pkt_size), 10449 edma_queues_id[dcore] + edma_idx * 4, 10450 chunk_size, cur_addr, val); 10451 if (rc) 10452 goto end; 10453 10454 dma_num++; 10455 cur_addr += chunk_size; 10456 if (cur_addr == end_addr) 10457 goto edma_wait; 10458 } 10459 } 10460 } 10461 10462 edma_wait: 10463 rc = hl_poll_timeout(hdev, sob_addr, busy, (busy == dma_num), 1000, 1000000); 10464 if (rc) { 10465 dev_err(hdev->dev, "DMA Timeout during HBM scrubbing(sob: 0x%x, dma_num: 0x%x)\n", 10466 busy, dma_num); 10467 goto end; 10468 } 10469 end: 10470 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 10471 for (edma_idx = 0 ; edma_idx < NUM_OF_EDMA_PER_DCORE ; edma_idx++) { 10472 u32 edma_offset = dcore * DCORE_OFFSET + edma_idx * DCORE_EDMA_OFFSET; 10473 u32 edma_bit = dcore * NUM_OF_EDMA_PER_DCORE + edma_idx; 10474 10475 if (!(prop->edma_enabled_mask & BIT(edma_bit))) 10476 continue; 10477 10478 WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP + edma_offset, old_mmubp); 10479 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_LO + edma_offset, 0); 10480 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_HI + edma_offset, 0); 10481 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_WDATA + edma_offset, 0); 10482 gaudi2_qman_set_test_mode(hdev, 10483 edma_queues_id[dcore] + 4 * edma_idx, false); 10484 } 10485 } 10486 10487 memset(lin_dma_pkts_arr, 0, sizeof(u64)); 10488 10489 /* Zero the HBM area where we copied the CB */ 10490 for (i = 0; i < cb_len / sizeof(u64); i += sizeof(u64)) 10491 rc = hdev->asic_funcs->access_dev_mem(hdev, PCI_REGION_DRAM, 10492 prop->dram_user_base_address + i, 10493 (u64 *)(lin_dma_pkts_arr), DEBUGFS_WRITE64); 10494 WREG32(sob_addr, 0); 10495 10496 kfree(lin_dma_pkts_arr); 10497 10498 return rc; 10499 } 10500 10501 static int gaudi2_scrub_device_dram(struct hl_device *hdev, u64 val) 10502 { 10503 int rc; 10504 struct asic_fixed_properties *prop = &hdev->asic_prop; 10505 u64 size = prop->dram_end_address - prop->dram_user_base_address; 10506 10507 rc = gaudi2_memset_device_memory(hdev, prop->dram_user_base_address, size, val); 10508 10509 if (rc) 10510 dev_err(hdev->dev, "Failed to scrub dram, address: 0x%llx size: %llu\n", 10511 prop->dram_user_base_address, size); 10512 return rc; 10513 } 10514 10515 static int gaudi2_scrub_device_mem(struct hl_device *hdev) 10516 { 10517 int rc; 10518 struct asic_fixed_properties *prop = &hdev->asic_prop; 10519 u64 val = hdev->memory_scrub_val; 10520 u64 addr, size; 10521 10522 if (!hdev->memory_scrub) 10523 return 0; 10524 10525 /* scrub SRAM */ 10526 addr = prop->sram_user_base_address; 10527 size = hdev->pldm ? 0x10000 : (prop->sram_size - SRAM_USER_BASE_OFFSET); 10528 dev_dbg(hdev->dev, "Scrubbing SRAM: 0x%09llx - 0x%09llx, val: 0x%llx\n", 10529 addr, addr + size, val); 10530 rc = gaudi2_memset_device_memory(hdev, addr, size, val); 10531 if (rc) { 10532 dev_err(hdev->dev, "scrubbing SRAM failed (%d)\n", rc); 10533 return rc; 10534 } 10535 10536 /* scrub DRAM */ 10537 rc = gaudi2_scrub_device_dram(hdev, val); 10538 if (rc) { 10539 dev_err(hdev->dev, "scrubbing DRAM failed (%d)\n", rc); 10540 return rc; 10541 } 10542 return 0; 10543 } 10544 10545 static void gaudi2_restore_user_sm_registers(struct hl_device *hdev) 10546 { 10547 u64 addr, mon_sts_addr, mon_cfg_addr, cq_lbw_l_addr, cq_lbw_h_addr, 10548 cq_lbw_data_addr, cq_base_l_addr, cq_base_h_addr, cq_size_addr; 10549 u32 val, size, offset; 10550 int dcore_id; 10551 10552 offset = hdev->asic_prop.first_available_cq[0] * 4; 10553 cq_lbw_l_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0 + offset; 10554 cq_lbw_h_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 + offset; 10555 cq_lbw_data_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_DATA_0 + offset; 10556 cq_base_l_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_L_0 + offset; 10557 cq_base_h_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_H_0 + offset; 10558 cq_size_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_SIZE_LOG2_0 + offset; 10559 size = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 - 10560 (mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0 + offset); 10561 10562 /* memset dcore0 CQ registers */ 10563 gaudi2_memset_device_lbw(hdev, cq_lbw_l_addr, size, 0); 10564 gaudi2_memset_device_lbw(hdev, cq_lbw_h_addr, size, 0); 10565 gaudi2_memset_device_lbw(hdev, cq_lbw_data_addr, size, 0); 10566 gaudi2_memset_device_lbw(hdev, cq_base_l_addr, size, 0); 10567 gaudi2_memset_device_lbw(hdev, cq_base_h_addr, size, 0); 10568 gaudi2_memset_device_lbw(hdev, cq_size_addr, size, 0); 10569 10570 cq_lbw_l_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0 + DCORE_OFFSET; 10571 cq_lbw_h_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 + DCORE_OFFSET; 10572 cq_lbw_data_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_DATA_0 + DCORE_OFFSET; 10573 cq_base_l_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_L_0 + DCORE_OFFSET; 10574 cq_base_h_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_H_0 + DCORE_OFFSET; 10575 cq_size_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_SIZE_LOG2_0 + DCORE_OFFSET; 10576 size = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 - mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0; 10577 10578 for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) { 10579 gaudi2_memset_device_lbw(hdev, cq_lbw_l_addr, size, 0); 10580 gaudi2_memset_device_lbw(hdev, cq_lbw_h_addr, size, 0); 10581 gaudi2_memset_device_lbw(hdev, cq_lbw_data_addr, size, 0); 10582 gaudi2_memset_device_lbw(hdev, cq_base_l_addr, size, 0); 10583 gaudi2_memset_device_lbw(hdev, cq_base_h_addr, size, 0); 10584 gaudi2_memset_device_lbw(hdev, cq_size_addr, size, 0); 10585 10586 cq_lbw_l_addr += DCORE_OFFSET; 10587 cq_lbw_h_addr += DCORE_OFFSET; 10588 cq_lbw_data_addr += DCORE_OFFSET; 10589 cq_base_l_addr += DCORE_OFFSET; 10590 cq_base_h_addr += DCORE_OFFSET; 10591 cq_size_addr += DCORE_OFFSET; 10592 } 10593 10594 offset = hdev->asic_prop.first_available_user_mon[0] * 4; 10595 addr = mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + offset; 10596 val = 1 << DCORE0_SYNC_MNGR_OBJS_MON_STATUS_PROT_SHIFT; 10597 size = mmDCORE0_SYNC_MNGR_OBJS_SM_SEC_0 - (mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + offset); 10598 10599 /* memset dcore0 monitors */ 10600 gaudi2_memset_device_lbw(hdev, addr, size, val); 10601 10602 addr = mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + offset; 10603 gaudi2_memset_device_lbw(hdev, addr, size, 0); 10604 10605 mon_sts_addr = mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + DCORE_OFFSET; 10606 mon_cfg_addr = mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + DCORE_OFFSET; 10607 size = mmDCORE0_SYNC_MNGR_OBJS_SM_SEC_0 - mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0; 10608 10609 for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) { 10610 gaudi2_memset_device_lbw(hdev, mon_sts_addr, size, val); 10611 gaudi2_memset_device_lbw(hdev, mon_cfg_addr, size, 0); 10612 mon_sts_addr += DCORE_OFFSET; 10613 mon_cfg_addr += DCORE_OFFSET; 10614 } 10615 10616 offset = hdev->asic_prop.first_available_user_sob[0] * 4; 10617 addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + offset; 10618 val = 0; 10619 size = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 - 10620 (mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + offset); 10621 10622 /* memset dcore0 sobs */ 10623 gaudi2_memset_device_lbw(hdev, addr, size, val); 10624 10625 addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + DCORE_OFFSET; 10626 size = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 - mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0; 10627 10628 for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) { 10629 gaudi2_memset_device_lbw(hdev, addr, size, val); 10630 addr += DCORE_OFFSET; 10631 } 10632 10633 /* Flush all WREG to prevent race */ 10634 val = RREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + offset); 10635 } 10636 10637 static void gaudi2_restore_user_qm_registers(struct hl_device *hdev) 10638 { 10639 u32 reg_base, hw_queue_id; 10640 10641 for (hw_queue_id = GAUDI2_QUEUE_ID_PDMA_0_0 ; hw_queue_id <= GAUDI2_QUEUE_ID_ROT_1_0; 10642 hw_queue_id += NUM_OF_PQ_PER_QMAN) { 10643 if (!gaudi2_is_queue_enabled(hdev, hw_queue_id)) 10644 continue; 10645 10646 gaudi2_clear_qm_fence_counters_common(hdev, hw_queue_id, false); 10647 10648 reg_base = gaudi2_qm_blocks_bases[hw_queue_id]; 10649 WREG32(reg_base + QM_ARB_CFG_0_OFFSET, 0); 10650 } 10651 10652 /* Flush all WREG to prevent race */ 10653 RREG32(mmPDMA0_QM_ARB_CFG_0); 10654 } 10655 10656 static void gaudi2_restore_nic_qm_registers(struct hl_device *hdev) 10657 { 10658 u32 reg_base, hw_queue_id; 10659 10660 for (hw_queue_id = GAUDI2_QUEUE_ID_NIC_0_0 ; hw_queue_id <= GAUDI2_QUEUE_ID_NIC_23_3; 10661 hw_queue_id += NUM_OF_PQ_PER_QMAN) { 10662 if (!gaudi2_is_queue_enabled(hdev, hw_queue_id)) 10663 continue; 10664 10665 gaudi2_clear_qm_fence_counters_common(hdev, hw_queue_id, false); 10666 10667 reg_base = gaudi2_qm_blocks_bases[hw_queue_id]; 10668 WREG32(reg_base + QM_ARB_CFG_0_OFFSET, 0); 10669 } 10670 10671 /* Flush all WREG to prevent race */ 10672 RREG32(mmPDMA0_QM_ARB_CFG_0); 10673 } 10674 10675 static int gaudi2_context_switch(struct hl_device *hdev, u32 asid) 10676 { 10677 return 0; 10678 } 10679 10680 static void gaudi2_restore_phase_topology(struct hl_device *hdev) 10681 { 10682 } 10683 10684 static void gaudi2_init_block_instances(struct hl_device *hdev, u32 block_idx, 10685 struct dup_block_ctx *cfg_ctx) 10686 { 10687 u64 block_base = cfg_ctx->base + block_idx * cfg_ctx->block_off; 10688 u8 seq; 10689 int i; 10690 10691 for (i = 0 ; i < cfg_ctx->instances ; i++) { 10692 seq = block_idx * cfg_ctx->instances + i; 10693 10694 /* skip disabled instance */ 10695 if (!(cfg_ctx->enabled_mask & BIT_ULL(seq))) 10696 continue; 10697 10698 cfg_ctx->instance_cfg_fn(hdev, block_base + i * cfg_ctx->instance_off, 10699 cfg_ctx->data); 10700 } 10701 } 10702 10703 static void gaudi2_init_blocks_with_mask(struct hl_device *hdev, struct dup_block_ctx *cfg_ctx, 10704 u64 mask) 10705 { 10706 int i; 10707 10708 cfg_ctx->enabled_mask = mask; 10709 10710 for (i = 0 ; i < cfg_ctx->blocks ; i++) 10711 gaudi2_init_block_instances(hdev, i, cfg_ctx); 10712 } 10713 10714 void gaudi2_init_blocks(struct hl_device *hdev, struct dup_block_ctx *cfg_ctx) 10715 { 10716 gaudi2_init_blocks_with_mask(hdev, cfg_ctx, U64_MAX); 10717 } 10718 10719 static int gaudi2_debugfs_read_dma(struct hl_device *hdev, u64 addr, u32 size, void *blob_addr) 10720 { 10721 void *host_mem_virtual_addr; 10722 dma_addr_t host_mem_dma_addr; 10723 u64 reserved_va_base; 10724 u32 pos, size_left, size_to_dma; 10725 struct hl_ctx *ctx; 10726 int rc = 0; 10727 10728 /* Fetch the ctx */ 10729 ctx = hl_get_compute_ctx(hdev); 10730 if (!ctx) { 10731 dev_err(hdev->dev, "No ctx available\n"); 10732 return -EINVAL; 10733 } 10734 10735 /* Allocate buffers for read and for poll */ 10736 host_mem_virtual_addr = hl_asic_dma_alloc_coherent(hdev, SZ_2M, &host_mem_dma_addr, 10737 GFP_KERNEL | __GFP_ZERO); 10738 if (host_mem_virtual_addr == NULL) { 10739 dev_err(hdev->dev, "Failed to allocate memory for KDMA read\n"); 10740 rc = -ENOMEM; 10741 goto put_ctx; 10742 } 10743 10744 /* Reserve VM region on asic side */ 10745 reserved_va_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST, SZ_2M, 10746 HL_MMU_VA_ALIGNMENT_NOT_NEEDED); 10747 if (!reserved_va_base) { 10748 dev_err(hdev->dev, "Failed to reserve vmem on asic\n"); 10749 rc = -ENOMEM; 10750 goto free_data_buffer; 10751 } 10752 10753 /* Create mapping on asic side */ 10754 mutex_lock(&hdev->mmu_lock); 10755 10756 rc = hl_mmu_map_contiguous(ctx, reserved_va_base, host_mem_dma_addr, SZ_2M); 10757 if (rc) { 10758 dev_err(hdev->dev, "Failed to create mapping on asic mmu\n"); 10759 goto unreserve_va; 10760 } 10761 10762 rc = hl_mmu_invalidate_cache_range(hdev, false, 10763 MMU_OP_USERPTR | MMU_OP_SKIP_LOW_CACHE_INV, 10764 ctx->asid, reserved_va_base, SZ_2M); 10765 if (rc) { 10766 hl_mmu_unmap_contiguous(ctx, reserved_va_base, SZ_2M); 10767 goto unreserve_va; 10768 } 10769 10770 mutex_unlock(&hdev->mmu_lock); 10771 10772 /* Enable MMU on KDMA */ 10773 gaudi2_kdma_set_mmbp_asid(hdev, false, ctx->asid); 10774 10775 pos = 0; 10776 size_left = size; 10777 size_to_dma = SZ_2M; 10778 10779 while (size_left > 0) { 10780 if (size_left < SZ_2M) 10781 size_to_dma = size_left; 10782 10783 rc = gaudi2_send_job_to_kdma(hdev, addr, reserved_va_base, size_to_dma, false); 10784 if (rc) 10785 break; 10786 10787 memcpy(blob_addr + pos, host_mem_virtual_addr, size_to_dma); 10788 10789 if (size_left <= SZ_2M) 10790 break; 10791 10792 pos += SZ_2M; 10793 addr += SZ_2M; 10794 size_left -= SZ_2M; 10795 } 10796 10797 gaudi2_kdma_set_mmbp_asid(hdev, true, HL_KERNEL_ASID_ID); 10798 10799 mutex_lock(&hdev->mmu_lock); 10800 10801 rc = hl_mmu_unmap_contiguous(ctx, reserved_va_base, SZ_2M); 10802 if (rc) 10803 goto unreserve_va; 10804 10805 rc = hl_mmu_invalidate_cache_range(hdev, false, MMU_OP_USERPTR, 10806 ctx->asid, reserved_va_base, SZ_2M); 10807 10808 unreserve_va: 10809 mutex_unlock(&hdev->mmu_lock); 10810 hl_unreserve_va_block(hdev, ctx, reserved_va_base, SZ_2M); 10811 free_data_buffer: 10812 hl_asic_dma_free_coherent(hdev, SZ_2M, host_mem_virtual_addr, host_mem_dma_addr); 10813 put_ctx: 10814 hl_ctx_put(ctx); 10815 10816 return rc; 10817 } 10818 10819 static int gaudi2_internal_cb_pool_init(struct hl_device *hdev, struct hl_ctx *ctx) 10820 { 10821 struct gaudi2_device *gaudi2 = hdev->asic_specific; 10822 int min_alloc_order, rc; 10823 10824 if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) 10825 return 0; 10826 10827 hdev->internal_cb_pool_virt_addr = hl_asic_dma_alloc_coherent(hdev, 10828 HOST_SPACE_INTERNAL_CB_SZ, 10829 &hdev->internal_cb_pool_dma_addr, 10830 GFP_KERNEL | __GFP_ZERO); 10831 10832 if (!hdev->internal_cb_pool_virt_addr) 10833 return -ENOMEM; 10834 10835 min_alloc_order = ilog2(min(gaudi2_get_signal_cb_size(hdev), 10836 gaudi2_get_wait_cb_size(hdev))); 10837 10838 hdev->internal_cb_pool = gen_pool_create(min_alloc_order, -1); 10839 if (!hdev->internal_cb_pool) { 10840 dev_err(hdev->dev, "Failed to create internal CB pool\n"); 10841 rc = -ENOMEM; 10842 goto free_internal_cb_pool; 10843 } 10844 10845 rc = gen_pool_add(hdev->internal_cb_pool, (uintptr_t) hdev->internal_cb_pool_virt_addr, 10846 HOST_SPACE_INTERNAL_CB_SZ, -1); 10847 if (rc) { 10848 dev_err(hdev->dev, "Failed to add memory to internal CB pool\n"); 10849 rc = -EFAULT; 10850 goto destroy_internal_cb_pool; 10851 } 10852 10853 hdev->internal_cb_va_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST, 10854 HOST_SPACE_INTERNAL_CB_SZ, HL_MMU_VA_ALIGNMENT_NOT_NEEDED); 10855 10856 if (!hdev->internal_cb_va_base) { 10857 rc = -ENOMEM; 10858 goto destroy_internal_cb_pool; 10859 } 10860 10861 mutex_lock(&hdev->mmu_lock); 10862 10863 rc = hl_mmu_map_contiguous(ctx, hdev->internal_cb_va_base, hdev->internal_cb_pool_dma_addr, 10864 HOST_SPACE_INTERNAL_CB_SZ); 10865 if (rc) 10866 goto unreserve_internal_cb_pool; 10867 10868 rc = hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR); 10869 if (rc) 10870 goto unmap_internal_cb_pool; 10871 10872 mutex_unlock(&hdev->mmu_lock); 10873 10874 return 0; 10875 10876 unmap_internal_cb_pool: 10877 hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); 10878 unreserve_internal_cb_pool: 10879 mutex_unlock(&hdev->mmu_lock); 10880 hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); 10881 destroy_internal_cb_pool: 10882 gen_pool_destroy(hdev->internal_cb_pool); 10883 free_internal_cb_pool: 10884 hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr, 10885 hdev->internal_cb_pool_dma_addr); 10886 10887 return rc; 10888 } 10889 10890 static void gaudi2_internal_cb_pool_fini(struct hl_device *hdev, struct hl_ctx *ctx) 10891 { 10892 struct gaudi2_device *gaudi2 = hdev->asic_specific; 10893 10894 if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) 10895 return; 10896 10897 mutex_lock(&hdev->mmu_lock); 10898 hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); 10899 hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); 10900 hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR); 10901 mutex_unlock(&hdev->mmu_lock); 10902 10903 gen_pool_destroy(hdev->internal_cb_pool); 10904 10905 hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr, 10906 hdev->internal_cb_pool_dma_addr); 10907 } 10908 10909 static void gaudi2_restore_user_registers(struct hl_device *hdev) 10910 { 10911 gaudi2_restore_user_sm_registers(hdev); 10912 gaudi2_restore_user_qm_registers(hdev); 10913 } 10914 10915 static int gaudi2_map_virtual_msix_doorbell_memory(struct hl_ctx *ctx) 10916 { 10917 struct hl_device *hdev = ctx->hdev; 10918 struct asic_fixed_properties *prop = &hdev->asic_prop; 10919 struct gaudi2_device *gaudi2 = hdev->asic_specific; 10920 int rc; 10921 10922 rc = hl_mmu_map_page(ctx, RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START, 10923 gaudi2->virt_msix_db_dma_addr, prop->pmmu.page_size, true); 10924 if (rc) 10925 dev_err(hdev->dev, "Failed to map VA %#llx for virtual MSI-X doorbell memory\n", 10926 RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START); 10927 10928 return rc; 10929 } 10930 10931 static void gaudi2_unmap_virtual_msix_doorbell_memory(struct hl_ctx *ctx) 10932 { 10933 struct hl_device *hdev = ctx->hdev; 10934 struct asic_fixed_properties *prop = &hdev->asic_prop; 10935 int rc; 10936 10937 rc = hl_mmu_unmap_page(ctx, RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START, 10938 prop->pmmu.page_size, true); 10939 if (rc) 10940 dev_err(hdev->dev, "Failed to unmap VA %#llx of virtual MSI-X doorbell memory\n", 10941 RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START); 10942 } 10943 10944 static int gaudi2_ctx_init(struct hl_ctx *ctx) 10945 { 10946 int rc; 10947 10948 if (ctx->asid == HL_KERNEL_ASID_ID) 10949 return 0; 10950 10951 rc = gaudi2_mmu_prepare(ctx->hdev, ctx->asid); 10952 if (rc) 10953 return rc; 10954 10955 /* No need to clear user registers if the device has just 10956 * performed reset, we restore only nic qm registers 10957 */ 10958 if (ctx->hdev->reset_upon_device_release) 10959 gaudi2_restore_nic_qm_registers(ctx->hdev); 10960 else 10961 gaudi2_restore_user_registers(ctx->hdev); 10962 10963 rc = gaudi2_internal_cb_pool_init(ctx->hdev, ctx); 10964 if (rc) 10965 return rc; 10966 10967 rc = gaudi2_map_virtual_msix_doorbell_memory(ctx); 10968 if (rc) 10969 gaudi2_internal_cb_pool_fini(ctx->hdev, ctx); 10970 10971 return rc; 10972 } 10973 10974 static void gaudi2_ctx_fini(struct hl_ctx *ctx) 10975 { 10976 if (ctx->asid == HL_KERNEL_ASID_ID) 10977 return; 10978 10979 gaudi2_internal_cb_pool_fini(ctx->hdev, ctx); 10980 10981 gaudi2_unmap_virtual_msix_doorbell_memory(ctx); 10982 } 10983 10984 static int gaudi2_pre_schedule_cs(struct hl_cs *cs) 10985 { 10986 struct hl_device *hdev = cs->ctx->hdev; 10987 int index = cs->sequence & (hdev->asic_prop.max_pending_cs - 1); 10988 u32 mon_payload, sob_id, mon_id; 10989 10990 if (!cs_needs_completion(cs)) 10991 return 0; 10992 10993 /* 10994 * First 64 SOB/MON are reserved for driver for QMAN auto completion 10995 * mechanism. Each SOB/MON pair are used for a pending CS with the same 10996 * cyclic index. The SOB value is increased when each of the CS jobs is 10997 * completed. When the SOB reaches the number of CS jobs, the monitor 10998 * generates MSI-X interrupt. 10999 */ 11000 11001 sob_id = mon_id = index; 11002 mon_payload = (1 << CQ_ENTRY_SHADOW_INDEX_VALID_SHIFT) | 11003 (1 << CQ_ENTRY_READY_SHIFT) | index; 11004 11005 gaudi2_arm_cq_monitor(hdev, sob_id, mon_id, GAUDI2_RESERVED_CQ_CS_COMPLETION, mon_payload, 11006 cs->jobs_cnt); 11007 11008 return 0; 11009 } 11010 11011 static u32 gaudi2_get_queue_id_for_cq(struct hl_device *hdev, u32 cq_idx) 11012 { 11013 return HL_INVALID_QUEUE; 11014 } 11015 11016 static u32 gaudi2_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id, u32 size, bool eb) 11017 { 11018 struct hl_cb *cb = data; 11019 struct packet_msg_short *pkt; 11020 u32 value, ctl, pkt_size = sizeof(*pkt); 11021 11022 pkt = (struct packet_msg_short *) (uintptr_t) (cb->kernel_address + size); 11023 memset(pkt, 0, pkt_size); 11024 11025 /* Inc by 1, Mode ADD */ 11026 value = FIELD_PREP(GAUDI2_PKT_SHORT_VAL_SOB_SYNC_VAL_MASK, 1); 11027 value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_SOB_MOD_MASK, 1); 11028 11029 ctl = FIELD_PREP(GAUDI2_PKT_SHORT_CTL_ADDR_MASK, sob_id * 4); 11030 ctl |= FIELD_PREP(GAUDI2_PKT_SHORT_CTL_BASE_MASK, 1); /* SOB base */ 11031 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); 11032 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, eb); 11033 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1); 11034 11035 pkt->value = cpu_to_le32(value); 11036 pkt->ctl = cpu_to_le32(ctl); 11037 11038 return size + pkt_size; 11039 } 11040 11041 static u32 gaudi2_add_mon_msg_short(struct packet_msg_short *pkt, u32 value, u16 addr) 11042 { 11043 u32 ctl, pkt_size = sizeof(*pkt); 11044 11045 memset(pkt, 0, pkt_size); 11046 11047 ctl = FIELD_PREP(GAUDI2_PKT_SHORT_CTL_ADDR_MASK, addr); 11048 ctl |= FIELD_PREP(GAUDI2_PKT_SHORT_CTL_BASE_MASK, 0); /* MON base */ 11049 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); 11050 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 0); 11051 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 0); 11052 11053 pkt->value = cpu_to_le32(value); 11054 pkt->ctl = cpu_to_le32(ctl); 11055 11056 return pkt_size; 11057 } 11058 11059 static u32 gaudi2_add_arm_monitor_pkt(struct hl_device *hdev, struct packet_msg_short *pkt, 11060 u16 sob_base, u8 sob_mask, u16 sob_val, u16 addr) 11061 { 11062 u32 ctl, value, pkt_size = sizeof(*pkt); 11063 u8 mask; 11064 11065 if (hl_gen_sob_mask(sob_base, sob_mask, &mask)) { 11066 dev_err(hdev->dev, "sob_base %u (mask %#x) is not valid\n", sob_base, sob_mask); 11067 return 0; 11068 } 11069 11070 memset(pkt, 0, pkt_size); 11071 11072 value = FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_SYNC_GID_MASK, sob_base / 8); 11073 value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_SYNC_VAL_MASK, sob_val); 11074 value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_MODE_MASK, 0); /* GREATER OR EQUAL*/ 11075 value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_MASK_MASK, mask); 11076 11077 ctl = FIELD_PREP(GAUDI2_PKT_SHORT_CTL_ADDR_MASK, addr); 11078 ctl |= FIELD_PREP(GAUDI2_PKT_SHORT_CTL_BASE_MASK, 0); /* MON base */ 11079 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); 11080 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 0); 11081 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1); 11082 11083 pkt->value = cpu_to_le32(value); 11084 pkt->ctl = cpu_to_le32(ctl); 11085 11086 return pkt_size; 11087 } 11088 11089 static u32 gaudi2_add_fence_pkt(struct packet_fence *pkt) 11090 { 11091 u32 ctl, cfg, pkt_size = sizeof(*pkt); 11092 11093 memset(pkt, 0, pkt_size); 11094 11095 cfg = FIELD_PREP(GAUDI2_PKT_FENCE_CFG_DEC_VAL_MASK, 1); 11096 cfg |= FIELD_PREP(GAUDI2_PKT_FENCE_CFG_TARGET_VAL_MASK, 1); 11097 cfg |= FIELD_PREP(GAUDI2_PKT_FENCE_CFG_ID_MASK, 2); 11098 11099 ctl = FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_FENCE); 11100 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 0); 11101 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1); 11102 11103 pkt->cfg = cpu_to_le32(cfg); 11104 pkt->ctl = cpu_to_le32(ctl); 11105 11106 return pkt_size; 11107 } 11108 11109 static u32 gaudi2_gen_wait_cb(struct hl_device *hdev, struct hl_gen_wait_properties *prop) 11110 { 11111 struct hl_cb *cb = prop->data; 11112 void *buf = (void *) (uintptr_t) (cb->kernel_address); 11113 11114 u64 monitor_base, fence_addr = 0; 11115 u32 stream_index, size = prop->size; 11116 u16 msg_addr_offset; 11117 11118 stream_index = prop->q_idx % 4; 11119 fence_addr = CFG_BASE + gaudi2_qm_blocks_bases[prop->q_idx] + 11120 QM_FENCE2_OFFSET + stream_index * 4; 11121 11122 /* 11123 * monitor_base should be the content of the base0 address registers, 11124 * so it will be added to the msg short offsets 11125 */ 11126 monitor_base = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0; 11127 11128 /* First monitor config packet: low address of the sync */ 11129 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + prop->mon_id * 4) - 11130 monitor_base; 11131 11132 size += gaudi2_add_mon_msg_short(buf + size, (u32) fence_addr, msg_addr_offset); 11133 11134 /* Second monitor config packet: high address of the sync */ 11135 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + prop->mon_id * 4) - 11136 monitor_base; 11137 11138 size += gaudi2_add_mon_msg_short(buf + size, (u32) (fence_addr >> 32), msg_addr_offset); 11139 11140 /* 11141 * Third monitor config packet: the payload, i.e. what to write when the 11142 * sync triggers 11143 */ 11144 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + prop->mon_id * 4) - 11145 monitor_base; 11146 11147 size += gaudi2_add_mon_msg_short(buf + size, 1, msg_addr_offset); 11148 11149 /* Fourth monitor config packet: bind the monitor to a sync object */ 11150 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + prop->mon_id * 4) - monitor_base; 11151 11152 size += gaudi2_add_arm_monitor_pkt(hdev, buf + size, prop->sob_base, prop->sob_mask, 11153 prop->sob_val, msg_addr_offset); 11154 11155 /* Fence packet */ 11156 size += gaudi2_add_fence_pkt(buf + size); 11157 11158 return size; 11159 } 11160 11161 static void gaudi2_reset_sob(struct hl_device *hdev, void *data) 11162 { 11163 struct hl_hw_sob *hw_sob = data; 11164 11165 dev_dbg(hdev->dev, "reset SOB, q_idx: %d, sob_id: %d\n", hw_sob->q_idx, hw_sob->sob_id); 11166 11167 WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + hw_sob->sob_id * 4, 0); 11168 11169 kref_init(&hw_sob->kref); 11170 } 11171 11172 static void gaudi2_reset_sob_group(struct hl_device *hdev, u16 sob_group) 11173 { 11174 } 11175 11176 static u64 gaudi2_get_device_time(struct hl_device *hdev) 11177 { 11178 u64 device_time = ((u64) RREG32(mmPSOC_TIMESTAMP_CNTCVU)) << 32; 11179 11180 return device_time | RREG32(mmPSOC_TIMESTAMP_CNTCVL); 11181 } 11182 11183 static int gaudi2_collective_wait_init_cs(struct hl_cs *cs) 11184 { 11185 return 0; 11186 } 11187 11188 static int gaudi2_collective_wait_create_jobs(struct hl_device *hdev, struct hl_ctx *ctx, 11189 struct hl_cs *cs, u32 wait_queue_id, 11190 u32 collective_engine_id, u32 encaps_signal_offset) 11191 { 11192 return -EINVAL; 11193 } 11194 11195 /* 11196 * hl_mmu_scramble - converts a dram (non power of 2) page-size aligned address 11197 * to DMMU page-size address (64MB) before mapping it in 11198 * the MMU. 11199 * The operation is performed on both the virtual and physical addresses. 11200 * for device with 6 HBMs the scramble is: 11201 * (addr[47:0] / 48M) * 64M + addr % 48M + addr[63:48] 11202 * 11203 * Example: 11204 * ============================================================================= 11205 * Allocated DRAM Reserved VA scrambled VA for MMU mapping Scrambled PA 11206 * Phys address in MMU last 11207 * HOP 11208 * ============================================================================= 11209 * PA1 0x3000000 VA1 0x9C000000 SVA1= (VA1/48M)*64M 0xD0000000 <- PA1/48M 0x1 11210 * PA2 0x9000000 VA2 0x9F000000 SVA2= (VA2/48M)*64M 0xD4000000 <- PA2/48M 0x3 11211 * ============================================================================= 11212 */ 11213 static u64 gaudi2_mmu_scramble_addr(struct hl_device *hdev, u64 raw_addr) 11214 { 11215 struct asic_fixed_properties *prop = &hdev->asic_prop; 11216 u32 divisor, mod_va; 11217 u64 div_va; 11218 11219 /* accept any address in the DRAM address space */ 11220 if (hl_mem_area_inside_range(raw_addr, sizeof(raw_addr), DRAM_PHYS_BASE, 11221 VA_HBM_SPACE_END)) { 11222 11223 divisor = prop->num_functional_hbms * GAUDI2_HBM_MMU_SCRM_MEM_SIZE; 11224 div_va = div_u64_rem(raw_addr & GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK, divisor, &mod_va); 11225 return (raw_addr & ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK) | 11226 (div_va << GAUDI2_HBM_MMU_SCRM_DIV_SHIFT) | 11227 (mod_va << GAUDI2_HBM_MMU_SCRM_MOD_SHIFT); 11228 } 11229 11230 return raw_addr; 11231 } 11232 11233 static u64 gaudi2_mmu_descramble_addr(struct hl_device *hdev, u64 scrambled_addr) 11234 { 11235 struct asic_fixed_properties *prop = &hdev->asic_prop; 11236 u32 divisor, mod_va; 11237 u64 div_va; 11238 11239 /* accept any address in the DRAM address space */ 11240 if (hl_mem_area_inside_range(scrambled_addr, sizeof(scrambled_addr), DRAM_PHYS_BASE, 11241 VA_HBM_SPACE_END)) { 11242 11243 divisor = prop->num_functional_hbms * GAUDI2_HBM_MMU_SCRM_MEM_SIZE; 11244 div_va = div_u64_rem(scrambled_addr & GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK, 11245 PAGE_SIZE_64MB, &mod_va); 11246 11247 return ((scrambled_addr & ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK) + 11248 (div_va * divisor + mod_va)); 11249 } 11250 11251 return scrambled_addr; 11252 } 11253 11254 static u32 gaudi2_get_dec_base_addr(struct hl_device *hdev, u32 core_id) 11255 { 11256 u32 base = 0, dcore_id, dec_id; 11257 11258 if (core_id >= NUMBER_OF_DEC) { 11259 dev_err(hdev->dev, "Unexpected core number %d for DEC\n", core_id); 11260 goto out; 11261 } 11262 11263 if (core_id < 8) { 11264 dcore_id = core_id / NUM_OF_DEC_PER_DCORE; 11265 dec_id = core_id % NUM_OF_DEC_PER_DCORE; 11266 11267 base = mmDCORE0_DEC0_CMD_BASE + dcore_id * DCORE_OFFSET + 11268 dec_id * DCORE_VDEC_OFFSET; 11269 } else { 11270 /* PCIe Shared Decoder */ 11271 base = mmPCIE_DEC0_CMD_BASE + ((core_id % 8) * PCIE_VDEC_OFFSET); 11272 } 11273 out: 11274 return base; 11275 } 11276 11277 static int gaudi2_get_hw_block_id(struct hl_device *hdev, u64 block_addr, 11278 u32 *block_size, u32 *block_id) 11279 { 11280 struct gaudi2_device *gaudi2 = hdev->asic_specific; 11281 int i; 11282 11283 for (i = 0 ; i < NUM_USER_MAPPED_BLOCKS ; i++) { 11284 if (block_addr == CFG_BASE + gaudi2->mapped_blocks[i].address) { 11285 *block_id = i; 11286 if (block_size) 11287 *block_size = gaudi2->mapped_blocks[i].size; 11288 return 0; 11289 } 11290 } 11291 11292 dev_err(hdev->dev, "Invalid block address %#llx", block_addr); 11293 11294 return -EINVAL; 11295 } 11296 11297 static int gaudi2_block_mmap(struct hl_device *hdev, struct vm_area_struct *vma, 11298 u32 block_id, u32 block_size) 11299 { 11300 struct gaudi2_device *gaudi2 = hdev->asic_specific; 11301 u64 offset_in_bar; 11302 u64 address; 11303 int rc; 11304 11305 if (block_id >= NUM_USER_MAPPED_BLOCKS) { 11306 dev_err(hdev->dev, "Invalid block id %u", block_id); 11307 return -EINVAL; 11308 } 11309 11310 /* we allow mapping only an entire block */ 11311 if (block_size != gaudi2->mapped_blocks[block_id].size) { 11312 dev_err(hdev->dev, "Invalid block size %u", block_size); 11313 return -EINVAL; 11314 } 11315 11316 offset_in_bar = CFG_BASE + gaudi2->mapped_blocks[block_id].address - STM_FLASH_BASE_ADDR; 11317 11318 address = pci_resource_start(hdev->pdev, SRAM_CFG_BAR_ID) + offset_in_bar; 11319 11320 vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | 11321 VM_DONTCOPY | VM_NORESERVE); 11322 11323 rc = remap_pfn_range(vma, vma->vm_start, address >> PAGE_SHIFT, 11324 block_size, vma->vm_page_prot); 11325 if (rc) 11326 dev_err(hdev->dev, "remap_pfn_range error %d", rc); 11327 11328 return rc; 11329 } 11330 11331 static void gaudi2_enable_events_from_fw(struct hl_device *hdev) 11332 { 11333 struct gaudi2_device *gaudi2 = hdev->asic_specific; 11334 11335 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 11336 u32 irq_handler_offset = le32_to_cpu(dyn_regs->gic_host_ints_irq); 11337 11338 if (gaudi2->hw_cap_initialized & HW_CAP_CPU_Q) 11339 WREG32(irq_handler_offset, 11340 gaudi2_irq_map_table[GAUDI2_EVENT_CPU_INTS_REGISTER].cpu_id); 11341 } 11342 11343 static int gaudi2_get_mmu_base(struct hl_device *hdev, u64 mmu_id, u32 *mmu_base) 11344 { 11345 switch (mmu_id) { 11346 case HW_CAP_DCORE0_DMMU0: 11347 *mmu_base = mmDCORE0_HMMU0_MMU_BASE; 11348 break; 11349 case HW_CAP_DCORE0_DMMU1: 11350 *mmu_base = mmDCORE0_HMMU1_MMU_BASE; 11351 break; 11352 case HW_CAP_DCORE0_DMMU2: 11353 *mmu_base = mmDCORE0_HMMU2_MMU_BASE; 11354 break; 11355 case HW_CAP_DCORE0_DMMU3: 11356 *mmu_base = mmDCORE0_HMMU3_MMU_BASE; 11357 break; 11358 case HW_CAP_DCORE1_DMMU0: 11359 *mmu_base = mmDCORE1_HMMU0_MMU_BASE; 11360 break; 11361 case HW_CAP_DCORE1_DMMU1: 11362 *mmu_base = mmDCORE1_HMMU1_MMU_BASE; 11363 break; 11364 case HW_CAP_DCORE1_DMMU2: 11365 *mmu_base = mmDCORE1_HMMU2_MMU_BASE; 11366 break; 11367 case HW_CAP_DCORE1_DMMU3: 11368 *mmu_base = mmDCORE1_HMMU3_MMU_BASE; 11369 break; 11370 case HW_CAP_DCORE2_DMMU0: 11371 *mmu_base = mmDCORE2_HMMU0_MMU_BASE; 11372 break; 11373 case HW_CAP_DCORE2_DMMU1: 11374 *mmu_base = mmDCORE2_HMMU1_MMU_BASE; 11375 break; 11376 case HW_CAP_DCORE2_DMMU2: 11377 *mmu_base = mmDCORE2_HMMU2_MMU_BASE; 11378 break; 11379 case HW_CAP_DCORE2_DMMU3: 11380 *mmu_base = mmDCORE2_HMMU3_MMU_BASE; 11381 break; 11382 case HW_CAP_DCORE3_DMMU0: 11383 *mmu_base = mmDCORE3_HMMU0_MMU_BASE; 11384 break; 11385 case HW_CAP_DCORE3_DMMU1: 11386 *mmu_base = mmDCORE3_HMMU1_MMU_BASE; 11387 break; 11388 case HW_CAP_DCORE3_DMMU2: 11389 *mmu_base = mmDCORE3_HMMU2_MMU_BASE; 11390 break; 11391 case HW_CAP_DCORE3_DMMU3: 11392 *mmu_base = mmDCORE3_HMMU3_MMU_BASE; 11393 break; 11394 case HW_CAP_PMMU: 11395 *mmu_base = mmPMMU_HBW_MMU_BASE; 11396 break; 11397 default: 11398 return -EINVAL; 11399 } 11400 11401 return 0; 11402 } 11403 11404 static void gaudi2_ack_mmu_error(struct hl_device *hdev, u64 mmu_id) 11405 { 11406 bool is_pmmu = (mmu_id == HW_CAP_PMMU); 11407 struct gaudi2_device *gaudi2 = hdev->asic_specific; 11408 u32 mmu_base; 11409 11410 if (!(gaudi2->hw_cap_initialized & mmu_id)) 11411 return; 11412 11413 if (gaudi2_get_mmu_base(hdev, mmu_id, &mmu_base)) 11414 return; 11415 11416 gaudi2_handle_page_error(hdev, mmu_base, is_pmmu, NULL); 11417 gaudi2_handle_access_error(hdev, mmu_base, is_pmmu); 11418 } 11419 11420 static int gaudi2_ack_mmu_page_fault_or_access_error(struct hl_device *hdev, u64 mmu_cap_mask) 11421 { 11422 u32 i, mmu_id, num_of_hmmus = NUM_OF_HMMU_PER_DCORE * NUM_OF_DCORES; 11423 11424 /* check all HMMUs */ 11425 for (i = 0 ; i < num_of_hmmus ; i++) { 11426 mmu_id = HW_CAP_DCORE0_DMMU0 << i; 11427 11428 if (mmu_cap_mask & mmu_id) 11429 gaudi2_ack_mmu_error(hdev, mmu_id); 11430 } 11431 11432 /* check PMMU */ 11433 if (mmu_cap_mask & HW_CAP_PMMU) 11434 gaudi2_ack_mmu_error(hdev, HW_CAP_PMMU); 11435 11436 return 0; 11437 } 11438 11439 static void gaudi2_get_msi_info(__le32 *table) 11440 { 11441 table[CPUCP_EVENT_QUEUE_MSI_TYPE] = cpu_to_le32(GAUDI2_EVENT_QUEUE_MSIX_IDX); 11442 table[CPUCP_EVENT_QUEUE_ERR_MSI_TYPE] = cpu_to_le32(GAUDI2_IRQ_NUM_EQ_ERROR); 11443 } 11444 11445 static int gaudi2_map_pll_idx_to_fw_idx(u32 pll_idx) 11446 { 11447 switch (pll_idx) { 11448 case HL_GAUDI2_CPU_PLL: return CPU_PLL; 11449 case HL_GAUDI2_PCI_PLL: return PCI_PLL; 11450 case HL_GAUDI2_NIC_PLL: return NIC_PLL; 11451 case HL_GAUDI2_DMA_PLL: return DMA_PLL; 11452 case HL_GAUDI2_MESH_PLL: return MESH_PLL; 11453 case HL_GAUDI2_MME_PLL: return MME_PLL; 11454 case HL_GAUDI2_TPC_PLL: return TPC_PLL; 11455 case HL_GAUDI2_IF_PLL: return IF_PLL; 11456 case HL_GAUDI2_SRAM_PLL: return SRAM_PLL; 11457 case HL_GAUDI2_HBM_PLL: return HBM_PLL; 11458 case HL_GAUDI2_VID_PLL: return VID_PLL; 11459 case HL_GAUDI2_MSS_PLL: return MSS_PLL; 11460 default: return -EINVAL; 11461 } 11462 } 11463 11464 static int gaudi2_gen_sync_to_engine_map(struct hl_device *hdev, struct hl_sync_to_engine_map *map) 11465 { 11466 /* Not implemented */ 11467 return 0; 11468 } 11469 11470 static int gaudi2_monitor_valid(struct hl_mon_state_dump *mon) 11471 { 11472 /* Not implemented */ 11473 return 0; 11474 } 11475 11476 static int gaudi2_print_single_monitor(char **buf, size_t *size, size_t *offset, 11477 struct hl_device *hdev, struct hl_mon_state_dump *mon) 11478 { 11479 /* Not implemented */ 11480 return 0; 11481 } 11482 11483 11484 static int gaudi2_print_fences_single_engine(struct hl_device *hdev, u64 base_offset, 11485 u64 status_base_offset, enum hl_sync_engine_type engine_type, 11486 u32 engine_id, char **buf, size_t *size, size_t *offset) 11487 { 11488 /* Not implemented */ 11489 return 0; 11490 } 11491 11492 11493 static struct hl_state_dump_specs_funcs gaudi2_state_dump_funcs = { 11494 .monitor_valid = gaudi2_monitor_valid, 11495 .print_single_monitor = gaudi2_print_single_monitor, 11496 .gen_sync_to_engine_map = gaudi2_gen_sync_to_engine_map, 11497 .print_fences_single_engine = gaudi2_print_fences_single_engine, 11498 }; 11499 11500 static void gaudi2_state_dump_init(struct hl_device *hdev) 11501 { 11502 /* Not implemented */ 11503 hdev->state_dump_specs.props = gaudi2_state_dump_specs_props; 11504 hdev->state_dump_specs.funcs = gaudi2_state_dump_funcs; 11505 } 11506 11507 static u32 gaudi2_get_sob_addr(struct hl_device *hdev, u32 sob_id) 11508 { 11509 return 0; 11510 } 11511 11512 static u32 *gaudi2_get_stream_master_qid_arr(void) 11513 { 11514 return NULL; 11515 } 11516 11517 static void gaudi2_add_device_attr(struct hl_device *hdev, struct attribute_group *dev_clk_attr_grp, 11518 struct attribute_group *dev_vrm_attr_grp) 11519 { 11520 hl_sysfs_add_dev_clk_attr(hdev, dev_clk_attr_grp); 11521 hl_sysfs_add_dev_vrm_attr(hdev, dev_vrm_attr_grp); 11522 } 11523 11524 static int gaudi2_mmu_get_real_page_size(struct hl_device *hdev, struct hl_mmu_properties *mmu_prop, 11525 u32 page_size, u32 *real_page_size, bool is_dram_addr) 11526 { 11527 struct asic_fixed_properties *prop = &hdev->asic_prop; 11528 11529 /* for host pages the page size must be */ 11530 if (!is_dram_addr) { 11531 if (page_size % mmu_prop->page_size) 11532 goto page_size_err; 11533 11534 *real_page_size = mmu_prop->page_size; 11535 return 0; 11536 } 11537 11538 if ((page_size % prop->dram_page_size) || (prop->dram_page_size > mmu_prop->page_size)) 11539 goto page_size_err; 11540 11541 /* 11542 * MMU page size is different from DRAM page size (more precisely, DMMU page is greater 11543 * than DRAM page size). 11544 * for this reason work with the DRAM page size and let the MMU scrambling routine handle 11545 * this mismatch when calculating the address to place in the MMU page table. 11546 * (in that case also make sure that the dram_page_size is not greater than the 11547 * mmu page size) 11548 */ 11549 *real_page_size = prop->dram_page_size; 11550 11551 return 0; 11552 11553 page_size_err: 11554 dev_err(hdev->dev, "page size of 0x%X is not 0x%X aligned, can't map\n", 11555 page_size, mmu_prop->page_size >> 10); 11556 return -EFAULT; 11557 } 11558 11559 static int gaudi2_get_monitor_dump(struct hl_device *hdev, void *data) 11560 { 11561 return -EOPNOTSUPP; 11562 } 11563 11564 int gaudi2_send_device_activity(struct hl_device *hdev, bool open) 11565 { 11566 struct gaudi2_device *gaudi2 = hdev->asic_specific; 11567 11568 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 11569 return 0; 11570 11571 return hl_fw_send_device_activity(hdev, open); 11572 } 11573 11574 static u64 gaudi2_read_pte(struct hl_device *hdev, u64 addr) 11575 { 11576 struct gaudi2_device *gaudi2 = hdev->asic_specific; 11577 u64 val; 11578 11579 if (hdev->reset_info.hard_reset_pending) 11580 return U64_MAX; 11581 11582 val = readq(hdev->pcie_bar[DRAM_BAR_ID] + (addr - gaudi2->dram_bar_cur_addr)); 11583 11584 return val; 11585 } 11586 11587 static void gaudi2_write_pte(struct hl_device *hdev, u64 addr, u64 val) 11588 { 11589 struct gaudi2_device *gaudi2 = hdev->asic_specific; 11590 11591 if (hdev->reset_info.hard_reset_pending) 11592 return; 11593 11594 writeq(val, hdev->pcie_bar[DRAM_BAR_ID] + (addr - gaudi2->dram_bar_cur_addr)); 11595 } 11596 11597 static const struct hl_asic_funcs gaudi2_funcs = { 11598 .early_init = gaudi2_early_init, 11599 .early_fini = gaudi2_early_fini, 11600 .late_init = gaudi2_late_init, 11601 .late_fini = gaudi2_late_fini, 11602 .sw_init = gaudi2_sw_init, 11603 .sw_fini = gaudi2_sw_fini, 11604 .hw_init = gaudi2_hw_init, 11605 .hw_fini = gaudi2_hw_fini, 11606 .halt_engines = gaudi2_halt_engines, 11607 .suspend = gaudi2_suspend, 11608 .resume = gaudi2_resume, 11609 .mmap = gaudi2_mmap, 11610 .ring_doorbell = gaudi2_ring_doorbell, 11611 .pqe_write = gaudi2_pqe_write, 11612 .asic_dma_alloc_coherent = gaudi2_dma_alloc_coherent, 11613 .asic_dma_free_coherent = gaudi2_dma_free_coherent, 11614 .scrub_device_mem = gaudi2_scrub_device_mem, 11615 .scrub_device_dram = gaudi2_scrub_device_dram, 11616 .get_int_queue_base = NULL, 11617 .test_queues = gaudi2_test_queues, 11618 .asic_dma_pool_zalloc = gaudi2_dma_pool_zalloc, 11619 .asic_dma_pool_free = gaudi2_dma_pool_free, 11620 .cpu_accessible_dma_pool_alloc = gaudi2_cpu_accessible_dma_pool_alloc, 11621 .cpu_accessible_dma_pool_free = gaudi2_cpu_accessible_dma_pool_free, 11622 .dma_unmap_sgtable = hl_asic_dma_unmap_sgtable, 11623 .cs_parser = gaudi2_cs_parser, 11624 .dma_map_sgtable = hl_asic_dma_map_sgtable, 11625 .add_end_of_cb_packets = NULL, 11626 .update_eq_ci = gaudi2_update_eq_ci, 11627 .context_switch = gaudi2_context_switch, 11628 .restore_phase_topology = gaudi2_restore_phase_topology, 11629 .debugfs_read_dma = gaudi2_debugfs_read_dma, 11630 .add_device_attr = gaudi2_add_device_attr, 11631 .handle_eqe = gaudi2_handle_eqe, 11632 .get_events_stat = gaudi2_get_events_stat, 11633 .read_pte = gaudi2_read_pte, 11634 .write_pte = gaudi2_write_pte, 11635 .mmu_invalidate_cache = gaudi2_mmu_invalidate_cache, 11636 .mmu_invalidate_cache_range = gaudi2_mmu_invalidate_cache_range, 11637 .mmu_prefetch_cache_range = NULL, 11638 .send_heartbeat = gaudi2_send_heartbeat, 11639 .debug_coresight = gaudi2_debug_coresight, 11640 .is_device_idle = gaudi2_is_device_idle, 11641 .compute_reset_late_init = gaudi2_compute_reset_late_init, 11642 .hw_queues_lock = gaudi2_hw_queues_lock, 11643 .hw_queues_unlock = gaudi2_hw_queues_unlock, 11644 .get_pci_id = gaudi2_get_pci_id, 11645 .get_eeprom_data = gaudi2_get_eeprom_data, 11646 .get_monitor_dump = gaudi2_get_monitor_dump, 11647 .send_cpu_message = gaudi2_send_cpu_message, 11648 .pci_bars_map = gaudi2_pci_bars_map, 11649 .init_iatu = gaudi2_init_iatu, 11650 .rreg = hl_rreg, 11651 .wreg = hl_wreg, 11652 .halt_coresight = gaudi2_halt_coresight, 11653 .ctx_init = gaudi2_ctx_init, 11654 .ctx_fini = gaudi2_ctx_fini, 11655 .pre_schedule_cs = gaudi2_pre_schedule_cs, 11656 .get_queue_id_for_cq = gaudi2_get_queue_id_for_cq, 11657 .load_firmware_to_device = NULL, 11658 .load_boot_fit_to_device = NULL, 11659 .get_signal_cb_size = gaudi2_get_signal_cb_size, 11660 .get_wait_cb_size = gaudi2_get_wait_cb_size, 11661 .gen_signal_cb = gaudi2_gen_signal_cb, 11662 .gen_wait_cb = gaudi2_gen_wait_cb, 11663 .reset_sob = gaudi2_reset_sob, 11664 .reset_sob_group = gaudi2_reset_sob_group, 11665 .get_device_time = gaudi2_get_device_time, 11666 .pb_print_security_errors = gaudi2_pb_print_security_errors, 11667 .collective_wait_init_cs = gaudi2_collective_wait_init_cs, 11668 .collective_wait_create_jobs = gaudi2_collective_wait_create_jobs, 11669 .get_dec_base_addr = gaudi2_get_dec_base_addr, 11670 .scramble_addr = gaudi2_mmu_scramble_addr, 11671 .descramble_addr = gaudi2_mmu_descramble_addr, 11672 .ack_protection_bits_errors = gaudi2_ack_protection_bits_errors, 11673 .get_hw_block_id = gaudi2_get_hw_block_id, 11674 .hw_block_mmap = gaudi2_block_mmap, 11675 .enable_events_from_fw = gaudi2_enable_events_from_fw, 11676 .ack_mmu_errors = gaudi2_ack_mmu_page_fault_or_access_error, 11677 .get_msi_info = gaudi2_get_msi_info, 11678 .map_pll_idx_to_fw_idx = gaudi2_map_pll_idx_to_fw_idx, 11679 .init_firmware_preload_params = gaudi2_init_firmware_preload_params, 11680 .init_firmware_loader = gaudi2_init_firmware_loader, 11681 .init_cpu_scrambler_dram = gaudi2_init_scrambler_hbm, 11682 .state_dump_init = gaudi2_state_dump_init, 11683 .get_sob_addr = &gaudi2_get_sob_addr, 11684 .set_pci_memory_regions = gaudi2_set_pci_memory_regions, 11685 .get_stream_master_qid_arr = gaudi2_get_stream_master_qid_arr, 11686 .check_if_razwi_happened = gaudi2_check_if_razwi_happened, 11687 .mmu_get_real_page_size = gaudi2_mmu_get_real_page_size, 11688 .access_dev_mem = hl_access_dev_mem, 11689 .set_dram_bar_base = gaudi2_set_hbm_bar_base, 11690 .set_engine_cores = gaudi2_set_engine_cores, 11691 .set_engines = gaudi2_set_engines, 11692 .send_device_activity = gaudi2_send_device_activity, 11693 .set_dram_properties = gaudi2_set_dram_properties, 11694 .set_binning_masks = gaudi2_set_binning_masks, 11695 }; 11696 11697 void gaudi2_set_asic_funcs(struct hl_device *hdev) 11698 { 11699 hdev->asic_funcs = &gaudi2_funcs; 11700 } 11701