1 // SPDX-License-Identifier: GPL-2.0 2 3 /* 4 * Copyright 2020-2022 HabanaLabs, Ltd. 5 * All Rights Reserved. 6 */ 7 8 #include "gaudi2P.h" 9 #include "gaudi2_masks.h" 10 #include "../include/gaudi2/gaudi2_special_blocks.h" 11 #include "../include/hw_ip/mmu/mmu_general.h" 12 #include "../include/hw_ip/mmu/mmu_v2_0.h" 13 #include "../include/gaudi2/gaudi2_packets.h" 14 #include "../include/gaudi2/gaudi2_reg_map.h" 15 #include "../include/gaudi2/gaudi2_async_ids_map_extended.h" 16 #include "../include/gaudi2/arc/gaudi2_arc_common_packets.h" 17 18 #include <linux/module.h> 19 #include <linux/pci.h> 20 #include <linux/hwmon.h> 21 #include <linux/iommu.h> 22 23 #define GAUDI2_DMA_POOL_BLK_SIZE SZ_256 /* 256 bytes */ 24 25 #define GAUDI2_RESET_TIMEOUT_MSEC 2000 /* 2000ms */ 26 27 #define GAUDI2_RESET_POLL_TIMEOUT_USEC 500000 /* 500ms */ 28 #define GAUDI2_PLDM_HRESET_TIMEOUT_MSEC 25000 /* 25s */ 29 #define GAUDI2_PLDM_SRESET_TIMEOUT_MSEC 25000 /* 25s */ 30 #define GAUDI2_PLDM_RESET_POLL_TIMEOUT_USEC 3000000 /* 3s */ 31 #define GAUDI2_RESET_POLL_CNT 3 32 #define GAUDI2_RESET_WAIT_MSEC 1 /* 1ms */ 33 #define GAUDI2_CPU_RESET_WAIT_MSEC 100 /* 100ms */ 34 #define GAUDI2_PLDM_RESET_WAIT_MSEC 1000 /* 1s */ 35 #define GAUDI2_CB_POOL_CB_CNT 512 36 #define GAUDI2_CB_POOL_CB_SIZE SZ_128K /* 128KB */ 37 #define GAUDI2_MSG_TO_CPU_TIMEOUT_USEC 4000000 /* 4s */ 38 #define GAUDI2_WAIT_FOR_BL_TIMEOUT_USEC 25000000 /* 25s */ 39 #define GAUDI2_TEST_QUEUE_WAIT_USEC 100000 /* 100ms */ 40 #define GAUDI2_PLDM_TEST_QUEUE_WAIT_USEC 1000000 /* 1s */ 41 42 #define GAUDI2_ALLOC_CPU_MEM_RETRY_CNT 3 43 44 /* 45 * since the code already has built-in support for binning of up to MAX_FAULTY_TPCS TPCs 46 * and the code relies on that value (for array size etc..) we define another value 47 * for MAX faulty TPCs which reflects the cluster binning requirements 48 */ 49 #define MAX_CLUSTER_BINNING_FAULTY_TPCS 1 50 #define MAX_FAULTY_XBARS 1 51 #define MAX_FAULTY_EDMAS 1 52 #define MAX_FAULTY_DECODERS 1 53 54 #define GAUDI2_TPC_FULL_MASK 0x1FFFFFF 55 #define GAUDI2_HIF_HMMU_FULL_MASK 0xFFFF 56 #define GAUDI2_DECODER_FULL_MASK 0x3FF 57 58 #define GAUDI2_NA_EVENT_CAUSE 0xFF 59 #define GAUDI2_NUM_OF_QM_ERR_CAUSE 18 60 #define GAUDI2_NUM_OF_LOWER_QM_ERR_CAUSE 25 61 #define GAUDI2_NUM_OF_QM_ARB_ERR_CAUSE 3 62 #define GAUDI2_NUM_OF_ARC_SEI_ERR_CAUSE 14 63 #define GAUDI2_NUM_OF_CPU_SEI_ERR_CAUSE 3 64 #define GAUDI2_NUM_OF_QM_SEI_ERR_CAUSE 2 65 #define GAUDI2_NUM_OF_ROT_ERR_CAUSE 22 66 #define GAUDI2_NUM_OF_TPC_INTR_CAUSE 31 67 #define GAUDI2_NUM_OF_DEC_ERR_CAUSE 25 68 #define GAUDI2_NUM_OF_MME_ERR_CAUSE 16 69 #define GAUDI2_NUM_OF_MME_WAP_ERR_CAUSE 7 70 #define GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE 8 71 #define GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE 19 72 #define GAUDI2_NUM_OF_HBM_SEI_CAUSE 9 73 #define GAUDI2_NUM_OF_SM_SEI_ERR_CAUSE 3 74 #define GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE 3 75 #define GAUDI2_NUM_OF_PMMU_FATAL_ERR_CAUSE 2 76 #define GAUDI2_NUM_OF_HIF_FATAL_ERR_CAUSE 2 77 #define GAUDI2_NUM_OF_AXI_DRAIN_ERR_CAUSE 2 78 #define GAUDI2_NUM_OF_HBM_MC_SPI_CAUSE 5 79 80 #define GAUDI2_MMU_CACHE_INV_TIMEOUT_USEC (MMU_CONFIG_TIMEOUT_USEC * 10) 81 #define GAUDI2_PLDM_MMU_TIMEOUT_USEC (MMU_CONFIG_TIMEOUT_USEC * 200) 82 #define GAUDI2_ARB_WDT_TIMEOUT (0x1000000) 83 84 #define GAUDI2_VDEC_TIMEOUT_USEC 10000 /* 10ms */ 85 #define GAUDI2_PLDM_VDEC_TIMEOUT_USEC (GAUDI2_VDEC_TIMEOUT_USEC * 100) 86 87 #define KDMA_TIMEOUT_USEC USEC_PER_SEC 88 89 #define IS_DMA_IDLE(dma_core_sts0) \ 90 (!((dma_core_sts0) & (DCORE0_EDMA0_CORE_STS0_BUSY_MASK))) 91 92 #define IS_DMA_HALTED(dma_core_sts1) \ 93 ((dma_core_sts1) & (DCORE0_EDMA0_CORE_STS1_IS_HALT_MASK)) 94 95 #define IS_MME_IDLE(mme_arch_sts) (((mme_arch_sts) & MME_ARCH_IDLE_MASK) == MME_ARCH_IDLE_MASK) 96 97 #define IS_TPC_IDLE(tpc_cfg_sts) (((tpc_cfg_sts) & (TPC_IDLE_MASK)) == (TPC_IDLE_MASK)) 98 99 #define IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) \ 100 ((((qm_glbl_sts0) & (QM_IDLE_MASK)) == (QM_IDLE_MASK)) && \ 101 (((qm_glbl_sts1) & (QM_ARC_IDLE_MASK)) == (QM_ARC_IDLE_MASK)) && \ 102 (((qm_cgm_sts) & (CGM_IDLE_MASK)) == (CGM_IDLE_MASK))) 103 104 #define PCIE_DEC_EN_MASK 0x300 105 #define DEC_WORK_STATE_IDLE 0 106 #define DEC_WORK_STATE_PEND 3 107 #define IS_DEC_IDLE(dec_swreg15) \ 108 (((dec_swreg15) & DCORE0_DEC0_CMD_SWREG15_SW_WORK_STATE_MASK) == DEC_WORK_STATE_IDLE || \ 109 ((dec_swreg15) & DCORE0_DEC0_CMD_SWREG15_SW_WORK_STATE_MASK) == DEC_WORK_STATE_PEND) 110 111 /* HBM MMU address scrambling parameters */ 112 #define GAUDI2_HBM_MMU_SCRM_MEM_SIZE SZ_8M 113 #define GAUDI2_HBM_MMU_SCRM_DIV_SHIFT 26 114 #define GAUDI2_HBM_MMU_SCRM_MOD_SHIFT 0 115 #define GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK DRAM_VA_HINT_MASK 116 #define GAUDI2_COMPENSATE_TLB_PAGE_SIZE_FACTOR 16 117 #define MMU_RANGE_INV_VA_LSB_SHIFT 12 118 #define MMU_RANGE_INV_VA_MSB_SHIFT 44 119 #define MMU_RANGE_INV_EN_SHIFT 0 120 #define MMU_RANGE_INV_ASID_EN_SHIFT 1 121 #define MMU_RANGE_INV_ASID_SHIFT 2 122 123 /* The last SPI_SEI cause bit, "burst_fifo_full", is expected to be triggered in PMMU because it has 124 * a 2 entries FIFO, and hence it is not enabled for it. 125 */ 126 #define GAUDI2_PMMU_SPI_SEI_ENABLE_MASK GENMASK(GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE - 2, 0) 127 #define GAUDI2_HMMU_SPI_SEI_ENABLE_MASK GENMASK(GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE - 1, 0) 128 129 #define GAUDI2_MAX_STRING_LEN 64 130 131 #define GAUDI2_VDEC_MSIX_ENTRIES (GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM - \ 132 GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + 1) 133 134 #define ENGINE_ID_DCORE_OFFSET (GAUDI2_DCORE1_ENGINE_ID_EDMA_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0) 135 136 /* RAZWI initiator coordinates */ 137 #define RAZWI_GET_AXUSER_XY(x) \ 138 ((x & 0xF8001FF0) >> 4) 139 140 #define RAZWI_GET_AXUSER_LOW_XY(x) \ 141 ((x & 0x00001FF0) >> 4) 142 143 #define RAZWI_INITIATOR_AXUER_L_X_SHIFT 0 144 #define RAZWI_INITIATOR_AXUER_L_X_MASK 0x1F 145 #define RAZWI_INITIATOR_AXUER_L_Y_SHIFT 5 146 #define RAZWI_INITIATOR_AXUER_L_Y_MASK 0xF 147 148 #define RAZWI_INITIATOR_AXUER_H_X_SHIFT 23 149 #define RAZWI_INITIATOR_AXUER_H_X_MASK 0x1F 150 151 #define RAZWI_INITIATOR_ID_X_Y_LOW(x, y) \ 152 ((((y) & RAZWI_INITIATOR_AXUER_L_Y_MASK) << RAZWI_INITIATOR_AXUER_L_Y_SHIFT) | \ 153 (((x) & RAZWI_INITIATOR_AXUER_L_X_MASK) << RAZWI_INITIATOR_AXUER_L_X_SHIFT)) 154 155 #define RAZWI_INITIATOR_ID_X_HIGH(x) \ 156 (((x) & RAZWI_INITIATOR_AXUER_H_X_MASK) << RAZWI_INITIATOR_AXUER_H_X_SHIFT) 157 158 #define RAZWI_INITIATOR_ID_X_Y(xl, yl, xh) \ 159 (RAZWI_INITIATOR_ID_X_Y_LOW(xl, yl) | RAZWI_INITIATOR_ID_X_HIGH(xh)) 160 161 #define PSOC_RAZWI_ENG_STR_SIZE 128 162 #define PSOC_RAZWI_MAX_ENG_PER_RTR 5 163 164 /* HW scrambles only bits 0-25 */ 165 #define HW_UNSCRAMBLED_BITS_MASK GENMASK_ULL(63, 26) 166 167 struct gaudi2_razwi_info { 168 u32 axuser_xy; 169 u32 rtr_ctrl; 170 u16 eng_id; 171 char *eng_name; 172 }; 173 174 static struct gaudi2_razwi_info common_razwi_info[] = { 175 {RAZWI_INITIATOR_ID_X_Y(2, 4, 0), mmDCORE0_RTR0_CTRL_BASE, 176 GAUDI2_DCORE0_ENGINE_ID_DEC_0, "DEC0"}, 177 {RAZWI_INITIATOR_ID_X_Y(2, 4, 4), mmDCORE0_RTR0_CTRL_BASE, 178 GAUDI2_DCORE0_ENGINE_ID_DEC_1, "DEC1"}, 179 {RAZWI_INITIATOR_ID_X_Y(17, 4, 18), mmDCORE1_RTR7_CTRL_BASE, 180 GAUDI2_DCORE1_ENGINE_ID_DEC_0, "DEC2"}, 181 {RAZWI_INITIATOR_ID_X_Y(17, 4, 14), mmDCORE1_RTR7_CTRL_BASE, 182 GAUDI2_DCORE1_ENGINE_ID_DEC_1, "DEC3"}, 183 {RAZWI_INITIATOR_ID_X_Y(2, 11, 0), mmDCORE2_RTR0_CTRL_BASE, 184 GAUDI2_DCORE2_ENGINE_ID_DEC_0, "DEC4"}, 185 {RAZWI_INITIATOR_ID_X_Y(2, 11, 4), mmDCORE2_RTR0_CTRL_BASE, 186 GAUDI2_DCORE2_ENGINE_ID_DEC_1, "DEC5"}, 187 {RAZWI_INITIATOR_ID_X_Y(17, 11, 18), mmDCORE3_RTR7_CTRL_BASE, 188 GAUDI2_DCORE3_ENGINE_ID_DEC_0, "DEC6"}, 189 {RAZWI_INITIATOR_ID_X_Y(17, 11, 14), mmDCORE3_RTR7_CTRL_BASE, 190 GAUDI2_DCORE3_ENGINE_ID_DEC_1, "DEC7"}, 191 {RAZWI_INITIATOR_ID_X_Y(2, 4, 6), mmDCORE0_RTR0_CTRL_BASE, 192 GAUDI2_PCIE_ENGINE_ID_DEC_0, "DEC8"}, 193 {RAZWI_INITIATOR_ID_X_Y(2, 4, 7), mmDCORE0_RTR0_CTRL_BASE, 194 GAUDI2_PCIE_ENGINE_ID_DEC_0, "DEC9"}, 195 {RAZWI_INITIATOR_ID_X_Y(3, 4, 2), mmDCORE0_RTR1_CTRL_BASE, 196 GAUDI2_DCORE0_ENGINE_ID_TPC_0, "TPC0"}, 197 {RAZWI_INITIATOR_ID_X_Y(3, 4, 4), mmDCORE0_RTR1_CTRL_BASE, 198 GAUDI2_DCORE0_ENGINE_ID_TPC_1, "TPC1"}, 199 {RAZWI_INITIATOR_ID_X_Y(4, 4, 2), mmDCORE0_RTR2_CTRL_BASE, 200 GAUDI2_DCORE0_ENGINE_ID_TPC_2, "TPC2"}, 201 {RAZWI_INITIATOR_ID_X_Y(4, 4, 4), mmDCORE0_RTR2_CTRL_BASE, 202 GAUDI2_DCORE0_ENGINE_ID_TPC_3, "TPC3"}, 203 {RAZWI_INITIATOR_ID_X_Y(5, 4, 2), mmDCORE0_RTR3_CTRL_BASE, 204 GAUDI2_DCORE0_ENGINE_ID_TPC_4, "TPC4"}, 205 {RAZWI_INITIATOR_ID_X_Y(5, 4, 4), mmDCORE0_RTR3_CTRL_BASE, 206 GAUDI2_DCORE0_ENGINE_ID_TPC_5, "TPC5"}, 207 {RAZWI_INITIATOR_ID_X_Y(16, 4, 14), mmDCORE1_RTR6_CTRL_BASE, 208 GAUDI2_DCORE1_ENGINE_ID_TPC_0, "TPC6"}, 209 {RAZWI_INITIATOR_ID_X_Y(16, 4, 16), mmDCORE1_RTR6_CTRL_BASE, 210 GAUDI2_DCORE1_ENGINE_ID_TPC_1, "TPC7"}, 211 {RAZWI_INITIATOR_ID_X_Y(15, 4, 14), mmDCORE1_RTR5_CTRL_BASE, 212 GAUDI2_DCORE1_ENGINE_ID_TPC_2, "TPC8"}, 213 {RAZWI_INITIATOR_ID_X_Y(15, 4, 16), mmDCORE1_RTR5_CTRL_BASE, 214 GAUDI2_DCORE1_ENGINE_ID_TPC_3, "TPC9"}, 215 {RAZWI_INITIATOR_ID_X_Y(14, 4, 14), mmDCORE1_RTR4_CTRL_BASE, 216 GAUDI2_DCORE1_ENGINE_ID_TPC_4, "TPC10"}, 217 {RAZWI_INITIATOR_ID_X_Y(14, 4, 16), mmDCORE1_RTR4_CTRL_BASE, 218 GAUDI2_DCORE1_ENGINE_ID_TPC_5, "TPC11"}, 219 {RAZWI_INITIATOR_ID_X_Y(5, 11, 2), mmDCORE2_RTR3_CTRL_BASE, 220 GAUDI2_DCORE2_ENGINE_ID_TPC_0, "TPC12"}, 221 {RAZWI_INITIATOR_ID_X_Y(5, 11, 4), mmDCORE2_RTR3_CTRL_BASE, 222 GAUDI2_DCORE2_ENGINE_ID_TPC_1, "TPC13"}, 223 {RAZWI_INITIATOR_ID_X_Y(4, 11, 2), mmDCORE2_RTR2_CTRL_BASE, 224 GAUDI2_DCORE2_ENGINE_ID_TPC_2, "TPC14"}, 225 {RAZWI_INITIATOR_ID_X_Y(4, 11, 4), mmDCORE2_RTR2_CTRL_BASE, 226 GAUDI2_DCORE2_ENGINE_ID_TPC_3, "TPC15"}, 227 {RAZWI_INITIATOR_ID_X_Y(3, 11, 2), mmDCORE2_RTR1_CTRL_BASE, 228 GAUDI2_DCORE2_ENGINE_ID_TPC_4, "TPC16"}, 229 {RAZWI_INITIATOR_ID_X_Y(3, 11, 4), mmDCORE2_RTR1_CTRL_BASE, 230 GAUDI2_DCORE2_ENGINE_ID_TPC_5, "TPC17"}, 231 {RAZWI_INITIATOR_ID_X_Y(14, 11, 14), mmDCORE3_RTR4_CTRL_BASE, 232 GAUDI2_DCORE3_ENGINE_ID_TPC_0, "TPC18"}, 233 {RAZWI_INITIATOR_ID_X_Y(14, 11, 16), mmDCORE3_RTR4_CTRL_BASE, 234 GAUDI2_DCORE3_ENGINE_ID_TPC_1, "TPC19"}, 235 {RAZWI_INITIATOR_ID_X_Y(15, 11, 14), mmDCORE3_RTR5_CTRL_BASE, 236 GAUDI2_DCORE3_ENGINE_ID_TPC_2, "TPC20"}, 237 {RAZWI_INITIATOR_ID_X_Y(15, 11, 16), mmDCORE3_RTR5_CTRL_BASE, 238 GAUDI2_DCORE3_ENGINE_ID_TPC_3, "TPC21"}, 239 {RAZWI_INITIATOR_ID_X_Y(16, 11, 14), mmDCORE3_RTR6_CTRL_BASE, 240 GAUDI2_DCORE3_ENGINE_ID_TPC_4, "TPC22"}, 241 {RAZWI_INITIATOR_ID_X_Y(16, 11, 16), mmDCORE3_RTR6_CTRL_BASE, 242 GAUDI2_DCORE3_ENGINE_ID_TPC_5, "TPC23"}, 243 {RAZWI_INITIATOR_ID_X_Y(2, 4, 2), mmDCORE0_RTR0_CTRL_BASE, 244 GAUDI2_DCORE3_ENGINE_ID_TPC_5, "TPC24"}, 245 {RAZWI_INITIATOR_ID_X_Y(17, 4, 8), mmDCORE1_RTR7_CTRL_BASE, 246 GAUDI2_ENGINE_ID_NIC0_0, "NIC0"}, 247 {RAZWI_INITIATOR_ID_X_Y(17, 4, 10), mmDCORE1_RTR7_CTRL_BASE, 248 GAUDI2_ENGINE_ID_NIC0_1, "NIC1"}, 249 {RAZWI_INITIATOR_ID_X_Y(17, 4, 12), mmDCORE1_RTR7_CTRL_BASE, 250 GAUDI2_ENGINE_ID_NIC1_0, "NIC2"}, 251 {RAZWI_INITIATOR_ID_X_Y(17, 4, 14), mmDCORE1_RTR7_CTRL_BASE, 252 GAUDI2_ENGINE_ID_NIC1_1, "NIC3"}, 253 {RAZWI_INITIATOR_ID_X_Y(17, 4, 15), mmDCORE1_RTR7_CTRL_BASE, 254 GAUDI2_ENGINE_ID_NIC2_0, "NIC4"}, 255 {RAZWI_INITIATOR_ID_X_Y(2, 11, 2), mmDCORE2_RTR0_CTRL_BASE, 256 GAUDI2_ENGINE_ID_NIC2_1, "NIC5"}, 257 {RAZWI_INITIATOR_ID_X_Y(2, 11, 4), mmDCORE2_RTR0_CTRL_BASE, 258 GAUDI2_ENGINE_ID_NIC3_0, "NIC6"}, 259 {RAZWI_INITIATOR_ID_X_Y(2, 11, 6), mmDCORE2_RTR0_CTRL_BASE, 260 GAUDI2_ENGINE_ID_NIC3_1, "NIC7"}, 261 {RAZWI_INITIATOR_ID_X_Y(2, 11, 8), mmDCORE2_RTR0_CTRL_BASE, 262 GAUDI2_ENGINE_ID_NIC4_0, "NIC8"}, 263 {RAZWI_INITIATOR_ID_X_Y(17, 11, 12), mmDCORE3_RTR7_CTRL_BASE, 264 GAUDI2_ENGINE_ID_NIC4_1, "NIC9"}, 265 {RAZWI_INITIATOR_ID_X_Y(17, 11, 14), mmDCORE3_RTR7_CTRL_BASE, 266 GAUDI2_ENGINE_ID_NIC5_0, "NIC10"}, 267 {RAZWI_INITIATOR_ID_X_Y(17, 11, 16), mmDCORE3_RTR7_CTRL_BASE, 268 GAUDI2_ENGINE_ID_NIC5_1, "NIC11"}, 269 {RAZWI_INITIATOR_ID_X_Y(2, 4, 2), mmDCORE0_RTR0_CTRL_BASE, 270 GAUDI2_ENGINE_ID_PDMA_0, "PDMA0"}, 271 {RAZWI_INITIATOR_ID_X_Y(2, 4, 3), mmDCORE0_RTR0_CTRL_BASE, 272 GAUDI2_ENGINE_ID_PDMA_1, "PDMA1"}, 273 {RAZWI_INITIATOR_ID_X_Y(2, 4, 4), mmDCORE0_RTR0_CTRL_BASE, 274 GAUDI2_ENGINE_ID_SIZE, "PMMU"}, 275 {RAZWI_INITIATOR_ID_X_Y(2, 4, 5), mmDCORE0_RTR0_CTRL_BASE, 276 GAUDI2_ENGINE_ID_SIZE, "PCIE"}, 277 {RAZWI_INITIATOR_ID_X_Y(17, 4, 16), mmDCORE1_RTR7_CTRL_BASE, 278 GAUDI2_ENGINE_ID_ARC_FARM, "ARC_FARM"}, 279 {RAZWI_INITIATOR_ID_X_Y(17, 4, 17), mmDCORE1_RTR7_CTRL_BASE, 280 GAUDI2_ENGINE_ID_KDMA, "KDMA"}, 281 {RAZWI_INITIATOR_ID_X_Y(1, 5, 1), mmSFT0_HBW_RTR_IF1_RTR_CTRL_BASE, 282 GAUDI2_DCORE0_ENGINE_ID_EDMA_0, "EDMA0"}, 283 {RAZWI_INITIATOR_ID_X_Y(1, 5, 1), mmSFT0_HBW_RTR_IF0_RTR_CTRL_BASE, 284 GAUDI2_DCORE0_ENGINE_ID_EDMA_1, "EDMA1"}, 285 {RAZWI_INITIATOR_ID_X_Y(18, 5, 18), mmSFT1_HBW_RTR_IF1_RTR_CTRL_BASE, 286 GAUDI2_DCORE1_ENGINE_ID_EDMA_0, "EDMA2"}, 287 {RAZWI_INITIATOR_ID_X_Y(18, 5, 18), mmSFT1_HBW_RTR_IF0_RTR_CTRL_BASE, 288 GAUDI2_DCORE1_ENGINE_ID_EDMA_1, "EDMA3"}, 289 {RAZWI_INITIATOR_ID_X_Y(1, 10, 1), mmSFT2_HBW_RTR_IF0_RTR_CTRL_BASE, 290 GAUDI2_DCORE2_ENGINE_ID_EDMA_0, "EDMA4"}, 291 {RAZWI_INITIATOR_ID_X_Y(1, 10, 1), mmSFT2_HBW_RTR_IF1_RTR_CTRL_BASE, 292 GAUDI2_DCORE2_ENGINE_ID_EDMA_1, "EDMA5"}, 293 {RAZWI_INITIATOR_ID_X_Y(18, 10, 18), mmSFT2_HBW_RTR_IF0_RTR_CTRL_BASE, 294 GAUDI2_DCORE3_ENGINE_ID_EDMA_0, "EDMA6"}, 295 {RAZWI_INITIATOR_ID_X_Y(18, 10, 18), mmSFT2_HBW_RTR_IF1_RTR_CTRL_BASE, 296 GAUDI2_DCORE3_ENGINE_ID_EDMA_1, "EDMA7"}, 297 {RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE, 298 GAUDI2_ENGINE_ID_SIZE, "HMMU0"}, 299 {RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE, 300 GAUDI2_ENGINE_ID_SIZE, "HMMU1"}, 301 {RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE, 302 GAUDI2_ENGINE_ID_SIZE, "HMMU2"}, 303 {RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE, 304 GAUDI2_ENGINE_ID_SIZE, "HMMU3"}, 305 {RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE, 306 GAUDI2_ENGINE_ID_SIZE, "HMMU4"}, 307 {RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE, 308 GAUDI2_ENGINE_ID_SIZE, "HMMU5"}, 309 {RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE, 310 GAUDI2_ENGINE_ID_SIZE, "HMMU6"}, 311 {RAZWI_INITIATOR_ID_X_Y(18, 5, 19), mmDCORE1_RTR7_CTRL_BASE, 312 GAUDI2_ENGINE_ID_SIZE, "HMMU7"}, 313 {RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE, 314 GAUDI2_ENGINE_ID_SIZE, "HMMU8"}, 315 {RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE, 316 GAUDI2_ENGINE_ID_SIZE, "HMMU9"}, 317 {RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE, 318 GAUDI2_ENGINE_ID_SIZE, "HMMU10"}, 319 {RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE, 320 GAUDI2_ENGINE_ID_SIZE, "HMMU11"}, 321 {RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE, 322 GAUDI2_ENGINE_ID_SIZE, "HMMU12"}, 323 {RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE, 324 GAUDI2_ENGINE_ID_SIZE, "HMMU13"}, 325 {RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE, 326 GAUDI2_ENGINE_ID_SIZE, "HMMU14"}, 327 {RAZWI_INITIATOR_ID_X_Y(18, 10, 19), mmDCORE3_RTR7_CTRL_BASE, 328 GAUDI2_ENGINE_ID_SIZE, "HMMU15"}, 329 {RAZWI_INITIATOR_ID_X_Y(2, 11, 2), mmDCORE2_RTR0_CTRL_BASE, 330 GAUDI2_ENGINE_ID_ROT_0, "ROT0"}, 331 {RAZWI_INITIATOR_ID_X_Y(17, 11, 16), mmDCORE3_RTR7_CTRL_BASE, 332 GAUDI2_ENGINE_ID_ROT_1, "ROT1"}, 333 {RAZWI_INITIATOR_ID_X_Y(2, 11, 2), mmDCORE2_RTR0_CTRL_BASE, 334 GAUDI2_ENGINE_ID_PSOC, "CPU"}, 335 {RAZWI_INITIATOR_ID_X_Y(17, 11, 11), mmDCORE3_RTR7_CTRL_BASE, 336 GAUDI2_ENGINE_ID_PSOC, "PSOC"} 337 }; 338 339 static struct gaudi2_razwi_info mme_razwi_info[] = { 340 /* MME X high coordinate is N/A, hence using only low coordinates */ 341 {RAZWI_INITIATOR_ID_X_Y_LOW(7, 4), mmDCORE0_RTR5_CTRL_BASE, 342 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_WAP0"}, 343 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 4), mmDCORE0_RTR7_CTRL_BASE, 344 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_WAP1"}, 345 {RAZWI_INITIATOR_ID_X_Y_LOW(8, 4), mmDCORE0_RTR6_CTRL_BASE, 346 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_CTRL_WR"}, 347 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 4), mmDCORE0_RTR7_CTRL_BASE, 348 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_CTRL_RD"}, 349 {RAZWI_INITIATOR_ID_X_Y_LOW(6, 4), mmDCORE0_RTR4_CTRL_BASE, 350 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE0"}, 351 {RAZWI_INITIATOR_ID_X_Y_LOW(6, 4), mmDCORE0_RTR4_CTRL_BASE, 352 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE1"}, 353 {RAZWI_INITIATOR_ID_X_Y_LOW(7, 4), mmDCORE0_RTR5_CTRL_BASE, 354 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE2"}, 355 {RAZWI_INITIATOR_ID_X_Y_LOW(8, 4), mmDCORE0_RTR6_CTRL_BASE, 356 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE3"}, 357 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 4), mmDCORE0_RTR7_CTRL_BASE, 358 GAUDI2_DCORE0_ENGINE_ID_MME, "MME0_SBTE4"}, 359 {RAZWI_INITIATOR_ID_X_Y_LOW(12, 4), mmDCORE1_RTR2_CTRL_BASE, 360 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_WAP0"}, 361 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 4), mmDCORE1_RTR0_CTRL_BASE, 362 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_WAP1"}, 363 {RAZWI_INITIATOR_ID_X_Y_LOW(11, 4), mmDCORE1_RTR1_CTRL_BASE, 364 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_CTRL_WR"}, 365 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 4), mmDCORE1_RTR0_CTRL_BASE, 366 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_CTRL_RD"}, 367 {RAZWI_INITIATOR_ID_X_Y_LOW(13, 4), mmDCORE1_RTR3_CTRL_BASE, 368 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE0"}, 369 {RAZWI_INITIATOR_ID_X_Y_LOW(13, 4), mmDCORE1_RTR3_CTRL_BASE, 370 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE1"}, 371 {RAZWI_INITIATOR_ID_X_Y_LOW(12, 4), mmDCORE1_RTR2_CTRL_BASE, 372 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE2"}, 373 {RAZWI_INITIATOR_ID_X_Y_LOW(11, 4), mmDCORE1_RTR1_CTRL_BASE, 374 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE3"}, 375 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 4), mmDCORE1_RTR0_CTRL_BASE, 376 GAUDI2_DCORE1_ENGINE_ID_MME, "MME1_SBTE4"}, 377 {RAZWI_INITIATOR_ID_X_Y_LOW(7, 11), mmDCORE2_RTR5_CTRL_BASE, 378 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_WAP0"}, 379 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 11), mmDCORE2_RTR7_CTRL_BASE, 380 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_WAP1"}, 381 {RAZWI_INITIATOR_ID_X_Y_LOW(8, 11), mmDCORE2_RTR6_CTRL_BASE, 382 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_CTRL_WR"}, 383 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 11), mmDCORE2_RTR7_CTRL_BASE, 384 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_CTRL_RD"}, 385 {RAZWI_INITIATOR_ID_X_Y_LOW(6, 11), mmDCORE2_RTR4_CTRL_BASE, 386 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE0"}, 387 {RAZWI_INITIATOR_ID_X_Y_LOW(6, 11), mmDCORE2_RTR4_CTRL_BASE, 388 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE1"}, 389 {RAZWI_INITIATOR_ID_X_Y_LOW(7, 11), mmDCORE2_RTR5_CTRL_BASE, 390 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE2"}, 391 {RAZWI_INITIATOR_ID_X_Y_LOW(8, 11), mmDCORE2_RTR6_CTRL_BASE, 392 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE3"}, 393 {RAZWI_INITIATOR_ID_X_Y_LOW(9, 11), mmDCORE2_RTR7_CTRL_BASE, 394 GAUDI2_DCORE2_ENGINE_ID_MME, "MME2_SBTE4"}, 395 {RAZWI_INITIATOR_ID_X_Y_LOW(12, 11), mmDCORE3_RTR2_CTRL_BASE, 396 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_WAP0"}, 397 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 11), mmDCORE3_RTR0_CTRL_BASE, 398 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_WAP1"}, 399 {RAZWI_INITIATOR_ID_X_Y_LOW(11, 11), mmDCORE3_RTR1_CTRL_BASE, 400 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_CTRL_WR"}, 401 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 11), mmDCORE3_RTR0_CTRL_BASE, 402 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_CTRL_RD"}, 403 {RAZWI_INITIATOR_ID_X_Y_LOW(13, 11), mmDCORE3_RTR3_CTRL_BASE, 404 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE0"}, 405 {RAZWI_INITIATOR_ID_X_Y_LOW(13, 11), mmDCORE3_RTR3_CTRL_BASE, 406 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE1"}, 407 {RAZWI_INITIATOR_ID_X_Y_LOW(12, 11), mmDCORE3_RTR2_CTRL_BASE, 408 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE2"}, 409 {RAZWI_INITIATOR_ID_X_Y_LOW(11, 11), mmDCORE3_RTR1_CTRL_BASE, 410 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE3"}, 411 {RAZWI_INITIATOR_ID_X_Y_LOW(10, 11), mmDCORE3_RTR0_CTRL_BASE, 412 GAUDI2_DCORE3_ENGINE_ID_MME, "MME3_SBTE4"} 413 }; 414 415 enum hl_pmmu_fatal_cause { 416 LATENCY_RD_OUT_FIFO_OVERRUN, 417 LATENCY_WR_OUT_FIFO_OVERRUN, 418 }; 419 420 enum hl_pcie_drain_ind_cause { 421 LBW_AXI_DRAIN_IND, 422 HBW_AXI_DRAIN_IND 423 }; 424 425 static const u32 cluster_hmmu_hif_enabled_mask[GAUDI2_HBM_NUM] = { 426 [HBM_ID0] = 0xFFFC, 427 [HBM_ID1] = 0xFFCF, 428 [HBM_ID2] = 0xF7F7, 429 [HBM_ID3] = 0x7F7F, 430 [HBM_ID4] = 0xFCFF, 431 [HBM_ID5] = 0xCFFF, 432 }; 433 434 static const u8 xbar_edge_to_hbm_cluster[EDMA_ID_SIZE] = { 435 [0] = HBM_ID0, 436 [1] = HBM_ID1, 437 [2] = HBM_ID4, 438 [3] = HBM_ID5, 439 }; 440 441 static const u8 edma_to_hbm_cluster[EDMA_ID_SIZE] = { 442 [EDMA_ID_DCORE0_INSTANCE0] = HBM_ID0, 443 [EDMA_ID_DCORE0_INSTANCE1] = HBM_ID2, 444 [EDMA_ID_DCORE1_INSTANCE0] = HBM_ID1, 445 [EDMA_ID_DCORE1_INSTANCE1] = HBM_ID3, 446 [EDMA_ID_DCORE2_INSTANCE0] = HBM_ID2, 447 [EDMA_ID_DCORE2_INSTANCE1] = HBM_ID4, 448 [EDMA_ID_DCORE3_INSTANCE0] = HBM_ID3, 449 [EDMA_ID_DCORE3_INSTANCE1] = HBM_ID5, 450 }; 451 452 static const int gaudi2_qman_async_event_id[] = { 453 [GAUDI2_QUEUE_ID_PDMA_0_0] = GAUDI2_EVENT_PDMA0_QM, 454 [GAUDI2_QUEUE_ID_PDMA_0_1] = GAUDI2_EVENT_PDMA0_QM, 455 [GAUDI2_QUEUE_ID_PDMA_0_2] = GAUDI2_EVENT_PDMA0_QM, 456 [GAUDI2_QUEUE_ID_PDMA_0_3] = GAUDI2_EVENT_PDMA0_QM, 457 [GAUDI2_QUEUE_ID_PDMA_1_0] = GAUDI2_EVENT_PDMA1_QM, 458 [GAUDI2_QUEUE_ID_PDMA_1_1] = GAUDI2_EVENT_PDMA1_QM, 459 [GAUDI2_QUEUE_ID_PDMA_1_2] = GAUDI2_EVENT_PDMA1_QM, 460 [GAUDI2_QUEUE_ID_PDMA_1_3] = GAUDI2_EVENT_PDMA1_QM, 461 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0] = GAUDI2_EVENT_HDMA0_QM, 462 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_1] = GAUDI2_EVENT_HDMA0_QM, 463 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_2] = GAUDI2_EVENT_HDMA0_QM, 464 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] = GAUDI2_EVENT_HDMA0_QM, 465 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0] = GAUDI2_EVENT_HDMA1_QM, 466 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_1] = GAUDI2_EVENT_HDMA1_QM, 467 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_2] = GAUDI2_EVENT_HDMA1_QM, 468 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] = GAUDI2_EVENT_HDMA1_QM, 469 [GAUDI2_QUEUE_ID_DCORE0_MME_0_0] = GAUDI2_EVENT_MME0_QM, 470 [GAUDI2_QUEUE_ID_DCORE0_MME_0_1] = GAUDI2_EVENT_MME0_QM, 471 [GAUDI2_QUEUE_ID_DCORE0_MME_0_2] = GAUDI2_EVENT_MME0_QM, 472 [GAUDI2_QUEUE_ID_DCORE0_MME_0_3] = GAUDI2_EVENT_MME0_QM, 473 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_0] = GAUDI2_EVENT_TPC0_QM, 474 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_1] = GAUDI2_EVENT_TPC0_QM, 475 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_2] = GAUDI2_EVENT_TPC0_QM, 476 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] = GAUDI2_EVENT_TPC0_QM, 477 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_0] = GAUDI2_EVENT_TPC1_QM, 478 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_1] = GAUDI2_EVENT_TPC1_QM, 479 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_2] = GAUDI2_EVENT_TPC1_QM, 480 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] = GAUDI2_EVENT_TPC1_QM, 481 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_0] = GAUDI2_EVENT_TPC2_QM, 482 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_1] = GAUDI2_EVENT_TPC2_QM, 483 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_2] = GAUDI2_EVENT_TPC2_QM, 484 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] = GAUDI2_EVENT_TPC2_QM, 485 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_0] = GAUDI2_EVENT_TPC3_QM, 486 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_1] = GAUDI2_EVENT_TPC3_QM, 487 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_2] = GAUDI2_EVENT_TPC3_QM, 488 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] = GAUDI2_EVENT_TPC3_QM, 489 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_0] = GAUDI2_EVENT_TPC4_QM, 490 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_1] = GAUDI2_EVENT_TPC4_QM, 491 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_2] = GAUDI2_EVENT_TPC4_QM, 492 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] = GAUDI2_EVENT_TPC4_QM, 493 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_0] = GAUDI2_EVENT_TPC5_QM, 494 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_1] = GAUDI2_EVENT_TPC5_QM, 495 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_2] = GAUDI2_EVENT_TPC5_QM, 496 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] = GAUDI2_EVENT_TPC5_QM, 497 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_0] = GAUDI2_EVENT_TPC24_QM, 498 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_1] = GAUDI2_EVENT_TPC24_QM, 499 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_2] = GAUDI2_EVENT_TPC24_QM, 500 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] = GAUDI2_EVENT_TPC24_QM, 501 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0] = GAUDI2_EVENT_HDMA2_QM, 502 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_1] = GAUDI2_EVENT_HDMA2_QM, 503 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_2] = GAUDI2_EVENT_HDMA2_QM, 504 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] = GAUDI2_EVENT_HDMA2_QM, 505 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0] = GAUDI2_EVENT_HDMA3_QM, 506 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_1] = GAUDI2_EVENT_HDMA3_QM, 507 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_2] = GAUDI2_EVENT_HDMA3_QM, 508 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] = GAUDI2_EVENT_HDMA3_QM, 509 [GAUDI2_QUEUE_ID_DCORE1_MME_0_0] = GAUDI2_EVENT_MME1_QM, 510 [GAUDI2_QUEUE_ID_DCORE1_MME_0_1] = GAUDI2_EVENT_MME1_QM, 511 [GAUDI2_QUEUE_ID_DCORE1_MME_0_2] = GAUDI2_EVENT_MME1_QM, 512 [GAUDI2_QUEUE_ID_DCORE1_MME_0_3] = GAUDI2_EVENT_MME1_QM, 513 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_0] = GAUDI2_EVENT_TPC6_QM, 514 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_1] = GAUDI2_EVENT_TPC6_QM, 515 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_2] = GAUDI2_EVENT_TPC6_QM, 516 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] = GAUDI2_EVENT_TPC6_QM, 517 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_0] = GAUDI2_EVENT_TPC7_QM, 518 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_1] = GAUDI2_EVENT_TPC7_QM, 519 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_2] = GAUDI2_EVENT_TPC7_QM, 520 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] = GAUDI2_EVENT_TPC7_QM, 521 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_0] = GAUDI2_EVENT_TPC8_QM, 522 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_1] = GAUDI2_EVENT_TPC8_QM, 523 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_2] = GAUDI2_EVENT_TPC8_QM, 524 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] = GAUDI2_EVENT_TPC8_QM, 525 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_0] = GAUDI2_EVENT_TPC9_QM, 526 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_1] = GAUDI2_EVENT_TPC9_QM, 527 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_2] = GAUDI2_EVENT_TPC9_QM, 528 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] = GAUDI2_EVENT_TPC9_QM, 529 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_0] = GAUDI2_EVENT_TPC10_QM, 530 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_1] = GAUDI2_EVENT_TPC10_QM, 531 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_2] = GAUDI2_EVENT_TPC10_QM, 532 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] = GAUDI2_EVENT_TPC10_QM, 533 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_0] = GAUDI2_EVENT_TPC11_QM, 534 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_1] = GAUDI2_EVENT_TPC11_QM, 535 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_2] = GAUDI2_EVENT_TPC11_QM, 536 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] = GAUDI2_EVENT_TPC11_QM, 537 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0] = GAUDI2_EVENT_HDMA4_QM, 538 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_1] = GAUDI2_EVENT_HDMA4_QM, 539 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_2] = GAUDI2_EVENT_HDMA4_QM, 540 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] = GAUDI2_EVENT_HDMA4_QM, 541 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0] = GAUDI2_EVENT_HDMA5_QM, 542 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_1] = GAUDI2_EVENT_HDMA5_QM, 543 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_2] = GAUDI2_EVENT_HDMA5_QM, 544 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] = GAUDI2_EVENT_HDMA5_QM, 545 [GAUDI2_QUEUE_ID_DCORE2_MME_0_0] = GAUDI2_EVENT_MME2_QM, 546 [GAUDI2_QUEUE_ID_DCORE2_MME_0_1] = GAUDI2_EVENT_MME2_QM, 547 [GAUDI2_QUEUE_ID_DCORE2_MME_0_2] = GAUDI2_EVENT_MME2_QM, 548 [GAUDI2_QUEUE_ID_DCORE2_MME_0_3] = GAUDI2_EVENT_MME2_QM, 549 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_0] = GAUDI2_EVENT_TPC12_QM, 550 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_1] = GAUDI2_EVENT_TPC12_QM, 551 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_2] = GAUDI2_EVENT_TPC12_QM, 552 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] = GAUDI2_EVENT_TPC12_QM, 553 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_0] = GAUDI2_EVENT_TPC13_QM, 554 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_1] = GAUDI2_EVENT_TPC13_QM, 555 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_2] = GAUDI2_EVENT_TPC13_QM, 556 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] = GAUDI2_EVENT_TPC13_QM, 557 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_0] = GAUDI2_EVENT_TPC14_QM, 558 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_1] = GAUDI2_EVENT_TPC14_QM, 559 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_2] = GAUDI2_EVENT_TPC14_QM, 560 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] = GAUDI2_EVENT_TPC14_QM, 561 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_0] = GAUDI2_EVENT_TPC15_QM, 562 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_1] = GAUDI2_EVENT_TPC15_QM, 563 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_2] = GAUDI2_EVENT_TPC15_QM, 564 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] = GAUDI2_EVENT_TPC15_QM, 565 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_0] = GAUDI2_EVENT_TPC16_QM, 566 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_1] = GAUDI2_EVENT_TPC16_QM, 567 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_2] = GAUDI2_EVENT_TPC16_QM, 568 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] = GAUDI2_EVENT_TPC16_QM, 569 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_0] = GAUDI2_EVENT_TPC17_QM, 570 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_1] = GAUDI2_EVENT_TPC17_QM, 571 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_2] = GAUDI2_EVENT_TPC17_QM, 572 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] = GAUDI2_EVENT_TPC17_QM, 573 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0] = GAUDI2_EVENT_HDMA6_QM, 574 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_1] = GAUDI2_EVENT_HDMA6_QM, 575 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_2] = GAUDI2_EVENT_HDMA6_QM, 576 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] = GAUDI2_EVENT_HDMA6_QM, 577 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0] = GAUDI2_EVENT_HDMA7_QM, 578 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1] = GAUDI2_EVENT_HDMA7_QM, 579 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2] = GAUDI2_EVENT_HDMA7_QM, 580 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] = GAUDI2_EVENT_HDMA7_QM, 581 [GAUDI2_QUEUE_ID_DCORE3_MME_0_0] = GAUDI2_EVENT_MME3_QM, 582 [GAUDI2_QUEUE_ID_DCORE3_MME_0_1] = GAUDI2_EVENT_MME3_QM, 583 [GAUDI2_QUEUE_ID_DCORE3_MME_0_2] = GAUDI2_EVENT_MME3_QM, 584 [GAUDI2_QUEUE_ID_DCORE3_MME_0_3] = GAUDI2_EVENT_MME3_QM, 585 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_0] = GAUDI2_EVENT_TPC18_QM, 586 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_1] = GAUDI2_EVENT_TPC18_QM, 587 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_2] = GAUDI2_EVENT_TPC18_QM, 588 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] = GAUDI2_EVENT_TPC18_QM, 589 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_0] = GAUDI2_EVENT_TPC19_QM, 590 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_1] = GAUDI2_EVENT_TPC19_QM, 591 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_2] = GAUDI2_EVENT_TPC19_QM, 592 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] = GAUDI2_EVENT_TPC19_QM, 593 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_0] = GAUDI2_EVENT_TPC20_QM, 594 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_1] = GAUDI2_EVENT_TPC20_QM, 595 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_2] = GAUDI2_EVENT_TPC20_QM, 596 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] = GAUDI2_EVENT_TPC20_QM, 597 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_0] = GAUDI2_EVENT_TPC21_QM, 598 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_1] = GAUDI2_EVENT_TPC21_QM, 599 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_2] = GAUDI2_EVENT_TPC21_QM, 600 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] = GAUDI2_EVENT_TPC21_QM, 601 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_0] = GAUDI2_EVENT_TPC22_QM, 602 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_1] = GAUDI2_EVENT_TPC22_QM, 603 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_2] = GAUDI2_EVENT_TPC22_QM, 604 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] = GAUDI2_EVENT_TPC22_QM, 605 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_0] = GAUDI2_EVENT_TPC23_QM, 606 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_1] = GAUDI2_EVENT_TPC23_QM, 607 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_2] = GAUDI2_EVENT_TPC23_QM, 608 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] = GAUDI2_EVENT_TPC23_QM, 609 [GAUDI2_QUEUE_ID_NIC_0_0] = GAUDI2_EVENT_NIC0_QM0, 610 [GAUDI2_QUEUE_ID_NIC_0_1] = GAUDI2_EVENT_NIC0_QM0, 611 [GAUDI2_QUEUE_ID_NIC_0_2] = GAUDI2_EVENT_NIC0_QM0, 612 [GAUDI2_QUEUE_ID_NIC_0_3] = GAUDI2_EVENT_NIC0_QM0, 613 [GAUDI2_QUEUE_ID_NIC_1_0] = GAUDI2_EVENT_NIC0_QM1, 614 [GAUDI2_QUEUE_ID_NIC_1_1] = GAUDI2_EVENT_NIC0_QM1, 615 [GAUDI2_QUEUE_ID_NIC_1_2] = GAUDI2_EVENT_NIC0_QM1, 616 [GAUDI2_QUEUE_ID_NIC_1_3] = GAUDI2_EVENT_NIC0_QM1, 617 [GAUDI2_QUEUE_ID_NIC_2_0] = GAUDI2_EVENT_NIC1_QM0, 618 [GAUDI2_QUEUE_ID_NIC_2_1] = GAUDI2_EVENT_NIC1_QM0, 619 [GAUDI2_QUEUE_ID_NIC_2_2] = GAUDI2_EVENT_NIC1_QM0, 620 [GAUDI2_QUEUE_ID_NIC_2_3] = GAUDI2_EVENT_NIC1_QM0, 621 [GAUDI2_QUEUE_ID_NIC_3_0] = GAUDI2_EVENT_NIC1_QM1, 622 [GAUDI2_QUEUE_ID_NIC_3_1] = GAUDI2_EVENT_NIC1_QM1, 623 [GAUDI2_QUEUE_ID_NIC_3_2] = GAUDI2_EVENT_NIC1_QM1, 624 [GAUDI2_QUEUE_ID_NIC_3_3] = GAUDI2_EVENT_NIC1_QM1, 625 [GAUDI2_QUEUE_ID_NIC_4_0] = GAUDI2_EVENT_NIC2_QM0, 626 [GAUDI2_QUEUE_ID_NIC_4_1] = GAUDI2_EVENT_NIC2_QM0, 627 [GAUDI2_QUEUE_ID_NIC_4_2] = GAUDI2_EVENT_NIC2_QM0, 628 [GAUDI2_QUEUE_ID_NIC_4_3] = GAUDI2_EVENT_NIC2_QM0, 629 [GAUDI2_QUEUE_ID_NIC_5_0] = GAUDI2_EVENT_NIC2_QM1, 630 [GAUDI2_QUEUE_ID_NIC_5_1] = GAUDI2_EVENT_NIC2_QM1, 631 [GAUDI2_QUEUE_ID_NIC_5_2] = GAUDI2_EVENT_NIC2_QM1, 632 [GAUDI2_QUEUE_ID_NIC_5_3] = GAUDI2_EVENT_NIC2_QM1, 633 [GAUDI2_QUEUE_ID_NIC_6_0] = GAUDI2_EVENT_NIC3_QM0, 634 [GAUDI2_QUEUE_ID_NIC_6_1] = GAUDI2_EVENT_NIC3_QM0, 635 [GAUDI2_QUEUE_ID_NIC_6_2] = GAUDI2_EVENT_NIC3_QM0, 636 [GAUDI2_QUEUE_ID_NIC_6_3] = GAUDI2_EVENT_NIC3_QM0, 637 [GAUDI2_QUEUE_ID_NIC_7_0] = GAUDI2_EVENT_NIC3_QM1, 638 [GAUDI2_QUEUE_ID_NIC_7_1] = GAUDI2_EVENT_NIC3_QM1, 639 [GAUDI2_QUEUE_ID_NIC_7_2] = GAUDI2_EVENT_NIC3_QM1, 640 [GAUDI2_QUEUE_ID_NIC_7_3] = GAUDI2_EVENT_NIC3_QM1, 641 [GAUDI2_QUEUE_ID_NIC_8_0] = GAUDI2_EVENT_NIC4_QM0, 642 [GAUDI2_QUEUE_ID_NIC_8_1] = GAUDI2_EVENT_NIC4_QM0, 643 [GAUDI2_QUEUE_ID_NIC_8_2] = GAUDI2_EVENT_NIC4_QM0, 644 [GAUDI2_QUEUE_ID_NIC_8_3] = GAUDI2_EVENT_NIC4_QM0, 645 [GAUDI2_QUEUE_ID_NIC_9_0] = GAUDI2_EVENT_NIC4_QM1, 646 [GAUDI2_QUEUE_ID_NIC_9_1] = GAUDI2_EVENT_NIC4_QM1, 647 [GAUDI2_QUEUE_ID_NIC_9_2] = GAUDI2_EVENT_NIC4_QM1, 648 [GAUDI2_QUEUE_ID_NIC_9_3] = GAUDI2_EVENT_NIC4_QM1, 649 [GAUDI2_QUEUE_ID_NIC_10_0] = GAUDI2_EVENT_NIC5_QM0, 650 [GAUDI2_QUEUE_ID_NIC_10_1] = GAUDI2_EVENT_NIC5_QM0, 651 [GAUDI2_QUEUE_ID_NIC_10_2] = GAUDI2_EVENT_NIC5_QM0, 652 [GAUDI2_QUEUE_ID_NIC_10_3] = GAUDI2_EVENT_NIC5_QM0, 653 [GAUDI2_QUEUE_ID_NIC_11_0] = GAUDI2_EVENT_NIC5_QM1, 654 [GAUDI2_QUEUE_ID_NIC_11_1] = GAUDI2_EVENT_NIC5_QM1, 655 [GAUDI2_QUEUE_ID_NIC_11_2] = GAUDI2_EVENT_NIC5_QM1, 656 [GAUDI2_QUEUE_ID_NIC_11_3] = GAUDI2_EVENT_NIC5_QM1, 657 [GAUDI2_QUEUE_ID_NIC_12_0] = GAUDI2_EVENT_NIC6_QM0, 658 [GAUDI2_QUEUE_ID_NIC_12_1] = GAUDI2_EVENT_NIC6_QM0, 659 [GAUDI2_QUEUE_ID_NIC_12_2] = GAUDI2_EVENT_NIC6_QM0, 660 [GAUDI2_QUEUE_ID_NIC_12_3] = GAUDI2_EVENT_NIC6_QM0, 661 [GAUDI2_QUEUE_ID_NIC_13_0] = GAUDI2_EVENT_NIC6_QM1, 662 [GAUDI2_QUEUE_ID_NIC_13_1] = GAUDI2_EVENT_NIC6_QM1, 663 [GAUDI2_QUEUE_ID_NIC_13_2] = GAUDI2_EVENT_NIC6_QM1, 664 [GAUDI2_QUEUE_ID_NIC_13_3] = GAUDI2_EVENT_NIC6_QM1, 665 [GAUDI2_QUEUE_ID_NIC_14_0] = GAUDI2_EVENT_NIC7_QM0, 666 [GAUDI2_QUEUE_ID_NIC_14_1] = GAUDI2_EVENT_NIC7_QM0, 667 [GAUDI2_QUEUE_ID_NIC_14_2] = GAUDI2_EVENT_NIC7_QM0, 668 [GAUDI2_QUEUE_ID_NIC_14_3] = GAUDI2_EVENT_NIC7_QM0, 669 [GAUDI2_QUEUE_ID_NIC_15_0] = GAUDI2_EVENT_NIC7_QM1, 670 [GAUDI2_QUEUE_ID_NIC_15_1] = GAUDI2_EVENT_NIC7_QM1, 671 [GAUDI2_QUEUE_ID_NIC_15_2] = GAUDI2_EVENT_NIC7_QM1, 672 [GAUDI2_QUEUE_ID_NIC_15_3] = GAUDI2_EVENT_NIC7_QM1, 673 [GAUDI2_QUEUE_ID_NIC_16_0] = GAUDI2_EVENT_NIC8_QM0, 674 [GAUDI2_QUEUE_ID_NIC_16_1] = GAUDI2_EVENT_NIC8_QM0, 675 [GAUDI2_QUEUE_ID_NIC_16_2] = GAUDI2_EVENT_NIC8_QM0, 676 [GAUDI2_QUEUE_ID_NIC_16_3] = GAUDI2_EVENT_NIC8_QM0, 677 [GAUDI2_QUEUE_ID_NIC_17_0] = GAUDI2_EVENT_NIC8_QM1, 678 [GAUDI2_QUEUE_ID_NIC_17_1] = GAUDI2_EVENT_NIC8_QM1, 679 [GAUDI2_QUEUE_ID_NIC_17_2] = GAUDI2_EVENT_NIC8_QM1, 680 [GAUDI2_QUEUE_ID_NIC_17_3] = GAUDI2_EVENT_NIC8_QM1, 681 [GAUDI2_QUEUE_ID_NIC_18_0] = GAUDI2_EVENT_NIC9_QM0, 682 [GAUDI2_QUEUE_ID_NIC_18_1] = GAUDI2_EVENT_NIC9_QM0, 683 [GAUDI2_QUEUE_ID_NIC_18_2] = GAUDI2_EVENT_NIC9_QM0, 684 [GAUDI2_QUEUE_ID_NIC_18_3] = GAUDI2_EVENT_NIC9_QM0, 685 [GAUDI2_QUEUE_ID_NIC_19_0] = GAUDI2_EVENT_NIC9_QM1, 686 [GAUDI2_QUEUE_ID_NIC_19_1] = GAUDI2_EVENT_NIC9_QM1, 687 [GAUDI2_QUEUE_ID_NIC_19_2] = GAUDI2_EVENT_NIC9_QM1, 688 [GAUDI2_QUEUE_ID_NIC_19_3] = GAUDI2_EVENT_NIC9_QM1, 689 [GAUDI2_QUEUE_ID_NIC_20_0] = GAUDI2_EVENT_NIC10_QM0, 690 [GAUDI2_QUEUE_ID_NIC_20_1] = GAUDI2_EVENT_NIC10_QM0, 691 [GAUDI2_QUEUE_ID_NIC_20_2] = GAUDI2_EVENT_NIC10_QM0, 692 [GAUDI2_QUEUE_ID_NIC_20_3] = GAUDI2_EVENT_NIC10_QM0, 693 [GAUDI2_QUEUE_ID_NIC_21_0] = GAUDI2_EVENT_NIC10_QM1, 694 [GAUDI2_QUEUE_ID_NIC_21_1] = GAUDI2_EVENT_NIC10_QM1, 695 [GAUDI2_QUEUE_ID_NIC_21_2] = GAUDI2_EVENT_NIC10_QM1, 696 [GAUDI2_QUEUE_ID_NIC_21_3] = GAUDI2_EVENT_NIC10_QM1, 697 [GAUDI2_QUEUE_ID_NIC_22_0] = GAUDI2_EVENT_NIC11_QM0, 698 [GAUDI2_QUEUE_ID_NIC_22_1] = GAUDI2_EVENT_NIC11_QM0, 699 [GAUDI2_QUEUE_ID_NIC_22_2] = GAUDI2_EVENT_NIC11_QM0, 700 [GAUDI2_QUEUE_ID_NIC_22_3] = GAUDI2_EVENT_NIC11_QM0, 701 [GAUDI2_QUEUE_ID_NIC_23_0] = GAUDI2_EVENT_NIC11_QM1, 702 [GAUDI2_QUEUE_ID_NIC_23_1] = GAUDI2_EVENT_NIC11_QM1, 703 [GAUDI2_QUEUE_ID_NIC_23_2] = GAUDI2_EVENT_NIC11_QM1, 704 [GAUDI2_QUEUE_ID_NIC_23_3] = GAUDI2_EVENT_NIC11_QM1, 705 [GAUDI2_QUEUE_ID_ROT_0_0] = GAUDI2_EVENT_ROTATOR0_ROT0_QM, 706 [GAUDI2_QUEUE_ID_ROT_0_1] = GAUDI2_EVENT_ROTATOR0_ROT0_QM, 707 [GAUDI2_QUEUE_ID_ROT_0_2] = GAUDI2_EVENT_ROTATOR0_ROT0_QM, 708 [GAUDI2_QUEUE_ID_ROT_0_3] = GAUDI2_EVENT_ROTATOR0_ROT0_QM, 709 [GAUDI2_QUEUE_ID_ROT_1_0] = GAUDI2_EVENT_ROTATOR1_ROT1_QM, 710 [GAUDI2_QUEUE_ID_ROT_1_1] = GAUDI2_EVENT_ROTATOR1_ROT1_QM, 711 [GAUDI2_QUEUE_ID_ROT_1_2] = GAUDI2_EVENT_ROTATOR1_ROT1_QM, 712 [GAUDI2_QUEUE_ID_ROT_1_3] = GAUDI2_EVENT_ROTATOR1_ROT1_QM 713 }; 714 715 static const int gaudi2_dma_core_async_event_id[] = { 716 [DMA_CORE_ID_EDMA0] = GAUDI2_EVENT_HDMA0_CORE, 717 [DMA_CORE_ID_EDMA1] = GAUDI2_EVENT_HDMA1_CORE, 718 [DMA_CORE_ID_EDMA2] = GAUDI2_EVENT_HDMA2_CORE, 719 [DMA_CORE_ID_EDMA3] = GAUDI2_EVENT_HDMA3_CORE, 720 [DMA_CORE_ID_EDMA4] = GAUDI2_EVENT_HDMA4_CORE, 721 [DMA_CORE_ID_EDMA5] = GAUDI2_EVENT_HDMA5_CORE, 722 [DMA_CORE_ID_EDMA6] = GAUDI2_EVENT_HDMA6_CORE, 723 [DMA_CORE_ID_EDMA7] = GAUDI2_EVENT_HDMA7_CORE, 724 [DMA_CORE_ID_PDMA0] = GAUDI2_EVENT_PDMA0_CORE, 725 [DMA_CORE_ID_PDMA1] = GAUDI2_EVENT_PDMA1_CORE, 726 [DMA_CORE_ID_KDMA] = GAUDI2_EVENT_KDMA0_CORE, 727 }; 728 729 static const char * const gaudi2_qm_sei_error_cause[GAUDI2_NUM_OF_QM_SEI_ERR_CAUSE] = { 730 "qman sei intr", 731 "arc sei intr" 732 }; 733 734 static const char * const gaudi2_cpu_sei_error_cause[GAUDI2_NUM_OF_CPU_SEI_ERR_CAUSE] = { 735 "AXI_TERMINATOR WR", 736 "AXI_TERMINATOR RD", 737 "AXI SPLIT SEI Status" 738 }; 739 740 static const char * const gaudi2_arc_sei_error_cause[GAUDI2_NUM_OF_ARC_SEI_ERR_CAUSE] = { 741 "cbu_bresp_sei_intr_cause", 742 "cbu_rresp_sei_intr_cause", 743 "lbu_bresp_sei_intr_cause", 744 "lbu_rresp_sei_intr_cause", 745 "cbu_axi_split_intr_cause", 746 "lbu_axi_split_intr_cause", 747 "arc_ip_excptn_sei_intr_cause", 748 "dmi_bresp_sei_intr_cause", 749 "aux2apb_err_sei_intr_cause", 750 "cfg_lbw_wr_terminated_intr_cause", 751 "cfg_lbw_rd_terminated_intr_cause", 752 "cfg_dccm_wr_terminated_intr_cause", 753 "cfg_dccm_rd_terminated_intr_cause", 754 "cfg_hbw_rd_terminated_intr_cause" 755 }; 756 757 static const char * const gaudi2_dec_error_cause[GAUDI2_NUM_OF_DEC_ERR_CAUSE] = { 758 "msix_vcd_hbw_sei", 759 "msix_l2c_hbw_sei", 760 "msix_nrm_hbw_sei", 761 "msix_abnrm_hbw_sei", 762 "msix_vcd_lbw_sei", 763 "msix_l2c_lbw_sei", 764 "msix_nrm_lbw_sei", 765 "msix_abnrm_lbw_sei", 766 "apb_vcd_lbw_sei", 767 "apb_l2c_lbw_sei", 768 "apb_nrm_lbw_sei", 769 "apb_abnrm_lbw_sei", 770 "dec_sei", 771 "dec_apb_sei", 772 "trc_apb_sei", 773 "lbw_mstr_if_sei", 774 "axi_split_bresp_err_sei", 775 "hbw_axi_wr_viol_sei", 776 "hbw_axi_rd_viol_sei", 777 "lbw_axi_wr_viol_sei", 778 "lbw_axi_rd_viol_sei", 779 "vcd_spi", 780 "l2c_spi", 781 "nrm_spi", 782 "abnrm_spi", 783 }; 784 785 static const char * const gaudi2_qman_error_cause[GAUDI2_NUM_OF_QM_ERR_CAUSE] = { 786 "PQ AXI HBW error", 787 "CQ AXI HBW error", 788 "CP AXI HBW error", 789 "CP error due to undefined OPCODE", 790 "CP encountered STOP OPCODE", 791 "CP AXI LBW error", 792 "CP WRREG32 or WRBULK returned error", 793 "N/A", 794 "FENCE 0 inc over max value and clipped", 795 "FENCE 1 inc over max value and clipped", 796 "FENCE 2 inc over max value and clipped", 797 "FENCE 3 inc over max value and clipped", 798 "FENCE 0 dec under min value and clipped", 799 "FENCE 1 dec under min value and clipped", 800 "FENCE 2 dec under min value and clipped", 801 "FENCE 3 dec under min value and clipped", 802 "CPDMA Up overflow", 803 "PQC L2H error" 804 }; 805 806 static const char * const gaudi2_lower_qman_error_cause[GAUDI2_NUM_OF_LOWER_QM_ERR_CAUSE] = { 807 "RSVD0", 808 "CQ AXI HBW error", 809 "CP AXI HBW error", 810 "CP error due to undefined OPCODE", 811 "CP encountered STOP OPCODE", 812 "CP AXI LBW error", 813 "CP WRREG32 or WRBULK returned error", 814 "N/A", 815 "FENCE 0 inc over max value and clipped", 816 "FENCE 1 inc over max value and clipped", 817 "FENCE 2 inc over max value and clipped", 818 "FENCE 3 inc over max value and clipped", 819 "FENCE 0 dec under min value and clipped", 820 "FENCE 1 dec under min value and clipped", 821 "FENCE 2 dec under min value and clipped", 822 "FENCE 3 dec under min value and clipped", 823 "CPDMA Up overflow", 824 "RSVD17", 825 "CQ_WR_IFIFO_CI_ERR", 826 "CQ_WR_CTL_CI_ERR", 827 "ARC_CQF_RD_ERR", 828 "ARC_CQ_WR_IFIFO_CI_ERR", 829 "ARC_CQ_WR_CTL_CI_ERR", 830 "ARC_AXI_ERR", 831 "CP_SWITCH_WDT_ERR" 832 }; 833 834 static const char * const gaudi2_qman_arb_error_cause[GAUDI2_NUM_OF_QM_ARB_ERR_CAUSE] = { 835 "Choice push while full error", 836 "Choice Q watchdog error", 837 "MSG AXI LBW returned with error" 838 }; 839 840 static const char * const guadi2_rot_error_cause[GAUDI2_NUM_OF_ROT_ERR_CAUSE] = { 841 "qm_axi_err", 842 "qm_trace_fence_events", 843 "qm_sw_err", 844 "qm_cp_sw_stop", 845 "lbw_mstr_rresp_err", 846 "lbw_mstr_bresp_err", 847 "lbw_msg_slverr", 848 "hbw_msg_slverr", 849 "wbc_slverr", 850 "hbw_mstr_rresp_err", 851 "hbw_mstr_bresp_err", 852 "sb_resp_intr", 853 "mrsb_resp_intr", 854 "core_dw_status_0", 855 "core_dw_status_1", 856 "core_dw_status_2", 857 "core_dw_status_3", 858 "core_dw_status_4", 859 "core_dw_status_5", 860 "core_dw_status_6", 861 "core_dw_status_7", 862 "async_arc2cpu_sei_intr", 863 }; 864 865 static const char * const gaudi2_tpc_interrupts_cause[GAUDI2_NUM_OF_TPC_INTR_CAUSE] = { 866 "tpc_address_exceed_slm", 867 "tpc_div_by_0", 868 "tpc_spu_mac_overflow", 869 "tpc_spu_addsub_overflow", 870 "tpc_spu_abs_overflow", 871 "tpc_spu_fma_fp_dst_nan", 872 "tpc_spu_fma_fp_dst_inf", 873 "tpc_spu_convert_fp_dst_nan", 874 "tpc_spu_convert_fp_dst_inf", 875 "tpc_spu_fp_dst_denorm", 876 "tpc_vpu_mac_overflow", 877 "tpc_vpu_addsub_overflow", 878 "tpc_vpu_abs_overflow", 879 "tpc_vpu_convert_fp_dst_nan", 880 "tpc_vpu_convert_fp_dst_inf", 881 "tpc_vpu_fma_fp_dst_nan", 882 "tpc_vpu_fma_fp_dst_inf", 883 "tpc_vpu_fp_dst_denorm", 884 "tpc_assertions", 885 "tpc_illegal_instruction", 886 "tpc_pc_wrap_around", 887 "tpc_qm_sw_err", 888 "tpc_hbw_rresp_err", 889 "tpc_hbw_bresp_err", 890 "tpc_lbw_rresp_err", 891 "tpc_lbw_bresp_err", 892 "st_unlock_already_locked", 893 "invalid_lock_access", 894 "LD_L protection violation", 895 "ST_L protection violation", 896 "D$ L0CS mismatch", 897 }; 898 899 static const char * const guadi2_mme_error_cause[GAUDI2_NUM_OF_MME_ERR_CAUSE] = { 900 "agu_resp_intr", 901 "qman_axi_err", 902 "wap sei (wbc axi err)", 903 "arc sei", 904 "cfg access error", 905 "qm_sw_err", 906 "sbte_dbg_intr_0", 907 "sbte_dbg_intr_1", 908 "sbte_dbg_intr_2", 909 "sbte_dbg_intr_3", 910 "sbte_dbg_intr_4", 911 "sbte_prtn_intr_0", 912 "sbte_prtn_intr_1", 913 "sbte_prtn_intr_2", 914 "sbte_prtn_intr_3", 915 "sbte_prtn_intr_4", 916 }; 917 918 static const char * const guadi2_mme_wap_error_cause[GAUDI2_NUM_OF_MME_WAP_ERR_CAUSE] = { 919 "WBC ERR RESP_0", 920 "WBC ERR RESP_1", 921 "AP SOURCE POS INF", 922 "AP SOURCE NEG INF", 923 "AP SOURCE NAN", 924 "AP RESULT POS INF", 925 "AP RESULT NEG INF", 926 }; 927 928 static const char * const gaudi2_dma_core_interrupts_cause[GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE] = { 929 "HBW Read returned with error RRESP", 930 "HBW write returned with error BRESP", 931 "LBW write returned with error BRESP", 932 "descriptor_fifo_overflow", 933 "KDMA SB LBW Read returned with error", 934 "KDMA WBC LBW Write returned with error", 935 "TRANSPOSE ENGINE DESC FIFO OVERFLOW", 936 "WRONG CFG FOR COMMIT IN LIN DMA" 937 }; 938 939 static const char * const gaudi2_kdma_core_interrupts_cause[GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE] = { 940 "HBW/LBW Read returned with error RRESP", 941 "HBW/LBW write returned with error BRESP", 942 "LBW write returned with error BRESP", 943 "descriptor_fifo_overflow", 944 "KDMA SB LBW Read returned with error", 945 "KDMA WBC LBW Write returned with error", 946 "TRANSPOSE ENGINE DESC FIFO OVERFLOW", 947 "WRONG CFG FOR COMMIT IN LIN DMA" 948 }; 949 950 struct gaudi2_sm_sei_cause_data { 951 const char *cause_name; 952 const char *log_name; 953 }; 954 955 static const struct gaudi2_sm_sei_cause_data 956 gaudi2_sm_sei_cause[GAUDI2_NUM_OF_SM_SEI_ERR_CAUSE] = { 957 {"calculated SO value overflow/underflow", "SOB ID"}, 958 {"payload address of monitor is not aligned to 4B", "monitor addr"}, 959 {"armed monitor write got BRESP (SLVERR or DECERR)", "AXI id"}, 960 }; 961 962 static const char * const 963 gaudi2_pmmu_fatal_interrupts_cause[GAUDI2_NUM_OF_PMMU_FATAL_ERR_CAUSE] = { 964 "LATENCY_RD_OUT_FIFO_OVERRUN", 965 "LATENCY_WR_OUT_FIFO_OVERRUN", 966 }; 967 968 static const char * const 969 gaudi2_hif_fatal_interrupts_cause[GAUDI2_NUM_OF_HIF_FATAL_ERR_CAUSE] = { 970 "LATENCY_RD_OUT_FIFO_OVERRUN", 971 "LATENCY_WR_OUT_FIFO_OVERRUN", 972 }; 973 974 static const char * const 975 gaudi2_psoc_axi_drain_interrupts_cause[GAUDI2_NUM_OF_AXI_DRAIN_ERR_CAUSE] = { 976 "AXI drain HBW", 977 "AXI drain LBW", 978 }; 979 980 static const char * const 981 gaudi2_pcie_addr_dec_error_cause[GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE] = { 982 "HBW error response", 983 "LBW error response", 984 "TLP is blocked by RR" 985 }; 986 987 static const int gaudi2_queue_id_to_engine_id[] = { 988 [GAUDI2_QUEUE_ID_PDMA_0_0...GAUDI2_QUEUE_ID_PDMA_0_3] = GAUDI2_ENGINE_ID_PDMA_0, 989 [GAUDI2_QUEUE_ID_PDMA_1_0...GAUDI2_QUEUE_ID_PDMA_1_3] = GAUDI2_ENGINE_ID_PDMA_1, 990 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] = 991 GAUDI2_DCORE0_ENGINE_ID_EDMA_0, 992 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] = 993 GAUDI2_DCORE0_ENGINE_ID_EDMA_1, 994 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] = 995 GAUDI2_DCORE1_ENGINE_ID_EDMA_0, 996 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0...GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] = 997 GAUDI2_DCORE1_ENGINE_ID_EDMA_1, 998 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] = 999 GAUDI2_DCORE2_ENGINE_ID_EDMA_0, 1000 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0...GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] = 1001 GAUDI2_DCORE2_ENGINE_ID_EDMA_1, 1002 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] = 1003 GAUDI2_DCORE3_ENGINE_ID_EDMA_0, 1004 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0...GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] = 1005 GAUDI2_DCORE3_ENGINE_ID_EDMA_1, 1006 [GAUDI2_QUEUE_ID_DCORE0_MME_0_0...GAUDI2_QUEUE_ID_DCORE0_MME_0_3] = 1007 GAUDI2_DCORE0_ENGINE_ID_MME, 1008 [GAUDI2_QUEUE_ID_DCORE1_MME_0_0...GAUDI2_QUEUE_ID_DCORE1_MME_0_3] = 1009 GAUDI2_DCORE1_ENGINE_ID_MME, 1010 [GAUDI2_QUEUE_ID_DCORE2_MME_0_0...GAUDI2_QUEUE_ID_DCORE2_MME_0_3] = 1011 GAUDI2_DCORE2_ENGINE_ID_MME, 1012 [GAUDI2_QUEUE_ID_DCORE3_MME_0_0...GAUDI2_QUEUE_ID_DCORE3_MME_0_3] = 1013 GAUDI2_DCORE3_ENGINE_ID_MME, 1014 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_0...GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] = 1015 GAUDI2_DCORE0_ENGINE_ID_TPC_0, 1016 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_0...GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] = 1017 GAUDI2_DCORE0_ENGINE_ID_TPC_1, 1018 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_0...GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] = 1019 GAUDI2_DCORE0_ENGINE_ID_TPC_2, 1020 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_0...GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] = 1021 GAUDI2_DCORE0_ENGINE_ID_TPC_3, 1022 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_0...GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] = 1023 GAUDI2_DCORE0_ENGINE_ID_TPC_4, 1024 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_0...GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] = 1025 GAUDI2_DCORE0_ENGINE_ID_TPC_5, 1026 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_0...GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] = 1027 GAUDI2_DCORE0_ENGINE_ID_TPC_6, 1028 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_0...GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] = 1029 GAUDI2_DCORE1_ENGINE_ID_TPC_0, 1030 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_0...GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] = 1031 GAUDI2_DCORE1_ENGINE_ID_TPC_1, 1032 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_0...GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] = 1033 GAUDI2_DCORE1_ENGINE_ID_TPC_2, 1034 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_0...GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] = 1035 GAUDI2_DCORE1_ENGINE_ID_TPC_3, 1036 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_0...GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] = 1037 GAUDI2_DCORE1_ENGINE_ID_TPC_4, 1038 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_0...GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] = 1039 GAUDI2_DCORE1_ENGINE_ID_TPC_5, 1040 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_0...GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] = 1041 GAUDI2_DCORE2_ENGINE_ID_TPC_0, 1042 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_0...GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] = 1043 GAUDI2_DCORE2_ENGINE_ID_TPC_1, 1044 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_0...GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] = 1045 GAUDI2_DCORE2_ENGINE_ID_TPC_2, 1046 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_0...GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] = 1047 GAUDI2_DCORE2_ENGINE_ID_TPC_3, 1048 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_0...GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] = 1049 GAUDI2_DCORE2_ENGINE_ID_TPC_4, 1050 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_0...GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] = 1051 GAUDI2_DCORE2_ENGINE_ID_TPC_5, 1052 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_0...GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] = 1053 GAUDI2_DCORE3_ENGINE_ID_TPC_0, 1054 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_0...GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] = 1055 GAUDI2_DCORE3_ENGINE_ID_TPC_1, 1056 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_0...GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] = 1057 GAUDI2_DCORE3_ENGINE_ID_TPC_2, 1058 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_0...GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] = 1059 GAUDI2_DCORE3_ENGINE_ID_TPC_3, 1060 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_0...GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] = 1061 GAUDI2_DCORE3_ENGINE_ID_TPC_4, 1062 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_0...GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] = 1063 GAUDI2_DCORE3_ENGINE_ID_TPC_5, 1064 [GAUDI2_QUEUE_ID_NIC_0_0...GAUDI2_QUEUE_ID_NIC_0_3] = GAUDI2_ENGINE_ID_NIC0_0, 1065 [GAUDI2_QUEUE_ID_NIC_1_0...GAUDI2_QUEUE_ID_NIC_1_3] = GAUDI2_ENGINE_ID_NIC0_1, 1066 [GAUDI2_QUEUE_ID_NIC_2_0...GAUDI2_QUEUE_ID_NIC_2_3] = GAUDI2_ENGINE_ID_NIC1_0, 1067 [GAUDI2_QUEUE_ID_NIC_3_0...GAUDI2_QUEUE_ID_NIC_3_3] = GAUDI2_ENGINE_ID_NIC1_1, 1068 [GAUDI2_QUEUE_ID_NIC_4_0...GAUDI2_QUEUE_ID_NIC_4_3] = GAUDI2_ENGINE_ID_NIC2_0, 1069 [GAUDI2_QUEUE_ID_NIC_5_0...GAUDI2_QUEUE_ID_NIC_5_3] = GAUDI2_ENGINE_ID_NIC2_1, 1070 [GAUDI2_QUEUE_ID_NIC_6_0...GAUDI2_QUEUE_ID_NIC_6_3] = GAUDI2_ENGINE_ID_NIC3_0, 1071 [GAUDI2_QUEUE_ID_NIC_7_0...GAUDI2_QUEUE_ID_NIC_7_3] = GAUDI2_ENGINE_ID_NIC3_1, 1072 [GAUDI2_QUEUE_ID_NIC_8_0...GAUDI2_QUEUE_ID_NIC_8_3] = GAUDI2_ENGINE_ID_NIC4_0, 1073 [GAUDI2_QUEUE_ID_NIC_9_0...GAUDI2_QUEUE_ID_NIC_9_3] = GAUDI2_ENGINE_ID_NIC4_1, 1074 [GAUDI2_QUEUE_ID_NIC_10_0...GAUDI2_QUEUE_ID_NIC_10_3] = GAUDI2_ENGINE_ID_NIC5_0, 1075 [GAUDI2_QUEUE_ID_NIC_11_0...GAUDI2_QUEUE_ID_NIC_11_3] = GAUDI2_ENGINE_ID_NIC5_1, 1076 [GAUDI2_QUEUE_ID_NIC_12_0...GAUDI2_QUEUE_ID_NIC_12_3] = GAUDI2_ENGINE_ID_NIC6_0, 1077 [GAUDI2_QUEUE_ID_NIC_13_0...GAUDI2_QUEUE_ID_NIC_13_3] = GAUDI2_ENGINE_ID_NIC6_1, 1078 [GAUDI2_QUEUE_ID_NIC_14_0...GAUDI2_QUEUE_ID_NIC_14_3] = GAUDI2_ENGINE_ID_NIC7_0, 1079 [GAUDI2_QUEUE_ID_NIC_15_0...GAUDI2_QUEUE_ID_NIC_15_3] = GAUDI2_ENGINE_ID_NIC7_1, 1080 [GAUDI2_QUEUE_ID_NIC_16_0...GAUDI2_QUEUE_ID_NIC_16_3] = GAUDI2_ENGINE_ID_NIC8_0, 1081 [GAUDI2_QUEUE_ID_NIC_17_0...GAUDI2_QUEUE_ID_NIC_17_3] = GAUDI2_ENGINE_ID_NIC8_1, 1082 [GAUDI2_QUEUE_ID_NIC_18_0...GAUDI2_QUEUE_ID_NIC_18_3] = GAUDI2_ENGINE_ID_NIC9_0, 1083 [GAUDI2_QUEUE_ID_NIC_19_0...GAUDI2_QUEUE_ID_NIC_19_3] = GAUDI2_ENGINE_ID_NIC9_1, 1084 [GAUDI2_QUEUE_ID_NIC_20_0...GAUDI2_QUEUE_ID_NIC_20_3] = GAUDI2_ENGINE_ID_NIC10_0, 1085 [GAUDI2_QUEUE_ID_NIC_21_0...GAUDI2_QUEUE_ID_NIC_21_3] = GAUDI2_ENGINE_ID_NIC10_1, 1086 [GAUDI2_QUEUE_ID_NIC_22_0...GAUDI2_QUEUE_ID_NIC_22_3] = GAUDI2_ENGINE_ID_NIC11_0, 1087 [GAUDI2_QUEUE_ID_NIC_23_0...GAUDI2_QUEUE_ID_NIC_23_3] = GAUDI2_ENGINE_ID_NIC11_1, 1088 [GAUDI2_QUEUE_ID_ROT_0_0...GAUDI2_QUEUE_ID_ROT_0_3] = GAUDI2_ENGINE_ID_ROT_0, 1089 [GAUDI2_QUEUE_ID_ROT_1_0...GAUDI2_QUEUE_ID_ROT_1_3] = GAUDI2_ENGINE_ID_ROT_1, 1090 }; 1091 1092 const u32 gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_SIZE] = { 1093 [GAUDI2_QUEUE_ID_PDMA_0_0] = mmPDMA0_QM_BASE, 1094 [GAUDI2_QUEUE_ID_PDMA_0_1] = mmPDMA0_QM_BASE, 1095 [GAUDI2_QUEUE_ID_PDMA_0_2] = mmPDMA0_QM_BASE, 1096 [GAUDI2_QUEUE_ID_PDMA_0_3] = mmPDMA0_QM_BASE, 1097 [GAUDI2_QUEUE_ID_PDMA_1_0] = mmPDMA1_QM_BASE, 1098 [GAUDI2_QUEUE_ID_PDMA_1_1] = mmPDMA1_QM_BASE, 1099 [GAUDI2_QUEUE_ID_PDMA_1_2] = mmPDMA1_QM_BASE, 1100 [GAUDI2_QUEUE_ID_PDMA_1_3] = mmPDMA1_QM_BASE, 1101 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0] = mmDCORE0_EDMA0_QM_BASE, 1102 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_1] = mmDCORE0_EDMA0_QM_BASE, 1103 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_2] = mmDCORE0_EDMA0_QM_BASE, 1104 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] = mmDCORE0_EDMA0_QM_BASE, 1105 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0] = mmDCORE0_EDMA1_QM_BASE, 1106 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_1] = mmDCORE0_EDMA1_QM_BASE, 1107 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_2] = mmDCORE0_EDMA1_QM_BASE, 1108 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] = mmDCORE0_EDMA1_QM_BASE, 1109 [GAUDI2_QUEUE_ID_DCORE0_MME_0_0] = mmDCORE0_MME_QM_BASE, 1110 [GAUDI2_QUEUE_ID_DCORE0_MME_0_1] = mmDCORE0_MME_QM_BASE, 1111 [GAUDI2_QUEUE_ID_DCORE0_MME_0_2] = mmDCORE0_MME_QM_BASE, 1112 [GAUDI2_QUEUE_ID_DCORE0_MME_0_3] = mmDCORE0_MME_QM_BASE, 1113 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_0] = mmDCORE0_TPC0_QM_BASE, 1114 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_1] = mmDCORE0_TPC0_QM_BASE, 1115 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_2] = mmDCORE0_TPC0_QM_BASE, 1116 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] = mmDCORE0_TPC0_QM_BASE, 1117 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_0] = mmDCORE0_TPC1_QM_BASE, 1118 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_1] = mmDCORE0_TPC1_QM_BASE, 1119 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_2] = mmDCORE0_TPC1_QM_BASE, 1120 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] = mmDCORE0_TPC1_QM_BASE, 1121 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_0] = mmDCORE0_TPC2_QM_BASE, 1122 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_1] = mmDCORE0_TPC2_QM_BASE, 1123 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_2] = mmDCORE0_TPC2_QM_BASE, 1124 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] = mmDCORE0_TPC2_QM_BASE, 1125 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_0] = mmDCORE0_TPC3_QM_BASE, 1126 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_1] = mmDCORE0_TPC3_QM_BASE, 1127 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_2] = mmDCORE0_TPC3_QM_BASE, 1128 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] = mmDCORE0_TPC3_QM_BASE, 1129 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_0] = mmDCORE0_TPC4_QM_BASE, 1130 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_1] = mmDCORE0_TPC4_QM_BASE, 1131 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_2] = mmDCORE0_TPC4_QM_BASE, 1132 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] = mmDCORE0_TPC4_QM_BASE, 1133 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_0] = mmDCORE0_TPC5_QM_BASE, 1134 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_1] = mmDCORE0_TPC5_QM_BASE, 1135 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_2] = mmDCORE0_TPC5_QM_BASE, 1136 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] = mmDCORE0_TPC5_QM_BASE, 1137 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_0] = mmDCORE0_TPC6_QM_BASE, 1138 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_1] = mmDCORE0_TPC6_QM_BASE, 1139 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_2] = mmDCORE0_TPC6_QM_BASE, 1140 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] = mmDCORE0_TPC6_QM_BASE, 1141 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0] = mmDCORE1_EDMA0_QM_BASE, 1142 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_1] = mmDCORE1_EDMA0_QM_BASE, 1143 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_2] = mmDCORE1_EDMA0_QM_BASE, 1144 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] = mmDCORE1_EDMA0_QM_BASE, 1145 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0] = mmDCORE1_EDMA1_QM_BASE, 1146 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_1] = mmDCORE1_EDMA1_QM_BASE, 1147 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_2] = mmDCORE1_EDMA1_QM_BASE, 1148 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] = mmDCORE1_EDMA1_QM_BASE, 1149 [GAUDI2_QUEUE_ID_DCORE1_MME_0_0] = mmDCORE1_MME_QM_BASE, 1150 [GAUDI2_QUEUE_ID_DCORE1_MME_0_1] = mmDCORE1_MME_QM_BASE, 1151 [GAUDI2_QUEUE_ID_DCORE1_MME_0_2] = mmDCORE1_MME_QM_BASE, 1152 [GAUDI2_QUEUE_ID_DCORE1_MME_0_3] = mmDCORE1_MME_QM_BASE, 1153 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_0] = mmDCORE1_TPC0_QM_BASE, 1154 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_1] = mmDCORE1_TPC0_QM_BASE, 1155 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_2] = mmDCORE1_TPC0_QM_BASE, 1156 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] = mmDCORE1_TPC0_QM_BASE, 1157 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_0] = mmDCORE1_TPC1_QM_BASE, 1158 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_1] = mmDCORE1_TPC1_QM_BASE, 1159 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_2] = mmDCORE1_TPC1_QM_BASE, 1160 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] = mmDCORE1_TPC1_QM_BASE, 1161 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_0] = mmDCORE1_TPC2_QM_BASE, 1162 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_1] = mmDCORE1_TPC2_QM_BASE, 1163 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_2] = mmDCORE1_TPC2_QM_BASE, 1164 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] = mmDCORE1_TPC2_QM_BASE, 1165 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_0] = mmDCORE1_TPC3_QM_BASE, 1166 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_1] = mmDCORE1_TPC3_QM_BASE, 1167 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_2] = mmDCORE1_TPC3_QM_BASE, 1168 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] = mmDCORE1_TPC3_QM_BASE, 1169 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_0] = mmDCORE1_TPC4_QM_BASE, 1170 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_1] = mmDCORE1_TPC4_QM_BASE, 1171 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_2] = mmDCORE1_TPC4_QM_BASE, 1172 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] = mmDCORE1_TPC4_QM_BASE, 1173 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_0] = mmDCORE1_TPC5_QM_BASE, 1174 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_1] = mmDCORE1_TPC5_QM_BASE, 1175 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_2] = mmDCORE1_TPC5_QM_BASE, 1176 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] = mmDCORE1_TPC5_QM_BASE, 1177 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0] = mmDCORE2_EDMA0_QM_BASE, 1178 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_1] = mmDCORE2_EDMA0_QM_BASE, 1179 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_2] = mmDCORE2_EDMA0_QM_BASE, 1180 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] = mmDCORE2_EDMA0_QM_BASE, 1181 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0] = mmDCORE2_EDMA1_QM_BASE, 1182 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_1] = mmDCORE2_EDMA1_QM_BASE, 1183 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_2] = mmDCORE2_EDMA1_QM_BASE, 1184 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] = mmDCORE2_EDMA1_QM_BASE, 1185 [GAUDI2_QUEUE_ID_DCORE2_MME_0_0] = mmDCORE2_MME_QM_BASE, 1186 [GAUDI2_QUEUE_ID_DCORE2_MME_0_1] = mmDCORE2_MME_QM_BASE, 1187 [GAUDI2_QUEUE_ID_DCORE2_MME_0_2] = mmDCORE2_MME_QM_BASE, 1188 [GAUDI2_QUEUE_ID_DCORE2_MME_0_3] = mmDCORE2_MME_QM_BASE, 1189 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_0] = mmDCORE2_TPC0_QM_BASE, 1190 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_1] = mmDCORE2_TPC0_QM_BASE, 1191 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_2] = mmDCORE2_TPC0_QM_BASE, 1192 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] = mmDCORE2_TPC0_QM_BASE, 1193 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_0] = mmDCORE2_TPC1_QM_BASE, 1194 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_1] = mmDCORE2_TPC1_QM_BASE, 1195 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_2] = mmDCORE2_TPC1_QM_BASE, 1196 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] = mmDCORE2_TPC1_QM_BASE, 1197 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_0] = mmDCORE2_TPC2_QM_BASE, 1198 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_1] = mmDCORE2_TPC2_QM_BASE, 1199 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_2] = mmDCORE2_TPC2_QM_BASE, 1200 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] = mmDCORE2_TPC2_QM_BASE, 1201 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_0] = mmDCORE2_TPC3_QM_BASE, 1202 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_1] = mmDCORE2_TPC3_QM_BASE, 1203 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_2] = mmDCORE2_TPC3_QM_BASE, 1204 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] = mmDCORE2_TPC3_QM_BASE, 1205 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_0] = mmDCORE2_TPC4_QM_BASE, 1206 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_1] = mmDCORE2_TPC4_QM_BASE, 1207 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_2] = mmDCORE2_TPC4_QM_BASE, 1208 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] = mmDCORE2_TPC4_QM_BASE, 1209 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_0] = mmDCORE2_TPC5_QM_BASE, 1210 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_1] = mmDCORE2_TPC5_QM_BASE, 1211 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_2] = mmDCORE2_TPC5_QM_BASE, 1212 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] = mmDCORE2_TPC5_QM_BASE, 1213 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0] = mmDCORE3_EDMA0_QM_BASE, 1214 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_1] = mmDCORE3_EDMA0_QM_BASE, 1215 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_2] = mmDCORE3_EDMA0_QM_BASE, 1216 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] = mmDCORE3_EDMA0_QM_BASE, 1217 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0] = mmDCORE3_EDMA1_QM_BASE, 1218 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1] = mmDCORE3_EDMA1_QM_BASE, 1219 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2] = mmDCORE3_EDMA1_QM_BASE, 1220 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] = mmDCORE3_EDMA1_QM_BASE, 1221 [GAUDI2_QUEUE_ID_DCORE3_MME_0_0] = mmDCORE3_MME_QM_BASE, 1222 [GAUDI2_QUEUE_ID_DCORE3_MME_0_1] = mmDCORE3_MME_QM_BASE, 1223 [GAUDI2_QUEUE_ID_DCORE3_MME_0_2] = mmDCORE3_MME_QM_BASE, 1224 [GAUDI2_QUEUE_ID_DCORE3_MME_0_3] = mmDCORE3_MME_QM_BASE, 1225 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_0] = mmDCORE3_TPC0_QM_BASE, 1226 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_1] = mmDCORE3_TPC0_QM_BASE, 1227 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_2] = mmDCORE3_TPC0_QM_BASE, 1228 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] = mmDCORE3_TPC0_QM_BASE, 1229 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_0] = mmDCORE3_TPC1_QM_BASE, 1230 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_1] = mmDCORE3_TPC1_QM_BASE, 1231 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_2] = mmDCORE3_TPC1_QM_BASE, 1232 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] = mmDCORE3_TPC1_QM_BASE, 1233 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_0] = mmDCORE3_TPC2_QM_BASE, 1234 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_1] = mmDCORE3_TPC2_QM_BASE, 1235 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_2] = mmDCORE3_TPC2_QM_BASE, 1236 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] = mmDCORE3_TPC2_QM_BASE, 1237 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_0] = mmDCORE3_TPC3_QM_BASE, 1238 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_1] = mmDCORE3_TPC3_QM_BASE, 1239 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_2] = mmDCORE3_TPC3_QM_BASE, 1240 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] = mmDCORE3_TPC3_QM_BASE, 1241 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_0] = mmDCORE3_TPC4_QM_BASE, 1242 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_1] = mmDCORE3_TPC4_QM_BASE, 1243 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_2] = mmDCORE3_TPC4_QM_BASE, 1244 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] = mmDCORE3_TPC4_QM_BASE, 1245 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_0] = mmDCORE3_TPC5_QM_BASE, 1246 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_1] = mmDCORE3_TPC5_QM_BASE, 1247 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_2] = mmDCORE3_TPC5_QM_BASE, 1248 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] = mmDCORE3_TPC5_QM_BASE, 1249 [GAUDI2_QUEUE_ID_NIC_0_0] = mmNIC0_QM0_BASE, 1250 [GAUDI2_QUEUE_ID_NIC_0_1] = mmNIC0_QM0_BASE, 1251 [GAUDI2_QUEUE_ID_NIC_0_2] = mmNIC0_QM0_BASE, 1252 [GAUDI2_QUEUE_ID_NIC_0_3] = mmNIC0_QM0_BASE, 1253 [GAUDI2_QUEUE_ID_NIC_1_0] = mmNIC0_QM1_BASE, 1254 [GAUDI2_QUEUE_ID_NIC_1_1] = mmNIC0_QM1_BASE, 1255 [GAUDI2_QUEUE_ID_NIC_1_2] = mmNIC0_QM1_BASE, 1256 [GAUDI2_QUEUE_ID_NIC_1_3] = mmNIC0_QM1_BASE, 1257 [GAUDI2_QUEUE_ID_NIC_2_0] = mmNIC1_QM0_BASE, 1258 [GAUDI2_QUEUE_ID_NIC_2_1] = mmNIC1_QM0_BASE, 1259 [GAUDI2_QUEUE_ID_NIC_2_2] = mmNIC1_QM0_BASE, 1260 [GAUDI2_QUEUE_ID_NIC_2_3] = mmNIC1_QM0_BASE, 1261 [GAUDI2_QUEUE_ID_NIC_3_0] = mmNIC1_QM1_BASE, 1262 [GAUDI2_QUEUE_ID_NIC_3_1] = mmNIC1_QM1_BASE, 1263 [GAUDI2_QUEUE_ID_NIC_3_2] = mmNIC1_QM1_BASE, 1264 [GAUDI2_QUEUE_ID_NIC_3_3] = mmNIC1_QM1_BASE, 1265 [GAUDI2_QUEUE_ID_NIC_4_0] = mmNIC2_QM0_BASE, 1266 [GAUDI2_QUEUE_ID_NIC_4_1] = mmNIC2_QM0_BASE, 1267 [GAUDI2_QUEUE_ID_NIC_4_2] = mmNIC2_QM0_BASE, 1268 [GAUDI2_QUEUE_ID_NIC_4_3] = mmNIC2_QM0_BASE, 1269 [GAUDI2_QUEUE_ID_NIC_5_0] = mmNIC2_QM1_BASE, 1270 [GAUDI2_QUEUE_ID_NIC_5_1] = mmNIC2_QM1_BASE, 1271 [GAUDI2_QUEUE_ID_NIC_5_2] = mmNIC2_QM1_BASE, 1272 [GAUDI2_QUEUE_ID_NIC_5_3] = mmNIC2_QM1_BASE, 1273 [GAUDI2_QUEUE_ID_NIC_6_0] = mmNIC3_QM0_BASE, 1274 [GAUDI2_QUEUE_ID_NIC_6_1] = mmNIC3_QM0_BASE, 1275 [GAUDI2_QUEUE_ID_NIC_6_2] = mmNIC3_QM0_BASE, 1276 [GAUDI2_QUEUE_ID_NIC_6_3] = mmNIC3_QM0_BASE, 1277 [GAUDI2_QUEUE_ID_NIC_7_0] = mmNIC3_QM1_BASE, 1278 [GAUDI2_QUEUE_ID_NIC_7_1] = mmNIC3_QM1_BASE, 1279 [GAUDI2_QUEUE_ID_NIC_7_2] = mmNIC3_QM1_BASE, 1280 [GAUDI2_QUEUE_ID_NIC_7_3] = mmNIC3_QM1_BASE, 1281 [GAUDI2_QUEUE_ID_NIC_8_0] = mmNIC4_QM0_BASE, 1282 [GAUDI2_QUEUE_ID_NIC_8_1] = mmNIC4_QM0_BASE, 1283 [GAUDI2_QUEUE_ID_NIC_8_2] = mmNIC4_QM0_BASE, 1284 [GAUDI2_QUEUE_ID_NIC_8_3] = mmNIC4_QM0_BASE, 1285 [GAUDI2_QUEUE_ID_NIC_9_0] = mmNIC4_QM1_BASE, 1286 [GAUDI2_QUEUE_ID_NIC_9_1] = mmNIC4_QM1_BASE, 1287 [GAUDI2_QUEUE_ID_NIC_9_2] = mmNIC4_QM1_BASE, 1288 [GAUDI2_QUEUE_ID_NIC_9_3] = mmNIC4_QM1_BASE, 1289 [GAUDI2_QUEUE_ID_NIC_10_0] = mmNIC5_QM0_BASE, 1290 [GAUDI2_QUEUE_ID_NIC_10_1] = mmNIC5_QM0_BASE, 1291 [GAUDI2_QUEUE_ID_NIC_10_2] = mmNIC5_QM0_BASE, 1292 [GAUDI2_QUEUE_ID_NIC_10_3] = mmNIC5_QM0_BASE, 1293 [GAUDI2_QUEUE_ID_NIC_11_0] = mmNIC5_QM1_BASE, 1294 [GAUDI2_QUEUE_ID_NIC_11_1] = mmNIC5_QM1_BASE, 1295 [GAUDI2_QUEUE_ID_NIC_11_2] = mmNIC5_QM1_BASE, 1296 [GAUDI2_QUEUE_ID_NIC_11_3] = mmNIC5_QM1_BASE, 1297 [GAUDI2_QUEUE_ID_NIC_12_0] = mmNIC6_QM0_BASE, 1298 [GAUDI2_QUEUE_ID_NIC_12_1] = mmNIC6_QM0_BASE, 1299 [GAUDI2_QUEUE_ID_NIC_12_2] = mmNIC6_QM0_BASE, 1300 [GAUDI2_QUEUE_ID_NIC_12_3] = mmNIC6_QM0_BASE, 1301 [GAUDI2_QUEUE_ID_NIC_13_0] = mmNIC6_QM1_BASE, 1302 [GAUDI2_QUEUE_ID_NIC_13_1] = mmNIC6_QM1_BASE, 1303 [GAUDI2_QUEUE_ID_NIC_13_2] = mmNIC6_QM1_BASE, 1304 [GAUDI2_QUEUE_ID_NIC_13_3] = mmNIC6_QM1_BASE, 1305 [GAUDI2_QUEUE_ID_NIC_14_0] = mmNIC7_QM0_BASE, 1306 [GAUDI2_QUEUE_ID_NIC_14_1] = mmNIC7_QM0_BASE, 1307 [GAUDI2_QUEUE_ID_NIC_14_2] = mmNIC7_QM0_BASE, 1308 [GAUDI2_QUEUE_ID_NIC_14_3] = mmNIC7_QM0_BASE, 1309 [GAUDI2_QUEUE_ID_NIC_15_0] = mmNIC7_QM1_BASE, 1310 [GAUDI2_QUEUE_ID_NIC_15_1] = mmNIC7_QM1_BASE, 1311 [GAUDI2_QUEUE_ID_NIC_15_2] = mmNIC7_QM1_BASE, 1312 [GAUDI2_QUEUE_ID_NIC_15_3] = mmNIC7_QM1_BASE, 1313 [GAUDI2_QUEUE_ID_NIC_16_0] = mmNIC8_QM0_BASE, 1314 [GAUDI2_QUEUE_ID_NIC_16_1] = mmNIC8_QM0_BASE, 1315 [GAUDI2_QUEUE_ID_NIC_16_2] = mmNIC8_QM0_BASE, 1316 [GAUDI2_QUEUE_ID_NIC_16_3] = mmNIC8_QM0_BASE, 1317 [GAUDI2_QUEUE_ID_NIC_17_0] = mmNIC8_QM1_BASE, 1318 [GAUDI2_QUEUE_ID_NIC_17_1] = mmNIC8_QM1_BASE, 1319 [GAUDI2_QUEUE_ID_NIC_17_2] = mmNIC8_QM1_BASE, 1320 [GAUDI2_QUEUE_ID_NIC_17_3] = mmNIC8_QM1_BASE, 1321 [GAUDI2_QUEUE_ID_NIC_18_0] = mmNIC9_QM0_BASE, 1322 [GAUDI2_QUEUE_ID_NIC_18_1] = mmNIC9_QM0_BASE, 1323 [GAUDI2_QUEUE_ID_NIC_18_2] = mmNIC9_QM0_BASE, 1324 [GAUDI2_QUEUE_ID_NIC_18_3] = mmNIC9_QM0_BASE, 1325 [GAUDI2_QUEUE_ID_NIC_19_0] = mmNIC9_QM1_BASE, 1326 [GAUDI2_QUEUE_ID_NIC_19_1] = mmNIC9_QM1_BASE, 1327 [GAUDI2_QUEUE_ID_NIC_19_2] = mmNIC9_QM1_BASE, 1328 [GAUDI2_QUEUE_ID_NIC_19_3] = mmNIC9_QM1_BASE, 1329 [GAUDI2_QUEUE_ID_NIC_20_0] = mmNIC10_QM0_BASE, 1330 [GAUDI2_QUEUE_ID_NIC_20_1] = mmNIC10_QM0_BASE, 1331 [GAUDI2_QUEUE_ID_NIC_20_2] = mmNIC10_QM0_BASE, 1332 [GAUDI2_QUEUE_ID_NIC_20_3] = mmNIC10_QM0_BASE, 1333 [GAUDI2_QUEUE_ID_NIC_21_0] = mmNIC10_QM1_BASE, 1334 [GAUDI2_QUEUE_ID_NIC_21_1] = mmNIC10_QM1_BASE, 1335 [GAUDI2_QUEUE_ID_NIC_21_2] = mmNIC10_QM1_BASE, 1336 [GAUDI2_QUEUE_ID_NIC_21_3] = mmNIC10_QM1_BASE, 1337 [GAUDI2_QUEUE_ID_NIC_22_0] = mmNIC11_QM0_BASE, 1338 [GAUDI2_QUEUE_ID_NIC_22_1] = mmNIC11_QM0_BASE, 1339 [GAUDI2_QUEUE_ID_NIC_22_2] = mmNIC11_QM0_BASE, 1340 [GAUDI2_QUEUE_ID_NIC_22_3] = mmNIC11_QM0_BASE, 1341 [GAUDI2_QUEUE_ID_NIC_23_0] = mmNIC11_QM1_BASE, 1342 [GAUDI2_QUEUE_ID_NIC_23_1] = mmNIC11_QM1_BASE, 1343 [GAUDI2_QUEUE_ID_NIC_23_2] = mmNIC11_QM1_BASE, 1344 [GAUDI2_QUEUE_ID_NIC_23_3] = mmNIC11_QM1_BASE, 1345 [GAUDI2_QUEUE_ID_ROT_0_0] = mmROT0_QM_BASE, 1346 [GAUDI2_QUEUE_ID_ROT_0_1] = mmROT0_QM_BASE, 1347 [GAUDI2_QUEUE_ID_ROT_0_2] = mmROT0_QM_BASE, 1348 [GAUDI2_QUEUE_ID_ROT_0_3] = mmROT0_QM_BASE, 1349 [GAUDI2_QUEUE_ID_ROT_1_0] = mmROT1_QM_BASE, 1350 [GAUDI2_QUEUE_ID_ROT_1_1] = mmROT1_QM_BASE, 1351 [GAUDI2_QUEUE_ID_ROT_1_2] = mmROT1_QM_BASE, 1352 [GAUDI2_QUEUE_ID_ROT_1_3] = mmROT1_QM_BASE 1353 }; 1354 1355 static const u32 gaudi2_arc_blocks_bases[NUM_ARC_CPUS] = { 1356 [CPU_ID_SCHED_ARC0] = mmARC_FARM_ARC0_AUX_BASE, 1357 [CPU_ID_SCHED_ARC1] = mmARC_FARM_ARC1_AUX_BASE, 1358 [CPU_ID_SCHED_ARC2] = mmARC_FARM_ARC2_AUX_BASE, 1359 [CPU_ID_SCHED_ARC3] = mmARC_FARM_ARC3_AUX_BASE, 1360 [CPU_ID_SCHED_ARC4] = mmDCORE1_MME_QM_ARC_AUX_BASE, 1361 [CPU_ID_SCHED_ARC5] = mmDCORE3_MME_QM_ARC_AUX_BASE, 1362 [CPU_ID_TPC_QMAN_ARC0] = mmDCORE0_TPC0_QM_ARC_AUX_BASE, 1363 [CPU_ID_TPC_QMAN_ARC1] = mmDCORE0_TPC1_QM_ARC_AUX_BASE, 1364 [CPU_ID_TPC_QMAN_ARC2] = mmDCORE0_TPC2_QM_ARC_AUX_BASE, 1365 [CPU_ID_TPC_QMAN_ARC3] = mmDCORE0_TPC3_QM_ARC_AUX_BASE, 1366 [CPU_ID_TPC_QMAN_ARC4] = mmDCORE0_TPC4_QM_ARC_AUX_BASE, 1367 [CPU_ID_TPC_QMAN_ARC5] = mmDCORE0_TPC5_QM_ARC_AUX_BASE, 1368 [CPU_ID_TPC_QMAN_ARC6] = mmDCORE1_TPC0_QM_ARC_AUX_BASE, 1369 [CPU_ID_TPC_QMAN_ARC7] = mmDCORE1_TPC1_QM_ARC_AUX_BASE, 1370 [CPU_ID_TPC_QMAN_ARC8] = mmDCORE1_TPC2_QM_ARC_AUX_BASE, 1371 [CPU_ID_TPC_QMAN_ARC9] = mmDCORE1_TPC3_QM_ARC_AUX_BASE, 1372 [CPU_ID_TPC_QMAN_ARC10] = mmDCORE1_TPC4_QM_ARC_AUX_BASE, 1373 [CPU_ID_TPC_QMAN_ARC11] = mmDCORE1_TPC5_QM_ARC_AUX_BASE, 1374 [CPU_ID_TPC_QMAN_ARC12] = mmDCORE2_TPC0_QM_ARC_AUX_BASE, 1375 [CPU_ID_TPC_QMAN_ARC13] = mmDCORE2_TPC1_QM_ARC_AUX_BASE, 1376 [CPU_ID_TPC_QMAN_ARC14] = mmDCORE2_TPC2_QM_ARC_AUX_BASE, 1377 [CPU_ID_TPC_QMAN_ARC15] = mmDCORE2_TPC3_QM_ARC_AUX_BASE, 1378 [CPU_ID_TPC_QMAN_ARC16] = mmDCORE2_TPC4_QM_ARC_AUX_BASE, 1379 [CPU_ID_TPC_QMAN_ARC17] = mmDCORE2_TPC5_QM_ARC_AUX_BASE, 1380 [CPU_ID_TPC_QMAN_ARC18] = mmDCORE3_TPC0_QM_ARC_AUX_BASE, 1381 [CPU_ID_TPC_QMAN_ARC19] = mmDCORE3_TPC1_QM_ARC_AUX_BASE, 1382 [CPU_ID_TPC_QMAN_ARC20] = mmDCORE3_TPC2_QM_ARC_AUX_BASE, 1383 [CPU_ID_TPC_QMAN_ARC21] = mmDCORE3_TPC3_QM_ARC_AUX_BASE, 1384 [CPU_ID_TPC_QMAN_ARC22] = mmDCORE3_TPC4_QM_ARC_AUX_BASE, 1385 [CPU_ID_TPC_QMAN_ARC23] = mmDCORE3_TPC5_QM_ARC_AUX_BASE, 1386 [CPU_ID_TPC_QMAN_ARC24] = mmDCORE0_TPC6_QM_ARC_AUX_BASE, 1387 [CPU_ID_MME_QMAN_ARC0] = mmDCORE0_MME_QM_ARC_AUX_BASE, 1388 [CPU_ID_MME_QMAN_ARC1] = mmDCORE2_MME_QM_ARC_AUX_BASE, 1389 [CPU_ID_EDMA_QMAN_ARC0] = mmDCORE0_EDMA0_QM_ARC_AUX_BASE, 1390 [CPU_ID_EDMA_QMAN_ARC1] = mmDCORE0_EDMA1_QM_ARC_AUX_BASE, 1391 [CPU_ID_EDMA_QMAN_ARC2] = mmDCORE1_EDMA0_QM_ARC_AUX_BASE, 1392 [CPU_ID_EDMA_QMAN_ARC3] = mmDCORE1_EDMA1_QM_ARC_AUX_BASE, 1393 [CPU_ID_EDMA_QMAN_ARC4] = mmDCORE2_EDMA0_QM_ARC_AUX_BASE, 1394 [CPU_ID_EDMA_QMAN_ARC5] = mmDCORE2_EDMA1_QM_ARC_AUX_BASE, 1395 [CPU_ID_EDMA_QMAN_ARC6] = mmDCORE3_EDMA0_QM_ARC_AUX_BASE, 1396 [CPU_ID_EDMA_QMAN_ARC7] = mmDCORE3_EDMA1_QM_ARC_AUX_BASE, 1397 [CPU_ID_PDMA_QMAN_ARC0] = mmPDMA0_QM_ARC_AUX_BASE, 1398 [CPU_ID_PDMA_QMAN_ARC1] = mmPDMA1_QM_ARC_AUX_BASE, 1399 [CPU_ID_ROT_QMAN_ARC0] = mmROT0_QM_ARC_AUX_BASE, 1400 [CPU_ID_ROT_QMAN_ARC1] = mmROT1_QM_ARC_AUX_BASE, 1401 [CPU_ID_NIC_QMAN_ARC0] = mmNIC0_QM_ARC_AUX0_BASE, 1402 [CPU_ID_NIC_QMAN_ARC1] = mmNIC0_QM_ARC_AUX1_BASE, 1403 [CPU_ID_NIC_QMAN_ARC2] = mmNIC1_QM_ARC_AUX0_BASE, 1404 [CPU_ID_NIC_QMAN_ARC3] = mmNIC1_QM_ARC_AUX1_BASE, 1405 [CPU_ID_NIC_QMAN_ARC4] = mmNIC2_QM_ARC_AUX0_BASE, 1406 [CPU_ID_NIC_QMAN_ARC5] = mmNIC2_QM_ARC_AUX1_BASE, 1407 [CPU_ID_NIC_QMAN_ARC6] = mmNIC3_QM_ARC_AUX0_BASE, 1408 [CPU_ID_NIC_QMAN_ARC7] = mmNIC3_QM_ARC_AUX1_BASE, 1409 [CPU_ID_NIC_QMAN_ARC8] = mmNIC4_QM_ARC_AUX0_BASE, 1410 [CPU_ID_NIC_QMAN_ARC9] = mmNIC4_QM_ARC_AUX1_BASE, 1411 [CPU_ID_NIC_QMAN_ARC10] = mmNIC5_QM_ARC_AUX0_BASE, 1412 [CPU_ID_NIC_QMAN_ARC11] = mmNIC5_QM_ARC_AUX1_BASE, 1413 [CPU_ID_NIC_QMAN_ARC12] = mmNIC6_QM_ARC_AUX0_BASE, 1414 [CPU_ID_NIC_QMAN_ARC13] = mmNIC6_QM_ARC_AUX1_BASE, 1415 [CPU_ID_NIC_QMAN_ARC14] = mmNIC7_QM_ARC_AUX0_BASE, 1416 [CPU_ID_NIC_QMAN_ARC15] = mmNIC7_QM_ARC_AUX1_BASE, 1417 [CPU_ID_NIC_QMAN_ARC16] = mmNIC8_QM_ARC_AUX0_BASE, 1418 [CPU_ID_NIC_QMAN_ARC17] = mmNIC8_QM_ARC_AUX1_BASE, 1419 [CPU_ID_NIC_QMAN_ARC18] = mmNIC9_QM_ARC_AUX0_BASE, 1420 [CPU_ID_NIC_QMAN_ARC19] = mmNIC9_QM_ARC_AUX1_BASE, 1421 [CPU_ID_NIC_QMAN_ARC20] = mmNIC10_QM_ARC_AUX0_BASE, 1422 [CPU_ID_NIC_QMAN_ARC21] = mmNIC10_QM_ARC_AUX1_BASE, 1423 [CPU_ID_NIC_QMAN_ARC22] = mmNIC11_QM_ARC_AUX0_BASE, 1424 [CPU_ID_NIC_QMAN_ARC23] = mmNIC11_QM_ARC_AUX1_BASE, 1425 }; 1426 1427 static const u32 gaudi2_arc_dccm_bases[NUM_ARC_CPUS] = { 1428 [CPU_ID_SCHED_ARC0] = mmARC_FARM_ARC0_DCCM0_BASE, 1429 [CPU_ID_SCHED_ARC1] = mmARC_FARM_ARC1_DCCM0_BASE, 1430 [CPU_ID_SCHED_ARC2] = mmARC_FARM_ARC2_DCCM0_BASE, 1431 [CPU_ID_SCHED_ARC3] = mmARC_FARM_ARC3_DCCM0_BASE, 1432 [CPU_ID_SCHED_ARC4] = mmDCORE1_MME_QM_ARC_DCCM_BASE, 1433 [CPU_ID_SCHED_ARC5] = mmDCORE3_MME_QM_ARC_DCCM_BASE, 1434 [CPU_ID_TPC_QMAN_ARC0] = mmDCORE0_TPC0_QM_DCCM_BASE, 1435 [CPU_ID_TPC_QMAN_ARC1] = mmDCORE0_TPC1_QM_DCCM_BASE, 1436 [CPU_ID_TPC_QMAN_ARC2] = mmDCORE0_TPC2_QM_DCCM_BASE, 1437 [CPU_ID_TPC_QMAN_ARC3] = mmDCORE0_TPC3_QM_DCCM_BASE, 1438 [CPU_ID_TPC_QMAN_ARC4] = mmDCORE0_TPC4_QM_DCCM_BASE, 1439 [CPU_ID_TPC_QMAN_ARC5] = mmDCORE0_TPC5_QM_DCCM_BASE, 1440 [CPU_ID_TPC_QMAN_ARC6] = mmDCORE1_TPC0_QM_DCCM_BASE, 1441 [CPU_ID_TPC_QMAN_ARC7] = mmDCORE1_TPC1_QM_DCCM_BASE, 1442 [CPU_ID_TPC_QMAN_ARC8] = mmDCORE1_TPC2_QM_DCCM_BASE, 1443 [CPU_ID_TPC_QMAN_ARC9] = mmDCORE1_TPC3_QM_DCCM_BASE, 1444 [CPU_ID_TPC_QMAN_ARC10] = mmDCORE1_TPC4_QM_DCCM_BASE, 1445 [CPU_ID_TPC_QMAN_ARC11] = mmDCORE1_TPC5_QM_DCCM_BASE, 1446 [CPU_ID_TPC_QMAN_ARC12] = mmDCORE2_TPC0_QM_DCCM_BASE, 1447 [CPU_ID_TPC_QMAN_ARC13] = mmDCORE2_TPC1_QM_DCCM_BASE, 1448 [CPU_ID_TPC_QMAN_ARC14] = mmDCORE2_TPC2_QM_DCCM_BASE, 1449 [CPU_ID_TPC_QMAN_ARC15] = mmDCORE2_TPC3_QM_DCCM_BASE, 1450 [CPU_ID_TPC_QMAN_ARC16] = mmDCORE2_TPC4_QM_DCCM_BASE, 1451 [CPU_ID_TPC_QMAN_ARC17] = mmDCORE2_TPC5_QM_DCCM_BASE, 1452 [CPU_ID_TPC_QMAN_ARC18] = mmDCORE3_TPC0_QM_DCCM_BASE, 1453 [CPU_ID_TPC_QMAN_ARC19] = mmDCORE3_TPC1_QM_DCCM_BASE, 1454 [CPU_ID_TPC_QMAN_ARC20] = mmDCORE3_TPC2_QM_DCCM_BASE, 1455 [CPU_ID_TPC_QMAN_ARC21] = mmDCORE3_TPC3_QM_DCCM_BASE, 1456 [CPU_ID_TPC_QMAN_ARC22] = mmDCORE3_TPC4_QM_DCCM_BASE, 1457 [CPU_ID_TPC_QMAN_ARC23] = mmDCORE3_TPC5_QM_DCCM_BASE, 1458 [CPU_ID_TPC_QMAN_ARC24] = mmDCORE0_TPC6_QM_DCCM_BASE, 1459 [CPU_ID_MME_QMAN_ARC0] = mmDCORE0_MME_QM_ARC_DCCM_BASE, 1460 [CPU_ID_MME_QMAN_ARC1] = mmDCORE2_MME_QM_ARC_DCCM_BASE, 1461 [CPU_ID_EDMA_QMAN_ARC0] = mmDCORE0_EDMA0_QM_DCCM_BASE, 1462 [CPU_ID_EDMA_QMAN_ARC1] = mmDCORE0_EDMA1_QM_DCCM_BASE, 1463 [CPU_ID_EDMA_QMAN_ARC2] = mmDCORE1_EDMA0_QM_DCCM_BASE, 1464 [CPU_ID_EDMA_QMAN_ARC3] = mmDCORE1_EDMA1_QM_DCCM_BASE, 1465 [CPU_ID_EDMA_QMAN_ARC4] = mmDCORE2_EDMA0_QM_DCCM_BASE, 1466 [CPU_ID_EDMA_QMAN_ARC5] = mmDCORE2_EDMA1_QM_DCCM_BASE, 1467 [CPU_ID_EDMA_QMAN_ARC6] = mmDCORE3_EDMA0_QM_DCCM_BASE, 1468 [CPU_ID_EDMA_QMAN_ARC7] = mmDCORE3_EDMA1_QM_DCCM_BASE, 1469 [CPU_ID_PDMA_QMAN_ARC0] = mmPDMA0_QM_ARC_DCCM_BASE, 1470 [CPU_ID_PDMA_QMAN_ARC1] = mmPDMA1_QM_ARC_DCCM_BASE, 1471 [CPU_ID_ROT_QMAN_ARC0] = mmROT0_QM_ARC_DCCM_BASE, 1472 [CPU_ID_ROT_QMAN_ARC1] = mmROT1_QM_ARC_DCCM_BASE, 1473 [CPU_ID_NIC_QMAN_ARC0] = mmNIC0_QM_DCCM0_BASE, 1474 [CPU_ID_NIC_QMAN_ARC1] = mmNIC0_QM_DCCM1_BASE, 1475 [CPU_ID_NIC_QMAN_ARC2] = mmNIC1_QM_DCCM0_BASE, 1476 [CPU_ID_NIC_QMAN_ARC3] = mmNIC1_QM_DCCM1_BASE, 1477 [CPU_ID_NIC_QMAN_ARC4] = mmNIC2_QM_DCCM0_BASE, 1478 [CPU_ID_NIC_QMAN_ARC5] = mmNIC2_QM_DCCM1_BASE, 1479 [CPU_ID_NIC_QMAN_ARC6] = mmNIC3_QM_DCCM0_BASE, 1480 [CPU_ID_NIC_QMAN_ARC7] = mmNIC3_QM_DCCM1_BASE, 1481 [CPU_ID_NIC_QMAN_ARC8] = mmNIC4_QM_DCCM0_BASE, 1482 [CPU_ID_NIC_QMAN_ARC9] = mmNIC4_QM_DCCM1_BASE, 1483 [CPU_ID_NIC_QMAN_ARC10] = mmNIC5_QM_DCCM0_BASE, 1484 [CPU_ID_NIC_QMAN_ARC11] = mmNIC5_QM_DCCM1_BASE, 1485 [CPU_ID_NIC_QMAN_ARC12] = mmNIC6_QM_DCCM0_BASE, 1486 [CPU_ID_NIC_QMAN_ARC13] = mmNIC6_QM_DCCM1_BASE, 1487 [CPU_ID_NIC_QMAN_ARC14] = mmNIC7_QM_DCCM0_BASE, 1488 [CPU_ID_NIC_QMAN_ARC15] = mmNIC7_QM_DCCM1_BASE, 1489 [CPU_ID_NIC_QMAN_ARC16] = mmNIC8_QM_DCCM0_BASE, 1490 [CPU_ID_NIC_QMAN_ARC17] = mmNIC8_QM_DCCM1_BASE, 1491 [CPU_ID_NIC_QMAN_ARC18] = mmNIC9_QM_DCCM0_BASE, 1492 [CPU_ID_NIC_QMAN_ARC19] = mmNIC9_QM_DCCM1_BASE, 1493 [CPU_ID_NIC_QMAN_ARC20] = mmNIC10_QM_DCCM0_BASE, 1494 [CPU_ID_NIC_QMAN_ARC21] = mmNIC10_QM_DCCM1_BASE, 1495 [CPU_ID_NIC_QMAN_ARC22] = mmNIC11_QM_DCCM0_BASE, 1496 [CPU_ID_NIC_QMAN_ARC23] = mmNIC11_QM_DCCM1_BASE, 1497 }; 1498 1499 const u32 gaudi2_mme_ctrl_lo_blocks_bases[MME_ID_SIZE] = { 1500 [MME_ID_DCORE0] = mmDCORE0_MME_CTRL_LO_BASE, 1501 [MME_ID_DCORE1] = mmDCORE1_MME_CTRL_LO_BASE, 1502 [MME_ID_DCORE2] = mmDCORE2_MME_CTRL_LO_BASE, 1503 [MME_ID_DCORE3] = mmDCORE3_MME_CTRL_LO_BASE, 1504 }; 1505 1506 static const u32 gaudi2_queue_id_to_arc_id[GAUDI2_QUEUE_ID_SIZE] = { 1507 [GAUDI2_QUEUE_ID_PDMA_0_0] = CPU_ID_PDMA_QMAN_ARC0, 1508 [GAUDI2_QUEUE_ID_PDMA_0_1] = CPU_ID_PDMA_QMAN_ARC0, 1509 [GAUDI2_QUEUE_ID_PDMA_0_2] = CPU_ID_PDMA_QMAN_ARC0, 1510 [GAUDI2_QUEUE_ID_PDMA_0_3] = CPU_ID_PDMA_QMAN_ARC0, 1511 [GAUDI2_QUEUE_ID_PDMA_1_0] = CPU_ID_PDMA_QMAN_ARC1, 1512 [GAUDI2_QUEUE_ID_PDMA_1_1] = CPU_ID_PDMA_QMAN_ARC1, 1513 [GAUDI2_QUEUE_ID_PDMA_1_2] = CPU_ID_PDMA_QMAN_ARC1, 1514 [GAUDI2_QUEUE_ID_PDMA_1_3] = CPU_ID_PDMA_QMAN_ARC1, 1515 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC0, 1516 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC0, 1517 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC0, 1518 [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC0, 1519 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC1, 1520 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC1, 1521 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC1, 1522 [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC1, 1523 [GAUDI2_QUEUE_ID_DCORE0_MME_0_0] = CPU_ID_MME_QMAN_ARC0, 1524 [GAUDI2_QUEUE_ID_DCORE0_MME_0_1] = CPU_ID_MME_QMAN_ARC0, 1525 [GAUDI2_QUEUE_ID_DCORE0_MME_0_2] = CPU_ID_MME_QMAN_ARC0, 1526 [GAUDI2_QUEUE_ID_DCORE0_MME_0_3] = CPU_ID_MME_QMAN_ARC0, 1527 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_0] = CPU_ID_TPC_QMAN_ARC0, 1528 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_1] = CPU_ID_TPC_QMAN_ARC0, 1529 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_2] = CPU_ID_TPC_QMAN_ARC0, 1530 [GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] = CPU_ID_TPC_QMAN_ARC0, 1531 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_0] = CPU_ID_TPC_QMAN_ARC1, 1532 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_1] = CPU_ID_TPC_QMAN_ARC1, 1533 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_2] = CPU_ID_TPC_QMAN_ARC1, 1534 [GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] = CPU_ID_TPC_QMAN_ARC1, 1535 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_0] = CPU_ID_TPC_QMAN_ARC2, 1536 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_1] = CPU_ID_TPC_QMAN_ARC2, 1537 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_2] = CPU_ID_TPC_QMAN_ARC2, 1538 [GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] = CPU_ID_TPC_QMAN_ARC2, 1539 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_0] = CPU_ID_TPC_QMAN_ARC3, 1540 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_1] = CPU_ID_TPC_QMAN_ARC3, 1541 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_2] = CPU_ID_TPC_QMAN_ARC3, 1542 [GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] = CPU_ID_TPC_QMAN_ARC3, 1543 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_0] = CPU_ID_TPC_QMAN_ARC4, 1544 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_1] = CPU_ID_TPC_QMAN_ARC4, 1545 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_2] = CPU_ID_TPC_QMAN_ARC4, 1546 [GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] = CPU_ID_TPC_QMAN_ARC4, 1547 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_0] = CPU_ID_TPC_QMAN_ARC5, 1548 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_1] = CPU_ID_TPC_QMAN_ARC5, 1549 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_2] = CPU_ID_TPC_QMAN_ARC5, 1550 [GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] = CPU_ID_TPC_QMAN_ARC5, 1551 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_0] = CPU_ID_TPC_QMAN_ARC24, 1552 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_1] = CPU_ID_TPC_QMAN_ARC24, 1553 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_2] = CPU_ID_TPC_QMAN_ARC24, 1554 [GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] = CPU_ID_TPC_QMAN_ARC24, 1555 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC2, 1556 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC2, 1557 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC2, 1558 [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC2, 1559 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC3, 1560 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC3, 1561 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC3, 1562 [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC3, 1563 [GAUDI2_QUEUE_ID_DCORE1_MME_0_0] = CPU_ID_SCHED_ARC4, 1564 [GAUDI2_QUEUE_ID_DCORE1_MME_0_1] = CPU_ID_SCHED_ARC4, 1565 [GAUDI2_QUEUE_ID_DCORE1_MME_0_2] = CPU_ID_SCHED_ARC4, 1566 [GAUDI2_QUEUE_ID_DCORE1_MME_0_3] = CPU_ID_SCHED_ARC4, 1567 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_0] = CPU_ID_TPC_QMAN_ARC6, 1568 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_1] = CPU_ID_TPC_QMAN_ARC6, 1569 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_2] = CPU_ID_TPC_QMAN_ARC6, 1570 [GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] = CPU_ID_TPC_QMAN_ARC6, 1571 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_0] = CPU_ID_TPC_QMAN_ARC7, 1572 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_1] = CPU_ID_TPC_QMAN_ARC7, 1573 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_2] = CPU_ID_TPC_QMAN_ARC7, 1574 [GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] = CPU_ID_TPC_QMAN_ARC7, 1575 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_0] = CPU_ID_TPC_QMAN_ARC8, 1576 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_1] = CPU_ID_TPC_QMAN_ARC8, 1577 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_2] = CPU_ID_TPC_QMAN_ARC8, 1578 [GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] = CPU_ID_TPC_QMAN_ARC8, 1579 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_0] = CPU_ID_TPC_QMAN_ARC9, 1580 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_1] = CPU_ID_TPC_QMAN_ARC9, 1581 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_2] = CPU_ID_TPC_QMAN_ARC9, 1582 [GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] = CPU_ID_TPC_QMAN_ARC9, 1583 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_0] = CPU_ID_TPC_QMAN_ARC10, 1584 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_1] = CPU_ID_TPC_QMAN_ARC10, 1585 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_2] = CPU_ID_TPC_QMAN_ARC10, 1586 [GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] = CPU_ID_TPC_QMAN_ARC10, 1587 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_0] = CPU_ID_TPC_QMAN_ARC11, 1588 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_1] = CPU_ID_TPC_QMAN_ARC11, 1589 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_2] = CPU_ID_TPC_QMAN_ARC11, 1590 [GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] = CPU_ID_TPC_QMAN_ARC11, 1591 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC4, 1592 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC4, 1593 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC4, 1594 [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC4, 1595 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC5, 1596 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC5, 1597 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC5, 1598 [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC5, 1599 [GAUDI2_QUEUE_ID_DCORE2_MME_0_0] = CPU_ID_MME_QMAN_ARC1, 1600 [GAUDI2_QUEUE_ID_DCORE2_MME_0_1] = CPU_ID_MME_QMAN_ARC1, 1601 [GAUDI2_QUEUE_ID_DCORE2_MME_0_2] = CPU_ID_MME_QMAN_ARC1, 1602 [GAUDI2_QUEUE_ID_DCORE2_MME_0_3] = CPU_ID_MME_QMAN_ARC1, 1603 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_0] = CPU_ID_TPC_QMAN_ARC12, 1604 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_1] = CPU_ID_TPC_QMAN_ARC12, 1605 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_2] = CPU_ID_TPC_QMAN_ARC12, 1606 [GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] = CPU_ID_TPC_QMAN_ARC12, 1607 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_0] = CPU_ID_TPC_QMAN_ARC13, 1608 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_1] = CPU_ID_TPC_QMAN_ARC13, 1609 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_2] = CPU_ID_TPC_QMAN_ARC13, 1610 [GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] = CPU_ID_TPC_QMAN_ARC13, 1611 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_0] = CPU_ID_TPC_QMAN_ARC14, 1612 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_1] = CPU_ID_TPC_QMAN_ARC14, 1613 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_2] = CPU_ID_TPC_QMAN_ARC14, 1614 [GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] = CPU_ID_TPC_QMAN_ARC14, 1615 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_0] = CPU_ID_TPC_QMAN_ARC15, 1616 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_1] = CPU_ID_TPC_QMAN_ARC15, 1617 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_2] = CPU_ID_TPC_QMAN_ARC15, 1618 [GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] = CPU_ID_TPC_QMAN_ARC15, 1619 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_0] = CPU_ID_TPC_QMAN_ARC16, 1620 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_1] = CPU_ID_TPC_QMAN_ARC16, 1621 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_2] = CPU_ID_TPC_QMAN_ARC16, 1622 [GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] = CPU_ID_TPC_QMAN_ARC16, 1623 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_0] = CPU_ID_TPC_QMAN_ARC17, 1624 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_1] = CPU_ID_TPC_QMAN_ARC17, 1625 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_2] = CPU_ID_TPC_QMAN_ARC17, 1626 [GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] = CPU_ID_TPC_QMAN_ARC17, 1627 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0] = CPU_ID_EDMA_QMAN_ARC6, 1628 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_1] = CPU_ID_EDMA_QMAN_ARC6, 1629 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_2] = CPU_ID_EDMA_QMAN_ARC6, 1630 [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] = CPU_ID_EDMA_QMAN_ARC6, 1631 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0] = CPU_ID_EDMA_QMAN_ARC7, 1632 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1] = CPU_ID_EDMA_QMAN_ARC7, 1633 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2] = CPU_ID_EDMA_QMAN_ARC7, 1634 [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] = CPU_ID_EDMA_QMAN_ARC7, 1635 [GAUDI2_QUEUE_ID_DCORE3_MME_0_0] = CPU_ID_SCHED_ARC5, 1636 [GAUDI2_QUEUE_ID_DCORE3_MME_0_1] = CPU_ID_SCHED_ARC5, 1637 [GAUDI2_QUEUE_ID_DCORE3_MME_0_2] = CPU_ID_SCHED_ARC5, 1638 [GAUDI2_QUEUE_ID_DCORE3_MME_0_3] = CPU_ID_SCHED_ARC5, 1639 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_0] = CPU_ID_TPC_QMAN_ARC18, 1640 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_1] = CPU_ID_TPC_QMAN_ARC18, 1641 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_2] = CPU_ID_TPC_QMAN_ARC18, 1642 [GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] = CPU_ID_TPC_QMAN_ARC18, 1643 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_0] = CPU_ID_TPC_QMAN_ARC19, 1644 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_1] = CPU_ID_TPC_QMAN_ARC19, 1645 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_2] = CPU_ID_TPC_QMAN_ARC19, 1646 [GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] = CPU_ID_TPC_QMAN_ARC19, 1647 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_0] = CPU_ID_TPC_QMAN_ARC20, 1648 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_1] = CPU_ID_TPC_QMAN_ARC20, 1649 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_2] = CPU_ID_TPC_QMAN_ARC20, 1650 [GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] = CPU_ID_TPC_QMAN_ARC20, 1651 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_0] = CPU_ID_TPC_QMAN_ARC21, 1652 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_1] = CPU_ID_TPC_QMAN_ARC21, 1653 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_2] = CPU_ID_TPC_QMAN_ARC21, 1654 [GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] = CPU_ID_TPC_QMAN_ARC21, 1655 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_0] = CPU_ID_TPC_QMAN_ARC22, 1656 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_1] = CPU_ID_TPC_QMAN_ARC22, 1657 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_2] = CPU_ID_TPC_QMAN_ARC22, 1658 [GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] = CPU_ID_TPC_QMAN_ARC22, 1659 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_0] = CPU_ID_TPC_QMAN_ARC23, 1660 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_1] = CPU_ID_TPC_QMAN_ARC23, 1661 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_2] = CPU_ID_TPC_QMAN_ARC23, 1662 [GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] = CPU_ID_TPC_QMAN_ARC23, 1663 [GAUDI2_QUEUE_ID_NIC_0_0] = CPU_ID_NIC_QMAN_ARC0, 1664 [GAUDI2_QUEUE_ID_NIC_0_1] = CPU_ID_NIC_QMAN_ARC0, 1665 [GAUDI2_QUEUE_ID_NIC_0_2] = CPU_ID_NIC_QMAN_ARC0, 1666 [GAUDI2_QUEUE_ID_NIC_0_3] = CPU_ID_NIC_QMAN_ARC0, 1667 [GAUDI2_QUEUE_ID_NIC_1_0] = CPU_ID_NIC_QMAN_ARC1, 1668 [GAUDI2_QUEUE_ID_NIC_1_1] = CPU_ID_NIC_QMAN_ARC1, 1669 [GAUDI2_QUEUE_ID_NIC_1_2] = CPU_ID_NIC_QMAN_ARC1, 1670 [GAUDI2_QUEUE_ID_NIC_1_3] = CPU_ID_NIC_QMAN_ARC1, 1671 [GAUDI2_QUEUE_ID_NIC_2_0] = CPU_ID_NIC_QMAN_ARC2, 1672 [GAUDI2_QUEUE_ID_NIC_2_1] = CPU_ID_NIC_QMAN_ARC2, 1673 [GAUDI2_QUEUE_ID_NIC_2_2] = CPU_ID_NIC_QMAN_ARC2, 1674 [GAUDI2_QUEUE_ID_NIC_2_3] = CPU_ID_NIC_QMAN_ARC2, 1675 [GAUDI2_QUEUE_ID_NIC_3_0] = CPU_ID_NIC_QMAN_ARC3, 1676 [GAUDI2_QUEUE_ID_NIC_3_1] = CPU_ID_NIC_QMAN_ARC3, 1677 [GAUDI2_QUEUE_ID_NIC_3_2] = CPU_ID_NIC_QMAN_ARC3, 1678 [GAUDI2_QUEUE_ID_NIC_3_3] = CPU_ID_NIC_QMAN_ARC3, 1679 [GAUDI2_QUEUE_ID_NIC_4_0] = CPU_ID_NIC_QMAN_ARC4, 1680 [GAUDI2_QUEUE_ID_NIC_4_1] = CPU_ID_NIC_QMAN_ARC4, 1681 [GAUDI2_QUEUE_ID_NIC_4_2] = CPU_ID_NIC_QMAN_ARC4, 1682 [GAUDI2_QUEUE_ID_NIC_4_3] = CPU_ID_NIC_QMAN_ARC4, 1683 [GAUDI2_QUEUE_ID_NIC_5_0] = CPU_ID_NIC_QMAN_ARC5, 1684 [GAUDI2_QUEUE_ID_NIC_5_1] = CPU_ID_NIC_QMAN_ARC5, 1685 [GAUDI2_QUEUE_ID_NIC_5_2] = CPU_ID_NIC_QMAN_ARC5, 1686 [GAUDI2_QUEUE_ID_NIC_5_3] = CPU_ID_NIC_QMAN_ARC5, 1687 [GAUDI2_QUEUE_ID_NIC_6_0] = CPU_ID_NIC_QMAN_ARC6, 1688 [GAUDI2_QUEUE_ID_NIC_6_1] = CPU_ID_NIC_QMAN_ARC6, 1689 [GAUDI2_QUEUE_ID_NIC_6_2] = CPU_ID_NIC_QMAN_ARC6, 1690 [GAUDI2_QUEUE_ID_NIC_6_3] = CPU_ID_NIC_QMAN_ARC6, 1691 [GAUDI2_QUEUE_ID_NIC_7_0] = CPU_ID_NIC_QMAN_ARC7, 1692 [GAUDI2_QUEUE_ID_NIC_7_1] = CPU_ID_NIC_QMAN_ARC7, 1693 [GAUDI2_QUEUE_ID_NIC_7_2] = CPU_ID_NIC_QMAN_ARC7, 1694 [GAUDI2_QUEUE_ID_NIC_7_3] = CPU_ID_NIC_QMAN_ARC7, 1695 [GAUDI2_QUEUE_ID_NIC_8_0] = CPU_ID_NIC_QMAN_ARC8, 1696 [GAUDI2_QUEUE_ID_NIC_8_1] = CPU_ID_NIC_QMAN_ARC8, 1697 [GAUDI2_QUEUE_ID_NIC_8_2] = CPU_ID_NIC_QMAN_ARC8, 1698 [GAUDI2_QUEUE_ID_NIC_8_3] = CPU_ID_NIC_QMAN_ARC8, 1699 [GAUDI2_QUEUE_ID_NIC_9_0] = CPU_ID_NIC_QMAN_ARC9, 1700 [GAUDI2_QUEUE_ID_NIC_9_1] = CPU_ID_NIC_QMAN_ARC9, 1701 [GAUDI2_QUEUE_ID_NIC_9_2] = CPU_ID_NIC_QMAN_ARC9, 1702 [GAUDI2_QUEUE_ID_NIC_9_3] = CPU_ID_NIC_QMAN_ARC9, 1703 [GAUDI2_QUEUE_ID_NIC_10_0] = CPU_ID_NIC_QMAN_ARC10, 1704 [GAUDI2_QUEUE_ID_NIC_10_1] = CPU_ID_NIC_QMAN_ARC10, 1705 [GAUDI2_QUEUE_ID_NIC_10_2] = CPU_ID_NIC_QMAN_ARC10, 1706 [GAUDI2_QUEUE_ID_NIC_10_3] = CPU_ID_NIC_QMAN_ARC10, 1707 [GAUDI2_QUEUE_ID_NIC_11_0] = CPU_ID_NIC_QMAN_ARC11, 1708 [GAUDI2_QUEUE_ID_NIC_11_1] = CPU_ID_NIC_QMAN_ARC11, 1709 [GAUDI2_QUEUE_ID_NIC_11_2] = CPU_ID_NIC_QMAN_ARC11, 1710 [GAUDI2_QUEUE_ID_NIC_11_3] = CPU_ID_NIC_QMAN_ARC11, 1711 [GAUDI2_QUEUE_ID_NIC_12_0] = CPU_ID_NIC_QMAN_ARC12, 1712 [GAUDI2_QUEUE_ID_NIC_12_1] = CPU_ID_NIC_QMAN_ARC12, 1713 [GAUDI2_QUEUE_ID_NIC_12_2] = CPU_ID_NIC_QMAN_ARC12, 1714 [GAUDI2_QUEUE_ID_NIC_12_3] = CPU_ID_NIC_QMAN_ARC12, 1715 [GAUDI2_QUEUE_ID_NIC_13_0] = CPU_ID_NIC_QMAN_ARC13, 1716 [GAUDI2_QUEUE_ID_NIC_13_1] = CPU_ID_NIC_QMAN_ARC13, 1717 [GAUDI2_QUEUE_ID_NIC_13_2] = CPU_ID_NIC_QMAN_ARC13, 1718 [GAUDI2_QUEUE_ID_NIC_13_3] = CPU_ID_NIC_QMAN_ARC13, 1719 [GAUDI2_QUEUE_ID_NIC_14_0] = CPU_ID_NIC_QMAN_ARC14, 1720 [GAUDI2_QUEUE_ID_NIC_14_1] = CPU_ID_NIC_QMAN_ARC14, 1721 [GAUDI2_QUEUE_ID_NIC_14_2] = CPU_ID_NIC_QMAN_ARC14, 1722 [GAUDI2_QUEUE_ID_NIC_14_3] = CPU_ID_NIC_QMAN_ARC14, 1723 [GAUDI2_QUEUE_ID_NIC_15_0] = CPU_ID_NIC_QMAN_ARC15, 1724 [GAUDI2_QUEUE_ID_NIC_15_1] = CPU_ID_NIC_QMAN_ARC15, 1725 [GAUDI2_QUEUE_ID_NIC_15_2] = CPU_ID_NIC_QMAN_ARC15, 1726 [GAUDI2_QUEUE_ID_NIC_15_3] = CPU_ID_NIC_QMAN_ARC15, 1727 [GAUDI2_QUEUE_ID_NIC_16_0] = CPU_ID_NIC_QMAN_ARC16, 1728 [GAUDI2_QUEUE_ID_NIC_16_1] = CPU_ID_NIC_QMAN_ARC16, 1729 [GAUDI2_QUEUE_ID_NIC_16_2] = CPU_ID_NIC_QMAN_ARC16, 1730 [GAUDI2_QUEUE_ID_NIC_16_3] = CPU_ID_NIC_QMAN_ARC16, 1731 [GAUDI2_QUEUE_ID_NIC_17_0] = CPU_ID_NIC_QMAN_ARC17, 1732 [GAUDI2_QUEUE_ID_NIC_17_1] = CPU_ID_NIC_QMAN_ARC17, 1733 [GAUDI2_QUEUE_ID_NIC_17_2] = CPU_ID_NIC_QMAN_ARC17, 1734 [GAUDI2_QUEUE_ID_NIC_17_3] = CPU_ID_NIC_QMAN_ARC17, 1735 [GAUDI2_QUEUE_ID_NIC_18_0] = CPU_ID_NIC_QMAN_ARC18, 1736 [GAUDI2_QUEUE_ID_NIC_18_1] = CPU_ID_NIC_QMAN_ARC18, 1737 [GAUDI2_QUEUE_ID_NIC_18_2] = CPU_ID_NIC_QMAN_ARC18, 1738 [GAUDI2_QUEUE_ID_NIC_18_3] = CPU_ID_NIC_QMAN_ARC18, 1739 [GAUDI2_QUEUE_ID_NIC_19_0] = CPU_ID_NIC_QMAN_ARC19, 1740 [GAUDI2_QUEUE_ID_NIC_19_1] = CPU_ID_NIC_QMAN_ARC19, 1741 [GAUDI2_QUEUE_ID_NIC_19_2] = CPU_ID_NIC_QMAN_ARC19, 1742 [GAUDI2_QUEUE_ID_NIC_19_3] = CPU_ID_NIC_QMAN_ARC19, 1743 [GAUDI2_QUEUE_ID_NIC_20_0] = CPU_ID_NIC_QMAN_ARC20, 1744 [GAUDI2_QUEUE_ID_NIC_20_1] = CPU_ID_NIC_QMAN_ARC20, 1745 [GAUDI2_QUEUE_ID_NIC_20_2] = CPU_ID_NIC_QMAN_ARC20, 1746 [GAUDI2_QUEUE_ID_NIC_20_3] = CPU_ID_NIC_QMAN_ARC20, 1747 [GAUDI2_QUEUE_ID_NIC_21_0] = CPU_ID_NIC_QMAN_ARC21, 1748 [GAUDI2_QUEUE_ID_NIC_21_1] = CPU_ID_NIC_QMAN_ARC21, 1749 [GAUDI2_QUEUE_ID_NIC_21_2] = CPU_ID_NIC_QMAN_ARC21, 1750 [GAUDI2_QUEUE_ID_NIC_21_3] = CPU_ID_NIC_QMAN_ARC21, 1751 [GAUDI2_QUEUE_ID_NIC_22_0] = CPU_ID_NIC_QMAN_ARC22, 1752 [GAUDI2_QUEUE_ID_NIC_22_1] = CPU_ID_NIC_QMAN_ARC22, 1753 [GAUDI2_QUEUE_ID_NIC_22_2] = CPU_ID_NIC_QMAN_ARC22, 1754 [GAUDI2_QUEUE_ID_NIC_22_3] = CPU_ID_NIC_QMAN_ARC22, 1755 [GAUDI2_QUEUE_ID_NIC_23_0] = CPU_ID_NIC_QMAN_ARC23, 1756 [GAUDI2_QUEUE_ID_NIC_23_1] = CPU_ID_NIC_QMAN_ARC23, 1757 [GAUDI2_QUEUE_ID_NIC_23_2] = CPU_ID_NIC_QMAN_ARC23, 1758 [GAUDI2_QUEUE_ID_NIC_23_3] = CPU_ID_NIC_QMAN_ARC23, 1759 [GAUDI2_QUEUE_ID_ROT_0_0] = CPU_ID_ROT_QMAN_ARC0, 1760 [GAUDI2_QUEUE_ID_ROT_0_1] = CPU_ID_ROT_QMAN_ARC0, 1761 [GAUDI2_QUEUE_ID_ROT_0_2] = CPU_ID_ROT_QMAN_ARC0, 1762 [GAUDI2_QUEUE_ID_ROT_0_3] = CPU_ID_ROT_QMAN_ARC0, 1763 [GAUDI2_QUEUE_ID_ROT_1_0] = CPU_ID_ROT_QMAN_ARC1, 1764 [GAUDI2_QUEUE_ID_ROT_1_1] = CPU_ID_ROT_QMAN_ARC1, 1765 [GAUDI2_QUEUE_ID_ROT_1_2] = CPU_ID_ROT_QMAN_ARC1, 1766 [GAUDI2_QUEUE_ID_ROT_1_3] = CPU_ID_ROT_QMAN_ARC1 1767 }; 1768 1769 const u32 gaudi2_dma_core_blocks_bases[DMA_CORE_ID_SIZE] = { 1770 [DMA_CORE_ID_PDMA0] = mmPDMA0_CORE_BASE, 1771 [DMA_CORE_ID_PDMA1] = mmPDMA1_CORE_BASE, 1772 [DMA_CORE_ID_EDMA0] = mmDCORE0_EDMA0_CORE_BASE, 1773 [DMA_CORE_ID_EDMA1] = mmDCORE0_EDMA1_CORE_BASE, 1774 [DMA_CORE_ID_EDMA2] = mmDCORE1_EDMA0_CORE_BASE, 1775 [DMA_CORE_ID_EDMA3] = mmDCORE1_EDMA1_CORE_BASE, 1776 [DMA_CORE_ID_EDMA4] = mmDCORE2_EDMA0_CORE_BASE, 1777 [DMA_CORE_ID_EDMA5] = mmDCORE2_EDMA1_CORE_BASE, 1778 [DMA_CORE_ID_EDMA6] = mmDCORE3_EDMA0_CORE_BASE, 1779 [DMA_CORE_ID_EDMA7] = mmDCORE3_EDMA1_CORE_BASE, 1780 [DMA_CORE_ID_KDMA] = mmARC_FARM_KDMA_BASE 1781 }; 1782 1783 const u32 gaudi2_mme_acc_blocks_bases[MME_ID_SIZE] = { 1784 [MME_ID_DCORE0] = mmDCORE0_MME_ACC_BASE, 1785 [MME_ID_DCORE1] = mmDCORE1_MME_ACC_BASE, 1786 [MME_ID_DCORE2] = mmDCORE2_MME_ACC_BASE, 1787 [MME_ID_DCORE3] = mmDCORE3_MME_ACC_BASE 1788 }; 1789 1790 static const u32 gaudi2_tpc_cfg_blocks_bases[TPC_ID_SIZE] = { 1791 [TPC_ID_DCORE0_TPC0] = mmDCORE0_TPC0_CFG_BASE, 1792 [TPC_ID_DCORE0_TPC1] = mmDCORE0_TPC1_CFG_BASE, 1793 [TPC_ID_DCORE0_TPC2] = mmDCORE0_TPC2_CFG_BASE, 1794 [TPC_ID_DCORE0_TPC3] = mmDCORE0_TPC3_CFG_BASE, 1795 [TPC_ID_DCORE0_TPC4] = mmDCORE0_TPC4_CFG_BASE, 1796 [TPC_ID_DCORE0_TPC5] = mmDCORE0_TPC5_CFG_BASE, 1797 [TPC_ID_DCORE1_TPC0] = mmDCORE1_TPC0_CFG_BASE, 1798 [TPC_ID_DCORE1_TPC1] = mmDCORE1_TPC1_CFG_BASE, 1799 [TPC_ID_DCORE1_TPC2] = mmDCORE1_TPC2_CFG_BASE, 1800 [TPC_ID_DCORE1_TPC3] = mmDCORE1_TPC3_CFG_BASE, 1801 [TPC_ID_DCORE1_TPC4] = mmDCORE1_TPC4_CFG_BASE, 1802 [TPC_ID_DCORE1_TPC5] = mmDCORE1_TPC5_CFG_BASE, 1803 [TPC_ID_DCORE2_TPC0] = mmDCORE2_TPC0_CFG_BASE, 1804 [TPC_ID_DCORE2_TPC1] = mmDCORE2_TPC1_CFG_BASE, 1805 [TPC_ID_DCORE2_TPC2] = mmDCORE2_TPC2_CFG_BASE, 1806 [TPC_ID_DCORE2_TPC3] = mmDCORE2_TPC3_CFG_BASE, 1807 [TPC_ID_DCORE2_TPC4] = mmDCORE2_TPC4_CFG_BASE, 1808 [TPC_ID_DCORE2_TPC5] = mmDCORE2_TPC5_CFG_BASE, 1809 [TPC_ID_DCORE3_TPC0] = mmDCORE3_TPC0_CFG_BASE, 1810 [TPC_ID_DCORE3_TPC1] = mmDCORE3_TPC1_CFG_BASE, 1811 [TPC_ID_DCORE3_TPC2] = mmDCORE3_TPC2_CFG_BASE, 1812 [TPC_ID_DCORE3_TPC3] = mmDCORE3_TPC3_CFG_BASE, 1813 [TPC_ID_DCORE3_TPC4] = mmDCORE3_TPC4_CFG_BASE, 1814 [TPC_ID_DCORE3_TPC5] = mmDCORE3_TPC5_CFG_BASE, 1815 [TPC_ID_DCORE0_TPC6] = mmDCORE0_TPC6_CFG_BASE, 1816 }; 1817 1818 static const u32 gaudi2_tpc_eml_cfg_blocks_bases[TPC_ID_SIZE] = { 1819 [TPC_ID_DCORE0_TPC0] = mmDCORE0_TPC0_EML_CFG_BASE, 1820 [TPC_ID_DCORE0_TPC1] = mmDCORE0_TPC1_EML_CFG_BASE, 1821 [TPC_ID_DCORE0_TPC2] = mmDCORE0_TPC2_EML_CFG_BASE, 1822 [TPC_ID_DCORE0_TPC3] = mmDCORE0_TPC3_EML_CFG_BASE, 1823 [TPC_ID_DCORE0_TPC4] = mmDCORE0_TPC4_EML_CFG_BASE, 1824 [TPC_ID_DCORE0_TPC5] = mmDCORE0_TPC5_EML_CFG_BASE, 1825 [TPC_ID_DCORE1_TPC0] = mmDCORE1_TPC0_EML_CFG_BASE, 1826 [TPC_ID_DCORE1_TPC1] = mmDCORE1_TPC1_EML_CFG_BASE, 1827 [TPC_ID_DCORE1_TPC2] = mmDCORE1_TPC2_EML_CFG_BASE, 1828 [TPC_ID_DCORE1_TPC3] = mmDCORE1_TPC3_EML_CFG_BASE, 1829 [TPC_ID_DCORE1_TPC4] = mmDCORE1_TPC4_EML_CFG_BASE, 1830 [TPC_ID_DCORE1_TPC5] = mmDCORE1_TPC5_EML_CFG_BASE, 1831 [TPC_ID_DCORE2_TPC0] = mmDCORE2_TPC0_EML_CFG_BASE, 1832 [TPC_ID_DCORE2_TPC1] = mmDCORE2_TPC1_EML_CFG_BASE, 1833 [TPC_ID_DCORE2_TPC2] = mmDCORE2_TPC2_EML_CFG_BASE, 1834 [TPC_ID_DCORE2_TPC3] = mmDCORE2_TPC3_EML_CFG_BASE, 1835 [TPC_ID_DCORE2_TPC4] = mmDCORE2_TPC4_EML_CFG_BASE, 1836 [TPC_ID_DCORE2_TPC5] = mmDCORE2_TPC5_EML_CFG_BASE, 1837 [TPC_ID_DCORE3_TPC0] = mmDCORE3_TPC0_EML_CFG_BASE, 1838 [TPC_ID_DCORE3_TPC1] = mmDCORE3_TPC1_EML_CFG_BASE, 1839 [TPC_ID_DCORE3_TPC2] = mmDCORE3_TPC2_EML_CFG_BASE, 1840 [TPC_ID_DCORE3_TPC3] = mmDCORE3_TPC3_EML_CFG_BASE, 1841 [TPC_ID_DCORE3_TPC4] = mmDCORE3_TPC4_EML_CFG_BASE, 1842 [TPC_ID_DCORE3_TPC5] = mmDCORE3_TPC5_EML_CFG_BASE, 1843 [TPC_ID_DCORE0_TPC6] = mmDCORE0_TPC6_EML_CFG_BASE, 1844 }; 1845 1846 const u32 gaudi2_rot_blocks_bases[ROTATOR_ID_SIZE] = { 1847 [ROTATOR_ID_0] = mmROT0_BASE, 1848 [ROTATOR_ID_1] = mmROT1_BASE 1849 }; 1850 1851 static const u32 gaudi2_tpc_id_to_queue_id[TPC_ID_SIZE] = { 1852 [TPC_ID_DCORE0_TPC0] = GAUDI2_QUEUE_ID_DCORE0_TPC_0_0, 1853 [TPC_ID_DCORE0_TPC1] = GAUDI2_QUEUE_ID_DCORE0_TPC_1_0, 1854 [TPC_ID_DCORE0_TPC2] = GAUDI2_QUEUE_ID_DCORE0_TPC_2_0, 1855 [TPC_ID_DCORE0_TPC3] = GAUDI2_QUEUE_ID_DCORE0_TPC_3_0, 1856 [TPC_ID_DCORE0_TPC4] = GAUDI2_QUEUE_ID_DCORE0_TPC_4_0, 1857 [TPC_ID_DCORE0_TPC5] = GAUDI2_QUEUE_ID_DCORE0_TPC_5_0, 1858 [TPC_ID_DCORE1_TPC0] = GAUDI2_QUEUE_ID_DCORE1_TPC_0_0, 1859 [TPC_ID_DCORE1_TPC1] = GAUDI2_QUEUE_ID_DCORE1_TPC_1_0, 1860 [TPC_ID_DCORE1_TPC2] = GAUDI2_QUEUE_ID_DCORE1_TPC_2_0, 1861 [TPC_ID_DCORE1_TPC3] = GAUDI2_QUEUE_ID_DCORE1_TPC_3_0, 1862 [TPC_ID_DCORE1_TPC4] = GAUDI2_QUEUE_ID_DCORE1_TPC_4_0, 1863 [TPC_ID_DCORE1_TPC5] = GAUDI2_QUEUE_ID_DCORE1_TPC_5_0, 1864 [TPC_ID_DCORE2_TPC0] = GAUDI2_QUEUE_ID_DCORE2_TPC_0_0, 1865 [TPC_ID_DCORE2_TPC1] = GAUDI2_QUEUE_ID_DCORE2_TPC_1_0, 1866 [TPC_ID_DCORE2_TPC2] = GAUDI2_QUEUE_ID_DCORE2_TPC_2_0, 1867 [TPC_ID_DCORE2_TPC3] = GAUDI2_QUEUE_ID_DCORE2_TPC_3_0, 1868 [TPC_ID_DCORE2_TPC4] = GAUDI2_QUEUE_ID_DCORE2_TPC_4_0, 1869 [TPC_ID_DCORE2_TPC5] = GAUDI2_QUEUE_ID_DCORE2_TPC_5_0, 1870 [TPC_ID_DCORE3_TPC0] = GAUDI2_QUEUE_ID_DCORE3_TPC_0_0, 1871 [TPC_ID_DCORE3_TPC1] = GAUDI2_QUEUE_ID_DCORE3_TPC_1_0, 1872 [TPC_ID_DCORE3_TPC2] = GAUDI2_QUEUE_ID_DCORE3_TPC_2_0, 1873 [TPC_ID_DCORE3_TPC3] = GAUDI2_QUEUE_ID_DCORE3_TPC_3_0, 1874 [TPC_ID_DCORE3_TPC4] = GAUDI2_QUEUE_ID_DCORE3_TPC_4_0, 1875 [TPC_ID_DCORE3_TPC5] = GAUDI2_QUEUE_ID_DCORE3_TPC_5_0, 1876 [TPC_ID_DCORE0_TPC6] = GAUDI2_QUEUE_ID_DCORE0_TPC_6_0, 1877 }; 1878 1879 static const u32 gaudi2_rot_id_to_queue_id[ROTATOR_ID_SIZE] = { 1880 [ROTATOR_ID_0] = GAUDI2_QUEUE_ID_ROT_0_0, 1881 [ROTATOR_ID_1] = GAUDI2_QUEUE_ID_ROT_1_0, 1882 }; 1883 1884 static const u32 gaudi2_tpc_engine_id_to_tpc_id[] = { 1885 [GAUDI2_DCORE0_ENGINE_ID_TPC_0] = TPC_ID_DCORE0_TPC0, 1886 [GAUDI2_DCORE0_ENGINE_ID_TPC_1] = TPC_ID_DCORE0_TPC1, 1887 [GAUDI2_DCORE0_ENGINE_ID_TPC_2] = TPC_ID_DCORE0_TPC2, 1888 [GAUDI2_DCORE0_ENGINE_ID_TPC_3] = TPC_ID_DCORE0_TPC3, 1889 [GAUDI2_DCORE0_ENGINE_ID_TPC_4] = TPC_ID_DCORE0_TPC4, 1890 [GAUDI2_DCORE0_ENGINE_ID_TPC_5] = TPC_ID_DCORE0_TPC5, 1891 [GAUDI2_DCORE1_ENGINE_ID_TPC_0] = TPC_ID_DCORE1_TPC0, 1892 [GAUDI2_DCORE1_ENGINE_ID_TPC_1] = TPC_ID_DCORE1_TPC1, 1893 [GAUDI2_DCORE1_ENGINE_ID_TPC_2] = TPC_ID_DCORE1_TPC2, 1894 [GAUDI2_DCORE1_ENGINE_ID_TPC_3] = TPC_ID_DCORE1_TPC3, 1895 [GAUDI2_DCORE1_ENGINE_ID_TPC_4] = TPC_ID_DCORE1_TPC4, 1896 [GAUDI2_DCORE1_ENGINE_ID_TPC_5] = TPC_ID_DCORE1_TPC5, 1897 [GAUDI2_DCORE2_ENGINE_ID_TPC_0] = TPC_ID_DCORE2_TPC0, 1898 [GAUDI2_DCORE2_ENGINE_ID_TPC_1] = TPC_ID_DCORE2_TPC1, 1899 [GAUDI2_DCORE2_ENGINE_ID_TPC_2] = TPC_ID_DCORE2_TPC2, 1900 [GAUDI2_DCORE2_ENGINE_ID_TPC_3] = TPC_ID_DCORE2_TPC3, 1901 [GAUDI2_DCORE2_ENGINE_ID_TPC_4] = TPC_ID_DCORE2_TPC4, 1902 [GAUDI2_DCORE2_ENGINE_ID_TPC_5] = TPC_ID_DCORE2_TPC5, 1903 [GAUDI2_DCORE3_ENGINE_ID_TPC_0] = TPC_ID_DCORE3_TPC0, 1904 [GAUDI2_DCORE3_ENGINE_ID_TPC_1] = TPC_ID_DCORE3_TPC1, 1905 [GAUDI2_DCORE3_ENGINE_ID_TPC_2] = TPC_ID_DCORE3_TPC2, 1906 [GAUDI2_DCORE3_ENGINE_ID_TPC_3] = TPC_ID_DCORE3_TPC3, 1907 [GAUDI2_DCORE3_ENGINE_ID_TPC_4] = TPC_ID_DCORE3_TPC4, 1908 [GAUDI2_DCORE3_ENGINE_ID_TPC_5] = TPC_ID_DCORE3_TPC5, 1909 /* the PCI TPC is placed last (mapped liked HW) */ 1910 [GAUDI2_DCORE0_ENGINE_ID_TPC_6] = TPC_ID_DCORE0_TPC6, 1911 }; 1912 1913 static const u32 gaudi2_mme_engine_id_to_mme_id[] = { 1914 [GAUDI2_DCORE0_ENGINE_ID_MME] = MME_ID_DCORE0, 1915 [GAUDI2_DCORE1_ENGINE_ID_MME] = MME_ID_DCORE1, 1916 [GAUDI2_DCORE2_ENGINE_ID_MME] = MME_ID_DCORE2, 1917 [GAUDI2_DCORE3_ENGINE_ID_MME] = MME_ID_DCORE3, 1918 }; 1919 1920 static const u32 gaudi2_edma_engine_id_to_edma_id[] = { 1921 [GAUDI2_ENGINE_ID_PDMA_0] = DMA_CORE_ID_PDMA0, 1922 [GAUDI2_ENGINE_ID_PDMA_1] = DMA_CORE_ID_PDMA1, 1923 [GAUDI2_DCORE0_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA0, 1924 [GAUDI2_DCORE0_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA1, 1925 [GAUDI2_DCORE1_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA2, 1926 [GAUDI2_DCORE1_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA3, 1927 [GAUDI2_DCORE2_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA4, 1928 [GAUDI2_DCORE2_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA5, 1929 [GAUDI2_DCORE3_ENGINE_ID_EDMA_0] = DMA_CORE_ID_EDMA6, 1930 [GAUDI2_DCORE3_ENGINE_ID_EDMA_1] = DMA_CORE_ID_EDMA7, 1931 [GAUDI2_ENGINE_ID_KDMA] = DMA_CORE_ID_KDMA, 1932 }; 1933 1934 const u32 edma_stream_base[NUM_OF_EDMA_PER_DCORE * NUM_OF_DCORES] = { 1935 GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0, 1936 GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0, 1937 GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0, 1938 GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0, 1939 GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0, 1940 GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0, 1941 GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0, 1942 GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0, 1943 }; 1944 1945 static const char gaudi2_vdec_irq_name[GAUDI2_VDEC_MSIX_ENTRIES][GAUDI2_MAX_STRING_LEN] = { 1946 "gaudi2 vdec 0_0", "gaudi2 vdec 0_0 abnormal", 1947 "gaudi2 vdec 0_1", "gaudi2 vdec 0_1 abnormal", 1948 "gaudi2 vdec 1_0", "gaudi2 vdec 1_0 abnormal", 1949 "gaudi2 vdec 1_1", "gaudi2 vdec 1_1 abnormal", 1950 "gaudi2 vdec 2_0", "gaudi2 vdec 2_0 abnormal", 1951 "gaudi2 vdec 2_1", "gaudi2 vdec 2_1 abnormal", 1952 "gaudi2 vdec 3_0", "gaudi2 vdec 3_0 abnormal", 1953 "gaudi2 vdec 3_1", "gaudi2 vdec 3_1 abnormal", 1954 "gaudi2 vdec s_0", "gaudi2 vdec s_0 abnormal", 1955 "gaudi2 vdec s_1", "gaudi2 vdec s_1 abnormal" 1956 }; 1957 1958 enum rtr_id { 1959 DCORE0_RTR0, 1960 DCORE0_RTR1, 1961 DCORE0_RTR2, 1962 DCORE0_RTR3, 1963 DCORE0_RTR4, 1964 DCORE0_RTR5, 1965 DCORE0_RTR6, 1966 DCORE0_RTR7, 1967 DCORE1_RTR0, 1968 DCORE1_RTR1, 1969 DCORE1_RTR2, 1970 DCORE1_RTR3, 1971 DCORE1_RTR4, 1972 DCORE1_RTR5, 1973 DCORE1_RTR6, 1974 DCORE1_RTR7, 1975 DCORE2_RTR0, 1976 DCORE2_RTR1, 1977 DCORE2_RTR2, 1978 DCORE2_RTR3, 1979 DCORE2_RTR4, 1980 DCORE2_RTR5, 1981 DCORE2_RTR6, 1982 DCORE2_RTR7, 1983 DCORE3_RTR0, 1984 DCORE3_RTR1, 1985 DCORE3_RTR2, 1986 DCORE3_RTR3, 1987 DCORE3_RTR4, 1988 DCORE3_RTR5, 1989 DCORE3_RTR6, 1990 DCORE3_RTR7, 1991 }; 1992 1993 static const u32 gaudi2_tpc_initiator_hbw_rtr_id[NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1] = { 1994 DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR2, DCORE0_RTR2, DCORE0_RTR3, DCORE0_RTR3, 1995 DCORE1_RTR6, DCORE1_RTR6, DCORE1_RTR5, DCORE1_RTR5, DCORE1_RTR4, DCORE1_RTR4, 1996 DCORE2_RTR3, DCORE2_RTR3, DCORE2_RTR2, DCORE2_RTR2, DCORE2_RTR1, DCORE2_RTR1, 1997 DCORE3_RTR4, DCORE3_RTR4, DCORE3_RTR5, DCORE3_RTR5, DCORE3_RTR6, DCORE3_RTR6, 1998 DCORE0_RTR0 1999 }; 2000 2001 static const u32 gaudi2_tpc_initiator_lbw_rtr_id[NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1] = { 2002 DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR2, DCORE0_RTR2, 2003 DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR6, DCORE1_RTR6, DCORE1_RTR5, DCORE1_RTR5, 2004 DCORE2_RTR2, DCORE2_RTR2, DCORE2_RTR1, DCORE2_RTR1, DCORE2_RTR0, DCORE2_RTR0, 2005 DCORE3_RTR5, DCORE3_RTR5, DCORE3_RTR6, DCORE3_RTR6, DCORE3_RTR7, DCORE3_RTR7, 2006 DCORE0_RTR0 2007 }; 2008 2009 static const u32 gaudi2_dec_initiator_hbw_rtr_id[NUMBER_OF_DEC] = { 2010 DCORE0_RTR0, DCORE0_RTR0, DCORE1_RTR7, DCORE1_RTR7, DCORE2_RTR0, DCORE2_RTR0, 2011 DCORE3_RTR7, DCORE3_RTR7, DCORE0_RTR0, DCORE0_RTR0 2012 }; 2013 2014 static const u32 gaudi2_dec_initiator_lbw_rtr_id[NUMBER_OF_DEC] = { 2015 DCORE0_RTR1, DCORE0_RTR1, DCORE1_RTR6, DCORE1_RTR6, DCORE2_RTR1, DCORE2_RTR1, 2016 DCORE3_RTR6, DCORE3_RTR6, DCORE0_RTR0, DCORE0_RTR0 2017 }; 2018 2019 static const u32 gaudi2_nic_initiator_hbw_rtr_id[NIC_NUMBER_OF_MACROS] = { 2020 DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE2_RTR0, 2021 DCORE2_RTR0, DCORE2_RTR0, DCORE2_RTR0, DCORE3_RTR7, DCORE3_RTR7, DCORE3_RTR7 2022 }; 2023 2024 static const u32 gaudi2_nic_initiator_lbw_rtr_id[NIC_NUMBER_OF_MACROS] = { 2025 DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE2_RTR0, 2026 DCORE2_RTR0, DCORE2_RTR0, DCORE2_RTR0, DCORE3_RTR7, DCORE3_RTR7, DCORE3_RTR7 2027 }; 2028 2029 static const u32 gaudi2_edma_initiator_hbw_sft[NUM_OF_EDMA_PER_DCORE * NUM_OF_DCORES] = { 2030 mmSFT0_HBW_RTR_IF1_MSTR_IF_RR_SHRD_HBW_BASE, 2031 mmSFT0_HBW_RTR_IF0_MSTR_IF_RR_SHRD_HBW_BASE, 2032 mmSFT1_HBW_RTR_IF1_MSTR_IF_RR_SHRD_HBW_BASE, 2033 mmSFT1_HBW_RTR_IF0_MSTR_IF_RR_SHRD_HBW_BASE, 2034 mmSFT2_HBW_RTR_IF0_MSTR_IF_RR_SHRD_HBW_BASE, 2035 mmSFT2_HBW_RTR_IF1_MSTR_IF_RR_SHRD_HBW_BASE, 2036 mmSFT3_HBW_RTR_IF0_MSTR_IF_RR_SHRD_HBW_BASE, 2037 mmSFT3_HBW_RTR_IF1_MSTR_IF_RR_SHRD_HBW_BASE 2038 }; 2039 2040 static const u32 gaudi2_pdma_initiator_hbw_rtr_id[NUM_OF_PDMA] = { 2041 DCORE0_RTR0, DCORE0_RTR0 2042 }; 2043 2044 static const u32 gaudi2_pdma_initiator_lbw_rtr_id[NUM_OF_PDMA] = { 2045 DCORE0_RTR2, DCORE0_RTR2 2046 }; 2047 2048 static const u32 gaudi2_rot_initiator_hbw_rtr_id[NUM_OF_ROT] = { 2049 DCORE2_RTR0, DCORE3_RTR7 2050 }; 2051 2052 static const u32 gaudi2_rot_initiator_lbw_rtr_id[NUM_OF_ROT] = { 2053 DCORE2_RTR2, DCORE3_RTR5 2054 }; 2055 2056 struct mme_initiators_rtr_id { 2057 u32 wap0; 2058 u32 wap1; 2059 u32 write; 2060 u32 read; 2061 u32 sbte0; 2062 u32 sbte1; 2063 u32 sbte2; 2064 u32 sbte3; 2065 u32 sbte4; 2066 }; 2067 2068 enum mme_initiators { 2069 MME_WAP0 = 0, 2070 MME_WAP1, 2071 MME_WRITE, 2072 MME_READ, 2073 MME_SBTE0, 2074 MME_SBTE1, 2075 MME_SBTE2, 2076 MME_SBTE3, 2077 MME_SBTE4, 2078 MME_INITIATORS_MAX 2079 }; 2080 2081 static const struct mme_initiators_rtr_id 2082 gaudi2_mme_initiator_rtr_id[NUM_OF_MME_PER_DCORE * NUM_OF_DCORES] = { 2083 { .wap0 = 5, .wap1 = 7, .write = 6, .read = 7, 2084 .sbte0 = 7, .sbte1 = 4, .sbte2 = 4, .sbte3 = 5, .sbte4 = 6}, 2085 { .wap0 = 10, .wap1 = 8, .write = 9, .read = 8, 2086 .sbte0 = 11, .sbte1 = 11, .sbte2 = 10, .sbte3 = 9, .sbte4 = 8}, 2087 { .wap0 = 21, .wap1 = 23, .write = 22, .read = 23, 2088 .sbte0 = 20, .sbte1 = 20, .sbte2 = 21, .sbte3 = 22, .sbte4 = 23}, 2089 { .wap0 = 30, .wap1 = 28, .write = 29, .read = 30, 2090 .sbte0 = 31, .sbte1 = 31, .sbte2 = 30, .sbte3 = 29, .sbte4 = 28}, 2091 }; 2092 2093 enum razwi_event_sources { 2094 RAZWI_TPC, 2095 RAZWI_MME, 2096 RAZWI_EDMA, 2097 RAZWI_PDMA, 2098 RAZWI_NIC, 2099 RAZWI_DEC, 2100 RAZWI_ROT, 2101 RAZWI_ARC_FARM 2102 }; 2103 2104 struct hbm_mc_error_causes { 2105 u32 mask; 2106 char cause[50]; 2107 }; 2108 2109 static struct hl_special_block_info gaudi2_special_blocks[] = GAUDI2_SPECIAL_BLOCKS; 2110 2111 /* Special blocks iterator is currently used to configure security protection bits, 2112 * and read global errors. Most HW blocks are addressable and those who aren't (N/A)- 2113 * must be skipped. Following configurations are commonly used for both PB config 2114 * and global error reading, since currently they both share the same settings. 2115 * Once it changes, we must remember to use separate configurations for either one. 2116 */ 2117 static int gaudi2_iterator_skip_block_types[] = { 2118 GAUDI2_BLOCK_TYPE_PLL, 2119 GAUDI2_BLOCK_TYPE_EU_BIST, 2120 GAUDI2_BLOCK_TYPE_HBM, 2121 GAUDI2_BLOCK_TYPE_XFT 2122 }; 2123 2124 static struct range gaudi2_iterator_skip_block_ranges[] = { 2125 /* Skip all PSOC blocks except for PSOC_GLOBAL_CONF */ 2126 {mmPSOC_I2C_M0_BASE, mmPSOC_EFUSE_BASE}, 2127 {mmPSOC_BTL_BASE, mmPSOC_MSTR_IF_RR_SHRD_HBW_BASE}, 2128 /* Skip all CPU blocks except for CPU_IF */ 2129 {mmCPU_CA53_CFG_BASE, mmCPU_CA53_CFG_BASE}, 2130 {mmCPU_TIMESTAMP_BASE, mmCPU_MSTR_IF_RR_SHRD_HBW_BASE} 2131 }; 2132 2133 static struct hbm_mc_error_causes hbm_mc_spi[GAUDI2_NUM_OF_HBM_MC_SPI_CAUSE] = { 2134 {HBM_MC_SPI_TEMP_PIN_CHG_MASK, "temperature pins changed"}, 2135 {HBM_MC_SPI_THR_ENG_MASK, "temperature-based throttling engaged"}, 2136 {HBM_MC_SPI_THR_DIS_ENG_MASK, "temperature-based throttling disengaged"}, 2137 {HBM_MC_SPI_IEEE1500_COMP_MASK, "IEEE1500 op comp"}, 2138 {HBM_MC_SPI_IEEE1500_PAUSED_MASK, "IEEE1500 op paused"}, 2139 }; 2140 2141 static const char * const hbm_mc_sei_cause[GAUDI2_NUM_OF_HBM_SEI_CAUSE] = { 2142 [HBM_SEI_CMD_PARITY_EVEN] = "SEI C/A parity even", 2143 [HBM_SEI_CMD_PARITY_ODD] = "SEI C/A parity odd", 2144 [HBM_SEI_READ_ERR] = "SEI read data error", 2145 [HBM_SEI_WRITE_DATA_PARITY_ERR] = "SEI write data parity error", 2146 [HBM_SEI_CATTRIP] = "SEI CATTRIP asserted", 2147 [HBM_SEI_MEM_BIST_FAIL] = "SEI memory BIST fail", 2148 [HBM_SEI_DFI] = "SEI DFI error", 2149 [HBM_SEI_INV_TEMP_READ_OUT] = "SEI invalid temp read", 2150 [HBM_SEI_BIST_FAIL] = "SEI BIST fail" 2151 }; 2152 2153 struct mmu_spi_sei_cause { 2154 char cause[50]; 2155 int clear_bit; 2156 }; 2157 2158 static const struct mmu_spi_sei_cause gaudi2_mmu_spi_sei[GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE] = { 2159 {"page fault", 1}, /* INTERRUPT_CLR[1] */ 2160 {"page access", 1}, /* INTERRUPT_CLR[1] */ 2161 {"bypass ddr", 2}, /* INTERRUPT_CLR[2] */ 2162 {"multi hit", 2}, /* INTERRUPT_CLR[2] */ 2163 {"mmu rei0", -1}, /* no clear register bit */ 2164 {"mmu rei1", -1}, /* no clear register bit */ 2165 {"stlb rei0", -1}, /* no clear register bit */ 2166 {"stlb rei1", -1}, /* no clear register bit */ 2167 {"rr privileged write hit", 2}, /* INTERRUPT_CLR[2] */ 2168 {"rr privileged read hit", 2}, /* INTERRUPT_CLR[2] */ 2169 {"rr secure write hit", 2}, /* INTERRUPT_CLR[2] */ 2170 {"rr secure read hit", 2}, /* INTERRUPT_CLR[2] */ 2171 {"bist_fail no use", 2}, /* INTERRUPT_CLR[2] */ 2172 {"bist_fail no use", 2}, /* INTERRUPT_CLR[2] */ 2173 {"bist_fail no use", 2}, /* INTERRUPT_CLR[2] */ 2174 {"bist_fail no use", 2}, /* INTERRUPT_CLR[2] */ 2175 {"slave error", 16}, /* INTERRUPT_CLR[16] */ 2176 {"dec error", 17}, /* INTERRUPT_CLR[17] */ 2177 {"burst fifo full", 2} /* INTERRUPT_CLR[2] */ 2178 }; 2179 2180 struct gaudi2_cache_invld_params { 2181 u64 start_va; 2182 u64 end_va; 2183 u32 inv_start_val; 2184 u32 flags; 2185 bool range_invalidation; 2186 }; 2187 2188 struct gaudi2_tpc_idle_data { 2189 struct engines_data *e; 2190 unsigned long *mask; 2191 bool *is_idle; 2192 const char *tpc_fmt; 2193 }; 2194 2195 struct gaudi2_tpc_mmu_data { 2196 u32 rw_asid; 2197 }; 2198 2199 static s64 gaudi2_state_dump_specs_props[SP_MAX] = {0}; 2200 2201 static int gaudi2_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, u64 val); 2202 static bool gaudi2_is_queue_enabled(struct hl_device *hdev, u32 hw_queue_id); 2203 static bool gaudi2_is_arc_enabled(struct hl_device *hdev, u64 arc_id); 2204 static void gaudi2_clr_arc_id_cap(struct hl_device *hdev, u64 arc_id); 2205 static void gaudi2_set_arc_id_cap(struct hl_device *hdev, u64 arc_id); 2206 static void gaudi2_memset_device_lbw(struct hl_device *hdev, u32 addr, u32 size, u32 val); 2207 static int gaudi2_send_job_to_kdma(struct hl_device *hdev, u64 src_addr, u64 dst_addr, u32 size, 2208 bool is_memset); 2209 static bool gaudi2_get_tpc_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 2210 struct engines_data *e); 2211 static bool gaudi2_get_mme_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 2212 struct engines_data *e); 2213 static bool gaudi2_get_edma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 2214 struct engines_data *e); 2215 static u64 gaudi2_mmu_scramble_addr(struct hl_device *hdev, u64 raw_addr); 2216 static u64 gaudi2_mmu_descramble_addr(struct hl_device *hdev, u64 scrambled_addr); 2217 2218 static void gaudi2_init_scrambler_hbm(struct hl_device *hdev) 2219 { 2220 2221 } 2222 2223 static u32 gaudi2_get_signal_cb_size(struct hl_device *hdev) 2224 { 2225 return sizeof(struct packet_msg_short); 2226 } 2227 2228 static u32 gaudi2_get_wait_cb_size(struct hl_device *hdev) 2229 { 2230 return sizeof(struct packet_msg_short) * 4 + sizeof(struct packet_fence); 2231 } 2232 2233 void gaudi2_iterate_tpcs(struct hl_device *hdev, struct iterate_module_ctx *ctx) 2234 { 2235 struct asic_fixed_properties *prop = &hdev->asic_prop; 2236 int dcore, inst, tpc_seq; 2237 u32 offset; 2238 2239 /* init the return code */ 2240 ctx->rc = 0; 2241 2242 for (dcore = 0; dcore < NUM_OF_DCORES; dcore++) { 2243 for (inst = 0; inst < NUM_OF_TPC_PER_DCORE; inst++) { 2244 tpc_seq = dcore * NUM_OF_TPC_PER_DCORE + inst; 2245 2246 if (!(prop->tpc_enabled_mask & BIT(tpc_seq))) 2247 continue; 2248 2249 offset = (DCORE_OFFSET * dcore) + (DCORE_TPC_OFFSET * inst); 2250 2251 ctx->fn(hdev, dcore, inst, offset, ctx); 2252 if (ctx->rc) { 2253 dev_err(hdev->dev, "TPC iterator failed for DCORE%d TPC%d\n", 2254 dcore, inst); 2255 return; 2256 } 2257 } 2258 } 2259 2260 if (!(prop->tpc_enabled_mask & BIT(TPC_ID_DCORE0_TPC6))) 2261 return; 2262 2263 /* special check for PCI TPC (DCORE0_TPC6) */ 2264 offset = DCORE_TPC_OFFSET * (NUM_DCORE0_TPC - 1); 2265 ctx->fn(hdev, 0, NUM_DCORE0_TPC - 1, offset, ctx); 2266 if (ctx->rc) 2267 dev_err(hdev->dev, "TPC iterator failed for DCORE0 TPC6\n"); 2268 } 2269 2270 static bool gaudi2_host_phys_addr_valid(u64 addr) 2271 { 2272 if ((addr < HOST_PHYS_BASE_0 + HOST_PHYS_SIZE_0) || (addr >= HOST_PHYS_BASE_1)) 2273 return true; 2274 2275 return false; 2276 } 2277 2278 static int set_number_of_functional_hbms(struct hl_device *hdev) 2279 { 2280 struct asic_fixed_properties *prop = &hdev->asic_prop; 2281 u8 faulty_hbms = hweight64(hdev->dram_binning); 2282 2283 /* check if all HBMs should be used */ 2284 if (!faulty_hbms) { 2285 dev_dbg(hdev->dev, "All HBM are in use (no binning)\n"); 2286 prop->num_functional_hbms = GAUDI2_HBM_NUM; 2287 return 0; 2288 } 2289 2290 /* 2291 * check for error condition in which number of binning 2292 * candidates is higher than the maximum supported by the 2293 * driver (in which case binning mask shall be ignored and driver will 2294 * set the default) 2295 */ 2296 if (faulty_hbms > MAX_FAULTY_HBMS) { 2297 dev_err(hdev->dev, 2298 "HBM binning supports max of %d faulty HBMs, supplied mask 0x%llx.\n", 2299 MAX_FAULTY_HBMS, hdev->dram_binning); 2300 return -EINVAL; 2301 } 2302 2303 /* 2304 * by default, number of functional HBMs in Gaudi2 is always 2305 * GAUDI2_HBM_NUM - 1. 2306 */ 2307 prop->num_functional_hbms = GAUDI2_HBM_NUM - faulty_hbms; 2308 return 0; 2309 } 2310 2311 static int gaudi2_set_dram_properties(struct hl_device *hdev) 2312 { 2313 struct asic_fixed_properties *prop = &hdev->asic_prop; 2314 u32 basic_hbm_page_size; 2315 int rc; 2316 2317 rc = set_number_of_functional_hbms(hdev); 2318 if (rc) 2319 return -EINVAL; 2320 2321 /* 2322 * Due to HW bug in which TLB size is x16 smaller than expected we use a workaround 2323 * in which we are using x16 bigger page size to be able to populate the entire 2324 * HBM mappings in the TLB 2325 */ 2326 basic_hbm_page_size = prop->num_functional_hbms * SZ_8M; 2327 prop->dram_page_size = GAUDI2_COMPENSATE_TLB_PAGE_SIZE_FACTOR * basic_hbm_page_size; 2328 prop->device_mem_alloc_default_page_size = prop->dram_page_size; 2329 prop->dram_size = prop->num_functional_hbms * SZ_16G; 2330 prop->dram_base_address = DRAM_PHYS_BASE; 2331 prop->dram_end_address = prop->dram_base_address + prop->dram_size; 2332 prop->dram_supports_virtual_memory = true; 2333 2334 prop->dram_user_base_address = DRAM_PHYS_BASE + prop->dram_page_size; 2335 prop->dram_hints_align_mask = ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK; 2336 prop->hints_dram_reserved_va_range.start_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HBM_START; 2337 prop->hints_dram_reserved_va_range.end_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HBM_END; 2338 2339 /* since DRAM page size differs from DMMU page size we need to allocate 2340 * DRAM memory in units of dram_page size and mapping this memory in 2341 * units of DMMU page size. we overcome this size mismatch using a 2342 * scrambling routine which takes a DRAM page and converts it to a DMMU 2343 * page. 2344 * We therefore: 2345 * 1. partition the virtual address space to DRAM-page (whole) pages. 2346 * (suppose we get n such pages) 2347 * 2. limit the amount of virtual address space we got from 1 above to 2348 * a multiple of 64M as we don't want the scrambled address to cross 2349 * the DRAM virtual address space. 2350 * ( m = (n * DRAM_page_size) / DMMU_page_size). 2351 * 3. determine the and address accordingly 2352 * end_addr = start_addr + m * 48M 2353 * 2354 * the DRAM address MSBs (63:48) are not part of the roundup calculation 2355 */ 2356 prop->dmmu.start_addr = prop->dram_base_address + 2357 (prop->dram_page_size * 2358 DIV_ROUND_UP_SECTOR_T(prop->dram_size, prop->dram_page_size)); 2359 2360 prop->dmmu.end_addr = prop->dmmu.start_addr + prop->dram_page_size * 2361 div_u64((VA_HBM_SPACE_END - prop->dmmu.start_addr), prop->dmmu.page_size); 2362 2363 return 0; 2364 } 2365 2366 static int gaudi2_set_fixed_properties(struct hl_device *hdev) 2367 { 2368 struct asic_fixed_properties *prop = &hdev->asic_prop; 2369 struct hw_queue_properties *q_props; 2370 u32 num_sync_stream_queues = 0; 2371 int i; 2372 2373 prop->max_queues = GAUDI2_QUEUE_ID_SIZE; 2374 prop->hw_queues_props = kcalloc(prop->max_queues, sizeof(struct hw_queue_properties), 2375 GFP_KERNEL); 2376 2377 if (!prop->hw_queues_props) 2378 return -ENOMEM; 2379 2380 q_props = prop->hw_queues_props; 2381 2382 for (i = 0 ; i < GAUDI2_QUEUE_ID_CPU_PQ ; i++) { 2383 q_props[i].type = QUEUE_TYPE_HW; 2384 q_props[i].driver_only = 0; 2385 2386 if (i >= GAUDI2_QUEUE_ID_NIC_0_0 && i <= GAUDI2_QUEUE_ID_NIC_23_3) { 2387 q_props[i].supports_sync_stream = 0; 2388 } else { 2389 q_props[i].supports_sync_stream = 1; 2390 num_sync_stream_queues++; 2391 } 2392 2393 q_props[i].cb_alloc_flags = CB_ALLOC_USER; 2394 } 2395 2396 q_props[GAUDI2_QUEUE_ID_CPU_PQ].type = QUEUE_TYPE_CPU; 2397 q_props[GAUDI2_QUEUE_ID_CPU_PQ].driver_only = 1; 2398 q_props[GAUDI2_QUEUE_ID_CPU_PQ].cb_alloc_flags = CB_ALLOC_KERNEL; 2399 2400 prop->cache_line_size = DEVICE_CACHE_LINE_SIZE; 2401 prop->cfg_base_address = CFG_BASE; 2402 prop->device_dma_offset_for_host_access = HOST_PHYS_BASE_0; 2403 prop->host_base_address = HOST_PHYS_BASE_0; 2404 prop->host_end_address = prop->host_base_address + HOST_PHYS_SIZE_0; 2405 prop->max_pending_cs = GAUDI2_MAX_PENDING_CS; 2406 prop->completion_queues_count = GAUDI2_RESERVED_CQ_NUMBER; 2407 prop->user_dec_intr_count = NUMBER_OF_DEC; 2408 prop->user_interrupt_count = GAUDI2_IRQ_NUM_USER_LAST - GAUDI2_IRQ_NUM_USER_FIRST + 1; 2409 prop->completion_mode = HL_COMPLETION_MODE_CS; 2410 prop->sync_stream_first_sob = GAUDI2_RESERVED_SOB_NUMBER; 2411 prop->sync_stream_first_mon = GAUDI2_RESERVED_MON_NUMBER; 2412 2413 prop->sram_base_address = SRAM_BASE_ADDR; 2414 prop->sram_size = SRAM_SIZE; 2415 prop->sram_end_address = prop->sram_base_address + prop->sram_size; 2416 prop->sram_user_base_address = prop->sram_base_address + SRAM_USER_BASE_OFFSET; 2417 2418 prop->hints_range_reservation = true; 2419 2420 prop->rotator_enabled_mask = BIT(NUM_OF_ROT) - 1; 2421 2422 if (hdev->pldm) 2423 prop->mmu_pgt_size = 0x800000; /* 8MB */ 2424 else 2425 prop->mmu_pgt_size = MMU_PAGE_TABLES_INITIAL_SIZE; 2426 2427 prop->mmu_pte_size = HL_PTE_SIZE; 2428 prop->mmu_hop_table_size = HOP_TABLE_SIZE_512_PTE; 2429 prop->mmu_hop0_tables_total_size = HOP0_512_PTE_TABLES_TOTAL_SIZE; 2430 2431 prop->dmmu.hop_shifts[MMU_HOP0] = DHOP0_SHIFT; 2432 prop->dmmu.hop_shifts[MMU_HOP1] = DHOP1_SHIFT; 2433 prop->dmmu.hop_shifts[MMU_HOP2] = DHOP2_SHIFT; 2434 prop->dmmu.hop_shifts[MMU_HOP3] = DHOP3_SHIFT; 2435 prop->dmmu.hop_shifts[MMU_HOP4] = DHOP4_SHIFT; 2436 prop->dmmu.hop_masks[MMU_HOP0] = DHOP0_MASK; 2437 prop->dmmu.hop_masks[MMU_HOP1] = DHOP1_MASK; 2438 prop->dmmu.hop_masks[MMU_HOP2] = DHOP2_MASK; 2439 prop->dmmu.hop_masks[MMU_HOP3] = DHOP3_MASK; 2440 prop->dmmu.hop_masks[MMU_HOP4] = DHOP4_MASK; 2441 prop->dmmu.page_size = PAGE_SIZE_1GB; 2442 prop->dmmu.num_hops = MMU_ARCH_6_HOPS; 2443 prop->dmmu.last_mask = LAST_MASK; 2444 prop->dmmu.host_resident = 1; 2445 prop->dmmu.hop_table_size = prop->mmu_hop_table_size; 2446 prop->dmmu.hop0_tables_total_size = prop->mmu_hop0_tables_total_size; 2447 2448 /* 2449 * this is done in order to be able to validate FW descriptor (i.e. validating that 2450 * the addresses and allocated space for FW image does not cross memory bounds). 2451 * for this reason we set the DRAM size to the minimum possible and later it will 2452 * be modified according to what reported in the cpucp info packet 2453 */ 2454 prop->dram_size = (GAUDI2_HBM_NUM - 1) * SZ_16G; 2455 2456 hdev->pmmu_huge_range = true; 2457 prop->pmmu.host_resident = 1; 2458 prop->pmmu.num_hops = MMU_ARCH_6_HOPS; 2459 prop->pmmu.last_mask = LAST_MASK; 2460 prop->pmmu.hop_table_size = prop->mmu_hop_table_size; 2461 prop->pmmu.hop0_tables_total_size = prop->mmu_hop0_tables_total_size; 2462 2463 prop->hints_host_reserved_va_range.start_addr = RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START; 2464 prop->hints_host_reserved_va_range.end_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HOST_END; 2465 prop->hints_host_hpage_reserved_va_range.start_addr = 2466 RESERVED_VA_RANGE_FOR_ARC_ON_HOST_HPAGE_START; 2467 prop->hints_host_hpage_reserved_va_range.end_addr = 2468 RESERVED_VA_RANGE_FOR_ARC_ON_HOST_HPAGE_END; 2469 2470 if (PAGE_SIZE == SZ_64K) { 2471 prop->pmmu.hop_shifts[MMU_HOP0] = HOP0_SHIFT_64K; 2472 prop->pmmu.hop_shifts[MMU_HOP1] = HOP1_SHIFT_64K; 2473 prop->pmmu.hop_shifts[MMU_HOP2] = HOP2_SHIFT_64K; 2474 prop->pmmu.hop_shifts[MMU_HOP3] = HOP3_SHIFT_64K; 2475 prop->pmmu.hop_shifts[MMU_HOP4] = HOP4_SHIFT_64K; 2476 prop->pmmu.hop_shifts[MMU_HOP5] = HOP5_SHIFT_64K; 2477 prop->pmmu.hop_masks[MMU_HOP0] = HOP0_MASK_64K; 2478 prop->pmmu.hop_masks[MMU_HOP1] = HOP1_MASK_64K; 2479 prop->pmmu.hop_masks[MMU_HOP2] = HOP2_MASK_64K; 2480 prop->pmmu.hop_masks[MMU_HOP3] = HOP3_MASK_64K; 2481 prop->pmmu.hop_masks[MMU_HOP4] = HOP4_MASK_64K; 2482 prop->pmmu.hop_masks[MMU_HOP5] = HOP5_MASK_64K; 2483 prop->pmmu.start_addr = VA_HOST_SPACE_PAGE_START; 2484 prop->pmmu.end_addr = VA_HOST_SPACE_PAGE_END; 2485 prop->pmmu.page_size = PAGE_SIZE_64KB; 2486 2487 /* shifts and masks are the same in PMMU and HPMMU */ 2488 memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu)); 2489 prop->pmmu_huge.page_size = PAGE_SIZE_16MB; 2490 prop->pmmu_huge.start_addr = VA_HOST_SPACE_HPAGE_START; 2491 prop->pmmu_huge.end_addr = VA_HOST_SPACE_HPAGE_END; 2492 } else { 2493 prop->pmmu.hop_shifts[MMU_HOP0] = HOP0_SHIFT_4K; 2494 prop->pmmu.hop_shifts[MMU_HOP1] = HOP1_SHIFT_4K; 2495 prop->pmmu.hop_shifts[MMU_HOP2] = HOP2_SHIFT_4K; 2496 prop->pmmu.hop_shifts[MMU_HOP3] = HOP3_SHIFT_4K; 2497 prop->pmmu.hop_shifts[MMU_HOP4] = HOP4_SHIFT_4K; 2498 prop->pmmu.hop_shifts[MMU_HOP5] = HOP5_SHIFT_4K; 2499 prop->pmmu.hop_masks[MMU_HOP0] = HOP0_MASK_4K; 2500 prop->pmmu.hop_masks[MMU_HOP1] = HOP1_MASK_4K; 2501 prop->pmmu.hop_masks[MMU_HOP2] = HOP2_MASK_4K; 2502 prop->pmmu.hop_masks[MMU_HOP3] = HOP3_MASK_4K; 2503 prop->pmmu.hop_masks[MMU_HOP4] = HOP4_MASK_4K; 2504 prop->pmmu.hop_masks[MMU_HOP5] = HOP5_MASK_4K; 2505 prop->pmmu.start_addr = VA_HOST_SPACE_PAGE_START; 2506 prop->pmmu.end_addr = VA_HOST_SPACE_PAGE_END; 2507 prop->pmmu.page_size = PAGE_SIZE_4KB; 2508 2509 /* shifts and masks are the same in PMMU and HPMMU */ 2510 memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu)); 2511 prop->pmmu_huge.page_size = PAGE_SIZE_2MB; 2512 prop->pmmu_huge.start_addr = VA_HOST_SPACE_HPAGE_START; 2513 prop->pmmu_huge.end_addr = VA_HOST_SPACE_HPAGE_END; 2514 } 2515 2516 prop->max_num_of_engines = GAUDI2_ENGINE_ID_SIZE; 2517 prop->num_engine_cores = CPU_ID_MAX; 2518 prop->cfg_size = CFG_SIZE; 2519 prop->max_asid = MAX_ASID; 2520 prop->num_of_events = GAUDI2_EVENT_SIZE; 2521 2522 prop->supports_engine_modes = true; 2523 2524 prop->dc_power_default = DC_POWER_DEFAULT; 2525 2526 prop->cb_pool_cb_cnt = GAUDI2_CB_POOL_CB_CNT; 2527 prop->cb_pool_cb_size = GAUDI2_CB_POOL_CB_SIZE; 2528 prop->pcie_dbi_base_address = CFG_BASE + mmPCIE_DBI_BASE; 2529 prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI; 2530 2531 strscpy_pad(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME, CARD_NAME_MAX_LEN); 2532 2533 prop->mme_master_slave_mode = 1; 2534 2535 prop->first_available_user_sob[0] = GAUDI2_RESERVED_SOB_NUMBER + 2536 (num_sync_stream_queues * HL_RSVD_SOBS); 2537 2538 prop->first_available_user_mon[0] = GAUDI2_RESERVED_MON_NUMBER + 2539 (num_sync_stream_queues * HL_RSVD_MONS); 2540 2541 prop->first_available_user_interrupt = GAUDI2_IRQ_NUM_USER_FIRST; 2542 prop->tpc_interrupt_id = GAUDI2_IRQ_NUM_TPC_ASSERT; 2543 prop->eq_interrupt_id = GAUDI2_IRQ_NUM_EVENT_QUEUE; 2544 2545 prop->first_available_cq[0] = GAUDI2_RESERVED_CQ_NUMBER; 2546 2547 prop->fw_cpu_boot_dev_sts0_valid = false; 2548 prop->fw_cpu_boot_dev_sts1_valid = false; 2549 prop->hard_reset_done_by_fw = false; 2550 prop->gic_interrupts_enable = true; 2551 2552 prop->server_type = HL_SERVER_TYPE_UNKNOWN; 2553 2554 prop->max_dec = NUMBER_OF_DEC; 2555 2556 prop->clk_pll_index = HL_GAUDI2_MME_PLL; 2557 2558 prop->dma_mask = 64; 2559 2560 prop->hbw_flush_reg = mmPCIE_WRAP_SPECIAL_GLBL_SPARE_0; 2561 2562 return 0; 2563 } 2564 2565 static int gaudi2_pci_bars_map(struct hl_device *hdev) 2566 { 2567 static const char * const name[] = {"CFG_SRAM", "MSIX", "DRAM"}; 2568 bool is_wc[3] = {false, false, true}; 2569 int rc; 2570 2571 rc = hl_pci_bars_map(hdev, name, is_wc); 2572 if (rc) 2573 return rc; 2574 2575 hdev->rmmio = hdev->pcie_bar[SRAM_CFG_BAR_ID] + (CFG_BASE - STM_FLASH_BASE_ADDR); 2576 2577 return 0; 2578 } 2579 2580 static u64 gaudi2_set_hbm_bar_base(struct hl_device *hdev, u64 addr) 2581 { 2582 struct gaudi2_device *gaudi2 = hdev->asic_specific; 2583 struct hl_inbound_pci_region pci_region; 2584 u64 old_addr = addr; 2585 int rc; 2586 2587 if ((gaudi2) && (gaudi2->dram_bar_cur_addr == addr)) 2588 return old_addr; 2589 2590 if (hdev->asic_prop.iatu_done_by_fw) 2591 return U64_MAX; 2592 2593 /* Inbound Region 2 - Bar 4 - Point to DRAM */ 2594 pci_region.mode = PCI_BAR_MATCH_MODE; 2595 pci_region.bar = DRAM_BAR_ID; 2596 pci_region.addr = addr; 2597 rc = hl_pci_set_inbound_region(hdev, 2, &pci_region); 2598 if (rc) 2599 return U64_MAX; 2600 2601 if (gaudi2) { 2602 old_addr = gaudi2->dram_bar_cur_addr; 2603 gaudi2->dram_bar_cur_addr = addr; 2604 } 2605 2606 return old_addr; 2607 } 2608 2609 static int gaudi2_init_iatu(struct hl_device *hdev) 2610 { 2611 struct hl_inbound_pci_region inbound_region; 2612 struct hl_outbound_pci_region outbound_region; 2613 u32 bar_addr_low, bar_addr_high; 2614 int rc; 2615 2616 if (hdev->asic_prop.iatu_done_by_fw) 2617 return 0; 2618 2619 /* Temporary inbound Region 0 - Bar 0 - Point to CFG 2620 * We must map this region in BAR match mode in order to 2621 * fetch BAR physical base address 2622 */ 2623 inbound_region.mode = PCI_BAR_MATCH_MODE; 2624 inbound_region.bar = SRAM_CFG_BAR_ID; 2625 /* Base address must be aligned to Bar size which is 256 MB */ 2626 inbound_region.addr = STM_FLASH_BASE_ADDR - STM_FLASH_ALIGNED_OFF; 2627 rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region); 2628 if (rc) 2629 return rc; 2630 2631 /* Fetch physical BAR address */ 2632 bar_addr_high = RREG32(mmPCIE_DBI_BAR1_REG + STM_FLASH_ALIGNED_OFF); 2633 bar_addr_low = RREG32(mmPCIE_DBI_BAR0_REG + STM_FLASH_ALIGNED_OFF) & ~0xF; 2634 2635 hdev->pcie_bar_phys[SRAM_CFG_BAR_ID] = (u64)bar_addr_high << 32 | bar_addr_low; 2636 2637 /* Inbound Region 0 - Bar 0 - Point to CFG */ 2638 inbound_region.mode = PCI_ADDRESS_MATCH_MODE; 2639 inbound_region.bar = SRAM_CFG_BAR_ID; 2640 inbound_region.offset_in_bar = 0; 2641 inbound_region.addr = STM_FLASH_BASE_ADDR; 2642 inbound_region.size = CFG_REGION_SIZE; 2643 rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region); 2644 if (rc) 2645 return rc; 2646 2647 /* Inbound Region 1 - Bar 0 - Point to BAR0_RESERVED + SRAM */ 2648 inbound_region.mode = PCI_ADDRESS_MATCH_MODE; 2649 inbound_region.bar = SRAM_CFG_BAR_ID; 2650 inbound_region.offset_in_bar = CFG_REGION_SIZE; 2651 inbound_region.addr = BAR0_RSRVD_BASE_ADDR; 2652 inbound_region.size = BAR0_RSRVD_SIZE + SRAM_SIZE; 2653 rc = hl_pci_set_inbound_region(hdev, 1, &inbound_region); 2654 if (rc) 2655 return rc; 2656 2657 /* Inbound Region 2 - Bar 4 - Point to DRAM */ 2658 inbound_region.mode = PCI_BAR_MATCH_MODE; 2659 inbound_region.bar = DRAM_BAR_ID; 2660 inbound_region.addr = DRAM_PHYS_BASE; 2661 rc = hl_pci_set_inbound_region(hdev, 2, &inbound_region); 2662 if (rc) 2663 return rc; 2664 2665 /* Outbound Region 0 - Point to Host */ 2666 outbound_region.addr = HOST_PHYS_BASE_0; 2667 outbound_region.size = HOST_PHYS_SIZE_0; 2668 rc = hl_pci_set_outbound_region(hdev, &outbound_region); 2669 2670 return rc; 2671 } 2672 2673 static enum hl_device_hw_state gaudi2_get_hw_state(struct hl_device *hdev) 2674 { 2675 return RREG32(mmHW_STATE); 2676 } 2677 2678 static int gaudi2_tpc_binning_init_prop(struct hl_device *hdev) 2679 { 2680 struct asic_fixed_properties *prop = &hdev->asic_prop; 2681 2682 /* 2683 * check for error condition in which number of binning candidates 2684 * is higher than the maximum supported by the driver 2685 */ 2686 if (hweight64(hdev->tpc_binning) > MAX_CLUSTER_BINNING_FAULTY_TPCS) { 2687 dev_err(hdev->dev, "TPC binning is supported for max of %d faulty TPCs, provided mask 0x%llx\n", 2688 MAX_CLUSTER_BINNING_FAULTY_TPCS, 2689 hdev->tpc_binning); 2690 return -EINVAL; 2691 } 2692 2693 prop->tpc_binning_mask = hdev->tpc_binning; 2694 prop->tpc_enabled_mask = GAUDI2_TPC_FULL_MASK; 2695 2696 return 0; 2697 } 2698 2699 static int gaudi2_set_tpc_binning_masks(struct hl_device *hdev) 2700 { 2701 struct asic_fixed_properties *prop = &hdev->asic_prop; 2702 struct hw_queue_properties *q_props = prop->hw_queues_props; 2703 u64 tpc_binning_mask; 2704 u8 subst_idx = 0; 2705 int i, rc; 2706 2707 rc = gaudi2_tpc_binning_init_prop(hdev); 2708 if (rc) 2709 return rc; 2710 2711 tpc_binning_mask = prop->tpc_binning_mask; 2712 2713 for (i = 0 ; i < MAX_FAULTY_TPCS ; i++) { 2714 u8 subst_seq, binned, qid_base; 2715 2716 if (tpc_binning_mask == 0) 2717 break; 2718 2719 if (subst_idx == 0) { 2720 subst_seq = TPC_ID_DCORE0_TPC6; 2721 qid_base = GAUDI2_QUEUE_ID_DCORE0_TPC_6_0; 2722 } else { 2723 subst_seq = TPC_ID_DCORE3_TPC5; 2724 qid_base = GAUDI2_QUEUE_ID_DCORE3_TPC_5_0; 2725 } 2726 2727 2728 /* clear bit from mask */ 2729 binned = __ffs(tpc_binning_mask); 2730 /* 2731 * Coverity complains about possible out-of-bound access in 2732 * clear_bit 2733 */ 2734 if (binned >= TPC_ID_SIZE) { 2735 dev_err(hdev->dev, 2736 "Invalid binned TPC (binning mask: %llx)\n", 2737 tpc_binning_mask); 2738 return -EINVAL; 2739 } 2740 clear_bit(binned, (unsigned long *)&tpc_binning_mask); 2741 2742 /* also clear replacing TPC bit from enabled mask */ 2743 clear_bit(subst_seq, (unsigned long *)&prop->tpc_enabled_mask); 2744 2745 /* bin substite TPC's Qs */ 2746 q_props[qid_base].binned = 1; 2747 q_props[qid_base + 1].binned = 1; 2748 q_props[qid_base + 2].binned = 1; 2749 q_props[qid_base + 3].binned = 1; 2750 2751 subst_idx++; 2752 } 2753 2754 return 0; 2755 } 2756 2757 static int gaudi2_set_dec_binning_masks(struct hl_device *hdev) 2758 { 2759 struct asic_fixed_properties *prop = &hdev->asic_prop; 2760 u8 num_faulty; 2761 2762 num_faulty = hweight32(hdev->decoder_binning); 2763 2764 /* 2765 * check for error condition in which number of binning candidates 2766 * is higher than the maximum supported by the driver 2767 */ 2768 if (num_faulty > MAX_FAULTY_DECODERS) { 2769 dev_err(hdev->dev, "decoder binning is supported for max of single faulty decoder, provided mask 0x%x\n", 2770 hdev->decoder_binning); 2771 return -EINVAL; 2772 } 2773 2774 prop->decoder_binning_mask = (hdev->decoder_binning & GAUDI2_DECODER_FULL_MASK); 2775 2776 if (prop->decoder_binning_mask) 2777 prop->decoder_enabled_mask = (GAUDI2_DECODER_FULL_MASK & ~BIT(DEC_ID_PCIE_VDEC1)); 2778 else 2779 prop->decoder_enabled_mask = GAUDI2_DECODER_FULL_MASK; 2780 2781 return 0; 2782 } 2783 2784 static void gaudi2_set_dram_binning_masks(struct hl_device *hdev) 2785 { 2786 struct asic_fixed_properties *prop = &hdev->asic_prop; 2787 2788 /* check if we should override default binning */ 2789 if (!hdev->dram_binning) { 2790 prop->dram_binning_mask = 0; 2791 prop->dram_enabled_mask = GAUDI2_DRAM_FULL_MASK; 2792 return; 2793 } 2794 2795 /* set DRAM binning constraints */ 2796 prop->faulty_dram_cluster_map |= hdev->dram_binning; 2797 prop->dram_binning_mask = hdev->dram_binning; 2798 prop->dram_enabled_mask = GAUDI2_DRAM_FULL_MASK & ~BIT(HBM_ID5); 2799 } 2800 2801 static int gaudi2_set_edma_binning_masks(struct hl_device *hdev) 2802 { 2803 struct asic_fixed_properties *prop = &hdev->asic_prop; 2804 struct hw_queue_properties *q_props; 2805 u8 seq, num_faulty; 2806 2807 num_faulty = hweight32(hdev->edma_binning); 2808 2809 /* 2810 * check for error condition in which number of binning candidates 2811 * is higher than the maximum supported by the driver 2812 */ 2813 if (num_faulty > MAX_FAULTY_EDMAS) { 2814 dev_err(hdev->dev, 2815 "EDMA binning is supported for max of single faulty EDMA, provided mask 0x%x\n", 2816 hdev->edma_binning); 2817 return -EINVAL; 2818 } 2819 2820 if (!hdev->edma_binning) { 2821 prop->edma_binning_mask = 0; 2822 prop->edma_enabled_mask = GAUDI2_EDMA_FULL_MASK; 2823 return 0; 2824 } 2825 2826 seq = __ffs((unsigned long)hdev->edma_binning); 2827 2828 /* set binning constraints */ 2829 prop->faulty_dram_cluster_map |= BIT(edma_to_hbm_cluster[seq]); 2830 prop->edma_binning_mask = hdev->edma_binning; 2831 prop->edma_enabled_mask = GAUDI2_EDMA_FULL_MASK & ~BIT(EDMA_ID_DCORE3_INSTANCE1); 2832 2833 /* bin substitute EDMA's queue */ 2834 q_props = prop->hw_queues_props; 2835 q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0].binned = 1; 2836 q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1].binned = 1; 2837 q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2].binned = 1; 2838 q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3].binned = 1; 2839 2840 return 0; 2841 } 2842 2843 static int gaudi2_set_xbar_edge_enable_mask(struct hl_device *hdev, u32 xbar_edge_iso_mask) 2844 { 2845 struct asic_fixed_properties *prop = &hdev->asic_prop; 2846 u8 num_faulty, seq; 2847 2848 /* check if we should override default binning */ 2849 if (!xbar_edge_iso_mask) { 2850 prop->xbar_edge_enabled_mask = GAUDI2_XBAR_EDGE_FULL_MASK; 2851 return 0; 2852 } 2853 2854 /* 2855 * note that it can be set to value other than 0 only after cpucp packet (i.e. 2856 * only the FW can set a redundancy value). for user it'll always be 0. 2857 */ 2858 num_faulty = hweight32(xbar_edge_iso_mask); 2859 2860 /* 2861 * check for error condition in which number of binning candidates 2862 * is higher than the maximum supported by the driver 2863 */ 2864 if (num_faulty > MAX_FAULTY_XBARS) { 2865 dev_err(hdev->dev, "we cannot have more than %d faulty XBAR EDGE\n", 2866 MAX_FAULTY_XBARS); 2867 return -EINVAL; 2868 } 2869 2870 seq = __ffs((unsigned long)xbar_edge_iso_mask); 2871 2872 /* set binning constraints */ 2873 prop->faulty_dram_cluster_map |= BIT(xbar_edge_to_hbm_cluster[seq]); 2874 prop->xbar_edge_enabled_mask = (~xbar_edge_iso_mask) & GAUDI2_XBAR_EDGE_FULL_MASK; 2875 2876 return 0; 2877 } 2878 2879 static int gaudi2_set_cluster_binning_masks_common(struct hl_device *hdev, u8 xbar_edge_iso_mask) 2880 { 2881 int rc; 2882 2883 /* 2884 * mark all clusters as good, each component will "fail" cluster 2885 * based on eFuse/user values. 2886 * If more than single cluster is faulty- the chip is unusable 2887 */ 2888 hdev->asic_prop.faulty_dram_cluster_map = 0; 2889 2890 gaudi2_set_dram_binning_masks(hdev); 2891 2892 rc = gaudi2_set_edma_binning_masks(hdev); 2893 if (rc) 2894 return rc; 2895 2896 rc = gaudi2_set_xbar_edge_enable_mask(hdev, xbar_edge_iso_mask); 2897 if (rc) 2898 return rc; 2899 2900 2901 /* always initially set to full mask */ 2902 hdev->asic_prop.hmmu_hif_enabled_mask = GAUDI2_HIF_HMMU_FULL_MASK; 2903 2904 return 0; 2905 } 2906 2907 static int gaudi2_set_cluster_binning_masks(struct hl_device *hdev) 2908 { 2909 struct asic_fixed_properties *prop = &hdev->asic_prop; 2910 int rc; 2911 2912 rc = gaudi2_set_cluster_binning_masks_common(hdev, prop->cpucp_info.xbar_binning_mask); 2913 if (rc) 2914 return rc; 2915 2916 /* if we have DRAM binning reported by FW we should perform cluster config */ 2917 if (prop->faulty_dram_cluster_map) { 2918 u8 cluster_seq = __ffs((unsigned long)prop->faulty_dram_cluster_map); 2919 2920 prop->hmmu_hif_enabled_mask = cluster_hmmu_hif_enabled_mask[cluster_seq]; 2921 } 2922 2923 return 0; 2924 } 2925 2926 static int gaudi2_set_binning_masks(struct hl_device *hdev) 2927 { 2928 int rc; 2929 2930 rc = gaudi2_set_cluster_binning_masks(hdev); 2931 if (rc) 2932 return rc; 2933 2934 rc = gaudi2_set_tpc_binning_masks(hdev); 2935 if (rc) 2936 return rc; 2937 2938 rc = gaudi2_set_dec_binning_masks(hdev); 2939 if (rc) 2940 return rc; 2941 2942 return 0; 2943 } 2944 2945 static int gaudi2_cpucp_info_get(struct hl_device *hdev) 2946 { 2947 struct gaudi2_device *gaudi2 = hdev->asic_specific; 2948 struct asic_fixed_properties *prop = &hdev->asic_prop; 2949 long max_power; 2950 u64 dram_size; 2951 int rc; 2952 2953 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 2954 return 0; 2955 2956 /* No point of asking this information again when not doing hard reset, as the device 2957 * CPU hasn't been reset 2958 */ 2959 if (hdev->reset_info.in_compute_reset) 2960 return 0; 2961 2962 rc = hl_fw_cpucp_handshake(hdev, mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_DEV_STS1, mmCPU_BOOT_ERR0, 2963 mmCPU_BOOT_ERR1); 2964 if (rc) 2965 return rc; 2966 2967 dram_size = le64_to_cpu(prop->cpucp_info.dram_size); 2968 if (dram_size) { 2969 /* we can have wither 5 or 6 HBMs. other values are invalid */ 2970 2971 if ((dram_size != ((GAUDI2_HBM_NUM - 1) * SZ_16G)) && 2972 (dram_size != (GAUDI2_HBM_NUM * SZ_16G))) { 2973 dev_err(hdev->dev, 2974 "F/W reported invalid DRAM size %llu. Trying to use default size %llu\n", 2975 dram_size, prop->dram_size); 2976 dram_size = prop->dram_size; 2977 } 2978 2979 prop->dram_size = dram_size; 2980 prop->dram_end_address = prop->dram_base_address + dram_size; 2981 } 2982 2983 if (!strlen(prop->cpucp_info.card_name)) 2984 strscpy_pad(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME, 2985 CARD_NAME_MAX_LEN); 2986 2987 /* Overwrite binning masks with the actual binning values from F/W */ 2988 hdev->dram_binning = prop->cpucp_info.dram_binning_mask; 2989 hdev->edma_binning = prop->cpucp_info.edma_binning_mask; 2990 hdev->tpc_binning = le64_to_cpu(prop->cpucp_info.tpc_binning_mask); 2991 hdev->decoder_binning = lower_32_bits(le64_to_cpu(prop->cpucp_info.decoder_binning_mask)); 2992 2993 dev_dbg(hdev->dev, "Read binning masks: tpc: 0x%llx, dram: 0x%llx, edma: 0x%x, dec: 0x%x\n", 2994 hdev->tpc_binning, hdev->dram_binning, hdev->edma_binning, 2995 hdev->decoder_binning); 2996 2997 /* 2998 * at this point the DRAM parameters need to be updated according to data obtained 2999 * from the FW 3000 */ 3001 rc = hdev->asic_funcs->set_dram_properties(hdev); 3002 if (rc) 3003 return rc; 3004 3005 rc = hdev->asic_funcs->set_binning_masks(hdev); 3006 if (rc) 3007 return rc; 3008 3009 max_power = hl_fw_get_max_power(hdev); 3010 if (max_power < 0) 3011 return max_power; 3012 3013 prop->max_power_default = (u64) max_power; 3014 3015 return 0; 3016 } 3017 3018 static int gaudi2_fetch_psoc_frequency(struct hl_device *hdev) 3019 { 3020 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3021 u16 pll_freq_arr[HL_PLL_NUM_OUTPUTS]; 3022 int rc; 3023 3024 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 3025 return 0; 3026 3027 rc = hl_fw_cpucp_pll_info_get(hdev, HL_GAUDI2_CPU_PLL, pll_freq_arr); 3028 if (rc) 3029 return rc; 3030 3031 hdev->asic_prop.psoc_timestamp_frequency = pll_freq_arr[3]; 3032 3033 return 0; 3034 } 3035 3036 static int gaudi2_early_init(struct hl_device *hdev) 3037 { 3038 struct asic_fixed_properties *prop = &hdev->asic_prop; 3039 struct pci_dev *pdev = hdev->pdev; 3040 resource_size_t pci_bar_size; 3041 int rc; 3042 3043 rc = gaudi2_set_fixed_properties(hdev); 3044 if (rc) 3045 return rc; 3046 3047 /* Check BAR sizes */ 3048 pci_bar_size = pci_resource_len(pdev, SRAM_CFG_BAR_ID); 3049 3050 if (pci_bar_size != CFG_BAR_SIZE) { 3051 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n", 3052 SRAM_CFG_BAR_ID, &pci_bar_size, CFG_BAR_SIZE); 3053 rc = -ENODEV; 3054 goto free_queue_props; 3055 } 3056 3057 pci_bar_size = pci_resource_len(pdev, MSIX_BAR_ID); 3058 if (pci_bar_size != MSIX_BAR_SIZE) { 3059 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n", 3060 MSIX_BAR_ID, &pci_bar_size, MSIX_BAR_SIZE); 3061 rc = -ENODEV; 3062 goto free_queue_props; 3063 } 3064 3065 prop->dram_pci_bar_size = pci_resource_len(pdev, DRAM_BAR_ID); 3066 hdev->dram_pci_bar_start = pci_resource_start(pdev, DRAM_BAR_ID); 3067 3068 /* 3069 * Only in pldm driver config iATU 3070 */ 3071 if (hdev->pldm) 3072 hdev->asic_prop.iatu_done_by_fw = false; 3073 else 3074 hdev->asic_prop.iatu_done_by_fw = true; 3075 3076 rc = hl_pci_init(hdev); 3077 if (rc) 3078 goto free_queue_props; 3079 3080 /* Before continuing in the initialization, we need to read the preboot 3081 * version to determine whether we run with a security-enabled firmware 3082 */ 3083 rc = hl_fw_read_preboot_status(hdev); 3084 if (rc) { 3085 if (hdev->reset_on_preboot_fail) 3086 /* we are already on failure flow, so don't check if hw_fini fails. */ 3087 hdev->asic_funcs->hw_fini(hdev, true, false); 3088 goto pci_fini; 3089 } 3090 3091 if (gaudi2_get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) { 3092 dev_dbg(hdev->dev, "H/W state is dirty, must reset before initializing\n"); 3093 rc = hdev->asic_funcs->hw_fini(hdev, true, false); 3094 if (rc) { 3095 dev_err(hdev->dev, "failed to reset HW in dirty state (%d)\n", rc); 3096 goto pci_fini; 3097 } 3098 } 3099 3100 return 0; 3101 3102 pci_fini: 3103 hl_pci_fini(hdev); 3104 free_queue_props: 3105 kfree(hdev->asic_prop.hw_queues_props); 3106 return rc; 3107 } 3108 3109 static int gaudi2_early_fini(struct hl_device *hdev) 3110 { 3111 kfree(hdev->asic_prop.hw_queues_props); 3112 hl_pci_fini(hdev); 3113 3114 return 0; 3115 } 3116 3117 static bool gaudi2_is_arc_nic_owned(u64 arc_id) 3118 { 3119 switch (arc_id) { 3120 case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23: 3121 return true; 3122 default: 3123 return false; 3124 } 3125 } 3126 3127 static bool gaudi2_is_arc_tpc_owned(u64 arc_id) 3128 { 3129 switch (arc_id) { 3130 case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24: 3131 return true; 3132 default: 3133 return false; 3134 } 3135 } 3136 3137 static void gaudi2_init_arcs(struct hl_device *hdev) 3138 { 3139 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 3140 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3141 u64 arc_id; 3142 u32 i; 3143 3144 for (i = CPU_ID_SCHED_ARC0 ; i <= CPU_ID_SCHED_ARC3 ; i++) { 3145 if (gaudi2_is_arc_enabled(hdev, i)) 3146 continue; 3147 3148 gaudi2_set_arc_id_cap(hdev, i); 3149 } 3150 3151 for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ ; i += 4) { 3152 if (!gaudi2_is_queue_enabled(hdev, i)) 3153 continue; 3154 3155 arc_id = gaudi2_queue_id_to_arc_id[i]; 3156 if (gaudi2_is_arc_enabled(hdev, arc_id)) 3157 continue; 3158 3159 if (gaudi2_is_arc_nic_owned(arc_id) && 3160 !(hdev->nic_ports_mask & BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0))) 3161 continue; 3162 3163 if (gaudi2_is_arc_tpc_owned(arc_id) && !(gaudi2->tpc_hw_cap_initialized & 3164 BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0))) 3165 continue; 3166 3167 gaudi2_set_arc_id_cap(hdev, arc_id); 3168 } 3169 3170 /* Fetch ARC scratchpad address */ 3171 hdev->asic_prop.engine_core_interrupt_reg_addr = 3172 CFG_BASE + le32_to_cpu(dyn_regs->eng_arc_irq_ctrl); 3173 } 3174 3175 static int gaudi2_scrub_arc_dccm(struct hl_device *hdev, u32 cpu_id) 3176 { 3177 u32 reg_base, reg_val; 3178 int rc; 3179 3180 switch (cpu_id) { 3181 case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC3: 3182 /* Each ARC scheduler has 2 consecutive DCCM blocks */ 3183 rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id], 3184 ARC_DCCM_BLOCK_SIZE * 2, true); 3185 if (rc) 3186 return rc; 3187 break; 3188 case CPU_ID_SCHED_ARC4: 3189 case CPU_ID_SCHED_ARC5: 3190 case CPU_ID_MME_QMAN_ARC0: 3191 case CPU_ID_MME_QMAN_ARC1: 3192 reg_base = gaudi2_arc_blocks_bases[cpu_id]; 3193 3194 /* Scrub lower DCCM block */ 3195 rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id], 3196 ARC_DCCM_BLOCK_SIZE, true); 3197 if (rc) 3198 return rc; 3199 3200 /* Switch to upper DCCM block */ 3201 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_MME_ARC_UPPER_DCCM_EN_VAL_MASK, 1); 3202 WREG32(reg_base + ARC_DCCM_UPPER_EN_OFFSET, reg_val); 3203 3204 /* Scrub upper DCCM block */ 3205 rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id], 3206 ARC_DCCM_BLOCK_SIZE, true); 3207 if (rc) 3208 return rc; 3209 3210 /* Switch to lower DCCM block */ 3211 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_MME_ARC_UPPER_DCCM_EN_VAL_MASK, 0); 3212 WREG32(reg_base + ARC_DCCM_UPPER_EN_OFFSET, reg_val); 3213 break; 3214 default: 3215 rc = gaudi2_send_job_to_kdma(hdev, 0, CFG_BASE + gaudi2_arc_dccm_bases[cpu_id], 3216 ARC_DCCM_BLOCK_SIZE, true); 3217 if (rc) 3218 return rc; 3219 } 3220 3221 return 0; 3222 } 3223 3224 static int gaudi2_scrub_arcs_dccm(struct hl_device *hdev) 3225 { 3226 u16 arc_id; 3227 int rc; 3228 3229 for (arc_id = CPU_ID_SCHED_ARC0 ; arc_id < CPU_ID_MAX ; arc_id++) { 3230 if (!gaudi2_is_arc_enabled(hdev, arc_id)) 3231 continue; 3232 3233 rc = gaudi2_scrub_arc_dccm(hdev, arc_id); 3234 if (rc) 3235 return rc; 3236 } 3237 3238 return 0; 3239 } 3240 3241 static int gaudi2_late_init(struct hl_device *hdev) 3242 { 3243 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3244 int rc; 3245 3246 hdev->asic_prop.supports_advanced_cpucp_rc = true; 3247 3248 rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_ENABLE_PCI_ACCESS, 3249 gaudi2->virt_msix_db_dma_addr); 3250 if (rc) { 3251 dev_err(hdev->dev, "Failed to enable PCI access from CPU\n"); 3252 return rc; 3253 } 3254 3255 rc = gaudi2_fetch_psoc_frequency(hdev); 3256 if (rc) { 3257 dev_err(hdev->dev, "Failed to fetch psoc frequency\n"); 3258 goto disable_pci_access; 3259 } 3260 3261 gaudi2_init_arcs(hdev); 3262 3263 rc = gaudi2_scrub_arcs_dccm(hdev); 3264 if (rc) { 3265 dev_err(hdev->dev, "Failed to scrub arcs DCCM\n"); 3266 goto disable_pci_access; 3267 } 3268 3269 gaudi2_init_security(hdev); 3270 3271 return 0; 3272 3273 disable_pci_access: 3274 hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0); 3275 3276 return rc; 3277 } 3278 3279 static void gaudi2_late_fini(struct hl_device *hdev) 3280 { 3281 hl_hwmon_release_resources(hdev); 3282 } 3283 3284 static void gaudi2_user_mapped_dec_init(struct gaudi2_device *gaudi2, u32 start_idx) 3285 { 3286 struct user_mapped_block *blocks = gaudi2->mapped_blocks; 3287 3288 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE0_DEC0_CMD_BASE, HL_BLOCK_SIZE); 3289 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE0_DEC1_CMD_BASE, HL_BLOCK_SIZE); 3290 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE1_DEC0_CMD_BASE, HL_BLOCK_SIZE); 3291 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE1_DEC1_CMD_BASE, HL_BLOCK_SIZE); 3292 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE2_DEC0_CMD_BASE, HL_BLOCK_SIZE); 3293 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE2_DEC1_CMD_BASE, HL_BLOCK_SIZE); 3294 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE3_DEC0_CMD_BASE, HL_BLOCK_SIZE); 3295 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmDCORE3_DEC1_CMD_BASE, HL_BLOCK_SIZE); 3296 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx++], mmPCIE_DEC0_CMD_BASE, HL_BLOCK_SIZE); 3297 HL_USR_MAPPED_BLK_INIT(&blocks[start_idx], mmPCIE_DEC1_CMD_BASE, HL_BLOCK_SIZE); 3298 } 3299 3300 static void gaudi2_user_mapped_blocks_init(struct hl_device *hdev) 3301 { 3302 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3303 struct user_mapped_block *blocks = gaudi2->mapped_blocks; 3304 u32 block_size, umr_start_idx, num_umr_blocks; 3305 int i; 3306 3307 for (i = 0 ; i < NUM_ARC_CPUS ; i++) { 3308 if (i >= CPU_ID_SCHED_ARC0 && i <= CPU_ID_SCHED_ARC3) 3309 block_size = ARC_DCCM_BLOCK_SIZE * 2; 3310 else 3311 block_size = ARC_DCCM_BLOCK_SIZE; 3312 3313 blocks[i].address = gaudi2_arc_dccm_bases[i]; 3314 blocks[i].size = block_size; 3315 } 3316 3317 blocks[NUM_ARC_CPUS].address = mmARC_FARM_ARC0_ACP_ENG_BASE; 3318 blocks[NUM_ARC_CPUS].size = HL_BLOCK_SIZE; 3319 3320 blocks[NUM_ARC_CPUS + 1].address = mmARC_FARM_ARC1_ACP_ENG_BASE; 3321 blocks[NUM_ARC_CPUS + 1].size = HL_BLOCK_SIZE; 3322 3323 blocks[NUM_ARC_CPUS + 2].address = mmARC_FARM_ARC2_ACP_ENG_BASE; 3324 blocks[NUM_ARC_CPUS + 2].size = HL_BLOCK_SIZE; 3325 3326 blocks[NUM_ARC_CPUS + 3].address = mmARC_FARM_ARC3_ACP_ENG_BASE; 3327 blocks[NUM_ARC_CPUS + 3].size = HL_BLOCK_SIZE; 3328 3329 blocks[NUM_ARC_CPUS + 4].address = mmDCORE0_MME_QM_ARC_ACP_ENG_BASE; 3330 blocks[NUM_ARC_CPUS + 4].size = HL_BLOCK_SIZE; 3331 3332 blocks[NUM_ARC_CPUS + 5].address = mmDCORE1_MME_QM_ARC_ACP_ENG_BASE; 3333 blocks[NUM_ARC_CPUS + 5].size = HL_BLOCK_SIZE; 3334 3335 blocks[NUM_ARC_CPUS + 6].address = mmDCORE2_MME_QM_ARC_ACP_ENG_BASE; 3336 blocks[NUM_ARC_CPUS + 6].size = HL_BLOCK_SIZE; 3337 3338 blocks[NUM_ARC_CPUS + 7].address = mmDCORE3_MME_QM_ARC_ACP_ENG_BASE; 3339 blocks[NUM_ARC_CPUS + 7].size = HL_BLOCK_SIZE; 3340 3341 umr_start_idx = NUM_ARC_CPUS + NUM_OF_USER_ACP_BLOCKS; 3342 num_umr_blocks = NIC_NUMBER_OF_ENGINES * NUM_OF_USER_NIC_UMR_BLOCKS; 3343 for (i = 0 ; i < num_umr_blocks ; i++) { 3344 u8 nic_id, umr_block_id; 3345 3346 nic_id = i / NUM_OF_USER_NIC_UMR_BLOCKS; 3347 umr_block_id = i % NUM_OF_USER_NIC_UMR_BLOCKS; 3348 3349 blocks[umr_start_idx + i].address = 3350 mmNIC0_UMR0_0_UNSECURE_DOORBELL0_BASE + 3351 (nic_id / NIC_NUMBER_OF_QM_PER_MACRO) * NIC_OFFSET + 3352 (nic_id % NIC_NUMBER_OF_QM_PER_MACRO) * NIC_QM_OFFSET + 3353 umr_block_id * NIC_UMR_OFFSET; 3354 blocks[umr_start_idx + i].size = HL_BLOCK_SIZE; 3355 } 3356 3357 /* Expose decoder HW configuration block to user */ 3358 gaudi2_user_mapped_dec_init(gaudi2, USR_MAPPED_BLK_DEC_START_IDX); 3359 3360 for (i = 1; i < NUM_OF_DCORES; ++i) { 3361 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1)].size = SM_OBJS_BLOCK_SIZE; 3362 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1) + 1].size = HL_BLOCK_SIZE; 3363 3364 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1)].address = 3365 mmDCORE0_SYNC_MNGR_OBJS_BASE + i * DCORE_OFFSET; 3366 3367 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1) + 1].address = 3368 mmDCORE0_SYNC_MNGR_GLBL_BASE + i * DCORE_OFFSET; 3369 } 3370 } 3371 3372 static int gaudi2_alloc_cpu_accessible_dma_mem(struct hl_device *hdev) 3373 { 3374 dma_addr_t dma_addr_arr[GAUDI2_ALLOC_CPU_MEM_RETRY_CNT] = {}, end_addr; 3375 void *virt_addr_arr[GAUDI2_ALLOC_CPU_MEM_RETRY_CNT] = {}; 3376 int i, j, rc = 0; 3377 3378 /* The device ARC works with 32-bits addresses, and because there is a single HW register 3379 * that holds the extension bits (49..28), these bits must be identical in all the allocated 3380 * range. 3381 */ 3382 3383 for (i = 0 ; i < GAUDI2_ALLOC_CPU_MEM_RETRY_CNT ; i++) { 3384 virt_addr_arr[i] = hl_asic_dma_alloc_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, 3385 &dma_addr_arr[i], GFP_KERNEL | __GFP_ZERO); 3386 if (!virt_addr_arr[i]) { 3387 rc = -ENOMEM; 3388 goto free_dma_mem_arr; 3389 } 3390 3391 end_addr = dma_addr_arr[i] + HL_CPU_ACCESSIBLE_MEM_SIZE - 1; 3392 if (GAUDI2_ARC_PCI_MSB_ADDR(dma_addr_arr[i]) == GAUDI2_ARC_PCI_MSB_ADDR(end_addr)) 3393 break; 3394 } 3395 3396 if (i == GAUDI2_ALLOC_CPU_MEM_RETRY_CNT) { 3397 dev_err(hdev->dev, 3398 "MSB of ARC accessible DMA memory are not identical in all range\n"); 3399 rc = -EFAULT; 3400 goto free_dma_mem_arr; 3401 } 3402 3403 hdev->cpu_accessible_dma_mem = virt_addr_arr[i]; 3404 hdev->cpu_accessible_dma_address = dma_addr_arr[i]; 3405 3406 free_dma_mem_arr: 3407 for (j = 0 ; j < i ; j++) 3408 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, virt_addr_arr[j], 3409 dma_addr_arr[j]); 3410 3411 return rc; 3412 } 3413 3414 static void gaudi2_set_pci_memory_regions(struct hl_device *hdev) 3415 { 3416 struct asic_fixed_properties *prop = &hdev->asic_prop; 3417 struct pci_mem_region *region; 3418 3419 /* CFG */ 3420 region = &hdev->pci_mem_region[PCI_REGION_CFG]; 3421 region->region_base = CFG_BASE; 3422 region->region_size = CFG_SIZE; 3423 region->offset_in_bar = CFG_BASE - STM_FLASH_BASE_ADDR; 3424 region->bar_size = CFG_BAR_SIZE; 3425 region->bar_id = SRAM_CFG_BAR_ID; 3426 region->used = 1; 3427 3428 /* SRAM */ 3429 region = &hdev->pci_mem_region[PCI_REGION_SRAM]; 3430 region->region_base = SRAM_BASE_ADDR; 3431 region->region_size = SRAM_SIZE; 3432 region->offset_in_bar = CFG_REGION_SIZE + BAR0_RSRVD_SIZE; 3433 region->bar_size = CFG_BAR_SIZE; 3434 region->bar_id = SRAM_CFG_BAR_ID; 3435 region->used = 1; 3436 3437 /* DRAM */ 3438 region = &hdev->pci_mem_region[PCI_REGION_DRAM]; 3439 region->region_base = DRAM_PHYS_BASE; 3440 region->region_size = hdev->asic_prop.dram_size; 3441 region->offset_in_bar = 0; 3442 region->bar_size = prop->dram_pci_bar_size; 3443 region->bar_id = DRAM_BAR_ID; 3444 region->used = 1; 3445 } 3446 3447 static void gaudi2_user_interrupt_setup(struct hl_device *hdev) 3448 { 3449 struct asic_fixed_properties *prop = &hdev->asic_prop; 3450 int i, j, k; 3451 3452 /* Initialize TPC interrupt */ 3453 HL_USR_INTR_STRUCT_INIT(hdev->tpc_interrupt, hdev, 0, HL_USR_INTERRUPT_TPC); 3454 3455 /* Initialize unexpected error interrupt */ 3456 HL_USR_INTR_STRUCT_INIT(hdev->unexpected_error_interrupt, hdev, 0, 3457 HL_USR_INTERRUPT_UNEXPECTED); 3458 3459 /* Initialize common user CQ interrupt */ 3460 HL_USR_INTR_STRUCT_INIT(hdev->common_user_cq_interrupt, hdev, 3461 HL_COMMON_USER_CQ_INTERRUPT_ID, HL_USR_INTERRUPT_CQ); 3462 3463 /* Initialize common decoder interrupt */ 3464 HL_USR_INTR_STRUCT_INIT(hdev->common_decoder_interrupt, hdev, 3465 HL_COMMON_DEC_INTERRUPT_ID, HL_USR_INTERRUPT_DECODER); 3466 3467 /* User interrupts structure holds both decoder and user interrupts from various engines. 3468 * We first initialize the decoder interrupts and then we add the user interrupts. 3469 * The only limitation is that the last decoder interrupt id must be smaller 3470 * then GAUDI2_IRQ_NUM_USER_FIRST. This is checked at compilation time. 3471 */ 3472 3473 /* Initialize decoder interrupts, expose only normal interrupts, 3474 * error interrupts to be handled by driver 3475 */ 3476 for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM, j = 0 ; i <= GAUDI2_IRQ_NUM_SHARED_DEC1_NRM; 3477 i += 2, j++) 3478 HL_USR_INTR_STRUCT_INIT(hdev->user_interrupt[j], hdev, i, 3479 HL_USR_INTERRUPT_DECODER); 3480 3481 for (i = GAUDI2_IRQ_NUM_USER_FIRST, k = 0 ; k < prop->user_interrupt_count; i++, j++, k++) 3482 HL_USR_INTR_STRUCT_INIT(hdev->user_interrupt[j], hdev, i, HL_USR_INTERRUPT_CQ); 3483 } 3484 3485 static inline int gaudi2_get_non_zero_random_int(void) 3486 { 3487 int rand = get_random_u32(); 3488 3489 return rand ? rand : 1; 3490 } 3491 3492 static void gaudi2_special_blocks_free(struct hl_device *hdev) 3493 { 3494 struct asic_fixed_properties *prop = &hdev->asic_prop; 3495 struct hl_skip_blocks_cfg *skip_special_blocks_cfg = 3496 &prop->skip_special_blocks_cfg; 3497 3498 kfree(prop->special_blocks); 3499 kfree(skip_special_blocks_cfg->block_types); 3500 kfree(skip_special_blocks_cfg->block_ranges); 3501 } 3502 3503 static void gaudi2_special_blocks_iterator_free(struct hl_device *hdev) 3504 { 3505 gaudi2_special_blocks_free(hdev); 3506 } 3507 3508 static bool gaudi2_special_block_skip(struct hl_device *hdev, 3509 struct hl_special_blocks_cfg *special_blocks_cfg, 3510 u32 blk_idx, u32 major, u32 minor, u32 sub_minor) 3511 { 3512 return false; 3513 } 3514 3515 static int gaudi2_special_blocks_config(struct hl_device *hdev) 3516 { 3517 struct asic_fixed_properties *prop = &hdev->asic_prop; 3518 int i, rc; 3519 3520 /* Configure Special blocks */ 3521 prop->glbl_err_cause_num = GAUDI2_NUM_OF_GLBL_ERR_CAUSE; 3522 prop->num_of_special_blocks = ARRAY_SIZE(gaudi2_special_blocks); 3523 prop->special_blocks = kmalloc_array(prop->num_of_special_blocks, 3524 sizeof(*prop->special_blocks), GFP_KERNEL); 3525 if (!prop->special_blocks) 3526 return -ENOMEM; 3527 3528 for (i = 0 ; i < prop->num_of_special_blocks ; i++) 3529 memcpy(&prop->special_blocks[i], &gaudi2_special_blocks[i], 3530 sizeof(*prop->special_blocks)); 3531 3532 /* Configure when to skip Special blocks */ 3533 memset(&prop->skip_special_blocks_cfg, 0, sizeof(prop->skip_special_blocks_cfg)); 3534 prop->skip_special_blocks_cfg.skip_block_hook = gaudi2_special_block_skip; 3535 3536 if (ARRAY_SIZE(gaudi2_iterator_skip_block_types)) { 3537 prop->skip_special_blocks_cfg.block_types = 3538 kmalloc_array(ARRAY_SIZE(gaudi2_iterator_skip_block_types), 3539 sizeof(gaudi2_iterator_skip_block_types[0]), GFP_KERNEL); 3540 if (!prop->skip_special_blocks_cfg.block_types) { 3541 rc = -ENOMEM; 3542 goto free_special_blocks; 3543 } 3544 3545 memcpy(prop->skip_special_blocks_cfg.block_types, gaudi2_iterator_skip_block_types, 3546 sizeof(gaudi2_iterator_skip_block_types)); 3547 3548 prop->skip_special_blocks_cfg.block_types_len = 3549 ARRAY_SIZE(gaudi2_iterator_skip_block_types); 3550 } 3551 3552 if (ARRAY_SIZE(gaudi2_iterator_skip_block_ranges)) { 3553 prop->skip_special_blocks_cfg.block_ranges = 3554 kmalloc_array(ARRAY_SIZE(gaudi2_iterator_skip_block_ranges), 3555 sizeof(gaudi2_iterator_skip_block_ranges[0]), GFP_KERNEL); 3556 if (!prop->skip_special_blocks_cfg.block_ranges) { 3557 rc = -ENOMEM; 3558 goto free_skip_special_blocks_types; 3559 } 3560 3561 for (i = 0 ; i < ARRAY_SIZE(gaudi2_iterator_skip_block_ranges) ; i++) 3562 memcpy(&prop->skip_special_blocks_cfg.block_ranges[i], 3563 &gaudi2_iterator_skip_block_ranges[i], 3564 sizeof(struct range)); 3565 3566 prop->skip_special_blocks_cfg.block_ranges_len = 3567 ARRAY_SIZE(gaudi2_iterator_skip_block_ranges); 3568 } 3569 3570 return 0; 3571 3572 free_skip_special_blocks_types: 3573 kfree(prop->skip_special_blocks_cfg.block_types); 3574 free_special_blocks: 3575 kfree(prop->special_blocks); 3576 3577 return rc; 3578 } 3579 3580 static int gaudi2_special_blocks_iterator_config(struct hl_device *hdev) 3581 { 3582 return gaudi2_special_blocks_config(hdev); 3583 } 3584 3585 static void gaudi2_test_queues_msgs_free(struct hl_device *hdev) 3586 { 3587 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3588 struct gaudi2_queues_test_info *msg_info = gaudi2->queues_test_info; 3589 int i; 3590 3591 for (i = 0 ; i < GAUDI2_NUM_TESTED_QS ; i++) { 3592 /* bail-out if this is an allocation failure point */ 3593 if (!msg_info[i].kern_addr) 3594 break; 3595 3596 hl_asic_dma_pool_free(hdev, msg_info[i].kern_addr, msg_info[i].dma_addr); 3597 msg_info[i].kern_addr = NULL; 3598 } 3599 } 3600 3601 static int gaudi2_test_queues_msgs_alloc(struct hl_device *hdev) 3602 { 3603 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3604 struct gaudi2_queues_test_info *msg_info = gaudi2->queues_test_info; 3605 int i, rc; 3606 3607 /* allocate a message-short buf for each Q we intend to test */ 3608 for (i = 0 ; i < GAUDI2_NUM_TESTED_QS ; i++) { 3609 msg_info[i].kern_addr = 3610 (void *)hl_asic_dma_pool_zalloc(hdev, sizeof(struct packet_msg_short), 3611 GFP_KERNEL, &msg_info[i].dma_addr); 3612 if (!msg_info[i].kern_addr) { 3613 dev_err(hdev->dev, 3614 "Failed to allocate dma memory for H/W queue %d testing\n", i); 3615 rc = -ENOMEM; 3616 goto err_exit; 3617 } 3618 } 3619 3620 return 0; 3621 3622 err_exit: 3623 gaudi2_test_queues_msgs_free(hdev); 3624 return rc; 3625 } 3626 3627 static int gaudi2_sw_init(struct hl_device *hdev) 3628 { 3629 struct asic_fixed_properties *prop = &hdev->asic_prop; 3630 struct gaudi2_device *gaudi2; 3631 int i, rc; 3632 3633 /* Allocate device structure */ 3634 gaudi2 = kzalloc(sizeof(*gaudi2), GFP_KERNEL); 3635 if (!gaudi2) 3636 return -ENOMEM; 3637 3638 for (i = 0 ; i < ARRAY_SIZE(gaudi2_irq_map_table) ; i++) { 3639 if (gaudi2_irq_map_table[i].msg || !gaudi2_irq_map_table[i].valid) 3640 continue; 3641 3642 if (gaudi2->num_of_valid_hw_events == GAUDI2_EVENT_SIZE) { 3643 dev_err(hdev->dev, "H/W events array exceeds the limit of %u events\n", 3644 GAUDI2_EVENT_SIZE); 3645 rc = -EINVAL; 3646 goto free_gaudi2_device; 3647 } 3648 3649 gaudi2->hw_events[gaudi2->num_of_valid_hw_events++] = gaudi2_irq_map_table[i].fc_id; 3650 } 3651 3652 for (i = 0 ; i < MME_NUM_OF_LFSR_SEEDS ; i++) 3653 gaudi2->lfsr_rand_seeds[i] = gaudi2_get_non_zero_random_int(); 3654 3655 gaudi2->cpucp_info_get = gaudi2_cpucp_info_get; 3656 3657 hdev->asic_specific = gaudi2; 3658 3659 /* Create DMA pool for small allocations. 3660 * Use DEVICE_CACHE_LINE_SIZE for alignment since the NIC memory-mapped 3661 * PI/CI registers allocated from this pool have this restriction 3662 */ 3663 hdev->dma_pool = dma_pool_create(dev_name(hdev->dev), &hdev->pdev->dev, 3664 GAUDI2_DMA_POOL_BLK_SIZE, DEVICE_CACHE_LINE_SIZE, 0); 3665 if (!hdev->dma_pool) { 3666 dev_err(hdev->dev, "failed to create DMA pool\n"); 3667 rc = -ENOMEM; 3668 goto free_gaudi2_device; 3669 } 3670 3671 rc = gaudi2_alloc_cpu_accessible_dma_mem(hdev); 3672 if (rc) 3673 goto free_dma_pool; 3674 3675 hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1); 3676 if (!hdev->cpu_accessible_dma_pool) { 3677 dev_err(hdev->dev, "Failed to create CPU accessible DMA pool\n"); 3678 rc = -ENOMEM; 3679 goto free_cpu_dma_mem; 3680 } 3681 3682 rc = gen_pool_add(hdev->cpu_accessible_dma_pool, (uintptr_t) hdev->cpu_accessible_dma_mem, 3683 HL_CPU_ACCESSIBLE_MEM_SIZE, -1); 3684 if (rc) { 3685 dev_err(hdev->dev, "Failed to add memory to CPU accessible DMA pool\n"); 3686 rc = -EFAULT; 3687 goto free_cpu_accessible_dma_pool; 3688 } 3689 3690 gaudi2->virt_msix_db_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, prop->pmmu.page_size, 3691 &gaudi2->virt_msix_db_dma_addr); 3692 if (!gaudi2->virt_msix_db_cpu_addr) { 3693 dev_err(hdev->dev, "Failed to allocate DMA memory for virtual MSI-X doorbell\n"); 3694 rc = -ENOMEM; 3695 goto free_cpu_accessible_dma_pool; 3696 } 3697 3698 spin_lock_init(&gaudi2->hw_queues_lock); 3699 3700 gaudi2->scratchpad_kernel_address = hl_asic_dma_alloc_coherent(hdev, PAGE_SIZE, 3701 &gaudi2->scratchpad_bus_address, 3702 GFP_KERNEL | __GFP_ZERO); 3703 if (!gaudi2->scratchpad_kernel_address) { 3704 rc = -ENOMEM; 3705 goto free_virt_msix_db_mem; 3706 } 3707 3708 gaudi2_user_mapped_blocks_init(hdev); 3709 3710 /* Initialize user interrupts */ 3711 gaudi2_user_interrupt_setup(hdev); 3712 3713 hdev->supports_coresight = true; 3714 hdev->supports_sync_stream = true; 3715 hdev->supports_cb_mapping = true; 3716 hdev->supports_wait_for_multi_cs = false; 3717 3718 prop->supports_compute_reset = true; 3719 3720 /* Event queue sanity check added in FW version 1.11 */ 3721 if (hl_is_fw_sw_ver_below(hdev, 1, 11)) 3722 hdev->event_queue.check_eqe_index = false; 3723 else 3724 hdev->event_queue.check_eqe_index = true; 3725 3726 hdev->asic_funcs->set_pci_memory_regions(hdev); 3727 3728 rc = gaudi2_special_blocks_iterator_config(hdev); 3729 if (rc) 3730 goto free_scratchpad_mem; 3731 3732 rc = gaudi2_test_queues_msgs_alloc(hdev); 3733 if (rc) 3734 goto special_blocks_free; 3735 3736 return 0; 3737 3738 special_blocks_free: 3739 gaudi2_special_blocks_iterator_free(hdev); 3740 free_scratchpad_mem: 3741 hl_asic_dma_free_coherent(hdev, PAGE_SIZE, gaudi2->scratchpad_kernel_address, 3742 gaudi2->scratchpad_bus_address); 3743 free_virt_msix_db_mem: 3744 hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr); 3745 free_cpu_accessible_dma_pool: 3746 gen_pool_destroy(hdev->cpu_accessible_dma_pool); 3747 free_cpu_dma_mem: 3748 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem, 3749 hdev->cpu_accessible_dma_address); 3750 free_dma_pool: 3751 dma_pool_destroy(hdev->dma_pool); 3752 free_gaudi2_device: 3753 kfree(gaudi2); 3754 return rc; 3755 } 3756 3757 static int gaudi2_sw_fini(struct hl_device *hdev) 3758 { 3759 struct asic_fixed_properties *prop = &hdev->asic_prop; 3760 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3761 3762 gaudi2_test_queues_msgs_free(hdev); 3763 3764 gaudi2_special_blocks_iterator_free(hdev); 3765 3766 hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr); 3767 3768 gen_pool_destroy(hdev->cpu_accessible_dma_pool); 3769 3770 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem, 3771 hdev->cpu_accessible_dma_address); 3772 3773 hl_asic_dma_free_coherent(hdev, PAGE_SIZE, gaudi2->scratchpad_kernel_address, 3774 gaudi2->scratchpad_bus_address); 3775 3776 dma_pool_destroy(hdev->dma_pool); 3777 3778 kfree(gaudi2); 3779 3780 return 0; 3781 } 3782 3783 static void gaudi2_stop_qman_common(struct hl_device *hdev, u32 reg_base) 3784 { 3785 WREG32(reg_base + QM_GLBL_CFG1_OFFSET, QM_GLBL_CFG1_PQF_STOP | 3786 QM_GLBL_CFG1_CQF_STOP | 3787 QM_GLBL_CFG1_CP_STOP); 3788 3789 /* stop also the ARC */ 3790 WREG32(reg_base + QM_GLBL_CFG2_OFFSET, QM_GLBL_CFG2_ARC_CQF_STOP); 3791 } 3792 3793 static void gaudi2_flush_qman_common(struct hl_device *hdev, u32 reg_base) 3794 { 3795 WREG32(reg_base + QM_GLBL_CFG1_OFFSET, QM_GLBL_CFG1_PQF_FLUSH | 3796 QM_GLBL_CFG1_CQF_FLUSH | 3797 QM_GLBL_CFG1_CP_FLUSH); 3798 } 3799 3800 static void gaudi2_flush_qman_arc_common(struct hl_device *hdev, u32 reg_base) 3801 { 3802 WREG32(reg_base + QM_GLBL_CFG2_OFFSET, QM_GLBL_CFG2_ARC_CQF_FLUSH); 3803 } 3804 3805 /** 3806 * gaudi2_clear_qm_fence_counters_common - clear QM's fence counters 3807 * 3808 * @hdev: pointer to the habanalabs device structure 3809 * @queue_id: queue to clear fence counters to 3810 * @skip_fence: if true set maximum fence value to all fence counters to avoid 3811 * getting stuck on any fence value. otherwise set all fence 3812 * counters to 0 (standard clear of fence counters) 3813 */ 3814 static void gaudi2_clear_qm_fence_counters_common(struct hl_device *hdev, u32 queue_id, 3815 bool skip_fence) 3816 { 3817 u32 size, reg_base; 3818 u32 addr, val; 3819 3820 reg_base = gaudi2_qm_blocks_bases[queue_id]; 3821 3822 addr = reg_base + QM_CP_FENCE0_CNT_0_OFFSET; 3823 size = mmPDMA0_QM_CP_BARRIER_CFG - mmPDMA0_QM_CP_FENCE0_CNT_0; 3824 3825 /* 3826 * in case we want to make sure that QM that is stuck on a fence will 3827 * be released we should set the fence counter to a higher value that 3828 * the value the QM waiting for. to comply with any fence counter of 3829 * any value we set maximum fence value to all counters 3830 */ 3831 val = skip_fence ? U32_MAX : 0; 3832 gaudi2_memset_device_lbw(hdev, addr, size, val); 3833 } 3834 3835 static void gaudi2_qman_manual_flush_common(struct hl_device *hdev, u32 queue_id) 3836 { 3837 u32 reg_base = gaudi2_qm_blocks_bases[queue_id]; 3838 3839 gaudi2_clear_qm_fence_counters_common(hdev, queue_id, true); 3840 gaudi2_flush_qman_common(hdev, reg_base); 3841 gaudi2_flush_qman_arc_common(hdev, reg_base); 3842 } 3843 3844 static void gaudi2_stop_dma_qmans(struct hl_device *hdev) 3845 { 3846 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3847 int dcore, inst; 3848 3849 if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK)) 3850 goto stop_edma_qmans; 3851 3852 /* Stop CPs of PDMA QMANs */ 3853 gaudi2_stop_qman_common(hdev, mmPDMA0_QM_BASE); 3854 gaudi2_stop_qman_common(hdev, mmPDMA1_QM_BASE); 3855 3856 stop_edma_qmans: 3857 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) 3858 return; 3859 3860 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 3861 for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) { 3862 u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst; 3863 u32 qm_base; 3864 3865 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq))) 3866 continue; 3867 3868 qm_base = mmDCORE0_EDMA0_QM_BASE + dcore * DCORE_OFFSET + 3869 inst * DCORE_EDMA_OFFSET; 3870 3871 /* Stop CPs of EDMA QMANs */ 3872 gaudi2_stop_qman_common(hdev, qm_base); 3873 } 3874 } 3875 } 3876 3877 static void gaudi2_stop_mme_qmans(struct hl_device *hdev) 3878 { 3879 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3880 u32 offset, i; 3881 3882 offset = mmDCORE1_MME_QM_BASE - mmDCORE0_MME_QM_BASE; 3883 3884 for (i = 0 ; i < NUM_OF_DCORES ; i++) { 3885 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i))) 3886 continue; 3887 3888 gaudi2_stop_qman_common(hdev, mmDCORE0_MME_QM_BASE + (i * offset)); 3889 } 3890 } 3891 3892 static void gaudi2_stop_tpc_qmans(struct hl_device *hdev) 3893 { 3894 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3895 u32 reg_base; 3896 int i; 3897 3898 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) 3899 return; 3900 3901 for (i = 0 ; i < TPC_ID_SIZE ; i++) { 3902 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i))) 3903 continue; 3904 3905 reg_base = gaudi2_qm_blocks_bases[gaudi2_tpc_id_to_queue_id[i]]; 3906 gaudi2_stop_qman_common(hdev, reg_base); 3907 } 3908 } 3909 3910 static void gaudi2_stop_rot_qmans(struct hl_device *hdev) 3911 { 3912 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3913 u32 reg_base; 3914 int i; 3915 3916 if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK)) 3917 return; 3918 3919 for (i = 0 ; i < ROTATOR_ID_SIZE ; i++) { 3920 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i))) 3921 continue; 3922 3923 reg_base = gaudi2_qm_blocks_bases[gaudi2_rot_id_to_queue_id[i]]; 3924 gaudi2_stop_qman_common(hdev, reg_base); 3925 } 3926 } 3927 3928 static void gaudi2_stop_nic_qmans(struct hl_device *hdev) 3929 { 3930 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3931 u32 reg_base, queue_id; 3932 int i; 3933 3934 if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK)) 3935 return; 3936 3937 queue_id = GAUDI2_QUEUE_ID_NIC_0_0; 3938 3939 for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++, queue_id += NUM_OF_PQ_PER_QMAN) { 3940 if (!(hdev->nic_ports_mask & BIT(i))) 3941 continue; 3942 3943 reg_base = gaudi2_qm_blocks_bases[queue_id]; 3944 gaudi2_stop_qman_common(hdev, reg_base); 3945 } 3946 } 3947 3948 static void gaudi2_stall_dma_common(struct hl_device *hdev, u32 reg_base) 3949 { 3950 u32 reg_val; 3951 3952 reg_val = FIELD_PREP(PDMA0_CORE_CFG_1_HALT_MASK, 0x1); 3953 WREG32(reg_base + DMA_CORE_CFG_1_OFFSET, reg_val); 3954 } 3955 3956 static void gaudi2_dma_stall(struct hl_device *hdev) 3957 { 3958 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3959 int dcore, inst; 3960 3961 if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK)) 3962 goto stall_edma; 3963 3964 gaudi2_stall_dma_common(hdev, mmPDMA0_CORE_BASE); 3965 gaudi2_stall_dma_common(hdev, mmPDMA1_CORE_BASE); 3966 3967 stall_edma: 3968 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) 3969 return; 3970 3971 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 3972 for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) { 3973 u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst; 3974 u32 core_base; 3975 3976 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq))) 3977 continue; 3978 3979 core_base = mmDCORE0_EDMA0_CORE_BASE + dcore * DCORE_OFFSET + 3980 inst * DCORE_EDMA_OFFSET; 3981 3982 /* Stall CPs of EDMA QMANs */ 3983 gaudi2_stall_dma_common(hdev, core_base); 3984 } 3985 } 3986 } 3987 3988 static void gaudi2_mme_stall(struct hl_device *hdev) 3989 { 3990 struct gaudi2_device *gaudi2 = hdev->asic_specific; 3991 u32 offset, i; 3992 3993 offset = mmDCORE1_MME_CTRL_LO_QM_STALL - mmDCORE0_MME_CTRL_LO_QM_STALL; 3994 3995 for (i = 0 ; i < NUM_OF_DCORES ; i++) 3996 if (gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i)) 3997 WREG32(mmDCORE0_MME_CTRL_LO_QM_STALL + (i * offset), 1); 3998 } 3999 4000 static void gaudi2_tpc_stall(struct hl_device *hdev) 4001 { 4002 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4003 u32 reg_base; 4004 int i; 4005 4006 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) 4007 return; 4008 4009 for (i = 0 ; i < TPC_ID_SIZE ; i++) { 4010 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i))) 4011 continue; 4012 4013 reg_base = gaudi2_tpc_cfg_blocks_bases[i]; 4014 WREG32(reg_base + TPC_CFG_STALL_OFFSET, 1); 4015 } 4016 } 4017 4018 static void gaudi2_rotator_stall(struct hl_device *hdev) 4019 { 4020 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4021 u32 reg_val; 4022 int i; 4023 4024 if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK)) 4025 return; 4026 4027 reg_val = FIELD_PREP(ROT_MSS_HALT_WBC_MASK, 0x1) | 4028 FIELD_PREP(ROT_MSS_HALT_RSB_MASK, 0x1) | 4029 FIELD_PREP(ROT_MSS_HALT_MRSB_MASK, 0x1); 4030 4031 for (i = 0 ; i < ROTATOR_ID_SIZE ; i++) { 4032 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i))) 4033 continue; 4034 4035 WREG32(mmROT0_MSS_HALT + i * ROT_OFFSET, reg_val); 4036 } 4037 } 4038 4039 static void gaudi2_disable_qman_common(struct hl_device *hdev, u32 reg_base) 4040 { 4041 WREG32(reg_base + QM_GLBL_CFG0_OFFSET, 0); 4042 } 4043 4044 static void gaudi2_disable_dma_qmans(struct hl_device *hdev) 4045 { 4046 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4047 int dcore, inst; 4048 4049 if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK)) 4050 goto stop_edma_qmans; 4051 4052 gaudi2_disable_qman_common(hdev, mmPDMA0_QM_BASE); 4053 gaudi2_disable_qman_common(hdev, mmPDMA1_QM_BASE); 4054 4055 stop_edma_qmans: 4056 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) 4057 return; 4058 4059 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 4060 for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) { 4061 u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst; 4062 u32 qm_base; 4063 4064 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq))) 4065 continue; 4066 4067 qm_base = mmDCORE0_EDMA0_QM_BASE + dcore * DCORE_OFFSET + 4068 inst * DCORE_EDMA_OFFSET; 4069 4070 /* Disable CPs of EDMA QMANs */ 4071 gaudi2_disable_qman_common(hdev, qm_base); 4072 } 4073 } 4074 } 4075 4076 static void gaudi2_disable_mme_qmans(struct hl_device *hdev) 4077 { 4078 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4079 u32 offset, i; 4080 4081 offset = mmDCORE1_MME_QM_BASE - mmDCORE0_MME_QM_BASE; 4082 4083 for (i = 0 ; i < NUM_OF_DCORES ; i++) 4084 if (gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i)) 4085 gaudi2_disable_qman_common(hdev, mmDCORE0_MME_QM_BASE + (i * offset)); 4086 } 4087 4088 static void gaudi2_disable_tpc_qmans(struct hl_device *hdev) 4089 { 4090 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4091 u32 reg_base; 4092 int i; 4093 4094 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) 4095 return; 4096 4097 for (i = 0 ; i < TPC_ID_SIZE ; i++) { 4098 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i))) 4099 continue; 4100 4101 reg_base = gaudi2_qm_blocks_bases[gaudi2_tpc_id_to_queue_id[i]]; 4102 gaudi2_disable_qman_common(hdev, reg_base); 4103 } 4104 } 4105 4106 static void gaudi2_disable_rot_qmans(struct hl_device *hdev) 4107 { 4108 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4109 u32 reg_base; 4110 int i; 4111 4112 if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK)) 4113 return; 4114 4115 for (i = 0 ; i < ROTATOR_ID_SIZE ; i++) { 4116 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i))) 4117 continue; 4118 4119 reg_base = gaudi2_qm_blocks_bases[gaudi2_rot_id_to_queue_id[i]]; 4120 gaudi2_disable_qman_common(hdev, reg_base); 4121 } 4122 } 4123 4124 static void gaudi2_disable_nic_qmans(struct hl_device *hdev) 4125 { 4126 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4127 u32 reg_base, queue_id; 4128 int i; 4129 4130 if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK)) 4131 return; 4132 4133 queue_id = GAUDI2_QUEUE_ID_NIC_0_0; 4134 4135 for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++, queue_id += NUM_OF_PQ_PER_QMAN) { 4136 if (!(hdev->nic_ports_mask & BIT(i))) 4137 continue; 4138 4139 reg_base = gaudi2_qm_blocks_bases[queue_id]; 4140 gaudi2_disable_qman_common(hdev, reg_base); 4141 } 4142 } 4143 4144 static void gaudi2_enable_timestamp(struct hl_device *hdev) 4145 { 4146 /* Disable the timestamp counter */ 4147 WREG32(mmPSOC_TIMESTAMP_BASE, 0); 4148 4149 /* Zero the lower/upper parts of the 64-bit counter */ 4150 WREG32(mmPSOC_TIMESTAMP_BASE + 0xC, 0); 4151 WREG32(mmPSOC_TIMESTAMP_BASE + 0x8, 0); 4152 4153 /* Enable the counter */ 4154 WREG32(mmPSOC_TIMESTAMP_BASE, 1); 4155 } 4156 4157 static void gaudi2_disable_timestamp(struct hl_device *hdev) 4158 { 4159 /* Disable the timestamp counter */ 4160 WREG32(mmPSOC_TIMESTAMP_BASE, 0); 4161 } 4162 4163 static const char *gaudi2_irq_name(u16 irq_number) 4164 { 4165 switch (irq_number) { 4166 case GAUDI2_IRQ_NUM_EVENT_QUEUE: 4167 return "gaudi2 cpu eq"; 4168 case GAUDI2_IRQ_NUM_COMPLETION: 4169 return "gaudi2 completion"; 4170 case GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM ... GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM: 4171 return gaudi2_vdec_irq_name[irq_number - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM]; 4172 case GAUDI2_IRQ_NUM_TPC_ASSERT: 4173 return "gaudi2 tpc assert"; 4174 case GAUDI2_IRQ_NUM_UNEXPECTED_ERROR: 4175 return "gaudi2 unexpected error"; 4176 case GAUDI2_IRQ_NUM_USER_FIRST ... GAUDI2_IRQ_NUM_USER_LAST: 4177 return "gaudi2 user completion"; 4178 case GAUDI2_IRQ_NUM_EQ_ERROR: 4179 return "gaudi2 eq error"; 4180 default: 4181 return "invalid"; 4182 } 4183 } 4184 4185 static void gaudi2_dec_disable_msix(struct hl_device *hdev, u32 max_irq_num) 4186 { 4187 int i, irq, relative_idx; 4188 struct hl_dec *dec; 4189 4190 for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM ; i < max_irq_num ; i++) { 4191 irq = pci_irq_vector(hdev->pdev, i); 4192 relative_idx = i - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM; 4193 4194 dec = hdev->dec + relative_idx / 2; 4195 4196 /* We pass different structures depending on the irq handler. For the abnormal 4197 * interrupt we pass hl_dec and for the regular interrupt we pass the relevant 4198 * user_interrupt entry 4199 */ 4200 free_irq(irq, ((relative_idx % 2) ? 4201 (void *) dec : 4202 (void *) &hdev->user_interrupt[dec->core_id])); 4203 } 4204 } 4205 4206 static int gaudi2_dec_enable_msix(struct hl_device *hdev) 4207 { 4208 int rc, i, irq_init_cnt, irq, relative_idx; 4209 struct hl_dec *dec; 4210 4211 for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM, irq_init_cnt = 0; 4212 i <= GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM; 4213 i++, irq_init_cnt++) { 4214 4215 irq = pci_irq_vector(hdev->pdev, i); 4216 relative_idx = i - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM; 4217 4218 /* We pass different structures depending on the irq handler. For the abnormal 4219 * interrupt we pass hl_dec and for the regular interrupt we pass the relevant 4220 * user_interrupt entry 4221 * 4222 * TODO: change the dec abnrm to threaded irq 4223 */ 4224 4225 dec = hdev->dec + relative_idx / 2; 4226 if (relative_idx % 2) { 4227 rc = request_irq(irq, hl_irq_handler_dec_abnrm, 0, 4228 gaudi2_irq_name(i), (void *) dec); 4229 } else { 4230 rc = request_irq(irq, hl_irq_user_interrupt_handler, 0, gaudi2_irq_name(i), 4231 (void *) &hdev->user_interrupt[dec->core_id]); 4232 } 4233 4234 if (rc) { 4235 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4236 goto free_dec_irqs; 4237 } 4238 } 4239 4240 return 0; 4241 4242 free_dec_irqs: 4243 gaudi2_dec_disable_msix(hdev, (GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + irq_init_cnt)); 4244 return rc; 4245 } 4246 4247 static int gaudi2_enable_msix(struct hl_device *hdev) 4248 { 4249 struct asic_fixed_properties *prop = &hdev->asic_prop; 4250 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4251 int rc, irq, i, j, user_irq_init_cnt; 4252 struct hl_cq *cq; 4253 4254 if (gaudi2->hw_cap_initialized & HW_CAP_MSIX) 4255 return 0; 4256 4257 hl_init_cpu_for_irq(hdev); 4258 4259 rc = pci_alloc_irq_vectors(hdev->pdev, GAUDI2_MSIX_ENTRIES, GAUDI2_MSIX_ENTRIES, 4260 PCI_IRQ_MSIX); 4261 if (rc < 0) { 4262 dev_err(hdev->dev, "MSI-X: Failed to enable support -- %d/%d\n", 4263 GAUDI2_MSIX_ENTRIES, rc); 4264 return rc; 4265 } 4266 4267 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION); 4268 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION]; 4269 rc = request_irq(irq, hl_irq_handler_cq, 0, gaudi2_irq_name(GAUDI2_IRQ_NUM_COMPLETION), cq); 4270 if (rc) { 4271 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4272 goto free_irq_vectors; 4273 } 4274 4275 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE); 4276 rc = request_irq(irq, hl_irq_handler_eq, 0, gaudi2_irq_name(GAUDI2_IRQ_NUM_EVENT_QUEUE), 4277 &hdev->event_queue); 4278 if (rc) { 4279 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4280 goto free_completion_irq; 4281 } 4282 4283 rc = gaudi2_dec_enable_msix(hdev); 4284 if (rc) { 4285 dev_err(hdev->dev, "Failed to enable decoder IRQ"); 4286 goto free_event_irq; 4287 } 4288 4289 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT); 4290 rc = request_threaded_irq(irq, NULL, hl_irq_user_interrupt_thread_handler, IRQF_ONESHOT, 4291 gaudi2_irq_name(GAUDI2_IRQ_NUM_TPC_ASSERT), 4292 &hdev->tpc_interrupt); 4293 if (rc) { 4294 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4295 goto free_dec_irq; 4296 } 4297 4298 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR); 4299 rc = request_threaded_irq(irq, NULL, hl_irq_user_interrupt_thread_handler, IRQF_ONESHOT, 4300 gaudi2_irq_name(GAUDI2_IRQ_NUM_UNEXPECTED_ERROR), 4301 &hdev->unexpected_error_interrupt); 4302 if (rc) { 4303 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4304 goto free_tpc_irq; 4305 } 4306 4307 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, user_irq_init_cnt = 0; 4308 user_irq_init_cnt < prop->user_interrupt_count; 4309 i++, j++, user_irq_init_cnt++) { 4310 4311 irq = pci_irq_vector(hdev->pdev, i); 4312 hl_set_irq_affinity(hdev, irq); 4313 rc = request_irq(irq, hl_irq_user_interrupt_handler, 0, gaudi2_irq_name(i), 4314 &hdev->user_interrupt[j]); 4315 if (rc) { 4316 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4317 goto free_user_irq; 4318 } 4319 } 4320 4321 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EQ_ERROR); 4322 rc = request_threaded_irq(irq, NULL, hl_irq_eq_error_interrupt_thread_handler, 4323 IRQF_ONESHOT, gaudi2_irq_name(GAUDI2_IRQ_NUM_EQ_ERROR), 4324 hdev); 4325 if (rc) { 4326 dev_err(hdev->dev, "Failed to request IRQ %d", irq); 4327 goto free_user_irq; 4328 } 4329 4330 gaudi2->hw_cap_initialized |= HW_CAP_MSIX; 4331 4332 return 0; 4333 4334 free_user_irq: 4335 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count; 4336 i < GAUDI2_IRQ_NUM_USER_FIRST + user_irq_init_cnt ; i++, j++) { 4337 4338 irq = pci_irq_vector(hdev->pdev, i); 4339 irq_set_affinity_hint(irq, NULL); 4340 free_irq(irq, &hdev->user_interrupt[j]); 4341 } 4342 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR); 4343 free_irq(irq, &hdev->unexpected_error_interrupt); 4344 free_tpc_irq: 4345 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT); 4346 free_irq(irq, &hdev->tpc_interrupt); 4347 free_dec_irq: 4348 gaudi2_dec_disable_msix(hdev, GAUDI2_IRQ_NUM_DEC_LAST + 1); 4349 free_event_irq: 4350 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE); 4351 free_irq(irq, cq); 4352 4353 free_completion_irq: 4354 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION); 4355 free_irq(irq, cq); 4356 4357 free_irq_vectors: 4358 pci_free_irq_vectors(hdev->pdev); 4359 4360 return rc; 4361 } 4362 4363 static void gaudi2_sync_irqs(struct hl_device *hdev) 4364 { 4365 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4366 int i, j; 4367 int irq; 4368 4369 if (!(gaudi2->hw_cap_initialized & HW_CAP_MSIX)) 4370 return; 4371 4372 /* Wait for all pending IRQs to be finished */ 4373 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION)); 4374 4375 for (i = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM ; i <= GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM ; i++) { 4376 irq = pci_irq_vector(hdev->pdev, i); 4377 synchronize_irq(irq); 4378 } 4379 4380 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT)); 4381 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR)); 4382 4383 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = 0 ; j < hdev->asic_prop.user_interrupt_count; 4384 i++, j++) { 4385 irq = pci_irq_vector(hdev->pdev, i); 4386 synchronize_irq(irq); 4387 } 4388 4389 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE)); 4390 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EQ_ERROR)); 4391 } 4392 4393 static void gaudi2_disable_msix(struct hl_device *hdev) 4394 { 4395 struct asic_fixed_properties *prop = &hdev->asic_prop; 4396 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4397 struct hl_cq *cq; 4398 int irq, i, j, k; 4399 4400 if (!(gaudi2->hw_cap_initialized & HW_CAP_MSIX)) 4401 return; 4402 4403 gaudi2_sync_irqs(hdev); 4404 4405 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE); 4406 free_irq(irq, &hdev->event_queue); 4407 4408 gaudi2_dec_disable_msix(hdev, GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM + 1); 4409 4410 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT); 4411 free_irq(irq, &hdev->tpc_interrupt); 4412 4413 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR); 4414 free_irq(irq, &hdev->unexpected_error_interrupt); 4415 4416 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, k = 0; 4417 k < hdev->asic_prop.user_interrupt_count ; i++, j++, k++) { 4418 4419 irq = pci_irq_vector(hdev->pdev, i); 4420 irq_set_affinity_hint(irq, NULL); 4421 free_irq(irq, &hdev->user_interrupt[j]); 4422 } 4423 4424 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION); 4425 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION]; 4426 free_irq(irq, cq); 4427 4428 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EQ_ERROR); 4429 free_irq(irq, hdev); 4430 4431 pci_free_irq_vectors(hdev->pdev); 4432 4433 gaudi2->hw_cap_initialized &= ~HW_CAP_MSIX; 4434 } 4435 4436 static void gaudi2_stop_dcore_dec(struct hl_device *hdev, int dcore_id) 4437 { 4438 u32 reg_val = FIELD_PREP(DCORE0_VDEC0_BRDG_CTRL_GRACEFUL_STOP_MASK, 0x1); 4439 u32 graceful_pend_mask = DCORE0_VDEC0_BRDG_CTRL_GRACEFUL_PEND_MASK; 4440 u32 timeout_usec, dec_id, dec_bit, offset, graceful; 4441 int rc; 4442 4443 if (hdev->pldm) 4444 timeout_usec = GAUDI2_PLDM_VDEC_TIMEOUT_USEC; 4445 else 4446 timeout_usec = GAUDI2_VDEC_TIMEOUT_USEC; 4447 4448 for (dec_id = 0 ; dec_id < NUM_OF_DEC_PER_DCORE ; dec_id++) { 4449 dec_bit = dcore_id * NUM_OF_DEC_PER_DCORE + dec_id; 4450 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) 4451 continue; 4452 4453 offset = dcore_id * DCORE_OFFSET + dec_id * DCORE_VDEC_OFFSET; 4454 4455 WREG32(mmDCORE0_DEC0_CMD_SWREG16 + offset, 0); 4456 4457 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_GRACEFUL + offset, reg_val); 4458 4459 /* Wait till all traffic from decoder stops 4460 * before apply core reset. 4461 */ 4462 rc = hl_poll_timeout( 4463 hdev, 4464 mmDCORE0_VDEC0_BRDG_CTRL_GRACEFUL + offset, 4465 graceful, 4466 (graceful & graceful_pend_mask), 4467 100, 4468 timeout_usec); 4469 if (rc) 4470 dev_err(hdev->dev, 4471 "Failed to stop traffic from DCORE%d Decoder %d\n", 4472 dcore_id, dec_id); 4473 } 4474 } 4475 4476 static void gaudi2_stop_pcie_dec(struct hl_device *hdev) 4477 { 4478 u32 reg_val = FIELD_PREP(DCORE0_VDEC0_BRDG_CTRL_GRACEFUL_STOP_MASK, 0x1); 4479 u32 graceful_pend_mask = PCIE_VDEC0_BRDG_CTRL_GRACEFUL_PEND_MASK; 4480 u32 timeout_usec, dec_id, dec_bit, offset, graceful; 4481 int rc; 4482 4483 if (hdev->pldm) 4484 timeout_usec = GAUDI2_PLDM_VDEC_TIMEOUT_USEC; 4485 else 4486 timeout_usec = GAUDI2_VDEC_TIMEOUT_USEC; 4487 4488 for (dec_id = 0 ; dec_id < NUM_OF_DEC_PER_DCORE ; dec_id++) { 4489 dec_bit = PCIE_DEC_SHIFT + dec_id; 4490 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) 4491 continue; 4492 4493 offset = dec_id * PCIE_VDEC_OFFSET; 4494 4495 WREG32(mmPCIE_DEC0_CMD_SWREG16 + offset, 0); 4496 4497 WREG32(mmPCIE_VDEC0_BRDG_CTRL_GRACEFUL + offset, reg_val); 4498 4499 /* Wait till all traffic from decoder stops 4500 * before apply core reset. 4501 */ 4502 rc = hl_poll_timeout( 4503 hdev, 4504 mmPCIE_VDEC0_BRDG_CTRL_GRACEFUL + offset, 4505 graceful, 4506 (graceful & graceful_pend_mask), 4507 100, 4508 timeout_usec); 4509 if (rc) 4510 dev_err(hdev->dev, 4511 "Failed to stop traffic from PCIe Decoder %d\n", 4512 dec_id); 4513 } 4514 } 4515 4516 static void gaudi2_stop_dec(struct hl_device *hdev) 4517 { 4518 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4519 int dcore_id; 4520 4521 if ((gaudi2->dec_hw_cap_initialized & HW_CAP_DEC_MASK) == 0) 4522 return; 4523 4524 for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) 4525 gaudi2_stop_dcore_dec(hdev, dcore_id); 4526 4527 gaudi2_stop_pcie_dec(hdev); 4528 } 4529 4530 static void gaudi2_set_arc_running_mode(struct hl_device *hdev, u32 cpu_id, u32 run_mode) 4531 { 4532 u32 reg_base, reg_val; 4533 4534 reg_base = gaudi2_arc_blocks_bases[cpu_id]; 4535 if (run_mode == HL_ENGINE_CORE_RUN) 4536 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_RUN_REQ_MASK, 1); 4537 else 4538 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_HALT_REQ_MASK, 1); 4539 4540 WREG32(reg_base + ARC_HALT_REQ_OFFSET, reg_val); 4541 } 4542 4543 static void gaudi2_halt_arcs(struct hl_device *hdev) 4544 { 4545 u16 arc_id; 4546 4547 for (arc_id = CPU_ID_SCHED_ARC0; arc_id < CPU_ID_MAX; arc_id++) { 4548 if (gaudi2_is_arc_enabled(hdev, arc_id)) 4549 gaudi2_set_arc_running_mode(hdev, arc_id, HL_ENGINE_CORE_HALT); 4550 } 4551 } 4552 4553 static int gaudi2_verify_arc_running_mode(struct hl_device *hdev, u32 cpu_id, u32 run_mode) 4554 { 4555 int rc; 4556 u32 reg_base, val, ack_mask, timeout_usec = 100000; 4557 4558 if (hdev->pldm) 4559 timeout_usec *= 100; 4560 4561 reg_base = gaudi2_arc_blocks_bases[cpu_id]; 4562 if (run_mode == HL_ENGINE_CORE_RUN) 4563 ack_mask = ARC_FARM_ARC0_AUX_RUN_HALT_ACK_RUN_ACK_MASK; 4564 else 4565 ack_mask = ARC_FARM_ARC0_AUX_RUN_HALT_ACK_HALT_ACK_MASK; 4566 4567 rc = hl_poll_timeout(hdev, reg_base + ARC_HALT_ACK_OFFSET, 4568 val, ((val & ack_mask) == ack_mask), 4569 1000, timeout_usec); 4570 4571 if (!rc) { 4572 /* Clear */ 4573 val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_RUN_REQ_MASK, 0); 4574 WREG32(reg_base + ARC_HALT_REQ_OFFSET, val); 4575 } 4576 4577 return rc; 4578 } 4579 4580 static void gaudi2_reset_arcs(struct hl_device *hdev) 4581 { 4582 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4583 u16 arc_id; 4584 4585 if (!gaudi2) 4586 return; 4587 4588 for (arc_id = CPU_ID_SCHED_ARC0; arc_id < CPU_ID_MAX; arc_id++) 4589 if (gaudi2_is_arc_enabled(hdev, arc_id)) 4590 gaudi2_clr_arc_id_cap(hdev, arc_id); 4591 } 4592 4593 static void gaudi2_nic_qmans_manual_flush(struct hl_device *hdev) 4594 { 4595 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4596 u32 queue_id; 4597 int i; 4598 4599 if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK)) 4600 return; 4601 4602 queue_id = GAUDI2_QUEUE_ID_NIC_0_0; 4603 4604 for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++, queue_id += NUM_OF_PQ_PER_QMAN) { 4605 if (!(hdev->nic_ports_mask & BIT(i))) 4606 continue; 4607 4608 gaudi2_qman_manual_flush_common(hdev, queue_id); 4609 } 4610 } 4611 4612 static int gaudi2_set_engine_cores(struct hl_device *hdev, u32 *core_ids, 4613 u32 num_cores, u32 core_command) 4614 { 4615 int i, rc; 4616 4617 for (i = 0 ; i < num_cores ; i++) { 4618 if (gaudi2_is_arc_enabled(hdev, core_ids[i])) 4619 gaudi2_set_arc_running_mode(hdev, core_ids[i], core_command); 4620 } 4621 4622 for (i = 0 ; i < num_cores ; i++) { 4623 if (gaudi2_is_arc_enabled(hdev, core_ids[i])) { 4624 rc = gaudi2_verify_arc_running_mode(hdev, core_ids[i], core_command); 4625 4626 if (rc) { 4627 dev_err(hdev->dev, "failed to %s arc: %d\n", 4628 (core_command == HL_ENGINE_CORE_HALT) ? 4629 "HALT" : "RUN", core_ids[i]); 4630 return -1; 4631 } 4632 } 4633 } 4634 4635 return 0; 4636 } 4637 4638 static int gaudi2_set_tpc_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command) 4639 { 4640 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4641 u32 reg_base, reg_addr, reg_val, tpc_id; 4642 4643 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) 4644 return 0; 4645 4646 tpc_id = gaudi2_tpc_engine_id_to_tpc_id[engine_id]; 4647 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + tpc_id))) 4648 return 0; 4649 4650 reg_base = gaudi2_tpc_cfg_blocks_bases[tpc_id]; 4651 reg_addr = reg_base + TPC_CFG_STALL_OFFSET; 4652 reg_val = FIELD_PREP(DCORE0_TPC0_CFG_TPC_STALL_V_MASK, 4653 (engine_command == HL_ENGINE_STALL) ? 1 : 0); 4654 WREG32(reg_addr, reg_val); 4655 4656 if (engine_command == HL_ENGINE_RESUME) { 4657 reg_base = gaudi2_tpc_eml_cfg_blocks_bases[tpc_id]; 4658 reg_addr = reg_base + TPC_EML_CFG_DBG_CNT_OFFSET; 4659 RMWREG32(reg_addr, 0x1, DCORE0_TPC0_EML_CFG_DBG_CNT_DBG_EXIT_MASK); 4660 } 4661 4662 return 0; 4663 } 4664 4665 static int gaudi2_set_mme_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command) 4666 { 4667 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4668 u32 reg_base, reg_addr, reg_val, mme_id; 4669 4670 mme_id = gaudi2_mme_engine_id_to_mme_id[engine_id]; 4671 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + mme_id))) 4672 return 0; 4673 4674 reg_base = gaudi2_mme_ctrl_lo_blocks_bases[mme_id]; 4675 reg_addr = reg_base + MME_CTRL_LO_QM_STALL_OFFSET; 4676 reg_val = FIELD_PREP(DCORE0_MME_CTRL_LO_QM_STALL_V_MASK, 4677 (engine_command == HL_ENGINE_STALL) ? 1 : 0); 4678 WREG32(reg_addr, reg_val); 4679 4680 return 0; 4681 } 4682 4683 static int gaudi2_set_edma_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command) 4684 { 4685 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4686 u32 reg_base, reg_addr, reg_val, edma_id; 4687 4688 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) 4689 return 0; 4690 4691 edma_id = gaudi2_edma_engine_id_to_edma_id[engine_id]; 4692 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + edma_id))) 4693 return 0; 4694 4695 reg_base = gaudi2_dma_core_blocks_bases[edma_id]; 4696 reg_addr = reg_base + EDMA_CORE_CFG_STALL_OFFSET; 4697 reg_val = FIELD_PREP(DCORE0_EDMA0_CORE_CFG_1_HALT_MASK, 4698 (engine_command == HL_ENGINE_STALL) ? 1 : 0); 4699 WREG32(reg_addr, reg_val); 4700 4701 if (engine_command == HL_ENGINE_STALL) { 4702 reg_val = FIELD_PREP(DCORE0_EDMA0_CORE_CFG_1_HALT_MASK, 0x1) | 4703 FIELD_PREP(DCORE0_EDMA0_CORE_CFG_1_FLUSH_MASK, 0x1); 4704 WREG32(reg_addr, reg_val); 4705 } 4706 4707 return 0; 4708 } 4709 4710 static int gaudi2_set_engine_modes(struct hl_device *hdev, 4711 u32 *engine_ids, u32 num_engines, u32 engine_command) 4712 { 4713 int i, rc; 4714 4715 for (i = 0 ; i < num_engines ; ++i) { 4716 switch (engine_ids[i]) { 4717 case GAUDI2_DCORE0_ENGINE_ID_TPC_0 ... GAUDI2_DCORE0_ENGINE_ID_TPC_5: 4718 case GAUDI2_DCORE1_ENGINE_ID_TPC_0 ... GAUDI2_DCORE1_ENGINE_ID_TPC_5: 4719 case GAUDI2_DCORE2_ENGINE_ID_TPC_0 ... GAUDI2_DCORE2_ENGINE_ID_TPC_5: 4720 case GAUDI2_DCORE3_ENGINE_ID_TPC_0 ... GAUDI2_DCORE3_ENGINE_ID_TPC_5: 4721 rc = gaudi2_set_tpc_engine_mode(hdev, engine_ids[i], engine_command); 4722 if (rc) 4723 return rc; 4724 4725 break; 4726 case GAUDI2_DCORE0_ENGINE_ID_MME: 4727 case GAUDI2_DCORE1_ENGINE_ID_MME: 4728 case GAUDI2_DCORE2_ENGINE_ID_MME: 4729 case GAUDI2_DCORE3_ENGINE_ID_MME: 4730 rc = gaudi2_set_mme_engine_mode(hdev, engine_ids[i], engine_command); 4731 if (rc) 4732 return rc; 4733 4734 break; 4735 case GAUDI2_DCORE0_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE0_ENGINE_ID_EDMA_1: 4736 case GAUDI2_DCORE1_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE1_ENGINE_ID_EDMA_1: 4737 case GAUDI2_DCORE2_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE2_ENGINE_ID_EDMA_1: 4738 case GAUDI2_DCORE3_ENGINE_ID_EDMA_0 ... GAUDI2_DCORE3_ENGINE_ID_EDMA_1: 4739 rc = gaudi2_set_edma_engine_mode(hdev, engine_ids[i], engine_command); 4740 if (rc) 4741 return rc; 4742 4743 break; 4744 default: 4745 dev_err(hdev->dev, "Invalid engine ID %u\n", engine_ids[i]); 4746 return -EINVAL; 4747 } 4748 } 4749 4750 return 0; 4751 } 4752 4753 static int gaudi2_set_engines(struct hl_device *hdev, u32 *engine_ids, 4754 u32 num_engines, u32 engine_command) 4755 { 4756 switch (engine_command) { 4757 case HL_ENGINE_CORE_HALT: 4758 case HL_ENGINE_CORE_RUN: 4759 return gaudi2_set_engine_cores(hdev, engine_ids, num_engines, engine_command); 4760 4761 case HL_ENGINE_STALL: 4762 case HL_ENGINE_RESUME: 4763 return gaudi2_set_engine_modes(hdev, engine_ids, num_engines, engine_command); 4764 4765 default: 4766 dev_err(hdev->dev, "failed to execute command id %u\n", engine_command); 4767 return -EINVAL; 4768 } 4769 } 4770 4771 static void gaudi2_halt_engines(struct hl_device *hdev, bool hard_reset, bool fw_reset) 4772 { 4773 u32 wait_timeout_ms; 4774 4775 if (hdev->pldm) 4776 wait_timeout_ms = GAUDI2_PLDM_RESET_WAIT_MSEC; 4777 else 4778 wait_timeout_ms = GAUDI2_RESET_WAIT_MSEC; 4779 4780 if (fw_reset) 4781 goto skip_engines; 4782 4783 gaudi2_stop_dma_qmans(hdev); 4784 gaudi2_stop_mme_qmans(hdev); 4785 gaudi2_stop_tpc_qmans(hdev); 4786 gaudi2_stop_rot_qmans(hdev); 4787 gaudi2_stop_nic_qmans(hdev); 4788 msleep(wait_timeout_ms); 4789 4790 gaudi2_halt_arcs(hdev); 4791 gaudi2_dma_stall(hdev); 4792 gaudi2_mme_stall(hdev); 4793 gaudi2_tpc_stall(hdev); 4794 gaudi2_rotator_stall(hdev); 4795 4796 msleep(wait_timeout_ms); 4797 4798 gaudi2_stop_dec(hdev); 4799 4800 /* 4801 * in case of soft reset do a manual flush for QMANs (currently called 4802 * only for NIC QMANs 4803 */ 4804 if (!hard_reset) 4805 gaudi2_nic_qmans_manual_flush(hdev); 4806 4807 gaudi2_disable_dma_qmans(hdev); 4808 gaudi2_disable_mme_qmans(hdev); 4809 gaudi2_disable_tpc_qmans(hdev); 4810 gaudi2_disable_rot_qmans(hdev); 4811 gaudi2_disable_nic_qmans(hdev); 4812 gaudi2_disable_timestamp(hdev); 4813 4814 skip_engines: 4815 if (hard_reset) { 4816 gaudi2_disable_msix(hdev); 4817 return; 4818 } 4819 4820 gaudi2_sync_irqs(hdev); 4821 } 4822 4823 static void gaudi2_init_firmware_preload_params(struct hl_device *hdev) 4824 { 4825 struct pre_fw_load_props *pre_fw_load = &hdev->fw_loader.pre_fw_load; 4826 4827 pre_fw_load->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS; 4828 pre_fw_load->sts_boot_dev_sts0_reg = mmCPU_BOOT_DEV_STS0; 4829 pre_fw_load->sts_boot_dev_sts1_reg = mmCPU_BOOT_DEV_STS1; 4830 pre_fw_load->boot_err0_reg = mmCPU_BOOT_ERR0; 4831 pre_fw_load->boot_err1_reg = mmCPU_BOOT_ERR1; 4832 pre_fw_load->wait_for_preboot_timeout = GAUDI2_PREBOOT_REQ_TIMEOUT_USEC; 4833 pre_fw_load->wait_for_preboot_extended_timeout = 4834 GAUDI2_PREBOOT_EXTENDED_REQ_TIMEOUT_USEC; 4835 } 4836 4837 static void gaudi2_init_firmware_loader(struct hl_device *hdev) 4838 { 4839 struct fw_load_mgr *fw_loader = &hdev->fw_loader; 4840 struct dynamic_fw_load_mgr *dynamic_loader; 4841 struct cpu_dyn_regs *dyn_regs; 4842 4843 /* fill common fields */ 4844 fw_loader->fw_comp_loaded = FW_TYPE_NONE; 4845 fw_loader->boot_fit_img.image_name = GAUDI2_BOOT_FIT_FILE; 4846 fw_loader->linux_img.image_name = GAUDI2_LINUX_FW_FILE; 4847 fw_loader->boot_fit_timeout = GAUDI2_BOOT_FIT_REQ_TIMEOUT_USEC; 4848 fw_loader->skip_bmc = false; 4849 fw_loader->sram_bar_id = SRAM_CFG_BAR_ID; 4850 fw_loader->dram_bar_id = DRAM_BAR_ID; 4851 fw_loader->cpu_timeout = GAUDI2_CPU_TIMEOUT_USEC; 4852 4853 /* here we update initial values for few specific dynamic regs (as 4854 * before reading the first descriptor from FW those value has to be 4855 * hard-coded). in later stages of the protocol those values will be 4856 * updated automatically by reading the FW descriptor so data there 4857 * will always be up-to-date 4858 */ 4859 dynamic_loader = &hdev->fw_loader.dynamic_loader; 4860 dyn_regs = &dynamic_loader->comm_desc.cpu_dyn_regs; 4861 dyn_regs->kmd_msg_to_cpu = cpu_to_le32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU); 4862 dyn_regs->cpu_cmd_status_to_host = cpu_to_le32(mmCPU_CMD_STATUS_TO_HOST); 4863 dynamic_loader->wait_for_bl_timeout = GAUDI2_WAIT_FOR_BL_TIMEOUT_USEC; 4864 } 4865 4866 static int gaudi2_init_cpu(struct hl_device *hdev) 4867 { 4868 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4869 int rc; 4870 4871 if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU)) 4872 return 0; 4873 4874 if (gaudi2->hw_cap_initialized & HW_CAP_CPU) 4875 return 0; 4876 4877 rc = hl_fw_init_cpu(hdev); 4878 if (rc) 4879 return rc; 4880 4881 gaudi2->hw_cap_initialized |= HW_CAP_CPU; 4882 4883 return 0; 4884 } 4885 4886 static int gaudi2_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout) 4887 { 4888 struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ]; 4889 struct asic_fixed_properties *prop = &hdev->asic_prop; 4890 struct gaudi2_device *gaudi2 = hdev->asic_specific; 4891 struct cpu_dyn_regs *dyn_regs; 4892 struct hl_eq *eq; 4893 u32 status; 4894 int err; 4895 4896 if (!hdev->cpu_queues_enable) 4897 return 0; 4898 4899 if (gaudi2->hw_cap_initialized & HW_CAP_CPU_Q) 4900 return 0; 4901 4902 eq = &hdev->event_queue; 4903 4904 dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 4905 4906 WREG32(mmCPU_IF_PQ_BASE_ADDR_LOW, lower_32_bits(cpu_pq->bus_address)); 4907 WREG32(mmCPU_IF_PQ_BASE_ADDR_HIGH, upper_32_bits(cpu_pq->bus_address)); 4908 4909 WREG32(mmCPU_IF_EQ_BASE_ADDR_LOW, lower_32_bits(eq->bus_address)); 4910 WREG32(mmCPU_IF_EQ_BASE_ADDR_HIGH, upper_32_bits(eq->bus_address)); 4911 4912 WREG32(mmCPU_IF_CQ_BASE_ADDR_LOW, lower_32_bits(hdev->cpu_accessible_dma_address)); 4913 WREG32(mmCPU_IF_CQ_BASE_ADDR_HIGH, upper_32_bits(hdev->cpu_accessible_dma_address)); 4914 4915 WREG32(mmCPU_IF_PQ_LENGTH, HL_QUEUE_SIZE_IN_BYTES); 4916 WREG32(mmCPU_IF_EQ_LENGTH, HL_EQ_SIZE_IN_BYTES); 4917 WREG32(mmCPU_IF_CQ_LENGTH, HL_CPU_ACCESSIBLE_MEM_SIZE); 4918 4919 /* Used for EQ CI */ 4920 WREG32(mmCPU_IF_EQ_RD_OFFS, 0); 4921 4922 WREG32(mmCPU_IF_PF_PQ_PI, 0); 4923 4924 WREG32(mmCPU_IF_QUEUE_INIT, PQ_INIT_STATUS_READY_FOR_CP); 4925 4926 /* Let the ARC know we are ready as it is now handling those queues */ 4927 4928 WREG32(le32_to_cpu(dyn_regs->gic_host_pi_upd_irq), 4929 gaudi2_irq_map_table[GAUDI2_EVENT_CPU_PI_UPDATE].cpu_id); 4930 4931 err = hl_poll_timeout( 4932 hdev, 4933 mmCPU_IF_QUEUE_INIT, 4934 status, 4935 (status == PQ_INIT_STATUS_READY_FOR_HOST), 4936 1000, 4937 cpu_timeout); 4938 4939 if (err) { 4940 dev_err(hdev->dev, "Failed to communicate with device CPU (timeout)\n"); 4941 return -EIO; 4942 } 4943 4944 /* update FW application security bits */ 4945 if (prop->fw_cpu_boot_dev_sts0_valid) 4946 prop->fw_app_cpu_boot_dev_sts0 = RREG32(mmCPU_BOOT_DEV_STS0); 4947 4948 if (prop->fw_cpu_boot_dev_sts1_valid) 4949 prop->fw_app_cpu_boot_dev_sts1 = RREG32(mmCPU_BOOT_DEV_STS1); 4950 4951 gaudi2->hw_cap_initialized |= HW_CAP_CPU_Q; 4952 return 0; 4953 } 4954 4955 static void gaudi2_init_qman_pq(struct hl_device *hdev, u32 reg_base, 4956 u32 queue_id_base) 4957 { 4958 struct hl_hw_queue *q; 4959 u32 pq_id, pq_offset; 4960 4961 for (pq_id = 0 ; pq_id < NUM_OF_PQ_PER_QMAN ; pq_id++) { 4962 q = &hdev->kernel_queues[queue_id_base + pq_id]; 4963 pq_offset = pq_id * 4; 4964 4965 WREG32(reg_base + QM_PQ_BASE_LO_0_OFFSET + pq_offset, 4966 lower_32_bits(q->bus_address)); 4967 WREG32(reg_base + QM_PQ_BASE_HI_0_OFFSET + pq_offset, 4968 upper_32_bits(q->bus_address)); 4969 WREG32(reg_base + QM_PQ_SIZE_0_OFFSET + pq_offset, ilog2(HL_QUEUE_LENGTH)); 4970 WREG32(reg_base + QM_PQ_PI_0_OFFSET + pq_offset, 0); 4971 WREG32(reg_base + QM_PQ_CI_0_OFFSET + pq_offset, 0); 4972 } 4973 } 4974 4975 static void gaudi2_init_qman_cp(struct hl_device *hdev, u32 reg_base) 4976 { 4977 u32 cp_id, cp_offset, mtr_base_lo, mtr_base_hi, so_base_lo, so_base_hi; 4978 4979 mtr_base_lo = lower_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); 4980 mtr_base_hi = upper_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); 4981 so_base_lo = lower_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0); 4982 so_base_hi = upper_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0); 4983 4984 for (cp_id = 0 ; cp_id < NUM_OF_CP_PER_QMAN; cp_id++) { 4985 cp_offset = cp_id * 4; 4986 4987 WREG32(reg_base + QM_CP_MSG_BASE0_ADDR_LO_0_OFFSET + cp_offset, mtr_base_lo); 4988 WREG32(reg_base + QM_CP_MSG_BASE0_ADDR_HI_0_OFFSET + cp_offset, mtr_base_hi); 4989 WREG32(reg_base + QM_CP_MSG_BASE1_ADDR_LO_0_OFFSET + cp_offset, so_base_lo); 4990 WREG32(reg_base + QM_CP_MSG_BASE1_ADDR_HI_0_OFFSET + cp_offset, so_base_hi); 4991 } 4992 4993 /* allow QMANs to accept work from ARC CQF */ 4994 WREG32(reg_base + QM_CP_CFG_OFFSET, FIELD_PREP(PDMA0_QM_CP_CFG_SWITCH_EN_MASK, 0x1)); 4995 } 4996 4997 static void gaudi2_init_qman_pqc(struct hl_device *hdev, u32 reg_base, 4998 u32 queue_id_base) 4999 { 5000 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5001 u32 pq_id, pq_offset, so_base_lo, so_base_hi; 5002 5003 so_base_lo = lower_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0); 5004 so_base_hi = upper_32_bits(CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0); 5005 5006 for (pq_id = 0 ; pq_id < NUM_OF_PQ_PER_QMAN ; pq_id++) { 5007 pq_offset = pq_id * 4; 5008 5009 /* Configure QMAN HBW to scratchpad as it is not needed */ 5010 WREG32(reg_base + QM_PQC_HBW_BASE_LO_0_OFFSET + pq_offset, 5011 lower_32_bits(gaudi2->scratchpad_bus_address)); 5012 WREG32(reg_base + QM_PQC_HBW_BASE_HI_0_OFFSET + pq_offset, 5013 upper_32_bits(gaudi2->scratchpad_bus_address)); 5014 WREG32(reg_base + QM_PQC_SIZE_0_OFFSET + pq_offset, 5015 ilog2(PAGE_SIZE / sizeof(struct hl_cq_entry))); 5016 5017 WREG32(reg_base + QM_PQC_PI_0_OFFSET + pq_offset, 0); 5018 WREG32(reg_base + QM_PQC_LBW_WDATA_0_OFFSET + pq_offset, QM_PQC_LBW_WDATA); 5019 WREG32(reg_base + QM_PQC_LBW_BASE_LO_0_OFFSET + pq_offset, so_base_lo); 5020 WREG32(reg_base + QM_PQC_LBW_BASE_HI_0_OFFSET + pq_offset, so_base_hi); 5021 } 5022 5023 /* Enable QMAN H/W completion */ 5024 WREG32(reg_base + QM_PQC_CFG_OFFSET, 1 << PDMA0_QM_PQC_CFG_EN_SHIFT); 5025 } 5026 5027 static u32 gaudi2_get_dyn_sp_reg(struct hl_device *hdev, u32 queue_id_base) 5028 { 5029 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 5030 u32 sp_reg_addr; 5031 5032 switch (queue_id_base) { 5033 case GAUDI2_QUEUE_ID_PDMA_0_0...GAUDI2_QUEUE_ID_PDMA_1_3: 5034 fallthrough; 5035 case GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3: 5036 fallthrough; 5037 case GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3: 5038 fallthrough; 5039 case GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3: 5040 fallthrough; 5041 case GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3: 5042 sp_reg_addr = le32_to_cpu(dyn_regs->gic_dma_qm_irq_ctrl); 5043 break; 5044 case GAUDI2_QUEUE_ID_DCORE0_MME_0_0...GAUDI2_QUEUE_ID_DCORE0_MME_0_3: 5045 fallthrough; 5046 case GAUDI2_QUEUE_ID_DCORE1_MME_0_0...GAUDI2_QUEUE_ID_DCORE1_MME_0_3: 5047 fallthrough; 5048 case GAUDI2_QUEUE_ID_DCORE2_MME_0_0...GAUDI2_QUEUE_ID_DCORE2_MME_0_3: 5049 fallthrough; 5050 case GAUDI2_QUEUE_ID_DCORE3_MME_0_0...GAUDI2_QUEUE_ID_DCORE3_MME_0_3: 5051 sp_reg_addr = le32_to_cpu(dyn_regs->gic_mme_qm_irq_ctrl); 5052 break; 5053 case GAUDI2_QUEUE_ID_DCORE0_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE0_TPC_6_3: 5054 fallthrough; 5055 case GAUDI2_QUEUE_ID_DCORE1_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE1_TPC_5_3: 5056 fallthrough; 5057 case GAUDI2_QUEUE_ID_DCORE2_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE2_TPC_5_3: 5058 fallthrough; 5059 case GAUDI2_QUEUE_ID_DCORE3_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE3_TPC_5_3: 5060 sp_reg_addr = le32_to_cpu(dyn_regs->gic_tpc_qm_irq_ctrl); 5061 break; 5062 case GAUDI2_QUEUE_ID_ROT_0_0...GAUDI2_QUEUE_ID_ROT_1_3: 5063 sp_reg_addr = le32_to_cpu(dyn_regs->gic_rot_qm_irq_ctrl); 5064 break; 5065 case GAUDI2_QUEUE_ID_NIC_0_0...GAUDI2_QUEUE_ID_NIC_23_3: 5066 sp_reg_addr = le32_to_cpu(dyn_regs->gic_nic_qm_irq_ctrl); 5067 break; 5068 default: 5069 dev_err(hdev->dev, "Unexpected h/w queue %d\n", queue_id_base); 5070 return 0; 5071 } 5072 5073 return sp_reg_addr; 5074 } 5075 5076 static void gaudi2_init_qman_common(struct hl_device *hdev, u32 reg_base, 5077 u32 queue_id_base) 5078 { 5079 u32 glbl_prot = QMAN_MAKE_TRUSTED, irq_handler_offset; 5080 int map_table_entry; 5081 5082 WREG32(reg_base + QM_GLBL_PROT_OFFSET, glbl_prot); 5083 5084 irq_handler_offset = gaudi2_get_dyn_sp_reg(hdev, queue_id_base); 5085 WREG32(reg_base + QM_GLBL_ERR_ADDR_LO_OFFSET, lower_32_bits(CFG_BASE + irq_handler_offset)); 5086 WREG32(reg_base + QM_GLBL_ERR_ADDR_HI_OFFSET, upper_32_bits(CFG_BASE + irq_handler_offset)); 5087 5088 map_table_entry = gaudi2_qman_async_event_id[queue_id_base]; 5089 WREG32(reg_base + QM_GLBL_ERR_WDATA_OFFSET, 5090 gaudi2_irq_map_table[map_table_entry].cpu_id); 5091 5092 WREG32(reg_base + QM_ARB_ERR_MSG_EN_OFFSET, QM_ARB_ERR_MSG_EN_MASK); 5093 5094 WREG32(reg_base + QM_ARB_SLV_CHOISE_WDT_OFFSET, GAUDI2_ARB_WDT_TIMEOUT); 5095 WREG32(reg_base + QM_GLBL_CFG1_OFFSET, 0); 5096 WREG32(reg_base + QM_GLBL_CFG2_OFFSET, 0); 5097 5098 /* Enable the QMAN channel. 5099 * PDMA QMAN configuration is different, as we do not allow user to 5100 * access some of the CPs. 5101 * PDMA0: CP2/3 are reserved for the ARC usage. 5102 * PDMA1: CP1/2/3 are reserved for the ARC usage. 5103 */ 5104 if (reg_base == gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_1_0]) 5105 WREG32(reg_base + QM_GLBL_CFG0_OFFSET, PDMA1_QMAN_ENABLE); 5106 else if (reg_base == gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_0_0]) 5107 WREG32(reg_base + QM_GLBL_CFG0_OFFSET, PDMA0_QMAN_ENABLE); 5108 else 5109 WREG32(reg_base + QM_GLBL_CFG0_OFFSET, QMAN_ENABLE); 5110 } 5111 5112 static void gaudi2_init_qman(struct hl_device *hdev, u32 reg_base, 5113 u32 queue_id_base) 5114 { 5115 u32 pq_id; 5116 5117 for (pq_id = 0 ; pq_id < NUM_OF_PQ_PER_QMAN ; pq_id++) 5118 hdev->kernel_queues[queue_id_base + pq_id].cq_id = GAUDI2_RESERVED_CQ_CS_COMPLETION; 5119 5120 gaudi2_init_qman_pq(hdev, reg_base, queue_id_base); 5121 gaudi2_init_qman_cp(hdev, reg_base); 5122 gaudi2_init_qman_pqc(hdev, reg_base, queue_id_base); 5123 gaudi2_init_qman_common(hdev, reg_base, queue_id_base); 5124 } 5125 5126 static void gaudi2_init_dma_core(struct hl_device *hdev, u32 reg_base, 5127 u32 dma_core_id, bool is_secure) 5128 { 5129 u32 prot, irq_handler_offset; 5130 struct cpu_dyn_regs *dyn_regs; 5131 int map_table_entry; 5132 5133 prot = 1 << ARC_FARM_KDMA_PROT_ERR_VAL_SHIFT; 5134 if (is_secure) 5135 prot |= 1 << ARC_FARM_KDMA_PROT_VAL_SHIFT; 5136 5137 WREG32(reg_base + DMA_CORE_PROT_OFFSET, prot); 5138 5139 dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 5140 irq_handler_offset = le32_to_cpu(dyn_regs->gic_dma_core_irq_ctrl); 5141 5142 WREG32(reg_base + DMA_CORE_ERRMSG_ADDR_LO_OFFSET, 5143 lower_32_bits(CFG_BASE + irq_handler_offset)); 5144 5145 WREG32(reg_base + DMA_CORE_ERRMSG_ADDR_HI_OFFSET, 5146 upper_32_bits(CFG_BASE + irq_handler_offset)); 5147 5148 map_table_entry = gaudi2_dma_core_async_event_id[dma_core_id]; 5149 WREG32(reg_base + DMA_CORE_ERRMSG_WDATA_OFFSET, 5150 gaudi2_irq_map_table[map_table_entry].cpu_id); 5151 5152 /* Enable the DMA channel */ 5153 WREG32(reg_base + DMA_CORE_CFG_0_OFFSET, 1 << ARC_FARM_KDMA_CFG_0_EN_SHIFT); 5154 } 5155 5156 static void gaudi2_init_kdma(struct hl_device *hdev) 5157 { 5158 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5159 u32 reg_base; 5160 5161 if ((gaudi2->hw_cap_initialized & HW_CAP_KDMA) == HW_CAP_KDMA) 5162 return; 5163 5164 reg_base = gaudi2_dma_core_blocks_bases[DMA_CORE_ID_KDMA]; 5165 5166 gaudi2_init_dma_core(hdev, reg_base, DMA_CORE_ID_KDMA, true); 5167 5168 gaudi2->hw_cap_initialized |= HW_CAP_KDMA; 5169 } 5170 5171 static void gaudi2_init_pdma(struct hl_device *hdev) 5172 { 5173 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5174 u32 reg_base; 5175 5176 if ((gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK) == HW_CAP_PDMA_MASK) 5177 return; 5178 5179 reg_base = gaudi2_dma_core_blocks_bases[DMA_CORE_ID_PDMA0]; 5180 gaudi2_init_dma_core(hdev, reg_base, DMA_CORE_ID_PDMA0, false); 5181 5182 reg_base = gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_0_0]; 5183 gaudi2_init_qman(hdev, reg_base, GAUDI2_QUEUE_ID_PDMA_0_0); 5184 5185 reg_base = gaudi2_dma_core_blocks_bases[DMA_CORE_ID_PDMA1]; 5186 gaudi2_init_dma_core(hdev, reg_base, DMA_CORE_ID_PDMA1, false); 5187 5188 reg_base = gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_1_0]; 5189 gaudi2_init_qman(hdev, reg_base, GAUDI2_QUEUE_ID_PDMA_1_0); 5190 5191 gaudi2->hw_cap_initialized |= HW_CAP_PDMA_MASK; 5192 } 5193 5194 static void gaudi2_init_edma_instance(struct hl_device *hdev, u8 seq) 5195 { 5196 u32 reg_base, base_edma_core_id, base_edma_qman_id; 5197 5198 base_edma_core_id = DMA_CORE_ID_EDMA0 + seq; 5199 base_edma_qman_id = edma_stream_base[seq]; 5200 5201 reg_base = gaudi2_dma_core_blocks_bases[base_edma_core_id]; 5202 gaudi2_init_dma_core(hdev, reg_base, base_edma_core_id, false); 5203 5204 reg_base = gaudi2_qm_blocks_bases[base_edma_qman_id]; 5205 gaudi2_init_qman(hdev, reg_base, base_edma_qman_id); 5206 } 5207 5208 static void gaudi2_init_edma(struct hl_device *hdev) 5209 { 5210 struct asic_fixed_properties *prop = &hdev->asic_prop; 5211 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5212 int dcore, inst; 5213 5214 if ((gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK) == HW_CAP_EDMA_MASK) 5215 return; 5216 5217 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 5218 for (inst = 0 ; inst < NUM_OF_EDMA_PER_DCORE ; inst++) { 5219 u8 seq = dcore * NUM_OF_EDMA_PER_DCORE + inst; 5220 5221 if (!(prop->edma_enabled_mask & BIT(seq))) 5222 continue; 5223 5224 gaudi2_init_edma_instance(hdev, seq); 5225 5226 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_EDMA_SHIFT + seq); 5227 } 5228 } 5229 } 5230 5231 /* 5232 * gaudi2_arm_monitors_for_virt_msix_db() - Arm monitors for writing to the virtual MSI-X doorbell. 5233 * @hdev: pointer to habanalabs device structure. 5234 * @sob_id: sync object ID. 5235 * @first_mon_id: ID of first monitor out of 3 consecutive monitors. 5236 * @interrupt_id: interrupt ID. 5237 * 5238 * Some initiators cannot have HBW address in their completion address registers, and thus cannot 5239 * write directly to the HBW host memory of the virtual MSI-X doorbell. 5240 * Instead, they are configured to LBW write to a sync object, and a monitor will do the HBW write. 5241 * 5242 * The mechanism in the sync manager block is composed of a master monitor with 3 messages. 5243 * In addition to the HBW write, the other 2 messages are for preparing the monitor to next 5244 * completion, by decrementing the sync object value and re-arming the monitor. 5245 */ 5246 static void gaudi2_arm_monitors_for_virt_msix_db(struct hl_device *hdev, u32 sob_id, 5247 u32 first_mon_id, u32 interrupt_id) 5248 { 5249 u32 sob_offset, first_mon_offset, mon_offset, payload, sob_group, mode, arm, config; 5250 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5251 u64 addr; 5252 u8 mask; 5253 5254 /* Reset the SOB value */ 5255 sob_offset = sob_id * sizeof(u32); 5256 WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset, 0); 5257 5258 /* Configure 3 monitors: 5259 * 1. Write interrupt ID to the virtual MSI-X doorbell (master monitor) 5260 * 2. Decrement SOB value by 1. 5261 * 3. Re-arm the master monitor. 5262 */ 5263 5264 first_mon_offset = first_mon_id * sizeof(u32); 5265 5266 /* 2nd monitor: Decrement SOB value by 1 */ 5267 mon_offset = first_mon_offset + sizeof(u32); 5268 5269 addr = CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset; 5270 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, lower_32_bits(addr)); 5271 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + mon_offset, upper_32_bits(addr)); 5272 5273 payload = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 0x7FFF) | /* "-1" */ 5274 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_SIGN_MASK, 1) | 5275 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1); 5276 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, payload); 5277 5278 /* 3rd monitor: Re-arm the master monitor */ 5279 mon_offset = first_mon_offset + 2 * sizeof(u32); 5280 5281 addr = CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + first_mon_offset; 5282 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, lower_32_bits(addr)); 5283 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + mon_offset, upper_32_bits(addr)); 5284 5285 sob_group = sob_id / 8; 5286 mask = ~BIT(sob_id & 0x7); 5287 mode = 0; /* comparison mode is "greater than or equal to" */ 5288 arm = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SID_MASK, sob_group) | 5289 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_MASK_MASK, mask) | 5290 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOP_MASK, mode) | 5291 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOD_MASK, 1); 5292 5293 payload = arm; 5294 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, payload); 5295 5296 /* 1st monitor (master): Write interrupt ID to the virtual MSI-X doorbell */ 5297 mon_offset = first_mon_offset; 5298 5299 config = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_WR_NUM_MASK, 2); /* "2": 3 writes */ 5300 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + mon_offset, config); 5301 5302 addr = gaudi2->virt_msix_db_dma_addr; 5303 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, lower_32_bits(addr)); 5304 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + mon_offset, upper_32_bits(addr)); 5305 5306 payload = interrupt_id; 5307 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, payload); 5308 5309 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + mon_offset, arm); 5310 } 5311 5312 static void gaudi2_prepare_sm_for_virt_msix_db(struct hl_device *hdev) 5313 { 5314 u32 decoder_id, sob_id, first_mon_id, interrupt_id; 5315 struct asic_fixed_properties *prop = &hdev->asic_prop; 5316 5317 /* Decoder normal/abnormal interrupts */ 5318 for (decoder_id = 0 ; decoder_id < NUMBER_OF_DEC ; ++decoder_id) { 5319 if (!(prop->decoder_enabled_mask & BIT(decoder_id))) 5320 continue; 5321 5322 sob_id = GAUDI2_RESERVED_SOB_DEC_NRM_FIRST + decoder_id; 5323 first_mon_id = GAUDI2_RESERVED_MON_DEC_NRM_FIRST + 3 * decoder_id; 5324 interrupt_id = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + 2 * decoder_id; 5325 gaudi2_arm_monitors_for_virt_msix_db(hdev, sob_id, first_mon_id, interrupt_id); 5326 5327 sob_id = GAUDI2_RESERVED_SOB_DEC_ABNRM_FIRST + decoder_id; 5328 first_mon_id = GAUDI2_RESERVED_MON_DEC_ABNRM_FIRST + 3 * decoder_id; 5329 interrupt_id += 1; 5330 gaudi2_arm_monitors_for_virt_msix_db(hdev, sob_id, first_mon_id, interrupt_id); 5331 } 5332 } 5333 5334 static void gaudi2_init_sm(struct hl_device *hdev) 5335 { 5336 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5337 u64 cq_address; 5338 u32 reg_val; 5339 int i; 5340 5341 /* Enable HBW/LBW CQ for completion monitors */ 5342 reg_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_CQ_EN_MASK, 1); 5343 reg_val |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_LBW_EN_MASK, 1); 5344 5345 for (i = 0 ; i < GAUDI2_MAX_PENDING_CS ; i++) 5346 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + (4 * i), reg_val); 5347 5348 /* Enable only HBW CQ for KDMA completion monitor */ 5349 reg_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_CQ_EN_MASK, 1); 5350 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + (4 * i), reg_val); 5351 5352 /* Init CQ0 DB - configure the monitor to trigger MSI-X interrupt */ 5353 WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0, lower_32_bits(gaudi2->virt_msix_db_dma_addr)); 5354 WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0, upper_32_bits(gaudi2->virt_msix_db_dma_addr)); 5355 WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_DATA_0, GAUDI2_IRQ_NUM_COMPLETION); 5356 5357 for (i = 0 ; i < GAUDI2_RESERVED_CQ_NUMBER ; i++) { 5358 cq_address = 5359 hdev->completion_queue[i].bus_address; 5360 5361 WREG32(mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_L_0 + (4 * i), 5362 lower_32_bits(cq_address)); 5363 WREG32(mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_H_0 + (4 * i), 5364 upper_32_bits(cq_address)); 5365 WREG32(mmDCORE0_SYNC_MNGR_GLBL_CQ_SIZE_LOG2_0 + (4 * i), 5366 ilog2(HL_CQ_SIZE_IN_BYTES)); 5367 } 5368 5369 /* Configure kernel ASID and MMU BP*/ 5370 WREG32(mmDCORE0_SYNC_MNGR_GLBL_ASID_SEC, 0x10000); 5371 WREG32(mmDCORE0_SYNC_MNGR_GLBL_ASID_NONE_SEC_PRIV, 0); 5372 5373 /* Initialize sync objects and monitors which are used for the virtual MSI-X doorbell */ 5374 gaudi2_prepare_sm_for_virt_msix_db(hdev); 5375 } 5376 5377 static void gaudi2_init_mme_acc(struct hl_device *hdev, u32 reg_base) 5378 { 5379 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5380 u32 reg_val; 5381 int i; 5382 5383 reg_val = FIELD_PREP(MME_ACC_INTR_MASK_WBC_ERR_RESP_MASK, 0); 5384 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_POS_INF_MASK, 1); 5385 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_NEG_INF_MASK, 1); 5386 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_NAN_MASK, 1); 5387 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_RESULT_POS_INF_MASK, 1); 5388 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_RESULT_NEG_INF_MASK, 1); 5389 5390 WREG32(reg_base + MME_ACC_INTR_MASK_OFFSET, reg_val); 5391 WREG32(reg_base + MME_ACC_AP_LFSR_POLY_OFFSET, 0x80DEADAF); 5392 5393 for (i = 0 ; i < MME_NUM_OF_LFSR_SEEDS ; i++) { 5394 WREG32(reg_base + MME_ACC_AP_LFSR_SEED_SEL_OFFSET, i); 5395 WREG32(reg_base + MME_ACC_AP_LFSR_SEED_WDATA_OFFSET, gaudi2->lfsr_rand_seeds[i]); 5396 } 5397 } 5398 5399 static void gaudi2_init_dcore_mme(struct hl_device *hdev, int dcore_id, 5400 bool config_qman_only) 5401 { 5402 u32 queue_id_base, reg_base; 5403 5404 switch (dcore_id) { 5405 case 0: 5406 queue_id_base = GAUDI2_QUEUE_ID_DCORE0_MME_0_0; 5407 break; 5408 case 1: 5409 queue_id_base = GAUDI2_QUEUE_ID_DCORE1_MME_0_0; 5410 break; 5411 case 2: 5412 queue_id_base = GAUDI2_QUEUE_ID_DCORE2_MME_0_0; 5413 break; 5414 case 3: 5415 queue_id_base = GAUDI2_QUEUE_ID_DCORE3_MME_0_0; 5416 break; 5417 default: 5418 dev_err(hdev->dev, "Invalid dcore id %u\n", dcore_id); 5419 return; 5420 } 5421 5422 if (!config_qman_only) { 5423 reg_base = gaudi2_mme_acc_blocks_bases[dcore_id]; 5424 gaudi2_init_mme_acc(hdev, reg_base); 5425 } 5426 5427 reg_base = gaudi2_qm_blocks_bases[queue_id_base]; 5428 gaudi2_init_qman(hdev, reg_base, queue_id_base); 5429 } 5430 5431 static void gaudi2_init_mme(struct hl_device *hdev) 5432 { 5433 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5434 int i; 5435 5436 if ((gaudi2->hw_cap_initialized & HW_CAP_MME_MASK) == HW_CAP_MME_MASK) 5437 return; 5438 5439 for (i = 0 ; i < NUM_OF_DCORES ; i++) { 5440 gaudi2_init_dcore_mme(hdev, i, false); 5441 5442 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_MME_SHIFT + i); 5443 } 5444 } 5445 5446 static void gaudi2_init_tpc_cfg(struct hl_device *hdev, u32 reg_base) 5447 { 5448 /* Mask arithmetic and QM interrupts in TPC */ 5449 WREG32(reg_base + TPC_CFG_TPC_INTR_MASK_OFFSET, 0x23FFFE); 5450 5451 /* Set 16 cache lines */ 5452 WREG32(reg_base + TPC_CFG_MSS_CONFIG_OFFSET, 5453 2 << DCORE0_TPC0_CFG_MSS_CONFIG_ICACHE_FETCH_LINE_NUM_SHIFT); 5454 } 5455 5456 struct gaudi2_tpc_init_cfg_data { 5457 enum gaudi2_queue_id dcore_tpc_qid_base[NUM_OF_DCORES]; 5458 }; 5459 5460 static void gaudi2_init_tpc_config(struct hl_device *hdev, int dcore, int inst, 5461 u32 offset, struct iterate_module_ctx *ctx) 5462 { 5463 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5464 struct gaudi2_tpc_init_cfg_data *cfg_data = ctx->data; 5465 u32 queue_id_base; 5466 u8 seq; 5467 5468 queue_id_base = cfg_data->dcore_tpc_qid_base[dcore] + (inst * NUM_OF_PQ_PER_QMAN); 5469 5470 if (dcore == 0 && inst == (NUM_DCORE0_TPC - 1)) 5471 /* gets last sequence number */ 5472 seq = NUM_OF_DCORES * NUM_OF_TPC_PER_DCORE; 5473 else 5474 seq = dcore * NUM_OF_TPC_PER_DCORE + inst; 5475 5476 gaudi2_init_tpc_cfg(hdev, mmDCORE0_TPC0_CFG_BASE + offset); 5477 gaudi2_init_qman(hdev, mmDCORE0_TPC0_QM_BASE + offset, queue_id_base); 5478 5479 gaudi2->tpc_hw_cap_initialized |= BIT_ULL(HW_CAP_TPC_SHIFT + seq); 5480 } 5481 5482 static void gaudi2_init_tpc(struct hl_device *hdev) 5483 { 5484 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5485 struct gaudi2_tpc_init_cfg_data init_cfg_data; 5486 struct iterate_module_ctx tpc_iter; 5487 5488 if (!hdev->asic_prop.tpc_enabled_mask) 5489 return; 5490 5491 if ((gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK) == HW_CAP_TPC_MASK) 5492 return; 5493 5494 init_cfg_data.dcore_tpc_qid_base[0] = GAUDI2_QUEUE_ID_DCORE0_TPC_0_0; 5495 init_cfg_data.dcore_tpc_qid_base[1] = GAUDI2_QUEUE_ID_DCORE1_TPC_0_0; 5496 init_cfg_data.dcore_tpc_qid_base[2] = GAUDI2_QUEUE_ID_DCORE2_TPC_0_0; 5497 init_cfg_data.dcore_tpc_qid_base[3] = GAUDI2_QUEUE_ID_DCORE3_TPC_0_0; 5498 tpc_iter.fn = &gaudi2_init_tpc_config; 5499 tpc_iter.data = &init_cfg_data; 5500 gaudi2_iterate_tpcs(hdev, &tpc_iter); 5501 } 5502 5503 static void gaudi2_init_rotator(struct hl_device *hdev) 5504 { 5505 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5506 u32 i, reg_base, queue_id; 5507 5508 queue_id = GAUDI2_QUEUE_ID_ROT_0_0; 5509 5510 for (i = 0 ; i < NUM_OF_ROT ; i++, queue_id += NUM_OF_PQ_PER_QMAN) { 5511 reg_base = gaudi2_qm_blocks_bases[queue_id]; 5512 gaudi2_init_qman(hdev, reg_base, queue_id); 5513 5514 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_ROT_SHIFT + i); 5515 } 5516 } 5517 5518 static void gaudi2_init_vdec_brdg_ctrl(struct hl_device *hdev, u64 base_addr, u32 decoder_id) 5519 { 5520 u32 sob_id; 5521 5522 /* VCMD normal interrupt */ 5523 sob_id = GAUDI2_RESERVED_SOB_DEC_NRM_FIRST + decoder_id; 5524 WREG32(base_addr + BRDG_CTRL_NRM_MSIX_LBW_AWADDR, 5525 mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_id * sizeof(u32)); 5526 WREG32(base_addr + BRDG_CTRL_NRM_MSIX_LBW_WDATA, GAUDI2_SOB_INCREMENT_BY_ONE); 5527 5528 /* VCMD abnormal interrupt */ 5529 sob_id = GAUDI2_RESERVED_SOB_DEC_ABNRM_FIRST + decoder_id; 5530 WREG32(base_addr + BRDG_CTRL_ABNRM_MSIX_LBW_AWADDR, 5531 mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_id * sizeof(u32)); 5532 WREG32(base_addr + BRDG_CTRL_ABNRM_MSIX_LBW_WDATA, GAUDI2_SOB_INCREMENT_BY_ONE); 5533 } 5534 5535 static void gaudi2_init_dec(struct hl_device *hdev) 5536 { 5537 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5538 u32 dcore_id, dec_id, dec_bit; 5539 u64 base_addr; 5540 5541 if (!hdev->asic_prop.decoder_enabled_mask) 5542 return; 5543 5544 if ((gaudi2->dec_hw_cap_initialized & HW_CAP_DEC_MASK) == HW_CAP_DEC_MASK) 5545 return; 5546 5547 for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) 5548 for (dec_id = 0 ; dec_id < NUM_OF_DEC_PER_DCORE ; dec_id++) { 5549 dec_bit = dcore_id * NUM_OF_DEC_PER_DCORE + dec_id; 5550 5551 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) 5552 continue; 5553 5554 base_addr = mmDCORE0_DEC0_CMD_BASE + 5555 BRDG_CTRL_BLOCK_OFFSET + 5556 dcore_id * DCORE_OFFSET + 5557 dec_id * DCORE_VDEC_OFFSET; 5558 5559 gaudi2_init_vdec_brdg_ctrl(hdev, base_addr, dec_bit); 5560 5561 gaudi2->dec_hw_cap_initialized |= BIT_ULL(HW_CAP_DEC_SHIFT + dec_bit); 5562 } 5563 5564 for (dec_id = 0 ; dec_id < NUM_OF_PCIE_VDEC ; dec_id++) { 5565 dec_bit = PCIE_DEC_SHIFT + dec_id; 5566 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) 5567 continue; 5568 5569 base_addr = mmPCIE_DEC0_CMD_BASE + BRDG_CTRL_BLOCK_OFFSET + 5570 dec_id * DCORE_VDEC_OFFSET; 5571 5572 gaudi2_init_vdec_brdg_ctrl(hdev, base_addr, dec_bit); 5573 5574 gaudi2->dec_hw_cap_initialized |= BIT_ULL(HW_CAP_DEC_SHIFT + dec_bit); 5575 } 5576 } 5577 5578 static int gaudi2_mmu_update_asid_hop0_addr(struct hl_device *hdev, 5579 u32 stlb_base, u32 asid, u64 phys_addr) 5580 { 5581 u32 status, timeout_usec; 5582 int rc; 5583 5584 if (hdev->pldm || !hdev->pdev) 5585 timeout_usec = GAUDI2_PLDM_MMU_TIMEOUT_USEC; 5586 else 5587 timeout_usec = MMU_CONFIG_TIMEOUT_USEC; 5588 5589 WREG32(stlb_base + STLB_ASID_OFFSET, asid); 5590 WREG32(stlb_base + STLB_HOP0_PA43_12_OFFSET, phys_addr >> MMU_HOP0_PA43_12_SHIFT); 5591 WREG32(stlb_base + STLB_HOP0_PA63_44_OFFSET, phys_addr >> MMU_HOP0_PA63_44_SHIFT); 5592 WREG32(stlb_base + STLB_BUSY_OFFSET, 0x80000000); 5593 5594 rc = hl_poll_timeout( 5595 hdev, 5596 stlb_base + STLB_BUSY_OFFSET, 5597 status, 5598 !(status & 0x80000000), 5599 1000, 5600 timeout_usec); 5601 5602 if (rc) { 5603 dev_err(hdev->dev, "Timeout during MMU hop0 config of asid %d\n", asid); 5604 return rc; 5605 } 5606 5607 return 0; 5608 } 5609 5610 static void gaudi2_mmu_send_invalidate_cache_cmd(struct hl_device *hdev, u32 stlb_base, 5611 u32 start_offset, u32 inv_start_val, 5612 u32 flags) 5613 { 5614 /* clear PMMU mem line cache (only needed in mmu range invalidation) */ 5615 if (flags & MMU_OP_CLEAR_MEMCACHE) 5616 WREG32(mmPMMU_HBW_STLB_MEM_CACHE_INVALIDATION, 0x1); 5617 5618 if (flags & MMU_OP_SKIP_LOW_CACHE_INV) 5619 return; 5620 5621 WREG32(stlb_base + start_offset, inv_start_val); 5622 } 5623 5624 static int gaudi2_mmu_invalidate_cache_status_poll(struct hl_device *hdev, u32 stlb_base, 5625 struct gaudi2_cache_invld_params *inv_params) 5626 { 5627 u32 status, timeout_usec, start_offset; 5628 int rc; 5629 5630 timeout_usec = (hdev->pldm) ? GAUDI2_PLDM_MMU_TIMEOUT_USEC : 5631 GAUDI2_MMU_CACHE_INV_TIMEOUT_USEC; 5632 5633 /* poll PMMU mem line cache (only needed in mmu range invalidation) */ 5634 if (inv_params->flags & MMU_OP_CLEAR_MEMCACHE) { 5635 rc = hl_poll_timeout( 5636 hdev, 5637 mmPMMU_HBW_STLB_MEM_CACHE_INV_STATUS, 5638 status, 5639 status & 0x1, 5640 1000, 5641 timeout_usec); 5642 5643 if (rc) 5644 return rc; 5645 5646 /* Need to manually reset the status to 0 */ 5647 WREG32(mmPMMU_HBW_STLB_MEM_CACHE_INV_STATUS, 0x0); 5648 } 5649 5650 /* Lower cache does not work with cache lines, hence we can skip its 5651 * invalidation upon map and invalidate only upon unmap 5652 */ 5653 if (inv_params->flags & MMU_OP_SKIP_LOW_CACHE_INV) 5654 return 0; 5655 5656 start_offset = inv_params->range_invalidation ? 5657 STLB_RANGE_CACHE_INVALIDATION_OFFSET : STLB_INV_ALL_START_OFFSET; 5658 5659 rc = hl_poll_timeout( 5660 hdev, 5661 stlb_base + start_offset, 5662 status, 5663 !(status & 0x1), 5664 1000, 5665 timeout_usec); 5666 5667 return rc; 5668 } 5669 5670 bool gaudi2_is_hmmu_enabled(struct hl_device *hdev, int dcore_id, int hmmu_id) 5671 { 5672 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5673 u32 hw_cap; 5674 5675 hw_cap = HW_CAP_DCORE0_DMMU0 << (NUM_OF_HMMU_PER_DCORE * dcore_id + hmmu_id); 5676 5677 if (gaudi2->hw_cap_initialized & hw_cap) 5678 return true; 5679 5680 return false; 5681 } 5682 5683 /* this function shall be called only for HMMUs for which capability bit is set */ 5684 static inline u32 get_hmmu_stlb_base(int dcore_id, int hmmu_id) 5685 { 5686 u32 offset; 5687 5688 offset = (u32) (dcore_id * DCORE_OFFSET + hmmu_id * DCORE_HMMU_OFFSET); 5689 return (u32)(mmDCORE0_HMMU0_STLB_BASE + offset); 5690 } 5691 5692 static void gaudi2_mmu_invalidate_cache_trigger(struct hl_device *hdev, u32 stlb_base, 5693 struct gaudi2_cache_invld_params *inv_params) 5694 { 5695 u32 start_offset; 5696 5697 if (inv_params->range_invalidation) { 5698 /* Set the addresses range 5699 * Note: that the start address we set in register, is not included in 5700 * the range of the invalidation, by design. 5701 * that's why we need to set lower address than the one we actually 5702 * want to be included in the range invalidation. 5703 */ 5704 u64 start = inv_params->start_va - 1; 5705 5706 start_offset = STLB_RANGE_CACHE_INVALIDATION_OFFSET; 5707 5708 WREG32(stlb_base + STLB_RANGE_INV_START_LSB_OFFSET, 5709 start >> MMU_RANGE_INV_VA_LSB_SHIFT); 5710 5711 WREG32(stlb_base + STLB_RANGE_INV_START_MSB_OFFSET, 5712 start >> MMU_RANGE_INV_VA_MSB_SHIFT); 5713 5714 WREG32(stlb_base + STLB_RANGE_INV_END_LSB_OFFSET, 5715 inv_params->end_va >> MMU_RANGE_INV_VA_LSB_SHIFT); 5716 5717 WREG32(stlb_base + STLB_RANGE_INV_END_MSB_OFFSET, 5718 inv_params->end_va >> MMU_RANGE_INV_VA_MSB_SHIFT); 5719 } else { 5720 start_offset = STLB_INV_ALL_START_OFFSET; 5721 } 5722 5723 gaudi2_mmu_send_invalidate_cache_cmd(hdev, stlb_base, start_offset, 5724 inv_params->inv_start_val, inv_params->flags); 5725 } 5726 5727 static inline void gaudi2_hmmu_invalidate_cache_trigger(struct hl_device *hdev, 5728 int dcore_id, int hmmu_id, 5729 struct gaudi2_cache_invld_params *inv_params) 5730 { 5731 u32 stlb_base = get_hmmu_stlb_base(dcore_id, hmmu_id); 5732 5733 gaudi2_mmu_invalidate_cache_trigger(hdev, stlb_base, inv_params); 5734 } 5735 5736 static inline int gaudi2_hmmu_invalidate_cache_status_poll(struct hl_device *hdev, 5737 int dcore_id, int hmmu_id, 5738 struct gaudi2_cache_invld_params *inv_params) 5739 { 5740 u32 stlb_base = get_hmmu_stlb_base(dcore_id, hmmu_id); 5741 5742 return gaudi2_mmu_invalidate_cache_status_poll(hdev, stlb_base, inv_params); 5743 } 5744 5745 static int gaudi2_hmmus_invalidate_cache(struct hl_device *hdev, 5746 struct gaudi2_cache_invld_params *inv_params) 5747 { 5748 int dcore_id, hmmu_id; 5749 5750 /* first send all invalidation commands */ 5751 for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) { 5752 for (hmmu_id = 0 ; hmmu_id < NUM_OF_HMMU_PER_DCORE ; hmmu_id++) { 5753 if (!gaudi2_is_hmmu_enabled(hdev, dcore_id, hmmu_id)) 5754 continue; 5755 5756 gaudi2_hmmu_invalidate_cache_trigger(hdev, dcore_id, hmmu_id, inv_params); 5757 } 5758 } 5759 5760 /* next, poll all invalidations status */ 5761 for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) { 5762 for (hmmu_id = 0 ; hmmu_id < NUM_OF_HMMU_PER_DCORE ; hmmu_id++) { 5763 int rc; 5764 5765 if (!gaudi2_is_hmmu_enabled(hdev, dcore_id, hmmu_id)) 5766 continue; 5767 5768 rc = gaudi2_hmmu_invalidate_cache_status_poll(hdev, dcore_id, hmmu_id, 5769 inv_params); 5770 if (rc) 5771 return rc; 5772 } 5773 } 5774 5775 return 0; 5776 } 5777 5778 static int gaudi2_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, u32 flags) 5779 { 5780 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5781 struct gaudi2_cache_invld_params invld_params; 5782 int rc = 0; 5783 5784 if (hdev->reset_info.hard_reset_pending) 5785 return rc; 5786 5787 invld_params.range_invalidation = false; 5788 invld_params.inv_start_val = 1; 5789 5790 if ((flags & MMU_OP_USERPTR) && (gaudi2->hw_cap_initialized & HW_CAP_PMMU)) { 5791 invld_params.flags = flags; 5792 gaudi2_mmu_invalidate_cache_trigger(hdev, mmPMMU_HBW_STLB_BASE, &invld_params); 5793 rc = gaudi2_mmu_invalidate_cache_status_poll(hdev, mmPMMU_HBW_STLB_BASE, 5794 &invld_params); 5795 } else if (flags & MMU_OP_PHYS_PACK) { 5796 invld_params.flags = 0; 5797 rc = gaudi2_hmmus_invalidate_cache(hdev, &invld_params); 5798 } 5799 5800 return rc; 5801 } 5802 5803 static int gaudi2_mmu_invalidate_cache_range(struct hl_device *hdev, bool is_hard, 5804 u32 flags, u32 asid, u64 va, u64 size) 5805 { 5806 struct gaudi2_cache_invld_params invld_params = {0}; 5807 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5808 u64 start_va, end_va; 5809 u32 inv_start_val; 5810 int rc = 0; 5811 5812 if (hdev->reset_info.hard_reset_pending) 5813 return 0; 5814 5815 inv_start_val = (1 << MMU_RANGE_INV_EN_SHIFT | 5816 1 << MMU_RANGE_INV_ASID_EN_SHIFT | 5817 asid << MMU_RANGE_INV_ASID_SHIFT); 5818 start_va = va; 5819 end_va = start_va + size; 5820 5821 if ((flags & MMU_OP_USERPTR) && (gaudi2->hw_cap_initialized & HW_CAP_PMMU)) { 5822 /* As range invalidation does not support zero address we will 5823 * do full invalidation in this case 5824 */ 5825 if (start_va) { 5826 invld_params.range_invalidation = true; 5827 invld_params.start_va = start_va; 5828 invld_params.end_va = end_va; 5829 invld_params.inv_start_val = inv_start_val; 5830 invld_params.flags = flags | MMU_OP_CLEAR_MEMCACHE; 5831 } else { 5832 invld_params.range_invalidation = false; 5833 invld_params.inv_start_val = 1; 5834 invld_params.flags = flags; 5835 } 5836 5837 5838 gaudi2_mmu_invalidate_cache_trigger(hdev, mmPMMU_HBW_STLB_BASE, &invld_params); 5839 rc = gaudi2_mmu_invalidate_cache_status_poll(hdev, mmPMMU_HBW_STLB_BASE, 5840 &invld_params); 5841 if (rc) 5842 return rc; 5843 5844 } else if (flags & MMU_OP_PHYS_PACK) { 5845 invld_params.start_va = gaudi2_mmu_scramble_addr(hdev, start_va); 5846 invld_params.end_va = gaudi2_mmu_scramble_addr(hdev, end_va); 5847 invld_params.inv_start_val = inv_start_val; 5848 invld_params.flags = flags; 5849 rc = gaudi2_hmmus_invalidate_cache(hdev, &invld_params); 5850 } 5851 5852 return rc; 5853 } 5854 5855 static int gaudi2_mmu_update_hop0_addr(struct hl_device *hdev, u32 stlb_base) 5856 { 5857 struct asic_fixed_properties *prop = &hdev->asic_prop; 5858 u64 hop0_addr; 5859 u32 asid, max_asid = prop->max_asid; 5860 int rc; 5861 5862 /* it takes too much time to init all of the ASIDs on palladium */ 5863 if (hdev->pldm) 5864 max_asid = min((u32) 8, max_asid); 5865 5866 for (asid = 0 ; asid < max_asid ; asid++) { 5867 hop0_addr = hdev->mmu_priv.hr.mmu_asid_hop0[asid].phys_addr; 5868 rc = gaudi2_mmu_update_asid_hop0_addr(hdev, stlb_base, asid, hop0_addr); 5869 if (rc) { 5870 dev_err(hdev->dev, "failed to set hop0 addr for asid %d\n", asid); 5871 return rc; 5872 } 5873 } 5874 5875 return 0; 5876 } 5877 5878 static int gaudi2_mmu_init_common(struct hl_device *hdev, u32 mmu_base, u32 stlb_base) 5879 { 5880 u32 status, timeout_usec; 5881 int rc; 5882 5883 if (hdev->pldm || !hdev->pdev) 5884 timeout_usec = GAUDI2_PLDM_MMU_TIMEOUT_USEC; 5885 else 5886 timeout_usec = GAUDI2_MMU_CACHE_INV_TIMEOUT_USEC; 5887 5888 WREG32(stlb_base + STLB_INV_ALL_START_OFFSET, 1); 5889 5890 rc = hl_poll_timeout( 5891 hdev, 5892 stlb_base + STLB_SRAM_INIT_OFFSET, 5893 status, 5894 !status, 5895 1000, 5896 timeout_usec); 5897 5898 if (rc) 5899 dev_notice_ratelimited(hdev->dev, "Timeout when waiting for MMU SRAM init\n"); 5900 5901 rc = gaudi2_mmu_update_hop0_addr(hdev, stlb_base); 5902 if (rc) 5903 return rc; 5904 5905 WREG32(mmu_base + MMU_BYPASS_OFFSET, 0); 5906 5907 rc = hl_poll_timeout( 5908 hdev, 5909 stlb_base + STLB_INV_ALL_START_OFFSET, 5910 status, 5911 !status, 5912 1000, 5913 timeout_usec); 5914 5915 if (rc) 5916 dev_notice_ratelimited(hdev->dev, "Timeout when waiting for MMU invalidate all\n"); 5917 5918 WREG32(mmu_base + MMU_ENABLE_OFFSET, 1); 5919 5920 return rc; 5921 } 5922 5923 static int gaudi2_pci_mmu_init(struct hl_device *hdev) 5924 { 5925 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5926 u32 mmu_base, stlb_base; 5927 int rc; 5928 5929 if (gaudi2->hw_cap_initialized & HW_CAP_PMMU) 5930 return 0; 5931 5932 mmu_base = mmPMMU_HBW_MMU_BASE; 5933 stlb_base = mmPMMU_HBW_STLB_BASE; 5934 5935 RMWREG32_SHIFTED(stlb_base + STLB_HOP_CONFIGURATION_OFFSET, 5936 (0 << PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_HOP_SHIFT) | 5937 (5 << PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_SHIFT) | 5938 (4 << PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_SHIFT) | 5939 (5 << PMMU_HBW_STLB_HOP_CONFIGURATION_LAST_HOP_SHIFT) | 5940 (5 << PMMU_HBW_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_SHIFT), 5941 PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_HOP_MASK | 5942 PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_MASK | 5943 PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_MASK | 5944 PMMU_HBW_STLB_HOP_CONFIGURATION_LAST_HOP_MASK | 5945 PMMU_HBW_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_MASK); 5946 5947 WREG32(stlb_base + STLB_LL_LOOKUP_MASK_63_32_OFFSET, 0); 5948 5949 if (PAGE_SIZE == SZ_64K) { 5950 /* Set page sizes to 64K on hop5 and 16M on hop4 + enable 8 bit hops */ 5951 RMWREG32_SHIFTED(mmu_base + MMU_STATIC_MULTI_PAGE_SIZE_OFFSET, 5952 FIELD_PREP(DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP5_PAGE_SIZE_MASK, 4) | 5953 FIELD_PREP(DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP4_PAGE_SIZE_MASK, 3) | 5954 FIELD_PREP( 5955 DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_CFG_8_BITS_HOP_MODE_EN_MASK, 5956 1), 5957 DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP5_PAGE_SIZE_MASK | 5958 DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP4_PAGE_SIZE_MASK | 5959 DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_CFG_8_BITS_HOP_MODE_EN_MASK); 5960 } 5961 5962 WREG32(mmu_base + MMU_SPI_SEI_MASK_OFFSET, GAUDI2_PMMU_SPI_SEI_ENABLE_MASK); 5963 5964 rc = gaudi2_mmu_init_common(hdev, mmu_base, stlb_base); 5965 if (rc) 5966 return rc; 5967 5968 gaudi2->hw_cap_initialized |= HW_CAP_PMMU; 5969 5970 return 0; 5971 } 5972 5973 static int gaudi2_dcore_hmmu_init(struct hl_device *hdev, int dcore_id, 5974 int hmmu_id) 5975 { 5976 struct asic_fixed_properties *prop = &hdev->asic_prop; 5977 struct gaudi2_device *gaudi2 = hdev->asic_specific; 5978 u32 offset, mmu_base, stlb_base, hw_cap; 5979 u8 dmmu_seq; 5980 int rc; 5981 5982 dmmu_seq = NUM_OF_HMMU_PER_DCORE * dcore_id + hmmu_id; 5983 hw_cap = HW_CAP_DCORE0_DMMU0 << dmmu_seq; 5984 5985 /* 5986 * return if DMMU is already initialized or if it's not out of 5987 * isolation (due to cluster binning) 5988 */ 5989 if ((gaudi2->hw_cap_initialized & hw_cap) || !(prop->hmmu_hif_enabled_mask & BIT(dmmu_seq))) 5990 return 0; 5991 5992 offset = (u32) (dcore_id * DCORE_OFFSET + hmmu_id * DCORE_HMMU_OFFSET); 5993 mmu_base = mmDCORE0_HMMU0_MMU_BASE + offset; 5994 stlb_base = mmDCORE0_HMMU0_STLB_BASE + offset; 5995 5996 RMWREG32(mmu_base + MMU_STATIC_MULTI_PAGE_SIZE_OFFSET, 5 /* 64MB */, 5997 MMU_STATIC_MULTI_PAGE_SIZE_HOP4_PAGE_SIZE_MASK); 5998 5999 RMWREG32_SHIFTED(stlb_base + STLB_HOP_CONFIGURATION_OFFSET, 6000 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_HOP_MASK, 0) | 6001 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_MASK, 3) | 6002 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_MASK, 3) | 6003 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_LAST_HOP_MASK, 3) | 6004 FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_MASK, 3), 6005 DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_HOP_MASK | 6006 DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_MASK | 6007 DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_MASK | 6008 DCORE0_HMMU0_STLB_HOP_CONFIGURATION_LAST_HOP_MASK | 6009 DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FOLLOWER_HOP_MASK); 6010 6011 RMWREG32(stlb_base + STLB_HOP_CONFIGURATION_OFFSET, 1, 6012 STLB_HOP_CONFIGURATION_ONLY_LARGE_PAGE_MASK); 6013 6014 WREG32(mmu_base + MMU_SPI_SEI_MASK_OFFSET, GAUDI2_HMMU_SPI_SEI_ENABLE_MASK); 6015 6016 rc = gaudi2_mmu_init_common(hdev, mmu_base, stlb_base); 6017 if (rc) 6018 return rc; 6019 6020 gaudi2->hw_cap_initialized |= hw_cap; 6021 6022 return 0; 6023 } 6024 6025 static int gaudi2_hbm_mmu_init(struct hl_device *hdev) 6026 { 6027 int rc, dcore_id, hmmu_id; 6028 6029 for (dcore_id = 0 ; dcore_id < NUM_OF_DCORES ; dcore_id++) 6030 for (hmmu_id = 0 ; hmmu_id < NUM_OF_HMMU_PER_DCORE; hmmu_id++) { 6031 rc = gaudi2_dcore_hmmu_init(hdev, dcore_id, hmmu_id); 6032 if (rc) 6033 return rc; 6034 } 6035 6036 return 0; 6037 } 6038 6039 static int gaudi2_mmu_init(struct hl_device *hdev) 6040 { 6041 int rc; 6042 6043 rc = gaudi2_pci_mmu_init(hdev); 6044 if (rc) 6045 return rc; 6046 6047 rc = gaudi2_hbm_mmu_init(hdev); 6048 if (rc) 6049 return rc; 6050 6051 return 0; 6052 } 6053 6054 static int gaudi2_hw_init(struct hl_device *hdev) 6055 { 6056 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6057 int rc; 6058 6059 /* Let's mark in the H/W that we have reached this point. We check 6060 * this value in the reset_before_init function to understand whether 6061 * we need to reset the chip before doing H/W init. This register is 6062 * cleared by the H/W upon H/W reset 6063 */ 6064 WREG32(mmHW_STATE, HL_DEVICE_HW_STATE_DIRTY); 6065 6066 /* Perform read from the device to make sure device is up */ 6067 RREG32(mmHW_STATE); 6068 6069 /* If iATU is done by FW, the HBM bar ALWAYS points to DRAM_PHYS_BASE. 6070 * So we set it here and if anyone tries to move it later to 6071 * a different address, there will be an error 6072 */ 6073 if (hdev->asic_prop.iatu_done_by_fw) 6074 gaudi2->dram_bar_cur_addr = DRAM_PHYS_BASE; 6075 6076 /* 6077 * Before pushing u-boot/linux to device, need to set the hbm bar to 6078 * base address of dram 6079 */ 6080 if (gaudi2_set_hbm_bar_base(hdev, DRAM_PHYS_BASE) == U64_MAX) { 6081 dev_err(hdev->dev, "failed to map HBM bar to DRAM base address\n"); 6082 return -EIO; 6083 } 6084 6085 rc = gaudi2_init_cpu(hdev); 6086 if (rc) { 6087 dev_err(hdev->dev, "failed to initialize CPU\n"); 6088 return rc; 6089 } 6090 6091 gaudi2_init_scrambler_hbm(hdev); 6092 gaudi2_init_kdma(hdev); 6093 6094 rc = gaudi2_init_cpu_queues(hdev, GAUDI2_CPU_TIMEOUT_USEC); 6095 if (rc) { 6096 dev_err(hdev->dev, "failed to initialize CPU H/W queues %d\n", rc); 6097 return rc; 6098 } 6099 6100 rc = gaudi2->cpucp_info_get(hdev); 6101 if (rc) { 6102 dev_err(hdev->dev, "Failed to get cpucp info\n"); 6103 return rc; 6104 } 6105 6106 rc = gaudi2_mmu_init(hdev); 6107 if (rc) 6108 return rc; 6109 6110 gaudi2_init_pdma(hdev); 6111 gaudi2_init_edma(hdev); 6112 gaudi2_init_sm(hdev); 6113 gaudi2_init_tpc(hdev); 6114 gaudi2_init_mme(hdev); 6115 gaudi2_init_rotator(hdev); 6116 gaudi2_init_dec(hdev); 6117 gaudi2_enable_timestamp(hdev); 6118 6119 rc = gaudi2_coresight_init(hdev); 6120 if (rc) 6121 goto disable_queues; 6122 6123 rc = gaudi2_enable_msix(hdev); 6124 if (rc) 6125 goto disable_queues; 6126 6127 /* Perform read from the device to flush all configuration */ 6128 RREG32(mmHW_STATE); 6129 6130 return 0; 6131 6132 disable_queues: 6133 gaudi2_disable_dma_qmans(hdev); 6134 gaudi2_disable_mme_qmans(hdev); 6135 gaudi2_disable_tpc_qmans(hdev); 6136 gaudi2_disable_rot_qmans(hdev); 6137 gaudi2_disable_nic_qmans(hdev); 6138 6139 gaudi2_disable_timestamp(hdev); 6140 6141 return rc; 6142 } 6143 6144 /** 6145 * gaudi2_send_hard_reset_cmd - common function to handle reset 6146 * 6147 * @hdev: pointer to the habanalabs device structure 6148 * 6149 * This function handles the various possible scenarios for reset. 6150 * It considers if reset is handled by driver\FW and what FW components are loaded 6151 */ 6152 static void gaudi2_send_hard_reset_cmd(struct hl_device *hdev) 6153 { 6154 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 6155 bool heartbeat_reset, preboot_only, cpu_initialized = false; 6156 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6157 u32 cpu_boot_status; 6158 6159 preboot_only = (hdev->fw_loader.fw_comp_loaded == FW_TYPE_PREBOOT_CPU); 6160 heartbeat_reset = (hdev->reset_info.curr_reset_cause == HL_RESET_CAUSE_HEARTBEAT); 6161 6162 /* 6163 * Handle corner case where failure was at cpu management app load, 6164 * and driver didn't detect any failure while loading the FW, 6165 * then at such scenario driver will send only HALT_MACHINE 6166 * and no one will respond to this request since FW already back to preboot 6167 * and it cannot handle such cmd. 6168 * In this case next time the management app loads it'll check on events register 6169 * which will still have the halt indication, and will reboot the device. 6170 * The solution is to let preboot clear all relevant registers before next boot 6171 * once driver send COMMS_RST_DEV. 6172 */ 6173 cpu_boot_status = RREG32(mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS); 6174 6175 if (gaudi2 && (gaudi2->hw_cap_initialized & HW_CAP_CPU) && 6176 (cpu_boot_status == CPU_BOOT_STATUS_SRAM_AVAIL)) 6177 cpu_initialized = true; 6178 6179 /* 6180 * when Linux/Bootfit exist this write to the SP can be interpreted in 2 ways: 6181 * 1. FW reset: FW initiate the reset sequence 6182 * 2. driver reset: FW will start HALT sequence (the preparations for the 6183 * reset but not the reset itself as it is not implemented 6184 * on their part) and LKD will wait to let FW complete the 6185 * sequence before issuing the reset 6186 */ 6187 if (!preboot_only && cpu_initialized) { 6188 WREG32(le32_to_cpu(dyn_regs->gic_host_halt_irq), 6189 gaudi2_irq_map_table[GAUDI2_EVENT_CPU_HALT_MACHINE].cpu_id); 6190 6191 msleep(GAUDI2_CPU_RESET_WAIT_MSEC); 6192 } 6193 6194 /* 6195 * When working with preboot (without Linux/Boot fit) we can 6196 * communicate only using the COMMS commands to issue halt/reset. 6197 * 6198 * For the case in which we are working with Linux/Bootfit this is a hail-mary 6199 * attempt to revive the card in the small chance that the f/w has 6200 * experienced a watchdog event, which caused it to return back to preboot. 6201 * In that case, triggering reset through GIC won't help. We need to 6202 * trigger the reset as if Linux wasn't loaded. 6203 * 6204 * We do it only if the reset cause was HB, because that would be the 6205 * indication of such an event. 6206 * 6207 * In case watchdog hasn't expired but we still got HB, then this won't 6208 * do any damage. 6209 */ 6210 6211 if (heartbeat_reset || preboot_only || !cpu_initialized) { 6212 if (hdev->asic_prop.hard_reset_done_by_fw) 6213 hl_fw_ask_hard_reset_without_linux(hdev); 6214 else 6215 hl_fw_ask_halt_machine_without_linux(hdev); 6216 } 6217 } 6218 6219 /** 6220 * gaudi2_execute_hard_reset - execute hard reset by driver/FW 6221 * 6222 * @hdev: pointer to the habanalabs device structure 6223 * 6224 * This function executes hard reset based on if driver/FW should do the reset 6225 */ 6226 static void gaudi2_execute_hard_reset(struct hl_device *hdev) 6227 { 6228 if (hdev->asic_prop.hard_reset_done_by_fw) { 6229 gaudi2_send_hard_reset_cmd(hdev); 6230 return; 6231 } 6232 6233 /* Set device to handle FLR by H/W as we will put the device 6234 * CPU to halt mode 6235 */ 6236 WREG32(mmPCIE_AUX_FLR_CTRL, 6237 (PCIE_AUX_FLR_CTRL_HW_CTRL_MASK | PCIE_AUX_FLR_CTRL_INT_MASK_MASK)); 6238 6239 gaudi2_send_hard_reset_cmd(hdev); 6240 6241 WREG32(mmPSOC_RESET_CONF_SW_ALL_RST, 1); 6242 } 6243 6244 static int gaudi2_get_soft_rst_done_indication(struct hl_device *hdev, u32 poll_timeout_us) 6245 { 6246 int i, rc = 0; 6247 u32 reg_val; 6248 6249 for (i = 0 ; i < GAUDI2_RESET_POLL_CNT ; i++) 6250 rc = hl_poll_timeout( 6251 hdev, 6252 mmCPU_RST_STATUS_TO_HOST, 6253 reg_val, 6254 reg_val == CPU_RST_STATUS_SOFT_RST_DONE, 6255 1000, 6256 poll_timeout_us); 6257 6258 if (rc) 6259 dev_err(hdev->dev, "Timeout while waiting for FW to complete soft reset (0x%x)\n", 6260 reg_val); 6261 return rc; 6262 } 6263 6264 /** 6265 * gaudi2_execute_soft_reset - execute soft reset by driver/FW 6266 * 6267 * @hdev: pointer to the habanalabs device structure 6268 * @driver_performs_reset: true if driver should perform reset instead of f/w. 6269 * @poll_timeout_us: time to wait for response from f/w. 6270 * 6271 * This function executes soft reset based on if driver/FW should do the reset 6272 */ 6273 static int gaudi2_execute_soft_reset(struct hl_device *hdev, bool driver_performs_reset, 6274 u32 poll_timeout_us) 6275 { 6276 int rc; 6277 6278 if (!driver_performs_reset) { 6279 if (hl_is_fw_sw_ver_below(hdev, 1, 10)) { 6280 /* set SP to indicate reset request sent to FW */ 6281 WREG32(mmCPU_RST_STATUS_TO_HOST, CPU_RST_STATUS_NA); 6282 6283 WREG32(mmGIC_HOST_SOFT_RST_IRQ_POLL_REG, 6284 gaudi2_irq_map_table[GAUDI2_EVENT_CPU_SOFT_RESET].cpu_id); 6285 6286 /* wait for f/w response */ 6287 rc = gaudi2_get_soft_rst_done_indication(hdev, poll_timeout_us); 6288 } else { 6289 rc = hl_fw_send_soft_reset(hdev); 6290 } 6291 return rc; 6292 } 6293 6294 /* Block access to engines, QMANs and SM during reset, these 6295 * RRs will be reconfigured after soft reset. 6296 * PCIE_MSIX is left unsecured to allow NIC packets processing during the reset. 6297 */ 6298 gaudi2_write_rr_to_all_lbw_rtrs(hdev, RR_TYPE_LONG, NUM_LONG_LBW_RR - 1, 6299 mmDCORE0_TPC0_QM_DCCM_BASE, mmPCIE_MSIX_BASE); 6300 6301 gaudi2_write_rr_to_all_lbw_rtrs(hdev, RR_TYPE_LONG, NUM_LONG_LBW_RR - 2, 6302 mmPCIE_MSIX_BASE + HL_BLOCK_SIZE, 6303 mmPCIE_VDEC1_MSTR_IF_RR_SHRD_HBW_BASE + HL_BLOCK_SIZE); 6304 6305 WREG32(mmPSOC_RESET_CONF_SOFT_RST, 1); 6306 return 0; 6307 } 6308 6309 static void gaudi2_poll_btm_indication(struct hl_device *hdev, u32 poll_timeout_us) 6310 { 6311 int i, rc = 0; 6312 u32 reg_val; 6313 6314 /* We poll the BTM done indication multiple times after reset due to 6315 * a HW errata 'GAUDI2_0300' 6316 */ 6317 for (i = 0 ; i < GAUDI2_RESET_POLL_CNT ; i++) 6318 rc = hl_poll_timeout( 6319 hdev, 6320 mmPSOC_GLOBAL_CONF_BTM_FSM, 6321 reg_val, 6322 reg_val == 0, 6323 1000, 6324 poll_timeout_us); 6325 6326 if (rc) 6327 dev_err(hdev->dev, "Timeout while waiting for device to reset 0x%x\n", reg_val); 6328 } 6329 6330 static int gaudi2_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset) 6331 { 6332 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6333 u32 poll_timeout_us, reset_sleep_ms; 6334 bool driver_performs_reset = false; 6335 int rc; 6336 6337 if (hdev->pldm) { 6338 reset_sleep_ms = hard_reset ? GAUDI2_PLDM_HRESET_TIMEOUT_MSEC : 6339 GAUDI2_PLDM_SRESET_TIMEOUT_MSEC; 6340 poll_timeout_us = GAUDI2_PLDM_RESET_POLL_TIMEOUT_USEC; 6341 } else { 6342 reset_sleep_ms = GAUDI2_RESET_TIMEOUT_MSEC; 6343 poll_timeout_us = GAUDI2_RESET_POLL_TIMEOUT_USEC; 6344 } 6345 6346 if (fw_reset) 6347 goto skip_reset; 6348 6349 gaudi2_reset_arcs(hdev); 6350 6351 if (hard_reset) { 6352 driver_performs_reset = !hdev->asic_prop.hard_reset_done_by_fw; 6353 gaudi2_execute_hard_reset(hdev); 6354 } else { 6355 /* 6356 * As we have to support also work with preboot only (which does not supports 6357 * soft reset) we have to make sure that security is disabled before letting driver 6358 * do the reset. user shall control the BFE flags to avoid asking soft reset in 6359 * secured device with preboot only. 6360 */ 6361 driver_performs_reset = (hdev->fw_components == FW_TYPE_PREBOOT_CPU && 6362 !hdev->asic_prop.fw_security_enabled); 6363 rc = gaudi2_execute_soft_reset(hdev, driver_performs_reset, poll_timeout_us); 6364 if (rc) 6365 return rc; 6366 } 6367 6368 skip_reset: 6369 if (driver_performs_reset || hard_reset) { 6370 /* 6371 * Instead of waiting for BTM indication we should wait for preboot ready: 6372 * Consider the below scenario: 6373 * 1. FW update is being triggered 6374 * - setting the dirty bit 6375 * 2. hard reset will be triggered due to the dirty bit 6376 * 3. FW initiates the reset: 6377 * - dirty bit cleared 6378 * - BTM indication cleared 6379 * - preboot ready indication cleared 6380 * 4. during hard reset: 6381 * - BTM indication will be set 6382 * - BIST test performed and another reset triggered 6383 * 5. only after this reset the preboot will set the preboot ready 6384 * 6385 * when polling on BTM indication alone we can lose sync with FW while trying to 6386 * communicate with FW that is during reset. 6387 * to overcome this we will always wait to preboot ready indication 6388 */ 6389 6390 /* without this sleep reset will not work */ 6391 msleep(reset_sleep_ms); 6392 6393 if (hdev->fw_components & FW_TYPE_PREBOOT_CPU) 6394 hl_fw_wait_preboot_ready(hdev); 6395 else 6396 gaudi2_poll_btm_indication(hdev, poll_timeout_us); 6397 } 6398 6399 if (!gaudi2) 6400 return 0; 6401 6402 gaudi2->dec_hw_cap_initialized &= ~(HW_CAP_DEC_MASK); 6403 gaudi2->tpc_hw_cap_initialized &= ~(HW_CAP_TPC_MASK); 6404 6405 /* 6406 * Clear NIC capability mask in order for driver to re-configure 6407 * NIC QMANs. NIC ports will not be re-configured during soft 6408 * reset as we call gaudi2_nic_init only during hard reset 6409 */ 6410 gaudi2->nic_hw_cap_initialized &= ~(HW_CAP_NIC_MASK); 6411 6412 if (hard_reset) { 6413 gaudi2->hw_cap_initialized &= 6414 ~(HW_CAP_DRAM | HW_CAP_CLK_GATE | HW_CAP_HBM_SCRAMBLER_MASK | 6415 HW_CAP_PMMU | HW_CAP_CPU | HW_CAP_CPU_Q | 6416 HW_CAP_SRAM_SCRAMBLER | HW_CAP_DMMU_MASK | 6417 HW_CAP_PDMA_MASK | HW_CAP_EDMA_MASK | HW_CAP_KDMA | 6418 HW_CAP_MME_MASK | HW_CAP_ROT_MASK); 6419 6420 memset(gaudi2->events_stat, 0, sizeof(gaudi2->events_stat)); 6421 } else { 6422 gaudi2->hw_cap_initialized &= 6423 ~(HW_CAP_CLK_GATE | HW_CAP_HBM_SCRAMBLER_SW_RESET | 6424 HW_CAP_PDMA_MASK | HW_CAP_EDMA_MASK | HW_CAP_MME_MASK | 6425 HW_CAP_ROT_MASK); 6426 } 6427 return 0; 6428 } 6429 6430 static int gaudi2_suspend(struct hl_device *hdev) 6431 { 6432 int rc; 6433 6434 rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0); 6435 if (rc) 6436 dev_err(hdev->dev, "Failed to disable PCI access from CPU\n"); 6437 6438 return rc; 6439 } 6440 6441 static int gaudi2_resume(struct hl_device *hdev) 6442 { 6443 return gaudi2_init_iatu(hdev); 6444 } 6445 6446 static int gaudi2_mmap(struct hl_device *hdev, struct vm_area_struct *vma, 6447 void *cpu_addr, dma_addr_t dma_addr, size_t size) 6448 { 6449 int rc; 6450 6451 vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | 6452 VM_DONTCOPY | VM_NORESERVE); 6453 6454 #ifdef _HAS_DMA_MMAP_COHERENT 6455 6456 rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr, dma_addr, size); 6457 if (rc) 6458 dev_err(hdev->dev, "dma_mmap_coherent error %d", rc); 6459 6460 #else 6461 6462 rc = remap_pfn_range(vma, vma->vm_start, 6463 virt_to_phys(cpu_addr) >> PAGE_SHIFT, 6464 size, vma->vm_page_prot); 6465 if (rc) 6466 dev_err(hdev->dev, "remap_pfn_range error %d", rc); 6467 6468 #endif 6469 6470 return rc; 6471 } 6472 6473 static bool gaudi2_is_queue_enabled(struct hl_device *hdev, u32 hw_queue_id) 6474 { 6475 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6476 u64 hw_cap_mask = 0; 6477 u64 hw_tpc_cap_bit = 0; 6478 u64 hw_nic_cap_bit = 0; 6479 u64 hw_test_cap_bit = 0; 6480 6481 switch (hw_queue_id) { 6482 case GAUDI2_QUEUE_ID_PDMA_0_0: 6483 case GAUDI2_QUEUE_ID_PDMA_0_1: 6484 case GAUDI2_QUEUE_ID_PDMA_1_0: 6485 hw_cap_mask = HW_CAP_PDMA_MASK; 6486 break; 6487 case GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3: 6488 hw_test_cap_bit = HW_CAP_EDMA_SHIFT + 6489 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0) >> 2); 6490 break; 6491 case GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3: 6492 hw_test_cap_bit = HW_CAP_EDMA_SHIFT + NUM_OF_EDMA_PER_DCORE + 6493 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0) >> 2); 6494 break; 6495 case GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3: 6496 hw_test_cap_bit = HW_CAP_EDMA_SHIFT + 2 * NUM_OF_EDMA_PER_DCORE + 6497 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0) >> 2); 6498 break; 6499 case GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3: 6500 hw_test_cap_bit = HW_CAP_EDMA_SHIFT + 3 * NUM_OF_EDMA_PER_DCORE + 6501 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0) >> 2); 6502 break; 6503 6504 case GAUDI2_QUEUE_ID_DCORE0_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE0_MME_0_3: 6505 hw_test_cap_bit = HW_CAP_MME_SHIFT; 6506 break; 6507 6508 case GAUDI2_QUEUE_ID_DCORE1_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE1_MME_0_3: 6509 hw_test_cap_bit = HW_CAP_MME_SHIFT + 1; 6510 break; 6511 6512 case GAUDI2_QUEUE_ID_DCORE2_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE2_MME_0_3: 6513 hw_test_cap_bit = HW_CAP_MME_SHIFT + 2; 6514 break; 6515 6516 case GAUDI2_QUEUE_ID_DCORE3_MME_0_0 ... GAUDI2_QUEUE_ID_DCORE3_MME_0_3: 6517 hw_test_cap_bit = HW_CAP_MME_SHIFT + 3; 6518 break; 6519 6520 case GAUDI2_QUEUE_ID_DCORE0_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE0_TPC_5_3: 6521 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + 6522 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE0_TPC_0_0) >> 2); 6523 6524 /* special case where cap bit refers to the first queue id */ 6525 if (!hw_tpc_cap_bit) 6526 return !!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(0)); 6527 break; 6528 6529 case GAUDI2_QUEUE_ID_DCORE1_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE1_TPC_5_3: 6530 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + NUM_OF_TPC_PER_DCORE + 6531 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE1_TPC_0_0) >> 2); 6532 break; 6533 6534 case GAUDI2_QUEUE_ID_DCORE2_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE2_TPC_5_3: 6535 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + (2 * NUM_OF_TPC_PER_DCORE) + 6536 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE2_TPC_0_0) >> 2); 6537 break; 6538 6539 case GAUDI2_QUEUE_ID_DCORE3_TPC_0_0 ... GAUDI2_QUEUE_ID_DCORE3_TPC_5_3: 6540 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + (3 * NUM_OF_TPC_PER_DCORE) + 6541 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE3_TPC_0_0) >> 2); 6542 break; 6543 6544 case GAUDI2_QUEUE_ID_DCORE0_TPC_6_0 ... GAUDI2_QUEUE_ID_DCORE0_TPC_6_3: 6545 hw_tpc_cap_bit = HW_CAP_TPC_SHIFT + (4 * NUM_OF_TPC_PER_DCORE); 6546 break; 6547 6548 case GAUDI2_QUEUE_ID_ROT_0_0 ... GAUDI2_QUEUE_ID_ROT_1_3: 6549 hw_test_cap_bit = HW_CAP_ROT_SHIFT + ((hw_queue_id - GAUDI2_QUEUE_ID_ROT_0_0) >> 2); 6550 break; 6551 6552 case GAUDI2_QUEUE_ID_NIC_0_0 ... GAUDI2_QUEUE_ID_NIC_23_3: 6553 hw_nic_cap_bit = HW_CAP_NIC_SHIFT + ((hw_queue_id - GAUDI2_QUEUE_ID_NIC_0_0) >> 2); 6554 6555 /* special case where cap bit refers to the first queue id */ 6556 if (!hw_nic_cap_bit) 6557 return !!(gaudi2->nic_hw_cap_initialized & BIT_ULL(0)); 6558 break; 6559 6560 case GAUDI2_QUEUE_ID_CPU_PQ: 6561 return !!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q); 6562 6563 default: 6564 return false; 6565 } 6566 6567 if (hw_tpc_cap_bit) 6568 return !!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(hw_tpc_cap_bit)); 6569 6570 if (hw_nic_cap_bit) 6571 return !!(gaudi2->nic_hw_cap_initialized & BIT_ULL(hw_nic_cap_bit)); 6572 6573 if (hw_test_cap_bit) 6574 hw_cap_mask = BIT_ULL(hw_test_cap_bit); 6575 6576 return !!(gaudi2->hw_cap_initialized & hw_cap_mask); 6577 } 6578 6579 static bool gaudi2_is_arc_enabled(struct hl_device *hdev, u64 arc_id) 6580 { 6581 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6582 6583 switch (arc_id) { 6584 case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC5: 6585 case CPU_ID_MME_QMAN_ARC0...CPU_ID_ROT_QMAN_ARC1: 6586 return !!(gaudi2->active_hw_arc & BIT_ULL(arc_id)); 6587 6588 case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24: 6589 return !!(gaudi2->active_tpc_arc & BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0)); 6590 6591 case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23: 6592 return !!(gaudi2->active_nic_arc & BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0)); 6593 6594 default: 6595 return false; 6596 } 6597 } 6598 6599 static void gaudi2_clr_arc_id_cap(struct hl_device *hdev, u64 arc_id) 6600 { 6601 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6602 6603 switch (arc_id) { 6604 case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC5: 6605 case CPU_ID_MME_QMAN_ARC0...CPU_ID_ROT_QMAN_ARC1: 6606 gaudi2->active_hw_arc &= ~(BIT_ULL(arc_id)); 6607 break; 6608 6609 case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24: 6610 gaudi2->active_tpc_arc &= ~(BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0)); 6611 break; 6612 6613 case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23: 6614 gaudi2->active_nic_arc &= ~(BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0)); 6615 break; 6616 6617 default: 6618 return; 6619 } 6620 } 6621 6622 static void gaudi2_set_arc_id_cap(struct hl_device *hdev, u64 arc_id) 6623 { 6624 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6625 6626 switch (arc_id) { 6627 case CPU_ID_SCHED_ARC0 ... CPU_ID_SCHED_ARC5: 6628 case CPU_ID_MME_QMAN_ARC0...CPU_ID_ROT_QMAN_ARC1: 6629 gaudi2->active_hw_arc |= BIT_ULL(arc_id); 6630 break; 6631 6632 case CPU_ID_TPC_QMAN_ARC0...CPU_ID_TPC_QMAN_ARC24: 6633 gaudi2->active_tpc_arc |= BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0); 6634 break; 6635 6636 case CPU_ID_NIC_QMAN_ARC0...CPU_ID_NIC_QMAN_ARC23: 6637 gaudi2->active_nic_arc |= BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0); 6638 break; 6639 6640 default: 6641 return; 6642 } 6643 } 6644 6645 static void gaudi2_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi) 6646 { 6647 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 6648 u32 pq_offset, reg_base, db_reg_offset, db_value; 6649 6650 if (hw_queue_id != GAUDI2_QUEUE_ID_CPU_PQ) { 6651 /* 6652 * QMAN has 4 successive PQ_PI registers, 1 for each of the QMAN PQs. 6653 * Masking the H/W queue ID with 0x3 extracts the QMAN internal PQ 6654 * number. 6655 */ 6656 pq_offset = (hw_queue_id & 0x3) * 4; 6657 reg_base = gaudi2_qm_blocks_bases[hw_queue_id]; 6658 db_reg_offset = reg_base + QM_PQ_PI_0_OFFSET + pq_offset; 6659 } else { 6660 db_reg_offset = mmCPU_IF_PF_PQ_PI; 6661 } 6662 6663 db_value = pi; 6664 6665 /* ring the doorbell */ 6666 WREG32(db_reg_offset, db_value); 6667 6668 if (hw_queue_id == GAUDI2_QUEUE_ID_CPU_PQ) { 6669 /* make sure device CPU will read latest data from host */ 6670 mb(); 6671 WREG32(le32_to_cpu(dyn_regs->gic_host_pi_upd_irq), 6672 gaudi2_irq_map_table[GAUDI2_EVENT_CPU_PI_UPDATE].cpu_id); 6673 } 6674 } 6675 6676 static void gaudi2_pqe_write(struct hl_device *hdev, __le64 *pqe, struct hl_bd *bd) 6677 { 6678 __le64 *pbd = (__le64 *) bd; 6679 6680 /* The QMANs are on the host memory so a simple copy suffice */ 6681 pqe[0] = pbd[0]; 6682 pqe[1] = pbd[1]; 6683 } 6684 6685 static void *gaudi2_dma_alloc_coherent(struct hl_device *hdev, size_t size, 6686 dma_addr_t *dma_handle, gfp_t flags) 6687 { 6688 return dma_alloc_coherent(&hdev->pdev->dev, size, dma_handle, flags); 6689 } 6690 6691 static void gaudi2_dma_free_coherent(struct hl_device *hdev, size_t size, 6692 void *cpu_addr, dma_addr_t dma_handle) 6693 { 6694 dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, dma_handle); 6695 } 6696 6697 static int gaudi2_send_cpu_message(struct hl_device *hdev, u32 *msg, u16 len, 6698 u32 timeout, u64 *result) 6699 { 6700 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6701 6702 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) { 6703 if (result) 6704 *result = 0; 6705 return 0; 6706 } 6707 6708 if (!timeout) 6709 timeout = GAUDI2_MSG_TO_CPU_TIMEOUT_USEC; 6710 6711 return hl_fw_send_cpu_message(hdev, GAUDI2_QUEUE_ID_CPU_PQ, msg, len, timeout, result); 6712 } 6713 6714 static void *gaudi2_dma_pool_zalloc(struct hl_device *hdev, size_t size, 6715 gfp_t mem_flags, dma_addr_t *dma_handle) 6716 { 6717 if (size > GAUDI2_DMA_POOL_BLK_SIZE) 6718 return NULL; 6719 6720 return dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle); 6721 } 6722 6723 static void gaudi2_dma_pool_free(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr) 6724 { 6725 dma_pool_free(hdev->dma_pool, vaddr, dma_addr); 6726 } 6727 6728 static void *gaudi2_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, 6729 dma_addr_t *dma_handle) 6730 { 6731 return hl_fw_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle); 6732 } 6733 6734 static void gaudi2_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr) 6735 { 6736 hl_fw_cpu_accessible_dma_pool_free(hdev, size, vaddr); 6737 } 6738 6739 static int gaudi2_validate_cb_address(struct hl_device *hdev, struct hl_cs_parser *parser) 6740 { 6741 struct asic_fixed_properties *asic_prop = &hdev->asic_prop; 6742 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6743 6744 if (!gaudi2_is_queue_enabled(hdev, parser->hw_queue_id)) { 6745 dev_err(hdev->dev, "h/w queue %d is disabled\n", parser->hw_queue_id); 6746 return -EINVAL; 6747 } 6748 6749 /* Just check if CB address is valid */ 6750 6751 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, 6752 parser->user_cb_size, 6753 asic_prop->sram_user_base_address, 6754 asic_prop->sram_end_address)) 6755 return 0; 6756 6757 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, 6758 parser->user_cb_size, 6759 asic_prop->dram_user_base_address, 6760 asic_prop->dram_end_address)) 6761 return 0; 6762 6763 if ((gaudi2->hw_cap_initialized & HW_CAP_DMMU_MASK) && 6764 hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, 6765 parser->user_cb_size, 6766 asic_prop->dmmu.start_addr, 6767 asic_prop->dmmu.end_addr)) 6768 return 0; 6769 6770 if (gaudi2->hw_cap_initialized & HW_CAP_PMMU) { 6771 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, 6772 parser->user_cb_size, 6773 asic_prop->pmmu.start_addr, 6774 asic_prop->pmmu.end_addr) || 6775 hl_mem_area_inside_range( 6776 (u64) (uintptr_t) parser->user_cb, 6777 parser->user_cb_size, 6778 asic_prop->pmmu_huge.start_addr, 6779 asic_prop->pmmu_huge.end_addr)) 6780 return 0; 6781 6782 } else if (gaudi2_host_phys_addr_valid((u64) (uintptr_t) parser->user_cb)) { 6783 if (!hdev->pdev) 6784 return 0; 6785 6786 if (!device_iommu_mapped(&hdev->pdev->dev)) 6787 return 0; 6788 } 6789 6790 dev_err(hdev->dev, "CB address %p + 0x%x for internal QMAN is not valid\n", 6791 parser->user_cb, parser->user_cb_size); 6792 6793 return -EFAULT; 6794 } 6795 6796 static int gaudi2_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser) 6797 { 6798 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6799 6800 if (!parser->is_kernel_allocated_cb) 6801 return gaudi2_validate_cb_address(hdev, parser); 6802 6803 if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) { 6804 dev_err(hdev->dev, "PMMU not initialized - Unsupported mode in Gaudi2\n"); 6805 return -EINVAL; 6806 } 6807 6808 return 0; 6809 } 6810 6811 static int gaudi2_send_heartbeat(struct hl_device *hdev) 6812 { 6813 struct gaudi2_device *gaudi2 = hdev->asic_specific; 6814 6815 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 6816 return 0; 6817 6818 return hl_fw_send_heartbeat(hdev); 6819 } 6820 6821 /* This is an internal helper function, used to update the KDMA mmu props. 6822 * Should be called with a proper kdma lock. 6823 */ 6824 static void gaudi2_kdma_set_mmbp_asid(struct hl_device *hdev, 6825 bool mmu_bypass, u32 asid) 6826 { 6827 u32 rw_asid, rw_mmu_bp; 6828 6829 rw_asid = (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_SHIFT) | 6830 (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_SHIFT); 6831 6832 rw_mmu_bp = (!!mmu_bypass << ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_RD_SHIFT) | 6833 (!!mmu_bypass << ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_WR_SHIFT); 6834 6835 WREG32(mmARC_FARM_KDMA_CTX_AXUSER_HB_ASID, rw_asid); 6836 WREG32(mmARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP, rw_mmu_bp); 6837 } 6838 6839 static void gaudi2_arm_cq_monitor(struct hl_device *hdev, u32 sob_id, u32 mon_id, u32 cq_id, 6840 u32 mon_payload, u32 sync_value) 6841 { 6842 u32 sob_offset, mon_offset, sync_group_id, mode, mon_arm; 6843 u8 mask; 6844 6845 sob_offset = sob_id * 4; 6846 mon_offset = mon_id * 4; 6847 6848 /* Reset the SOB value */ 6849 WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset, 0); 6850 6851 /* Configure this address with CQ_ID 0 because CQ_EN is set */ 6852 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_offset, cq_id); 6853 6854 /* Configure this address with CS index because CQ_EN is set */ 6855 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_offset, mon_payload); 6856 6857 sync_group_id = sob_id / 8; 6858 mask = ~(1 << (sob_id & 0x7)); 6859 mode = 1; /* comparison mode is "equal to" */ 6860 6861 mon_arm = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOD_MASK, sync_value); 6862 mon_arm |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOP_MASK, mode); 6863 mon_arm |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_MASK_MASK, mask); 6864 mon_arm |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SID_MASK, sync_group_id); 6865 WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + mon_offset, mon_arm); 6866 } 6867 6868 /* This is an internal helper function used by gaudi2_send_job_to_kdma only */ 6869 static int gaudi2_send_job_to_kdma(struct hl_device *hdev, 6870 u64 src_addr, u64 dst_addr, 6871 u32 size, bool is_memset) 6872 { 6873 u32 comp_val, commit_mask, *polling_addr, timeout, status = 0; 6874 struct hl_cq_entry *cq_base; 6875 struct hl_cq *cq; 6876 u64 comp_addr; 6877 int rc; 6878 6879 gaudi2_arm_cq_monitor(hdev, GAUDI2_RESERVED_SOB_KDMA_COMPLETION, 6880 GAUDI2_RESERVED_MON_KDMA_COMPLETION, 6881 GAUDI2_RESERVED_CQ_KDMA_COMPLETION, 1, 1); 6882 6883 comp_addr = CFG_BASE + mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + 6884 (GAUDI2_RESERVED_SOB_KDMA_COMPLETION * sizeof(u32)); 6885 6886 comp_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1) | 6887 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 1); 6888 6889 WREG32(mmARC_FARM_KDMA_CTX_SRC_BASE_LO, lower_32_bits(src_addr)); 6890 WREG32(mmARC_FARM_KDMA_CTX_SRC_BASE_HI, upper_32_bits(src_addr)); 6891 WREG32(mmARC_FARM_KDMA_CTX_DST_BASE_LO, lower_32_bits(dst_addr)); 6892 WREG32(mmARC_FARM_KDMA_CTX_DST_BASE_HI, upper_32_bits(dst_addr)); 6893 WREG32(mmARC_FARM_KDMA_CTX_WR_COMP_ADDR_LO, lower_32_bits(comp_addr)); 6894 WREG32(mmARC_FARM_KDMA_CTX_WR_COMP_ADDR_HI, upper_32_bits(comp_addr)); 6895 WREG32(mmARC_FARM_KDMA_CTX_WR_COMP_WDATA, comp_val); 6896 WREG32(mmARC_FARM_KDMA_CTX_DST_TSIZE_0, size); 6897 6898 commit_mask = FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_LIN_MASK, 1) | 6899 FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_WR_COMP_EN_MASK, 1); 6900 6901 if (is_memset) 6902 commit_mask |= FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_MEM_SET_MASK, 1); 6903 6904 WREG32(mmARC_FARM_KDMA_CTX_COMMIT, commit_mask); 6905 6906 /* Wait for completion */ 6907 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_KDMA_COMPLETION]; 6908 cq_base = cq->kernel_address; 6909 polling_addr = (u32 *)&cq_base[cq->ci]; 6910 6911 if (hdev->pldm) 6912 /* for each 1MB 20 second of timeout */ 6913 timeout = ((size / SZ_1M) + 1) * USEC_PER_SEC * 20; 6914 else 6915 timeout = KDMA_TIMEOUT_USEC; 6916 6917 /* Polling */ 6918 rc = hl_poll_timeout_memory( 6919 hdev, 6920 polling_addr, 6921 status, 6922 (status == 1), 6923 1000, 6924 timeout, 6925 true); 6926 6927 *polling_addr = 0; 6928 6929 if (rc) { 6930 dev_err(hdev->dev, "Timeout while waiting for KDMA to be idle\n"); 6931 WREG32(mmARC_FARM_KDMA_CFG_1, 1 << ARC_FARM_KDMA_CFG_1_HALT_SHIFT); 6932 return rc; 6933 } 6934 6935 cq->ci = hl_cq_inc_ptr(cq->ci); 6936 6937 return 0; 6938 } 6939 6940 static void gaudi2_memset_device_lbw(struct hl_device *hdev, u32 addr, u32 size, u32 val) 6941 { 6942 u32 i; 6943 6944 for (i = 0 ; i < size ; i += sizeof(u32)) 6945 WREG32(addr + i, val); 6946 } 6947 6948 static void gaudi2_qman_set_test_mode(struct hl_device *hdev, u32 hw_queue_id, bool enable) 6949 { 6950 u32 reg_base = gaudi2_qm_blocks_bases[hw_queue_id]; 6951 6952 if (enable) { 6953 WREG32(reg_base + QM_GLBL_PROT_OFFSET, QMAN_MAKE_TRUSTED_TEST_MODE); 6954 WREG32(reg_base + QM_PQC_CFG_OFFSET, 0); 6955 } else { 6956 WREG32(reg_base + QM_GLBL_PROT_OFFSET, QMAN_MAKE_TRUSTED); 6957 WREG32(reg_base + QM_PQC_CFG_OFFSET, 1 << PDMA0_QM_PQC_CFG_EN_SHIFT); 6958 } 6959 } 6960 6961 static inline u32 gaudi2_test_queue_hw_queue_id_to_sob_id(struct hl_device *hdev, u32 hw_queue_id) 6962 { 6963 return hdev->asic_prop.first_available_user_sob[0] + 6964 hw_queue_id - GAUDI2_QUEUE_ID_PDMA_0_0; 6965 } 6966 6967 static void gaudi2_test_queue_clear(struct hl_device *hdev, u32 hw_queue_id) 6968 { 6969 u32 sob_offset = gaudi2_test_queue_hw_queue_id_to_sob_id(hdev, hw_queue_id) * 4; 6970 u32 sob_addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset; 6971 6972 /* Reset the SOB value */ 6973 WREG32(sob_addr, 0); 6974 } 6975 6976 static int gaudi2_test_queue_send_msg_short(struct hl_device *hdev, u32 hw_queue_id, u32 sob_val, 6977 struct gaudi2_queues_test_info *msg_info) 6978 { 6979 u32 sob_offset = gaudi2_test_queue_hw_queue_id_to_sob_id(hdev, hw_queue_id) * 4; 6980 u32 tmp, sob_base = 1; 6981 struct packet_msg_short *msg_short_pkt = msg_info->kern_addr; 6982 size_t pkt_size = sizeof(struct packet_msg_short); 6983 int rc; 6984 6985 tmp = (PACKET_MSG_SHORT << GAUDI2_PKT_CTL_OPCODE_SHIFT) | 6986 (1 << GAUDI2_PKT_CTL_EB_SHIFT) | 6987 (1 << GAUDI2_PKT_CTL_MB_SHIFT) | 6988 (sob_base << GAUDI2_PKT_SHORT_CTL_BASE_SHIFT) | 6989 (sob_offset << GAUDI2_PKT_SHORT_CTL_ADDR_SHIFT); 6990 6991 msg_short_pkt->value = cpu_to_le32(sob_val); 6992 msg_short_pkt->ctl = cpu_to_le32(tmp); 6993 6994 rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, pkt_size, msg_info->dma_addr); 6995 if (rc) 6996 dev_err(hdev->dev, 6997 "Failed to send msg_short packet to H/W queue %d\n", hw_queue_id); 6998 6999 return rc; 7000 } 7001 7002 static int gaudi2_test_queue_wait_completion(struct hl_device *hdev, u32 hw_queue_id, u32 sob_val) 7003 { 7004 u32 sob_offset = gaudi2_test_queue_hw_queue_id_to_sob_id(hdev, hw_queue_id) * 4; 7005 u32 sob_addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset; 7006 u32 timeout_usec, tmp; 7007 int rc; 7008 7009 if (hdev->pldm) 7010 timeout_usec = GAUDI2_PLDM_TEST_QUEUE_WAIT_USEC; 7011 else 7012 timeout_usec = GAUDI2_TEST_QUEUE_WAIT_USEC; 7013 7014 rc = hl_poll_timeout( 7015 hdev, 7016 sob_addr, 7017 tmp, 7018 (tmp == sob_val), 7019 1000, 7020 timeout_usec); 7021 7022 if (rc == -ETIMEDOUT) { 7023 dev_err(hdev->dev, "H/W queue %d test failed (SOB_OBJ_0 == 0x%x)\n", 7024 hw_queue_id, tmp); 7025 rc = -EIO; 7026 } 7027 7028 return rc; 7029 } 7030 7031 static int gaudi2_test_cpu_queue(struct hl_device *hdev) 7032 { 7033 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7034 7035 /* 7036 * check capability here as send_cpu_message() won't update the result 7037 * value if no capability 7038 */ 7039 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 7040 return 0; 7041 7042 return hl_fw_test_cpu_queue(hdev); 7043 } 7044 7045 static int gaudi2_test_queues(struct hl_device *hdev) 7046 { 7047 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7048 struct gaudi2_queues_test_info *msg_info; 7049 u32 sob_val = 0x5a5a; 7050 int i, rc; 7051 7052 /* send test message on all enabled Qs */ 7053 for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ; i++) { 7054 if (!gaudi2_is_queue_enabled(hdev, i)) 7055 continue; 7056 7057 msg_info = &gaudi2->queues_test_info[i - GAUDI2_QUEUE_ID_PDMA_0_0]; 7058 gaudi2_qman_set_test_mode(hdev, i, true); 7059 gaudi2_test_queue_clear(hdev, i); 7060 rc = gaudi2_test_queue_send_msg_short(hdev, i, sob_val, msg_info); 7061 if (rc) 7062 goto done; 7063 } 7064 7065 rc = gaudi2_test_cpu_queue(hdev); 7066 if (rc) 7067 goto done; 7068 7069 /* verify that all messages were processed */ 7070 for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ; i++) { 7071 if (!gaudi2_is_queue_enabled(hdev, i)) 7072 continue; 7073 7074 rc = gaudi2_test_queue_wait_completion(hdev, i, sob_val); 7075 if (rc) 7076 /* chip is not usable, no need for cleanups, just bail-out with error */ 7077 goto done; 7078 7079 gaudi2_test_queue_clear(hdev, i); 7080 gaudi2_qman_set_test_mode(hdev, i, false); 7081 } 7082 7083 done: 7084 return rc; 7085 } 7086 7087 static int gaudi2_compute_reset_late_init(struct hl_device *hdev) 7088 { 7089 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7090 size_t irq_arr_size; 7091 int rc; 7092 7093 gaudi2_init_arcs(hdev); 7094 7095 rc = gaudi2_scrub_arcs_dccm(hdev); 7096 if (rc) { 7097 dev_err(hdev->dev, "Failed to scrub arcs DCCM\n"); 7098 return rc; 7099 } 7100 7101 gaudi2_init_security(hdev); 7102 7103 /* Unmask all IRQs since some could have been received during the soft reset */ 7104 irq_arr_size = gaudi2->num_of_valid_hw_events * sizeof(gaudi2->hw_events[0]); 7105 return hl_fw_unmask_irq_arr(hdev, gaudi2->hw_events, irq_arr_size); 7106 } 7107 7108 static bool gaudi2_get_edma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7109 struct engines_data *e) 7110 { 7111 u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts, dma_core_sts0, dma_core_sts1; 7112 struct asic_fixed_properties *prop = &hdev->asic_prop; 7113 unsigned long *mask = (unsigned long *) mask_arr; 7114 const char *edma_fmt = "%-6d%-6d%-9s%#-14x%#-15x%#x\n"; 7115 bool is_idle = true, is_eng_idle; 7116 int engine_idx, i, j; 7117 u64 offset; 7118 7119 if (e) 7120 hl_engine_data_sprintf(e, 7121 "\nCORE EDMA is_idle QM_GLBL_STS0 DMA_CORE_STS0 DMA_CORE_STS1\n" 7122 "---- ---- ------- ------------ ------------- -------------\n"); 7123 7124 for (i = 0; i < NUM_OF_DCORES; i++) { 7125 for (j = 0 ; j < NUM_OF_EDMA_PER_DCORE ; j++) { 7126 int seq = i * NUM_OF_EDMA_PER_DCORE + j; 7127 7128 if (!(prop->edma_enabled_mask & BIT(seq))) 7129 continue; 7130 7131 engine_idx = GAUDI2_DCORE0_ENGINE_ID_EDMA_0 + 7132 i * GAUDI2_ENGINE_ID_DCORE_OFFSET + j; 7133 offset = i * DCORE_OFFSET + j * DCORE_EDMA_OFFSET; 7134 7135 dma_core_sts0 = RREG32(mmDCORE0_EDMA0_CORE_STS0 + offset); 7136 dma_core_sts1 = RREG32(mmDCORE0_EDMA0_CORE_STS1 + offset); 7137 7138 qm_glbl_sts0 = RREG32(mmDCORE0_EDMA0_QM_GLBL_STS0 + offset); 7139 qm_glbl_sts1 = RREG32(mmDCORE0_EDMA0_QM_GLBL_STS1 + offset); 7140 qm_cgm_sts = RREG32(mmDCORE0_EDMA0_QM_CGM_STS + offset); 7141 7142 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) && 7143 IS_DMA_IDLE(dma_core_sts0) && !IS_DMA_HALTED(dma_core_sts1); 7144 is_idle &= is_eng_idle; 7145 7146 if (mask && !is_eng_idle) 7147 set_bit(engine_idx, mask); 7148 7149 if (e) 7150 hl_engine_data_sprintf(e, edma_fmt, i, j, is_eng_idle ? "Y" : "N", 7151 qm_glbl_sts0, dma_core_sts0, dma_core_sts1); 7152 } 7153 } 7154 7155 return is_idle; 7156 } 7157 7158 static bool gaudi2_get_pdma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7159 struct engines_data *e) 7160 { 7161 u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts, dma_core_sts0, dma_core_sts1; 7162 unsigned long *mask = (unsigned long *) mask_arr; 7163 const char *pdma_fmt = "%-6d%-9s%#-14x%#-15x%#x\n"; 7164 bool is_idle = true, is_eng_idle; 7165 int engine_idx, i; 7166 u64 offset; 7167 7168 if (e) 7169 hl_engine_data_sprintf(e, 7170 "\nPDMA is_idle QM_GLBL_STS0 DMA_CORE_STS0 DMA_CORE_STS1\n" 7171 "---- ------- ------------ ------------- -------------\n"); 7172 7173 for (i = 0 ; i < NUM_OF_PDMA ; i++) { 7174 engine_idx = GAUDI2_ENGINE_ID_PDMA_0 + i; 7175 offset = i * PDMA_OFFSET; 7176 dma_core_sts0 = RREG32(mmPDMA0_CORE_STS0 + offset); 7177 dma_core_sts1 = RREG32(mmPDMA0_CORE_STS1 + offset); 7178 7179 qm_glbl_sts0 = RREG32(mmPDMA0_QM_GLBL_STS0 + offset); 7180 qm_glbl_sts1 = RREG32(mmPDMA0_QM_GLBL_STS1 + offset); 7181 qm_cgm_sts = RREG32(mmPDMA0_QM_CGM_STS + offset); 7182 7183 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) && 7184 IS_DMA_IDLE(dma_core_sts0) && !IS_DMA_HALTED(dma_core_sts1); 7185 is_idle &= is_eng_idle; 7186 7187 if (mask && !is_eng_idle) 7188 set_bit(engine_idx, mask); 7189 7190 if (e) 7191 hl_engine_data_sprintf(e, pdma_fmt, i, is_eng_idle ? "Y" : "N", 7192 qm_glbl_sts0, dma_core_sts0, dma_core_sts1); 7193 } 7194 7195 return is_idle; 7196 } 7197 7198 static bool gaudi2_get_nic_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7199 struct engines_data *e) 7200 { 7201 unsigned long *mask = (unsigned long *) mask_arr; 7202 const char *nic_fmt = "%-5d%-9s%#-14x%#-12x\n"; 7203 u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts; 7204 bool is_idle = true, is_eng_idle; 7205 int engine_idx, i; 7206 u64 offset = 0; 7207 7208 /* NIC, twelve macros in Full chip */ 7209 if (e && hdev->nic_ports_mask) 7210 hl_engine_data_sprintf(e, 7211 "\nNIC is_idle QM_GLBL_STS0 QM_CGM_STS\n" 7212 "--- ------- ------------ ----------\n"); 7213 7214 for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++) { 7215 if (!(i & 1)) 7216 offset = i / 2 * NIC_OFFSET; 7217 else 7218 offset += NIC_QM_OFFSET; 7219 7220 if (!(hdev->nic_ports_mask & BIT(i))) 7221 continue; 7222 7223 engine_idx = GAUDI2_ENGINE_ID_NIC0_0 + i; 7224 7225 7226 qm_glbl_sts0 = RREG32(mmNIC0_QM0_GLBL_STS0 + offset); 7227 qm_glbl_sts1 = RREG32(mmNIC0_QM0_GLBL_STS1 + offset); 7228 qm_cgm_sts = RREG32(mmNIC0_QM0_CGM_STS + offset); 7229 7230 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts); 7231 is_idle &= is_eng_idle; 7232 7233 if (mask && !is_eng_idle) 7234 set_bit(engine_idx, mask); 7235 7236 if (e) 7237 hl_engine_data_sprintf(e, nic_fmt, i, is_eng_idle ? "Y" : "N", 7238 qm_glbl_sts0, qm_cgm_sts); 7239 } 7240 7241 return is_idle; 7242 } 7243 7244 static bool gaudi2_get_mme_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7245 struct engines_data *e) 7246 { 7247 u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts, mme_arch_sts; 7248 unsigned long *mask = (unsigned long *) mask_arr; 7249 const char *mme_fmt = "%-5d%-6s%-9s%#-14x%#x\n"; 7250 bool is_idle = true, is_eng_idle; 7251 int engine_idx, i; 7252 u64 offset; 7253 7254 if (e) 7255 hl_engine_data_sprintf(e, 7256 "\nMME Stub is_idle QM_GLBL_STS0 MME_ARCH_STATUS\n" 7257 "--- ---- ------- ------------ ---------------\n"); 7258 /* MME, one per Dcore */ 7259 for (i = 0 ; i < NUM_OF_DCORES ; i++) { 7260 engine_idx = GAUDI2_DCORE0_ENGINE_ID_MME + i * GAUDI2_ENGINE_ID_DCORE_OFFSET; 7261 offset = i * DCORE_OFFSET; 7262 7263 qm_glbl_sts0 = RREG32(mmDCORE0_MME_QM_GLBL_STS0 + offset); 7264 qm_glbl_sts1 = RREG32(mmDCORE0_MME_QM_GLBL_STS1 + offset); 7265 qm_cgm_sts = RREG32(mmDCORE0_MME_QM_CGM_STS + offset); 7266 7267 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts); 7268 is_idle &= is_eng_idle; 7269 7270 mme_arch_sts = RREG32(mmDCORE0_MME_CTRL_LO_ARCH_STATUS + offset); 7271 is_eng_idle &= IS_MME_IDLE(mme_arch_sts); 7272 is_idle &= is_eng_idle; 7273 7274 if (e) 7275 hl_engine_data_sprintf(e, mme_fmt, i, "N", 7276 is_eng_idle ? "Y" : "N", 7277 qm_glbl_sts0, 7278 mme_arch_sts); 7279 7280 if (mask && !is_eng_idle) 7281 set_bit(engine_idx, mask); 7282 } 7283 7284 return is_idle; 7285 } 7286 7287 static void gaudi2_is_tpc_engine_idle(struct hl_device *hdev, int dcore, int inst, u32 offset, 7288 struct iterate_module_ctx *ctx) 7289 { 7290 struct gaudi2_tpc_idle_data *idle_data = ctx->data; 7291 u32 tpc_cfg_sts, qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts; 7292 bool is_eng_idle; 7293 int engine_idx; 7294 7295 if ((dcore == 0) && (inst == (NUM_DCORE0_TPC - 1))) 7296 engine_idx = GAUDI2_DCORE0_ENGINE_ID_TPC_6; 7297 else 7298 engine_idx = GAUDI2_DCORE0_ENGINE_ID_TPC_0 + 7299 dcore * GAUDI2_ENGINE_ID_DCORE_OFFSET + inst; 7300 7301 tpc_cfg_sts = RREG32(mmDCORE0_TPC0_CFG_STATUS + offset); 7302 qm_glbl_sts0 = RREG32(mmDCORE0_TPC0_QM_GLBL_STS0 + offset); 7303 qm_glbl_sts1 = RREG32(mmDCORE0_TPC0_QM_GLBL_STS1 + offset); 7304 qm_cgm_sts = RREG32(mmDCORE0_TPC0_QM_CGM_STS + offset); 7305 7306 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts) && 7307 IS_TPC_IDLE(tpc_cfg_sts); 7308 *(idle_data->is_idle) &= is_eng_idle; 7309 7310 if (idle_data->mask && !is_eng_idle) 7311 set_bit(engine_idx, idle_data->mask); 7312 7313 if (idle_data->e) 7314 hl_engine_data_sprintf(idle_data->e, 7315 idle_data->tpc_fmt, dcore, inst, 7316 is_eng_idle ? "Y" : "N", 7317 qm_glbl_sts0, qm_cgm_sts, tpc_cfg_sts); 7318 } 7319 7320 static bool gaudi2_get_tpc_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7321 struct engines_data *e) 7322 { 7323 struct asic_fixed_properties *prop = &hdev->asic_prop; 7324 unsigned long *mask = (unsigned long *) mask_arr; 7325 bool is_idle = true; 7326 7327 struct gaudi2_tpc_idle_data tpc_idle_data = { 7328 .tpc_fmt = "%-6d%-5d%-9s%#-14x%#-12x%#x\n", 7329 .e = e, 7330 .mask = mask, 7331 .is_idle = &is_idle, 7332 }; 7333 struct iterate_module_ctx tpc_iter = { 7334 .fn = &gaudi2_is_tpc_engine_idle, 7335 .data = &tpc_idle_data, 7336 }; 7337 7338 if (e && prop->tpc_enabled_mask) 7339 hl_engine_data_sprintf(e, 7340 "\nCORE TPC is_idle QM_GLBL_STS0 QM_CGM_STS STATUS\n" 7341 "---- --- ------- ------------ ---------- ------\n"); 7342 7343 gaudi2_iterate_tpcs(hdev, &tpc_iter); 7344 7345 return *tpc_idle_data.is_idle; 7346 } 7347 7348 static bool gaudi2_get_decoder_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7349 struct engines_data *e) 7350 { 7351 struct asic_fixed_properties *prop = &hdev->asic_prop; 7352 unsigned long *mask = (unsigned long *) mask_arr; 7353 const char *pcie_dec_fmt = "%-10d%-9s%#x\n"; 7354 const char *dec_fmt = "%-6d%-5d%-9s%#x\n"; 7355 bool is_idle = true, is_eng_idle; 7356 u32 dec_swreg15, dec_enabled_bit; 7357 int engine_idx, i, j; 7358 u64 offset; 7359 7360 /* Decoders, two each Dcore and two shared PCIe decoders */ 7361 if (e && (prop->decoder_enabled_mask & (~PCIE_DEC_EN_MASK))) 7362 hl_engine_data_sprintf(e, 7363 "\nCORE DEC is_idle VSI_CMD_SWREG15\n" 7364 "---- --- ------- ---------------\n"); 7365 7366 for (i = 0 ; i < NUM_OF_DCORES ; i++) { 7367 for (j = 0 ; j < NUM_OF_DEC_PER_DCORE ; j++) { 7368 dec_enabled_bit = 1 << (i * NUM_OF_DEC_PER_DCORE + j); 7369 if (!(prop->decoder_enabled_mask & dec_enabled_bit)) 7370 continue; 7371 7372 engine_idx = GAUDI2_DCORE0_ENGINE_ID_DEC_0 + 7373 i * GAUDI2_ENGINE_ID_DCORE_OFFSET + j; 7374 offset = i * DCORE_OFFSET + j * DCORE_DEC_OFFSET; 7375 7376 dec_swreg15 = RREG32(mmDCORE0_DEC0_CMD_SWREG15 + offset); 7377 is_eng_idle = IS_DEC_IDLE(dec_swreg15); 7378 is_idle &= is_eng_idle; 7379 7380 if (mask && !is_eng_idle) 7381 set_bit(engine_idx, mask); 7382 7383 if (e) 7384 hl_engine_data_sprintf(e, dec_fmt, i, j, 7385 is_eng_idle ? "Y" : "N", dec_swreg15); 7386 } 7387 } 7388 7389 if (e && (prop->decoder_enabled_mask & PCIE_DEC_EN_MASK)) 7390 hl_engine_data_sprintf(e, 7391 "\nPCIe DEC is_idle VSI_CMD_SWREG15\n" 7392 "-------- ------- ---------------\n"); 7393 7394 /* Check shared(PCIe) decoders */ 7395 for (i = 0 ; i < NUM_OF_DEC_PER_DCORE ; i++) { 7396 dec_enabled_bit = PCIE_DEC_SHIFT + i; 7397 if (!(prop->decoder_enabled_mask & BIT(dec_enabled_bit))) 7398 continue; 7399 7400 engine_idx = GAUDI2_PCIE_ENGINE_ID_DEC_0 + i; 7401 offset = i * DCORE_DEC_OFFSET; 7402 dec_swreg15 = RREG32(mmPCIE_DEC0_CMD_SWREG15 + offset); 7403 is_eng_idle = IS_DEC_IDLE(dec_swreg15); 7404 is_idle &= is_eng_idle; 7405 7406 if (mask && !is_eng_idle) 7407 set_bit(engine_idx, mask); 7408 7409 if (e) 7410 hl_engine_data_sprintf(e, pcie_dec_fmt, i, 7411 is_eng_idle ? "Y" : "N", dec_swreg15); 7412 } 7413 7414 return is_idle; 7415 } 7416 7417 static bool gaudi2_get_rotator_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7418 struct engines_data *e) 7419 { 7420 const char *rot_fmt = "%-6d%-5d%-9s%#-14x%#-14x%#x\n"; 7421 unsigned long *mask = (unsigned long *) mask_arr; 7422 u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts; 7423 bool is_idle = true, is_eng_idle; 7424 int engine_idx, i; 7425 u64 offset; 7426 7427 if (e) 7428 hl_engine_data_sprintf(e, 7429 "\nCORE ROT is_idle QM_GLBL_STS0 QM_GLBL_STS1 QM_CGM_STS\n" 7430 "---- --- ------- ------------ ------------ ----------\n"); 7431 7432 for (i = 0 ; i < NUM_OF_ROT ; i++) { 7433 engine_idx = GAUDI2_ENGINE_ID_ROT_0 + i; 7434 7435 offset = i * ROT_OFFSET; 7436 7437 qm_glbl_sts0 = RREG32(mmROT0_QM_GLBL_STS0 + offset); 7438 qm_glbl_sts1 = RREG32(mmROT0_QM_GLBL_STS1 + offset); 7439 qm_cgm_sts = RREG32(mmROT0_QM_CGM_STS + offset); 7440 7441 is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts); 7442 is_idle &= is_eng_idle; 7443 7444 if (mask && !is_eng_idle) 7445 set_bit(engine_idx, mask); 7446 7447 if (e) 7448 hl_engine_data_sprintf(e, rot_fmt, i, 0, is_eng_idle ? "Y" : "N", 7449 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts); 7450 } 7451 7452 return is_idle; 7453 } 7454 7455 static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, 7456 struct engines_data *e) 7457 { 7458 bool is_idle = true; 7459 7460 is_idle &= gaudi2_get_edma_idle_status(hdev, mask_arr, mask_len, e); 7461 is_idle &= gaudi2_get_pdma_idle_status(hdev, mask_arr, mask_len, e); 7462 is_idle &= gaudi2_get_nic_idle_status(hdev, mask_arr, mask_len, e); 7463 is_idle &= gaudi2_get_mme_idle_status(hdev, mask_arr, mask_len, e); 7464 is_idle &= gaudi2_get_tpc_idle_status(hdev, mask_arr, mask_len, e); 7465 is_idle &= gaudi2_get_decoder_idle_status(hdev, mask_arr, mask_len, e); 7466 is_idle &= gaudi2_get_rotator_idle_status(hdev, mask_arr, mask_len, e); 7467 7468 return is_idle; 7469 } 7470 7471 static void gaudi2_hw_queues_lock(struct hl_device *hdev) 7472 __acquires(&gaudi2->hw_queues_lock) 7473 { 7474 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7475 7476 spin_lock(&gaudi2->hw_queues_lock); 7477 } 7478 7479 static void gaudi2_hw_queues_unlock(struct hl_device *hdev) 7480 __releases(&gaudi2->hw_queues_lock) 7481 { 7482 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7483 7484 spin_unlock(&gaudi2->hw_queues_lock); 7485 } 7486 7487 static u32 gaudi2_get_pci_id(struct hl_device *hdev) 7488 { 7489 return hdev->pdev->device; 7490 } 7491 7492 static int gaudi2_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size) 7493 { 7494 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7495 7496 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 7497 return 0; 7498 7499 return hl_fw_get_eeprom_data(hdev, data, max_size); 7500 } 7501 7502 static void gaudi2_update_eq_ci(struct hl_device *hdev, u32 val) 7503 { 7504 WREG32(mmCPU_IF_EQ_RD_OFFS, val); 7505 } 7506 7507 static void *gaudi2_get_events_stat(struct hl_device *hdev, bool aggregate, u32 *size) 7508 { 7509 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7510 7511 if (aggregate) { 7512 *size = (u32) sizeof(gaudi2->events_stat_aggregate); 7513 return gaudi2->events_stat_aggregate; 7514 } 7515 7516 *size = (u32) sizeof(gaudi2->events_stat); 7517 return gaudi2->events_stat; 7518 } 7519 7520 static void gaudi2_mmu_vdec_dcore_prepare(struct hl_device *hdev, int dcore_id, 7521 int dcore_vdec_id, u32 rw_asid, u32 rw_mmu_bp) 7522 { 7523 u32 offset = (mmDCORE0_VDEC1_BRDG_CTRL_BASE - mmDCORE0_VDEC0_BRDG_CTRL_BASE) * 7524 dcore_vdec_id + DCORE_OFFSET * dcore_id; 7525 7526 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_MMU_BP + offset, rw_mmu_bp); 7527 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_ASID + offset, rw_asid); 7528 7529 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_MMU_BP + offset, rw_mmu_bp); 7530 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_ASID + offset, rw_asid); 7531 7532 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_MMU_BP + offset, rw_mmu_bp); 7533 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_ASID + offset, rw_asid); 7534 7535 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_MMU_BP + offset, rw_mmu_bp); 7536 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_ASID + offset, rw_asid); 7537 7538 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_MMU_BP + offset, rw_mmu_bp); 7539 WREG32(mmDCORE0_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_ASID + offset, rw_asid); 7540 } 7541 7542 static void gaudi2_mmu_dcore_prepare(struct hl_device *hdev, int dcore_id, u32 asid) 7543 { 7544 u32 rw_asid = (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_SHIFT) | 7545 (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_SHIFT); 7546 struct asic_fixed_properties *prop = &hdev->asic_prop; 7547 u32 dcore_offset = dcore_id * DCORE_OFFSET; 7548 u32 vdec_id, i, ports_offset, reg_val; 7549 u8 edma_seq_base; 7550 7551 /* EDMA */ 7552 edma_seq_base = dcore_id * NUM_OF_EDMA_PER_DCORE; 7553 if (prop->edma_enabled_mask & BIT(edma_seq_base)) { 7554 WREG32(mmDCORE0_EDMA0_QM_AXUSER_NONSECURED_HB_MMU_BP + dcore_offset, 0); 7555 WREG32(mmDCORE0_EDMA0_QM_AXUSER_NONSECURED_HB_ASID + dcore_offset, rw_asid); 7556 WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP + dcore_offset, 0); 7557 WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_ASID + dcore_offset, rw_asid); 7558 } 7559 7560 if (prop->edma_enabled_mask & BIT(edma_seq_base + 1)) { 7561 WREG32(mmDCORE0_EDMA1_QM_AXUSER_NONSECURED_HB_MMU_BP + dcore_offset, 0); 7562 WREG32(mmDCORE0_EDMA1_QM_AXUSER_NONSECURED_HB_ASID + dcore_offset, rw_asid); 7563 WREG32(mmDCORE0_EDMA1_CORE_CTX_AXUSER_HB_ASID + dcore_offset, rw_asid); 7564 WREG32(mmDCORE0_EDMA1_CORE_CTX_AXUSER_HB_MMU_BP + dcore_offset, 0); 7565 } 7566 7567 /* Sync Mngr */ 7568 WREG32(mmDCORE0_SYNC_MNGR_GLBL_ASID_NONE_SEC_PRIV + dcore_offset, asid); 7569 /* 7570 * Sync Mngrs on dcores 1 - 3 are exposed to user, so must use user ASID 7571 * for any access type 7572 */ 7573 if (dcore_id > 0) { 7574 reg_val = (asid << DCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_ASID_RD_SHIFT) | 7575 (asid << DCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_ASID_WR_SHIFT); 7576 WREG32(mmDCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_ASID + dcore_offset, reg_val); 7577 WREG32(mmDCORE0_SYNC_MNGR_MSTR_IF_AXUSER_HB_MMU_BP + dcore_offset, 0); 7578 } 7579 7580 WREG32(mmDCORE0_MME_CTRL_LO_MME_AXUSER_HB_MMU_BP + dcore_offset, 0); 7581 WREG32(mmDCORE0_MME_CTRL_LO_MME_AXUSER_HB_ASID + dcore_offset, rw_asid); 7582 7583 for (i = 0 ; i < NUM_OF_MME_SBTE_PORTS ; i++) { 7584 ports_offset = i * DCORE_MME_SBTE_OFFSET; 7585 WREG32(mmDCORE0_MME_SBTE0_MSTR_IF_AXUSER_HB_MMU_BP + 7586 dcore_offset + ports_offset, 0); 7587 WREG32(mmDCORE0_MME_SBTE0_MSTR_IF_AXUSER_HB_ASID + 7588 dcore_offset + ports_offset, rw_asid); 7589 } 7590 7591 for (i = 0 ; i < NUM_OF_MME_WB_PORTS ; i++) { 7592 ports_offset = i * DCORE_MME_WB_OFFSET; 7593 WREG32(mmDCORE0_MME_WB0_MSTR_IF_AXUSER_HB_MMU_BP + 7594 dcore_offset + ports_offset, 0); 7595 WREG32(mmDCORE0_MME_WB0_MSTR_IF_AXUSER_HB_ASID + 7596 dcore_offset + ports_offset, rw_asid); 7597 } 7598 7599 WREG32(mmDCORE0_MME_QM_AXUSER_NONSECURED_HB_MMU_BP + dcore_offset, 0); 7600 WREG32(mmDCORE0_MME_QM_AXUSER_NONSECURED_HB_ASID + dcore_offset, rw_asid); 7601 7602 /* 7603 * Decoders 7604 */ 7605 for (vdec_id = 0 ; vdec_id < NUM_OF_DEC_PER_DCORE ; vdec_id++) { 7606 if (prop->decoder_enabled_mask & BIT(dcore_id * NUM_OF_DEC_PER_DCORE + vdec_id)) 7607 gaudi2_mmu_vdec_dcore_prepare(hdev, dcore_id, vdec_id, rw_asid, 0); 7608 } 7609 } 7610 7611 static void gudi2_mmu_vdec_shared_prepare(struct hl_device *hdev, 7612 int shared_vdec_id, u32 rw_asid, u32 rw_mmu_bp) 7613 { 7614 u32 offset = (mmPCIE_VDEC1_BRDG_CTRL_BASE - mmPCIE_VDEC0_BRDG_CTRL_BASE) * shared_vdec_id; 7615 7616 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_MMU_BP + offset, rw_mmu_bp); 7617 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_DEC_HB_ASID + offset, rw_asid); 7618 7619 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_MMU_BP + offset, rw_mmu_bp); 7620 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_ABNRM_HB_ASID + offset, rw_asid); 7621 7622 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_MMU_BP + offset, rw_mmu_bp); 7623 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_L2C_HB_ASID + offset, rw_asid); 7624 7625 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_MMU_BP + offset, rw_mmu_bp); 7626 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_NRM_HB_ASID + offset, rw_asid); 7627 7628 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_MMU_BP + offset, rw_mmu_bp); 7629 WREG32(mmPCIE_VDEC0_BRDG_CTRL_AXUSER_MSIX_VCD_HB_ASID + offset, rw_asid); 7630 } 7631 7632 static void gudi2_mmu_arc_farm_arc_dup_eng_prepare(struct hl_device *hdev, int arc_farm_id, 7633 u32 rw_asid, u32 rw_mmu_bp) 7634 { 7635 u32 offset = (mmARC_FARM_ARC1_DUP_ENG_BASE - mmARC_FARM_ARC0_DUP_ENG_BASE) * arc_farm_id; 7636 7637 WREG32(mmARC_FARM_ARC0_DUP_ENG_AXUSER_HB_MMU_BP + offset, rw_mmu_bp); 7638 WREG32(mmARC_FARM_ARC0_DUP_ENG_AXUSER_HB_ASID + offset, rw_asid); 7639 } 7640 7641 static void gaudi2_arc_mmu_prepare(struct hl_device *hdev, u32 cpu_id, u32 asid) 7642 { 7643 u32 reg_base, reg_offset, reg_val = 0; 7644 7645 reg_base = gaudi2_arc_blocks_bases[cpu_id]; 7646 7647 /* Enable MMU and configure asid for all relevant ARC regions */ 7648 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_ARC_REGION_CFG_MMU_BP_MASK, 0); 7649 reg_val |= FIELD_PREP(ARC_FARM_ARC0_AUX_ARC_REGION_CFG_0_ASID_MASK, asid); 7650 7651 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION3_GENERAL); 7652 WREG32(reg_base + reg_offset, reg_val); 7653 7654 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION4_HBM0_FW); 7655 WREG32(reg_base + reg_offset, reg_val); 7656 7657 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION5_HBM1_GC_DATA); 7658 WREG32(reg_base + reg_offset, reg_val); 7659 7660 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION6_HBM2_GC_DATA); 7661 WREG32(reg_base + reg_offset, reg_val); 7662 7663 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION7_HBM3_GC_DATA); 7664 WREG32(reg_base + reg_offset, reg_val); 7665 7666 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION9_PCIE); 7667 WREG32(reg_base + reg_offset, reg_val); 7668 7669 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION10_GENERAL); 7670 WREG32(reg_base + reg_offset, reg_val); 7671 7672 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION11_GENERAL); 7673 WREG32(reg_base + reg_offset, reg_val); 7674 7675 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION12_GENERAL); 7676 WREG32(reg_base + reg_offset, reg_val); 7677 7678 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION13_GENERAL); 7679 WREG32(reg_base + reg_offset, reg_val); 7680 7681 reg_offset = ARC_REGION_CFG_OFFSET(ARC_REGION14_GENERAL); 7682 WREG32(reg_base + reg_offset, reg_val); 7683 } 7684 7685 static int gaudi2_arc_mmu_prepare_all(struct hl_device *hdev, u32 asid) 7686 { 7687 int i; 7688 7689 if (hdev->fw_components & FW_TYPE_BOOT_CPU) 7690 return hl_fw_cpucp_engine_core_asid_set(hdev, asid); 7691 7692 for (i = CPU_ID_SCHED_ARC0 ; i < NUM_OF_ARC_FARMS_ARC ; i++) 7693 gaudi2_arc_mmu_prepare(hdev, i, asid); 7694 7695 for (i = GAUDI2_QUEUE_ID_PDMA_0_0 ; i < GAUDI2_QUEUE_ID_CPU_PQ ; i += 4) { 7696 if (!gaudi2_is_queue_enabled(hdev, i)) 7697 continue; 7698 7699 gaudi2_arc_mmu_prepare(hdev, gaudi2_queue_id_to_arc_id[i], asid); 7700 } 7701 7702 return 0; 7703 } 7704 7705 static int gaudi2_mmu_shared_prepare(struct hl_device *hdev, u32 asid) 7706 { 7707 struct asic_fixed_properties *prop = &hdev->asic_prop; 7708 u32 rw_asid, offset; 7709 int rc, i; 7710 7711 rw_asid = FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_MASK, asid) | 7712 FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_MASK, asid); 7713 7714 WREG32(mmPDMA0_QM_AXUSER_NONSECURED_HB_ASID, rw_asid); 7715 WREG32(mmPDMA0_QM_AXUSER_NONSECURED_HB_MMU_BP, 0); 7716 WREG32(mmPDMA0_CORE_CTX_AXUSER_HB_ASID, rw_asid); 7717 WREG32(mmPDMA0_CORE_CTX_AXUSER_HB_MMU_BP, 0); 7718 7719 WREG32(mmPDMA1_QM_AXUSER_NONSECURED_HB_ASID, rw_asid); 7720 WREG32(mmPDMA1_QM_AXUSER_NONSECURED_HB_MMU_BP, 0); 7721 WREG32(mmPDMA1_CORE_CTX_AXUSER_HB_ASID, rw_asid); 7722 WREG32(mmPDMA1_CORE_CTX_AXUSER_HB_MMU_BP, 0); 7723 7724 /* ROT */ 7725 for (i = 0 ; i < NUM_OF_ROT ; i++) { 7726 offset = i * ROT_OFFSET; 7727 WREG32(mmROT0_QM_AXUSER_NONSECURED_HB_ASID + offset, rw_asid); 7728 WREG32(mmROT0_QM_AXUSER_NONSECURED_HB_MMU_BP + offset, 0); 7729 RMWREG32(mmROT0_CPL_QUEUE_AWUSER + offset, asid, MMUBP_ASID_MASK); 7730 RMWREG32(mmROT0_DESC_HBW_ARUSER_LO + offset, asid, MMUBP_ASID_MASK); 7731 RMWREG32(mmROT0_DESC_HBW_AWUSER_LO + offset, asid, MMUBP_ASID_MASK); 7732 } 7733 7734 /* Shared Decoders are the last bits in the decoders mask */ 7735 if (prop->decoder_enabled_mask & BIT(NUM_OF_DCORES * NUM_OF_DEC_PER_DCORE + 0)) 7736 gudi2_mmu_vdec_shared_prepare(hdev, 0, rw_asid, 0); 7737 7738 if (prop->decoder_enabled_mask & BIT(NUM_OF_DCORES * NUM_OF_DEC_PER_DCORE + 1)) 7739 gudi2_mmu_vdec_shared_prepare(hdev, 1, rw_asid, 0); 7740 7741 /* arc farm arc dup eng */ 7742 for (i = 0 ; i < NUM_OF_ARC_FARMS_ARC ; i++) 7743 gudi2_mmu_arc_farm_arc_dup_eng_prepare(hdev, i, rw_asid, 0); 7744 7745 rc = gaudi2_arc_mmu_prepare_all(hdev, asid); 7746 if (rc) 7747 return rc; 7748 7749 return 0; 7750 } 7751 7752 static void gaudi2_tpc_mmu_prepare(struct hl_device *hdev, int dcore, int inst, u32 offset, 7753 struct iterate_module_ctx *ctx) 7754 { 7755 struct gaudi2_tpc_mmu_data *mmu_data = ctx->data; 7756 7757 WREG32(mmDCORE0_TPC0_CFG_AXUSER_HB_MMU_BP + offset, 0); 7758 WREG32(mmDCORE0_TPC0_CFG_AXUSER_HB_ASID + offset, mmu_data->rw_asid); 7759 WREG32(mmDCORE0_TPC0_QM_AXUSER_NONSECURED_HB_MMU_BP + offset, 0); 7760 WREG32(mmDCORE0_TPC0_QM_AXUSER_NONSECURED_HB_ASID + offset, mmu_data->rw_asid); 7761 } 7762 7763 /* zero the MMUBP and set the ASID */ 7764 static int gaudi2_mmu_prepare(struct hl_device *hdev, u32 asid) 7765 { 7766 struct gaudi2_device *gaudi2 = hdev->asic_specific; 7767 struct gaudi2_tpc_mmu_data tpc_mmu_data; 7768 struct iterate_module_ctx tpc_iter = { 7769 .fn = &gaudi2_tpc_mmu_prepare, 7770 .data = &tpc_mmu_data, 7771 }; 7772 int rc, i; 7773 7774 if (asid & ~DCORE0_HMMU0_STLB_ASID_ASID_MASK) { 7775 dev_crit(hdev->dev, "asid %u is too big\n", asid); 7776 return -EINVAL; 7777 } 7778 7779 if (!(gaudi2->hw_cap_initialized & HW_CAP_MMU_MASK)) 7780 return 0; 7781 7782 rc = gaudi2_mmu_shared_prepare(hdev, asid); 7783 if (rc) 7784 return rc; 7785 7786 /* configure DCORE MMUs */ 7787 tpc_mmu_data.rw_asid = (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_RD_SHIFT) | 7788 (asid << ARC_FARM_KDMA_CTX_AXUSER_HB_ASID_WR_SHIFT); 7789 gaudi2_iterate_tpcs(hdev, &tpc_iter); 7790 for (i = 0 ; i < NUM_OF_DCORES ; i++) 7791 gaudi2_mmu_dcore_prepare(hdev, i, asid); 7792 7793 return 0; 7794 } 7795 7796 static inline bool is_info_event(u32 event) 7797 { 7798 switch (event) { 7799 case GAUDI2_EVENT_CPU_CPLD_SHUTDOWN_CAUSE: 7800 case GAUDI2_EVENT_CPU_FIX_POWER_ENV_S ... GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_E: 7801 case GAUDI2_EVENT_ARC_PWR_BRK_ENTRY ... GAUDI2_EVENT_ARC_PWR_RD_MODE3: 7802 7803 /* return in case of NIC status event - these events are received periodically and not as 7804 * an indication to an error. 7805 */ 7806 case GAUDI2_EVENT_CPU0_STATUS_NIC0_ENG0 ... GAUDI2_EVENT_CPU11_STATUS_NIC11_ENG1: 7807 case GAUDI2_EVENT_ARC_EQ_HEARTBEAT: 7808 return true; 7809 default: 7810 return false; 7811 } 7812 } 7813 7814 static void gaudi2_print_event(struct hl_device *hdev, u16 event_type, 7815 bool ratelimited, const char *fmt, ...) 7816 { 7817 struct va_format vaf; 7818 va_list args; 7819 7820 va_start(args, fmt); 7821 vaf.fmt = fmt; 7822 vaf.va = &args; 7823 7824 if (ratelimited) 7825 dev_err_ratelimited(hdev->dev, "%s: %pV\n", 7826 gaudi2_irq_map_table[event_type].valid ? 7827 gaudi2_irq_map_table[event_type].name : "N/A Event", &vaf); 7828 else 7829 dev_err(hdev->dev, "%s: %pV\n", 7830 gaudi2_irq_map_table[event_type].valid ? 7831 gaudi2_irq_map_table[event_type].name : "N/A Event", &vaf); 7832 7833 va_end(args); 7834 } 7835 7836 static bool gaudi2_handle_ecc_event(struct hl_device *hdev, u16 event_type, 7837 struct hl_eq_ecc_data *ecc_data) 7838 { 7839 u64 ecc_address = 0, ecc_syndrome = 0; 7840 u8 memory_wrapper_idx = 0; 7841 bool has_block_id = false; 7842 u16 block_id; 7843 7844 if (!hl_is_fw_sw_ver_below(hdev, 1, 12)) 7845 has_block_id = true; 7846 7847 ecc_address = le64_to_cpu(ecc_data->ecc_address); 7848 ecc_syndrome = le64_to_cpu(ecc_data->ecc_syndrom); 7849 memory_wrapper_idx = ecc_data->memory_wrapper_idx; 7850 7851 if (has_block_id) { 7852 block_id = le16_to_cpu(ecc_data->block_id); 7853 gaudi2_print_event(hdev, event_type, !ecc_data->is_critical, 7854 "ECC error detected. address: %#llx. Syndrome: %#llx. wrapper id %u. block id %#x. critical %u.", 7855 ecc_address, ecc_syndrome, memory_wrapper_idx, block_id, 7856 ecc_data->is_critical); 7857 } else { 7858 gaudi2_print_event(hdev, event_type, !ecc_data->is_critical, 7859 "ECC error detected. address: %#llx. Syndrome: %#llx. wrapper id %u. critical %u.", 7860 ecc_address, ecc_syndrome, memory_wrapper_idx, ecc_data->is_critical); 7861 } 7862 7863 return !!ecc_data->is_critical; 7864 } 7865 7866 static void handle_lower_qman_data_on_err(struct hl_device *hdev, u64 qman_base, u32 engine_id) 7867 { 7868 struct undefined_opcode_info *undef_opcode = &hdev->captured_err_info.undef_opcode; 7869 u64 cq_ptr, cp_current_inst; 7870 u32 lo, hi, cq_size, cp_sts; 7871 bool is_arc_cq; 7872 7873 cp_sts = RREG32(qman_base + QM_CP_STS_4_OFFSET); 7874 is_arc_cq = FIELD_GET(PDMA0_QM_CP_STS_CUR_CQ_MASK, cp_sts); /* 0 - legacy CQ, 1 - ARC_CQ */ 7875 7876 if (is_arc_cq) { 7877 lo = RREG32(qman_base + QM_ARC_CQ_PTR_LO_STS_OFFSET); 7878 hi = RREG32(qman_base + QM_ARC_CQ_PTR_HI_STS_OFFSET); 7879 cq_ptr = ((u64) hi) << 32 | lo; 7880 cq_size = RREG32(qman_base + QM_ARC_CQ_TSIZE_STS_OFFSET); 7881 } else { 7882 lo = RREG32(qman_base + QM_CQ_PTR_LO_STS_4_OFFSET); 7883 hi = RREG32(qman_base + QM_CQ_PTR_HI_STS_4_OFFSET); 7884 cq_ptr = ((u64) hi) << 32 | lo; 7885 cq_size = RREG32(qman_base + QM_CQ_TSIZE_STS_4_OFFSET); 7886 } 7887 7888 lo = RREG32(qman_base + QM_CP_CURRENT_INST_LO_4_OFFSET); 7889 hi = RREG32(qman_base + QM_CP_CURRENT_INST_HI_4_OFFSET); 7890 cp_current_inst = ((u64) hi) << 32 | lo; 7891 7892 dev_info(hdev->dev, 7893 "LowerQM. %sCQ: {ptr %#llx, size %u}, CP: {instruction %#018llx}\n", 7894 is_arc_cq ? "ARC_" : "", cq_ptr, cq_size, cp_current_inst); 7895 7896 if (undef_opcode->write_enable) { 7897 memset(undef_opcode, 0, sizeof(*undef_opcode)); 7898 undef_opcode->timestamp = ktime_get(); 7899 undef_opcode->cq_addr = cq_ptr; 7900 undef_opcode->cq_size = cq_size; 7901 undef_opcode->engine_id = engine_id; 7902 undef_opcode->stream_id = QMAN_STREAMS; 7903 undef_opcode->write_enable = 0; 7904 } 7905 } 7906 7907 static int gaudi2_handle_qman_err_generic(struct hl_device *hdev, u16 event_type, 7908 u64 qman_base, u32 qid_base, u64 *event_mask) 7909 { 7910 u32 i, j, glbl_sts_val, arb_err_val, num_error_causes, error_count = 0; 7911 u64 glbl_sts_addr, arb_err_addr; 7912 char reg_desc[32]; 7913 7914 glbl_sts_addr = qman_base + (mmDCORE0_TPC0_QM_GLBL_ERR_STS_0 - mmDCORE0_TPC0_QM_BASE); 7915 arb_err_addr = qman_base + (mmDCORE0_TPC0_QM_ARB_ERR_CAUSE - mmDCORE0_TPC0_QM_BASE); 7916 7917 /* Iterate through all stream GLBL_ERR_STS registers + Lower CP */ 7918 for (i = 0 ; i < QMAN_STREAMS + 1 ; i++) { 7919 glbl_sts_val = RREG32(glbl_sts_addr + 4 * i); 7920 7921 if (!glbl_sts_val) 7922 continue; 7923 7924 if (i == QMAN_STREAMS) { 7925 snprintf(reg_desc, ARRAY_SIZE(reg_desc), "LowerQM"); 7926 num_error_causes = GAUDI2_NUM_OF_LOWER_QM_ERR_CAUSE; 7927 } else { 7928 snprintf(reg_desc, ARRAY_SIZE(reg_desc), "stream%u", i); 7929 num_error_causes = GAUDI2_NUM_OF_QM_ERR_CAUSE; 7930 } 7931 7932 for (j = 0 ; j < num_error_causes ; j++) 7933 if (glbl_sts_val & BIT(j)) { 7934 gaudi2_print_event(hdev, event_type, true, 7935 "%s. err cause: %s", reg_desc, 7936 i == QMAN_STREAMS ? 7937 gaudi2_lower_qman_error_cause[j] : 7938 gaudi2_qman_error_cause[j]); 7939 error_count++; 7940 } 7941 7942 /* Check for undefined opcode error in lower QM */ 7943 if ((i == QMAN_STREAMS) && 7944 (glbl_sts_val & PDMA0_QM_GLBL_ERR_STS_CP_UNDEF_CMD_ERR_MASK)) { 7945 handle_lower_qman_data_on_err(hdev, qman_base, 7946 gaudi2_queue_id_to_engine_id[qid_base]); 7947 *event_mask |= HL_NOTIFIER_EVENT_UNDEFINED_OPCODE; 7948 } 7949 } 7950 7951 arb_err_val = RREG32(arb_err_addr); 7952 7953 if (!arb_err_val) 7954 goto out; 7955 7956 for (j = 0 ; j < GAUDI2_NUM_OF_QM_ARB_ERR_CAUSE ; j++) { 7957 if (arb_err_val & BIT(j)) { 7958 gaudi2_print_event(hdev, event_type, true, 7959 "ARB_ERR. err cause: %s", 7960 gaudi2_qman_arb_error_cause[j]); 7961 error_count++; 7962 } 7963 } 7964 7965 out: 7966 return error_count; 7967 } 7968 7969 static void gaudi2_razwi_rr_hbw_shared_printf_info(struct hl_device *hdev, 7970 u64 rtr_mstr_if_base_addr, bool is_write, char *name, 7971 enum gaudi2_engine_id id, u64 *event_mask) 7972 { 7973 u32 razwi_hi, razwi_lo, razwi_xy; 7974 u16 eng_id = id; 7975 u8 rd_wr_flag; 7976 7977 if (is_write) { 7978 razwi_hi = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HI); 7979 razwi_lo = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_LO); 7980 razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_XY); 7981 rd_wr_flag = HL_RAZWI_WRITE; 7982 } else { 7983 razwi_hi = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HI); 7984 razwi_lo = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_LO); 7985 razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_XY); 7986 rd_wr_flag = HL_RAZWI_READ; 7987 } 7988 7989 hl_handle_razwi(hdev, (u64)razwi_hi << 32 | razwi_lo, &eng_id, 1, 7990 rd_wr_flag | HL_RAZWI_HBW, event_mask); 7991 7992 dev_err_ratelimited(hdev->dev, 7993 "%s-RAZWI SHARED RR HBW %s error, address %#llx, Initiator coordinates 0x%x\n", 7994 name, is_write ? "WR" : "RD", (u64)razwi_hi << 32 | razwi_lo, razwi_xy); 7995 } 7996 7997 static void gaudi2_razwi_rr_lbw_shared_printf_info(struct hl_device *hdev, 7998 u64 rtr_mstr_if_base_addr, bool is_write, char *name, 7999 enum gaudi2_engine_id id, u64 *event_mask) 8000 { 8001 u64 razwi_addr = CFG_BASE; 8002 u32 razwi_xy; 8003 u16 eng_id = id; 8004 u8 rd_wr_flag; 8005 8006 if (is_write) { 8007 razwi_addr += RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI); 8008 razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_XY); 8009 rd_wr_flag = HL_RAZWI_WRITE; 8010 } else { 8011 razwi_addr += RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI); 8012 razwi_xy = RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_XY); 8013 rd_wr_flag = HL_RAZWI_READ; 8014 } 8015 8016 hl_handle_razwi(hdev, razwi_addr, &eng_id, 1, rd_wr_flag | HL_RAZWI_LBW, event_mask); 8017 dev_err_ratelimited(hdev->dev, 8018 "%s-RAZWI SHARED RR LBW %s error, mstr_if 0x%llx, captured address 0x%llX Initiator coordinates 0x%x\n", 8019 name, is_write ? "WR" : "RD", rtr_mstr_if_base_addr, razwi_addr, 8020 razwi_xy); 8021 } 8022 8023 static enum gaudi2_engine_id gaudi2_razwi_calc_engine_id(struct hl_device *hdev, 8024 enum razwi_event_sources module, u8 module_idx) 8025 { 8026 switch (module) { 8027 case RAZWI_TPC: 8028 if (module_idx == (NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES)) 8029 return GAUDI2_DCORE0_ENGINE_ID_TPC_6; 8030 return (((module_idx / NUM_OF_TPC_PER_DCORE) * ENGINE_ID_DCORE_OFFSET) + 8031 (module_idx % NUM_OF_TPC_PER_DCORE) + 8032 (GAUDI2_DCORE0_ENGINE_ID_TPC_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0)); 8033 8034 case RAZWI_MME: 8035 return ((GAUDI2_DCORE0_ENGINE_ID_MME - GAUDI2_DCORE0_ENGINE_ID_EDMA_0) + 8036 (module_idx * ENGINE_ID_DCORE_OFFSET)); 8037 8038 case RAZWI_EDMA: 8039 return (((module_idx / NUM_OF_EDMA_PER_DCORE) * ENGINE_ID_DCORE_OFFSET) + 8040 (module_idx % NUM_OF_EDMA_PER_DCORE)); 8041 8042 case RAZWI_PDMA: 8043 return (GAUDI2_ENGINE_ID_PDMA_0 + module_idx); 8044 8045 case RAZWI_NIC: 8046 return (GAUDI2_ENGINE_ID_NIC0_0 + (NIC_NUMBER_OF_QM_PER_MACRO * module_idx)); 8047 8048 case RAZWI_DEC: 8049 if (module_idx == 8) 8050 return GAUDI2_PCIE_ENGINE_ID_DEC_0; 8051 8052 if (module_idx == 9) 8053 return GAUDI2_PCIE_ENGINE_ID_DEC_1; 8054 ; 8055 return (((module_idx / NUM_OF_DEC_PER_DCORE) * ENGINE_ID_DCORE_OFFSET) + 8056 (module_idx % NUM_OF_DEC_PER_DCORE) + 8057 (GAUDI2_DCORE0_ENGINE_ID_DEC_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0)); 8058 8059 case RAZWI_ROT: 8060 return GAUDI2_ENGINE_ID_ROT_0 + module_idx; 8061 8062 case RAZWI_ARC_FARM: 8063 return GAUDI2_ENGINE_ID_ARC_FARM; 8064 8065 default: 8066 return GAUDI2_ENGINE_ID_SIZE; 8067 } 8068 } 8069 8070 /* 8071 * This function handles RR(Range register) hit events. 8072 * raised be initiators not PSOC RAZWI. 8073 */ 8074 static void gaudi2_ack_module_razwi_event_handler(struct hl_device *hdev, 8075 enum razwi_event_sources module, u8 module_idx, 8076 u8 module_sub_idx, u64 *event_mask) 8077 { 8078 bool via_sft = false; 8079 u32 hbw_rtr_id, lbw_rtr_id, dcore_id, dcore_rtr_id, eng_id, binned_idx; 8080 u64 hbw_rtr_mstr_if_base_addr, lbw_rtr_mstr_if_base_addr; 8081 u32 hbw_shrd_aw = 0, hbw_shrd_ar = 0; 8082 u32 lbw_shrd_aw = 0, lbw_shrd_ar = 0; 8083 char initiator_name[64]; 8084 8085 switch (module) { 8086 case RAZWI_TPC: 8087 sprintf(initiator_name, "TPC_%u", module_idx); 8088 if (hdev->tpc_binning) { 8089 binned_idx = __ffs(hdev->tpc_binning); 8090 if (binned_idx == module_idx) 8091 module_idx = TPC_ID_DCORE0_TPC6; 8092 } 8093 8094 hbw_rtr_id = gaudi2_tpc_initiator_hbw_rtr_id[module_idx]; 8095 8096 if (hl_is_fw_sw_ver_below(hdev, 1, 9) && 8097 !hdev->asic_prop.fw_security_enabled && 8098 ((module_idx == 0) || (module_idx == 1))) 8099 lbw_rtr_id = DCORE0_RTR0; 8100 else 8101 lbw_rtr_id = gaudi2_tpc_initiator_lbw_rtr_id[module_idx]; 8102 break; 8103 case RAZWI_MME: 8104 sprintf(initiator_name, "MME_%u", module_idx); 8105 switch (module_sub_idx) { 8106 case MME_WAP0: 8107 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].wap0; 8108 break; 8109 case MME_WAP1: 8110 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].wap1; 8111 break; 8112 case MME_WRITE: 8113 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].write; 8114 break; 8115 case MME_READ: 8116 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].read; 8117 break; 8118 case MME_SBTE0: 8119 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte0; 8120 break; 8121 case MME_SBTE1: 8122 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte1; 8123 break; 8124 case MME_SBTE2: 8125 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte2; 8126 break; 8127 case MME_SBTE3: 8128 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte3; 8129 break; 8130 case MME_SBTE4: 8131 hbw_rtr_id = gaudi2_mme_initiator_rtr_id[module_idx].sbte4; 8132 break; 8133 default: 8134 return; 8135 } 8136 lbw_rtr_id = hbw_rtr_id; 8137 break; 8138 case RAZWI_EDMA: 8139 hbw_rtr_mstr_if_base_addr = gaudi2_edma_initiator_hbw_sft[module_idx]; 8140 dcore_id = module_idx / NUM_OF_EDMA_PER_DCORE; 8141 /* SFT has separate MSTR_IF for LBW, only there we can 8142 * read the LBW razwi related registers 8143 */ 8144 lbw_rtr_mstr_if_base_addr = mmSFT0_LBW_RTR_IF_MSTR_IF_RR_SHRD_HBW_BASE + 8145 dcore_id * SFT_DCORE_OFFSET; 8146 via_sft = true; 8147 sprintf(initiator_name, "EDMA_%u", module_idx); 8148 break; 8149 case RAZWI_PDMA: 8150 hbw_rtr_id = gaudi2_pdma_initiator_hbw_rtr_id[module_idx]; 8151 lbw_rtr_id = gaudi2_pdma_initiator_lbw_rtr_id[module_idx]; 8152 sprintf(initiator_name, "PDMA_%u", module_idx); 8153 break; 8154 case RAZWI_NIC: 8155 hbw_rtr_id = gaudi2_nic_initiator_hbw_rtr_id[module_idx]; 8156 lbw_rtr_id = gaudi2_nic_initiator_lbw_rtr_id[module_idx]; 8157 sprintf(initiator_name, "NIC_%u", module_idx); 8158 break; 8159 case RAZWI_DEC: 8160 sprintf(initiator_name, "DEC_%u", module_idx); 8161 if (hdev->decoder_binning) { 8162 binned_idx = __ffs(hdev->decoder_binning); 8163 if (binned_idx == module_idx) 8164 module_idx = DEC_ID_PCIE_VDEC1; 8165 } 8166 hbw_rtr_id = gaudi2_dec_initiator_hbw_rtr_id[module_idx]; 8167 lbw_rtr_id = gaudi2_dec_initiator_lbw_rtr_id[module_idx]; 8168 break; 8169 case RAZWI_ROT: 8170 hbw_rtr_id = gaudi2_rot_initiator_hbw_rtr_id[module_idx]; 8171 lbw_rtr_id = gaudi2_rot_initiator_lbw_rtr_id[module_idx]; 8172 sprintf(initiator_name, "ROT_%u", module_idx); 8173 break; 8174 case RAZWI_ARC_FARM: 8175 lbw_rtr_id = DCORE1_RTR5; 8176 hbw_rtr_id = DCORE1_RTR7; 8177 sprintf(initiator_name, "ARC_FARM_%u", module_idx); 8178 break; 8179 default: 8180 return; 8181 } 8182 8183 /* Find router mstr_if register base */ 8184 if (!via_sft) { 8185 dcore_id = hbw_rtr_id / NUM_OF_RTR_PER_DCORE; 8186 dcore_rtr_id = hbw_rtr_id % NUM_OF_RTR_PER_DCORE; 8187 hbw_rtr_mstr_if_base_addr = mmDCORE0_RTR0_CTRL_BASE + 8188 dcore_id * DCORE_OFFSET + 8189 dcore_rtr_id * DCORE_RTR_OFFSET + 8190 RTR_MSTR_IF_OFFSET; 8191 lbw_rtr_mstr_if_base_addr = hbw_rtr_mstr_if_base_addr + 8192 (((s32)lbw_rtr_id - hbw_rtr_id) * DCORE_RTR_OFFSET); 8193 } 8194 8195 /* Find out event cause by reading "RAZWI_HAPPENED" registers */ 8196 hbw_shrd_aw = RREG32(hbw_rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HAPPENED); 8197 hbw_shrd_ar = RREG32(hbw_rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HAPPENED); 8198 lbw_shrd_aw = RREG32(lbw_rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_HAPPENED); 8199 lbw_shrd_ar = RREG32(lbw_rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_HAPPENED); 8200 8201 eng_id = gaudi2_razwi_calc_engine_id(hdev, module, module_idx); 8202 if (hbw_shrd_aw) { 8203 gaudi2_razwi_rr_hbw_shared_printf_info(hdev, hbw_rtr_mstr_if_base_addr, true, 8204 initiator_name, eng_id, event_mask); 8205 8206 /* Clear event indication */ 8207 WREG32(hbw_rtr_mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HAPPENED, hbw_shrd_aw); 8208 } 8209 8210 if (hbw_shrd_ar) { 8211 gaudi2_razwi_rr_hbw_shared_printf_info(hdev, hbw_rtr_mstr_if_base_addr, false, 8212 initiator_name, eng_id, event_mask); 8213 8214 /* Clear event indication */ 8215 WREG32(hbw_rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HAPPENED, hbw_shrd_ar); 8216 } 8217 8218 if (lbw_shrd_aw) { 8219 gaudi2_razwi_rr_lbw_shared_printf_info(hdev, lbw_rtr_mstr_if_base_addr, true, 8220 initiator_name, eng_id, event_mask); 8221 8222 /* Clear event indication */ 8223 WREG32(lbw_rtr_mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_HAPPENED, lbw_shrd_aw); 8224 } 8225 8226 if (lbw_shrd_ar) { 8227 gaudi2_razwi_rr_lbw_shared_printf_info(hdev, lbw_rtr_mstr_if_base_addr, false, 8228 initiator_name, eng_id, event_mask); 8229 8230 /* Clear event indication */ 8231 WREG32(lbw_rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_HAPPENED, lbw_shrd_ar); 8232 } 8233 } 8234 8235 static void gaudi2_check_if_razwi_happened(struct hl_device *hdev) 8236 { 8237 struct asic_fixed_properties *prop = &hdev->asic_prop; 8238 u8 mod_idx, sub_mod; 8239 8240 /* check all TPCs */ 8241 for (mod_idx = 0 ; mod_idx < (NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1) ; mod_idx++) { 8242 if (prop->tpc_enabled_mask & BIT(mod_idx)) 8243 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_TPC, mod_idx, 0, NULL); 8244 } 8245 8246 /* check all MMEs */ 8247 for (mod_idx = 0 ; mod_idx < (NUM_OF_MME_PER_DCORE * NUM_OF_DCORES) ; mod_idx++) 8248 for (sub_mod = MME_WAP0 ; sub_mod < MME_INITIATORS_MAX ; sub_mod++) 8249 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mod_idx, 8250 sub_mod, NULL); 8251 8252 /* check all EDMAs */ 8253 for (mod_idx = 0 ; mod_idx < (NUM_OF_EDMA_PER_DCORE * NUM_OF_DCORES) ; mod_idx++) 8254 if (prop->edma_enabled_mask & BIT(mod_idx)) 8255 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_EDMA, mod_idx, 0, NULL); 8256 8257 /* check all PDMAs */ 8258 for (mod_idx = 0 ; mod_idx < NUM_OF_PDMA ; mod_idx++) 8259 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_PDMA, mod_idx, 0, NULL); 8260 8261 /* check all NICs */ 8262 for (mod_idx = 0 ; mod_idx < NIC_NUMBER_OF_PORTS ; mod_idx++) 8263 if (hdev->nic_ports_mask & BIT(mod_idx)) 8264 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_NIC, mod_idx >> 1, 0, 8265 NULL); 8266 8267 /* check all DECs */ 8268 for (mod_idx = 0 ; mod_idx < NUMBER_OF_DEC ; mod_idx++) 8269 if (prop->decoder_enabled_mask & BIT(mod_idx)) 8270 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_DEC, mod_idx, 0, NULL); 8271 8272 /* check all ROTs */ 8273 for (mod_idx = 0 ; mod_idx < NUM_OF_ROT ; mod_idx++) 8274 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_ROT, mod_idx, 0, NULL); 8275 } 8276 8277 static int gaudi2_psoc_razwi_get_engines(struct gaudi2_razwi_info *razwi_info, u32 array_size, 8278 u32 axuser_xy, u32 *base, u16 *eng_id, 8279 char *eng_name) 8280 { 8281 8282 int i, num_of_eng = 0; 8283 u16 str_size = 0; 8284 8285 for (i = 0 ; i < array_size ; i++) { 8286 if (axuser_xy != razwi_info[i].axuser_xy) 8287 continue; 8288 8289 eng_id[num_of_eng] = razwi_info[i].eng_id; 8290 base[num_of_eng] = razwi_info[i].rtr_ctrl; 8291 if (!num_of_eng) 8292 str_size += scnprintf(eng_name + str_size, 8293 PSOC_RAZWI_ENG_STR_SIZE - str_size, "%s", 8294 razwi_info[i].eng_name); 8295 else 8296 str_size += scnprintf(eng_name + str_size, 8297 PSOC_RAZWI_ENG_STR_SIZE - str_size, " or %s", 8298 razwi_info[i].eng_name); 8299 num_of_eng++; 8300 } 8301 8302 return num_of_eng; 8303 } 8304 8305 static bool gaudi2_handle_psoc_razwi_happened(struct hl_device *hdev, u32 razwi_reg, 8306 u64 *event_mask) 8307 { 8308 u32 axuser_xy = RAZWI_GET_AXUSER_XY(razwi_reg), addr_hi = 0, addr_lo = 0; 8309 u32 base[PSOC_RAZWI_MAX_ENG_PER_RTR]; 8310 u16 num_of_eng, eng_id[PSOC_RAZWI_MAX_ENG_PER_RTR]; 8311 char eng_name_str[PSOC_RAZWI_ENG_STR_SIZE]; 8312 bool razwi_happened = false; 8313 u64 addr; 8314 int i; 8315 8316 num_of_eng = gaudi2_psoc_razwi_get_engines(common_razwi_info, ARRAY_SIZE(common_razwi_info), 8317 axuser_xy, base, eng_id, eng_name_str); 8318 8319 /* If no match for XY coordinates, try to find it in MME razwi table */ 8320 if (!num_of_eng) { 8321 axuser_xy = RAZWI_GET_AXUSER_LOW_XY(razwi_reg); 8322 num_of_eng = gaudi2_psoc_razwi_get_engines(mme_razwi_info, 8323 ARRAY_SIZE(mme_razwi_info), 8324 axuser_xy, base, eng_id, 8325 eng_name_str); 8326 } 8327 8328 for (i = 0 ; i < num_of_eng ; i++) { 8329 if (RREG32(base[i] + DEC_RAZWI_HBW_AW_SET)) { 8330 addr_hi = RREG32(base[i] + DEC_RAZWI_HBW_AW_ADDR_HI); 8331 addr_lo = RREG32(base[i] + DEC_RAZWI_HBW_AW_ADDR_LO); 8332 addr = ((u64)addr_hi << 32) + addr_lo; 8333 if (addr) { 8334 dev_err(hdev->dev, 8335 "PSOC HBW AW RAZWI: %s, address (aligned to 128 byte): 0x%llX\n", 8336 eng_name_str, addr); 8337 hl_handle_razwi(hdev, addr, &eng_id[0], 8338 num_of_eng, HL_RAZWI_HBW | HL_RAZWI_WRITE, event_mask); 8339 razwi_happened = true; 8340 } 8341 } 8342 8343 if (RREG32(base[i] + DEC_RAZWI_HBW_AR_SET)) { 8344 addr_hi = RREG32(base[i] + DEC_RAZWI_HBW_AR_ADDR_HI); 8345 addr_lo = RREG32(base[i] + DEC_RAZWI_HBW_AR_ADDR_LO); 8346 addr = ((u64)addr_hi << 32) + addr_lo; 8347 if (addr) { 8348 dev_err(hdev->dev, 8349 "PSOC HBW AR RAZWI: %s, address (aligned to 128 byte): 0x%llX\n", 8350 eng_name_str, addr); 8351 hl_handle_razwi(hdev, addr, &eng_id[0], 8352 num_of_eng, HL_RAZWI_HBW | HL_RAZWI_READ, event_mask); 8353 razwi_happened = true; 8354 } 8355 } 8356 8357 if (RREG32(base[i] + DEC_RAZWI_LBW_AW_SET)) { 8358 addr_lo = RREG32(base[i] + DEC_RAZWI_LBW_AW_ADDR); 8359 if (addr_lo) { 8360 dev_err(hdev->dev, 8361 "PSOC LBW AW RAZWI: %s, address (aligned to 128 byte): 0x%X\n", 8362 eng_name_str, addr_lo); 8363 hl_handle_razwi(hdev, addr_lo, &eng_id[0], 8364 num_of_eng, HL_RAZWI_LBW | HL_RAZWI_WRITE, event_mask); 8365 razwi_happened = true; 8366 } 8367 } 8368 8369 if (RREG32(base[i] + DEC_RAZWI_LBW_AR_SET)) { 8370 addr_lo = RREG32(base[i] + DEC_RAZWI_LBW_AR_ADDR); 8371 if (addr_lo) { 8372 dev_err(hdev->dev, 8373 "PSOC LBW AR RAZWI: %s, address (aligned to 128 byte): 0x%X\n", 8374 eng_name_str, addr_lo); 8375 hl_handle_razwi(hdev, addr_lo, &eng_id[0], 8376 num_of_eng, HL_RAZWI_LBW | HL_RAZWI_READ, event_mask); 8377 razwi_happened = true; 8378 } 8379 } 8380 /* In common case the loop will break, when there is only one engine id, or 8381 * several engines with the same router. The exceptional case is with psoc razwi 8382 * from EDMA, where it's possible to get axuser id which fits 2 routers (2 8383 * interfaces of sft router). In this case, maybe the first router won't hold info 8384 * and we will need to iterate on the other router. 8385 */ 8386 if (razwi_happened) 8387 break; 8388 } 8389 8390 return razwi_happened; 8391 } 8392 8393 /* PSOC RAZWI interrupt occurs only when trying to access a bad address */ 8394 static int gaudi2_ack_psoc_razwi_event_handler(struct hl_device *hdev, u64 *event_mask) 8395 { 8396 u32 razwi_mask_info, razwi_intr = 0, error_count = 0; 8397 8398 if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX)) { 8399 razwi_intr = RREG32(mmPSOC_GLOBAL_CONF_RAZWI_INTERRUPT); 8400 if (!razwi_intr) 8401 return 0; 8402 } 8403 8404 razwi_mask_info = RREG32(mmPSOC_GLOBAL_CONF_RAZWI_MASK_INFO); 8405 8406 dev_err_ratelimited(hdev->dev, 8407 "PSOC RAZWI interrupt: Mask %d, AR %d, AW %d, AXUSER_L 0x%x AXUSER_H 0x%x\n", 8408 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_MASK_MASK, razwi_mask_info), 8409 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_WAS_AR_MASK, razwi_mask_info), 8410 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_WAS_AW_MASK, razwi_mask_info), 8411 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_AXUSER_L_MASK, razwi_mask_info), 8412 FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_AXUSER_H_MASK, razwi_mask_info)); 8413 8414 if (gaudi2_handle_psoc_razwi_happened(hdev, razwi_mask_info, event_mask)) 8415 error_count++; 8416 else 8417 dev_err_ratelimited(hdev->dev, 8418 "PSOC RAZWI interrupt: invalid razwi info (0x%x)\n", 8419 razwi_mask_info); 8420 8421 /* Clear Interrupts only on pldm or if f/w doesn't handle interrupts */ 8422 if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX)) 8423 WREG32(mmPSOC_GLOBAL_CONF_RAZWI_INTERRUPT, razwi_intr); 8424 8425 return error_count; 8426 } 8427 8428 static int _gaudi2_handle_qm_sei_err(struct hl_device *hdev, u64 qman_base, u16 event_type) 8429 { 8430 u32 i, sts_val, sts_clr_val = 0, error_count = 0; 8431 8432 sts_val = RREG32(qman_base + QM_SEI_STATUS_OFFSET); 8433 8434 for (i = 0 ; i < GAUDI2_NUM_OF_QM_SEI_ERR_CAUSE ; i++) { 8435 if (sts_val & BIT(i)) { 8436 gaudi2_print_event(hdev, event_type, true, 8437 "err cause: %s", gaudi2_qm_sei_error_cause[i]); 8438 sts_clr_val |= BIT(i); 8439 error_count++; 8440 } 8441 } 8442 8443 WREG32(qman_base + QM_SEI_STATUS_OFFSET, sts_clr_val); 8444 8445 return error_count; 8446 } 8447 8448 static int gaudi2_handle_qm_sei_err(struct hl_device *hdev, u16 event_type, 8449 bool extended_err_check, u64 *event_mask) 8450 { 8451 enum razwi_event_sources module; 8452 u32 error_count = 0; 8453 u64 qman_base; 8454 u8 index; 8455 8456 switch (event_type) { 8457 case GAUDI2_EVENT_TPC0_AXI_ERR_RSP ... GAUDI2_EVENT_TPC23_AXI_ERR_RSP: 8458 index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP; 8459 qman_base = mmDCORE0_TPC0_QM_BASE + 8460 (index / NUM_OF_TPC_PER_DCORE) * DCORE_OFFSET + 8461 (index % NUM_OF_TPC_PER_DCORE) * DCORE_TPC_OFFSET; 8462 module = RAZWI_TPC; 8463 break; 8464 case GAUDI2_EVENT_TPC24_AXI_ERR_RSP: 8465 qman_base = mmDCORE0_TPC6_QM_BASE; 8466 module = RAZWI_TPC; 8467 break; 8468 case GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE: 8469 case GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE: 8470 case GAUDI2_EVENT_MME2_CTRL_AXI_ERROR_RESPONSE: 8471 case GAUDI2_EVENT_MME3_CTRL_AXI_ERROR_RESPONSE: 8472 index = (event_type - GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE) / 8473 (GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE - 8474 GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE); 8475 qman_base = mmDCORE0_MME_QM_BASE + index * DCORE_OFFSET; 8476 module = RAZWI_MME; 8477 break; 8478 case GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP: 8479 case GAUDI2_EVENT_PDMA_CH1_AXI_ERR_RSP: 8480 index = event_type - GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP; 8481 qman_base = mmPDMA0_QM_BASE + index * PDMA_OFFSET; 8482 module = RAZWI_PDMA; 8483 break; 8484 case GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE: 8485 case GAUDI2_EVENT_ROTATOR1_AXI_ERROR_RESPONSE: 8486 index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE; 8487 qman_base = mmROT0_QM_BASE + index * ROT_OFFSET; 8488 module = RAZWI_ROT; 8489 break; 8490 default: 8491 return 0; 8492 } 8493 8494 error_count = _gaudi2_handle_qm_sei_err(hdev, qman_base, event_type); 8495 8496 /* There is a single event per NIC macro, so should check its both QMAN blocks */ 8497 if (event_type >= GAUDI2_EVENT_NIC0_AXI_ERROR_RESPONSE && 8498 event_type <= GAUDI2_EVENT_NIC11_AXI_ERROR_RESPONSE) 8499 error_count += _gaudi2_handle_qm_sei_err(hdev, 8500 qman_base + NIC_QM_OFFSET, event_type); 8501 8502 if (extended_err_check) { 8503 /* check if RAZWI happened */ 8504 gaudi2_ack_module_razwi_event_handler(hdev, module, 0, 0, event_mask); 8505 hl_check_for_glbl_errors(hdev); 8506 } 8507 8508 return error_count; 8509 } 8510 8511 static int gaudi2_handle_qman_err(struct hl_device *hdev, u16 event_type, u64 *event_mask) 8512 { 8513 u32 qid_base, error_count = 0; 8514 u64 qman_base; 8515 u8 index = 0; 8516 8517 switch (event_type) { 8518 case GAUDI2_EVENT_TPC0_QM ... GAUDI2_EVENT_TPC5_QM: 8519 index = event_type - GAUDI2_EVENT_TPC0_QM; 8520 qid_base = GAUDI2_QUEUE_ID_DCORE0_TPC_0_0 + index * QMAN_STREAMS; 8521 qman_base = mmDCORE0_TPC0_QM_BASE + index * DCORE_TPC_OFFSET; 8522 break; 8523 case GAUDI2_EVENT_TPC6_QM ... GAUDI2_EVENT_TPC11_QM: 8524 index = event_type - GAUDI2_EVENT_TPC6_QM; 8525 qid_base = GAUDI2_QUEUE_ID_DCORE1_TPC_0_0 + index * QMAN_STREAMS; 8526 qman_base = mmDCORE1_TPC0_QM_BASE + index * DCORE_TPC_OFFSET; 8527 break; 8528 case GAUDI2_EVENT_TPC12_QM ... GAUDI2_EVENT_TPC17_QM: 8529 index = event_type - GAUDI2_EVENT_TPC12_QM; 8530 qid_base = GAUDI2_QUEUE_ID_DCORE2_TPC_0_0 + index * QMAN_STREAMS; 8531 qman_base = mmDCORE2_TPC0_QM_BASE + index * DCORE_TPC_OFFSET; 8532 break; 8533 case GAUDI2_EVENT_TPC18_QM ... GAUDI2_EVENT_TPC23_QM: 8534 index = event_type - GAUDI2_EVENT_TPC18_QM; 8535 qid_base = GAUDI2_QUEUE_ID_DCORE3_TPC_0_0 + index * QMAN_STREAMS; 8536 qman_base = mmDCORE3_TPC0_QM_BASE + index * DCORE_TPC_OFFSET; 8537 break; 8538 case GAUDI2_EVENT_TPC24_QM: 8539 qid_base = GAUDI2_QUEUE_ID_DCORE0_TPC_6_0; 8540 qman_base = mmDCORE0_TPC6_QM_BASE; 8541 break; 8542 case GAUDI2_EVENT_MME0_QM: 8543 qid_base = GAUDI2_QUEUE_ID_DCORE0_MME_0_0; 8544 qman_base = mmDCORE0_MME_QM_BASE; 8545 break; 8546 case GAUDI2_EVENT_MME1_QM: 8547 qid_base = GAUDI2_QUEUE_ID_DCORE1_MME_0_0; 8548 qman_base = mmDCORE1_MME_QM_BASE; 8549 break; 8550 case GAUDI2_EVENT_MME2_QM: 8551 qid_base = GAUDI2_QUEUE_ID_DCORE2_MME_0_0; 8552 qman_base = mmDCORE2_MME_QM_BASE; 8553 break; 8554 case GAUDI2_EVENT_MME3_QM: 8555 qid_base = GAUDI2_QUEUE_ID_DCORE3_MME_0_0; 8556 qman_base = mmDCORE3_MME_QM_BASE; 8557 break; 8558 case GAUDI2_EVENT_HDMA0_QM: 8559 index = 0; 8560 qid_base = GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0; 8561 qman_base = mmDCORE0_EDMA0_QM_BASE; 8562 break; 8563 case GAUDI2_EVENT_HDMA1_QM: 8564 index = 1; 8565 qid_base = GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0; 8566 qman_base = mmDCORE0_EDMA1_QM_BASE; 8567 break; 8568 case GAUDI2_EVENT_HDMA2_QM: 8569 index = 2; 8570 qid_base = GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0; 8571 qman_base = mmDCORE1_EDMA0_QM_BASE; 8572 break; 8573 case GAUDI2_EVENT_HDMA3_QM: 8574 index = 3; 8575 qid_base = GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0; 8576 qman_base = mmDCORE1_EDMA1_QM_BASE; 8577 break; 8578 case GAUDI2_EVENT_HDMA4_QM: 8579 index = 4; 8580 qid_base = GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0; 8581 qman_base = mmDCORE2_EDMA0_QM_BASE; 8582 break; 8583 case GAUDI2_EVENT_HDMA5_QM: 8584 index = 5; 8585 qid_base = GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0; 8586 qman_base = mmDCORE2_EDMA1_QM_BASE; 8587 break; 8588 case GAUDI2_EVENT_HDMA6_QM: 8589 index = 6; 8590 qid_base = GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0; 8591 qman_base = mmDCORE3_EDMA0_QM_BASE; 8592 break; 8593 case GAUDI2_EVENT_HDMA7_QM: 8594 index = 7; 8595 qid_base = GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0; 8596 qman_base = mmDCORE3_EDMA1_QM_BASE; 8597 break; 8598 case GAUDI2_EVENT_PDMA0_QM: 8599 qid_base = GAUDI2_QUEUE_ID_PDMA_0_0; 8600 qman_base = mmPDMA0_QM_BASE; 8601 break; 8602 case GAUDI2_EVENT_PDMA1_QM: 8603 qid_base = GAUDI2_QUEUE_ID_PDMA_1_0; 8604 qman_base = mmPDMA1_QM_BASE; 8605 break; 8606 case GAUDI2_EVENT_ROTATOR0_ROT0_QM: 8607 qid_base = GAUDI2_QUEUE_ID_ROT_0_0; 8608 qman_base = mmROT0_QM_BASE; 8609 break; 8610 case GAUDI2_EVENT_ROTATOR1_ROT1_QM: 8611 qid_base = GAUDI2_QUEUE_ID_ROT_1_0; 8612 qman_base = mmROT1_QM_BASE; 8613 break; 8614 default: 8615 return 0; 8616 } 8617 8618 error_count = gaudi2_handle_qman_err_generic(hdev, event_type, qman_base, 8619 qid_base, event_mask); 8620 8621 /* Handle EDMA QM SEI here because there is no AXI error response event for EDMA */ 8622 if (event_type >= GAUDI2_EVENT_HDMA2_QM && event_type <= GAUDI2_EVENT_HDMA5_QM) { 8623 error_count += _gaudi2_handle_qm_sei_err(hdev, qman_base, event_type); 8624 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_EDMA, index, 0, event_mask); 8625 } 8626 8627 hl_check_for_glbl_errors(hdev); 8628 8629 return error_count; 8630 } 8631 8632 static int gaudi2_handle_arc_farm_sei_err(struct hl_device *hdev, u16 event_type, u64 *event_mask) 8633 { 8634 u32 i, sts_val, sts_clr_val, error_count = 0, arc_farm; 8635 8636 for (arc_farm = 0 ; arc_farm < NUM_OF_ARC_FARMS_ARC ; arc_farm++) { 8637 sts_clr_val = 0; 8638 sts_val = RREG32(mmARC_FARM_ARC0_AUX_ARC_SEI_INTR_STS + 8639 (arc_farm * ARC_FARM_OFFSET)); 8640 8641 for (i = 0 ; i < GAUDI2_NUM_OF_ARC_SEI_ERR_CAUSE ; i++) { 8642 if (sts_val & BIT(i)) { 8643 gaudi2_print_event(hdev, event_type, true, 8644 "ARC FARM ARC %u err cause: %s", 8645 arc_farm, gaudi2_arc_sei_error_cause[i]); 8646 sts_clr_val |= BIT(i); 8647 error_count++; 8648 } 8649 } 8650 WREG32(mmARC_FARM_ARC0_AUX_ARC_SEI_INTR_CLR + (arc_farm * ARC_FARM_OFFSET), 8651 sts_clr_val); 8652 } 8653 8654 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_ARC_FARM, 0, 0, event_mask); 8655 hl_check_for_glbl_errors(hdev); 8656 8657 return error_count; 8658 } 8659 8660 static int gaudi2_handle_cpu_sei_err(struct hl_device *hdev, u16 event_type) 8661 { 8662 u32 i, sts_val, sts_clr_val = 0, error_count = 0; 8663 8664 sts_val = RREG32(mmCPU_IF_CPU_SEI_INTR_STS); 8665 8666 for (i = 0 ; i < GAUDI2_NUM_OF_CPU_SEI_ERR_CAUSE ; i++) { 8667 if (sts_val & BIT(i)) { 8668 gaudi2_print_event(hdev, event_type, true, 8669 "err cause: %s", gaudi2_cpu_sei_error_cause[i]); 8670 sts_clr_val |= BIT(i); 8671 error_count++; 8672 } 8673 } 8674 8675 hl_check_for_glbl_errors(hdev); 8676 8677 WREG32(mmCPU_IF_CPU_SEI_INTR_CLR, sts_clr_val); 8678 8679 return error_count; 8680 } 8681 8682 static int gaudi2_handle_rot_err(struct hl_device *hdev, u8 rot_index, u16 event_type, 8683 struct hl_eq_razwi_with_intr_cause *razwi_with_intr_cause, 8684 u64 *event_mask) 8685 { 8686 u64 intr_cause_data = le64_to_cpu(razwi_with_intr_cause->intr_cause.intr_cause_data); 8687 u32 error_count = 0; 8688 int i; 8689 8690 for (i = 0 ; i < GAUDI2_NUM_OF_ROT_ERR_CAUSE ; i++) 8691 if (intr_cause_data & BIT(i)) { 8692 gaudi2_print_event(hdev, event_type, true, 8693 "err cause: %s", guadi2_rot_error_cause[i]); 8694 error_count++; 8695 } 8696 8697 /* check if RAZWI happened */ 8698 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_ROT, rot_index, 0, event_mask); 8699 hl_check_for_glbl_errors(hdev); 8700 8701 return error_count; 8702 } 8703 8704 static int gaudi2_tpc_ack_interrupts(struct hl_device *hdev, u8 tpc_index, u16 event_type, 8705 struct hl_eq_razwi_with_intr_cause *razwi_with_intr_cause, 8706 u64 *event_mask) 8707 { 8708 u64 intr_cause_data = le64_to_cpu(razwi_with_intr_cause->intr_cause.intr_cause_data); 8709 u32 error_count = 0; 8710 int i; 8711 8712 for (i = 0 ; i < GAUDI2_NUM_OF_TPC_INTR_CAUSE ; i++) 8713 if (intr_cause_data & BIT(i)) { 8714 gaudi2_print_event(hdev, event_type, true, 8715 "interrupt cause: %s", gaudi2_tpc_interrupts_cause[i]); 8716 error_count++; 8717 } 8718 8719 /* check if RAZWI happened */ 8720 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_TPC, tpc_index, 0, event_mask); 8721 hl_check_for_glbl_errors(hdev); 8722 8723 return error_count; 8724 } 8725 8726 static int gaudi2_handle_dec_err(struct hl_device *hdev, u8 dec_index, u16 event_type, 8727 u64 *event_mask) 8728 { 8729 u32 sts_addr, sts_val, sts_clr_val = 0, error_count = 0; 8730 int i; 8731 8732 if (dec_index < NUM_OF_VDEC_PER_DCORE * NUM_OF_DCORES) 8733 /* DCORE DEC */ 8734 sts_addr = mmDCORE0_VDEC0_BRDG_CTRL_CAUSE_INTR + 8735 DCORE_OFFSET * (dec_index / NUM_OF_DEC_PER_DCORE) + 8736 DCORE_VDEC_OFFSET * (dec_index % NUM_OF_DEC_PER_DCORE); 8737 else 8738 /* PCIE DEC */ 8739 sts_addr = mmPCIE_VDEC0_BRDG_CTRL_CAUSE_INTR + PCIE_VDEC_OFFSET * 8740 (dec_index - NUM_OF_VDEC_PER_DCORE * NUM_OF_DCORES); 8741 8742 sts_val = RREG32(sts_addr); 8743 8744 for (i = 0 ; i < GAUDI2_NUM_OF_DEC_ERR_CAUSE ; i++) { 8745 if (sts_val & BIT(i)) { 8746 gaudi2_print_event(hdev, event_type, true, 8747 "err cause: %s", gaudi2_dec_error_cause[i]); 8748 sts_clr_val |= BIT(i); 8749 error_count++; 8750 } 8751 } 8752 8753 /* check if RAZWI happened */ 8754 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_DEC, dec_index, 0, event_mask); 8755 hl_check_for_glbl_errors(hdev); 8756 8757 /* Write 1 clear errors */ 8758 WREG32(sts_addr, sts_clr_val); 8759 8760 return error_count; 8761 } 8762 8763 static int gaudi2_handle_mme_err(struct hl_device *hdev, u8 mme_index, u16 event_type, 8764 u64 *event_mask) 8765 { 8766 u32 sts_addr, sts_val, sts_clr_addr, sts_clr_val = 0, error_count = 0; 8767 int i; 8768 8769 sts_addr = mmDCORE0_MME_CTRL_LO_INTR_CAUSE + DCORE_OFFSET * mme_index; 8770 sts_clr_addr = mmDCORE0_MME_CTRL_LO_INTR_CLEAR + DCORE_OFFSET * mme_index; 8771 8772 sts_val = RREG32(sts_addr); 8773 8774 for (i = 0 ; i < GAUDI2_NUM_OF_MME_ERR_CAUSE ; i++) { 8775 if (sts_val & BIT(i)) { 8776 gaudi2_print_event(hdev, event_type, true, 8777 "err cause: %s", guadi2_mme_error_cause[i]); 8778 sts_clr_val |= BIT(i); 8779 error_count++; 8780 } 8781 } 8782 8783 /* check if RAZWI happened */ 8784 for (i = MME_WRITE ; i < MME_INITIATORS_MAX ; i++) 8785 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, i, event_mask); 8786 8787 hl_check_for_glbl_errors(hdev); 8788 8789 WREG32(sts_clr_addr, sts_clr_val); 8790 8791 return error_count; 8792 } 8793 8794 static int gaudi2_handle_mme_sbte_err(struct hl_device *hdev, u16 event_type) 8795 { 8796 /* 8797 * We have a single error cause here but the report mechanism is 8798 * buggy. Hence there is no good reason to fetch the cause so we 8799 * just check for glbl_errors and exit. 8800 */ 8801 hl_check_for_glbl_errors(hdev); 8802 8803 return GAUDI2_NA_EVENT_CAUSE; 8804 } 8805 8806 static int gaudi2_handle_mme_wap_err(struct hl_device *hdev, u8 mme_index, u16 event_type, 8807 u64 *event_mask) 8808 { 8809 u32 sts_addr, sts_val, sts_clr_addr, sts_clr_val = 0, error_count = 0; 8810 int i; 8811 8812 sts_addr = mmDCORE0_MME_ACC_INTR_CAUSE + DCORE_OFFSET * mme_index; 8813 sts_clr_addr = mmDCORE0_MME_ACC_INTR_CLEAR + DCORE_OFFSET * mme_index; 8814 8815 sts_val = RREG32(sts_addr); 8816 8817 for (i = 0 ; i < GAUDI2_NUM_OF_MME_WAP_ERR_CAUSE ; i++) { 8818 if (sts_val & BIT(i)) { 8819 gaudi2_print_event(hdev, event_type, true, 8820 "err cause: %s", guadi2_mme_wap_error_cause[i]); 8821 sts_clr_val |= BIT(i); 8822 error_count++; 8823 } 8824 } 8825 8826 /* check if RAZWI happened on WAP0/1 */ 8827 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, MME_WAP0, event_mask); 8828 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, MME_WAP1, event_mask); 8829 hl_check_for_glbl_errors(hdev); 8830 8831 WREG32(sts_clr_addr, sts_clr_val); 8832 8833 return error_count; 8834 } 8835 8836 static int gaudi2_handle_kdma_core_event(struct hl_device *hdev, u16 event_type, 8837 u64 intr_cause_data) 8838 { 8839 u32 error_count = 0; 8840 int i; 8841 8842 /* If an AXI read or write error is received, an error is reported and 8843 * interrupt message is sent. Due to an HW errata, when reading the cause 8844 * register of the KDMA engine, the reported error is always HBW even if 8845 * the actual error caused by a LBW KDMA transaction. 8846 */ 8847 for (i = 0 ; i < GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE ; i++) 8848 if (intr_cause_data & BIT(i)) { 8849 gaudi2_print_event(hdev, event_type, true, 8850 "err cause: %s", gaudi2_kdma_core_interrupts_cause[i]); 8851 error_count++; 8852 } 8853 8854 hl_check_for_glbl_errors(hdev); 8855 8856 return error_count; 8857 } 8858 8859 static int gaudi2_handle_dma_core_event(struct hl_device *hdev, u16 event_type, u64 intr_cause) 8860 { 8861 u32 error_count = 0; 8862 int i; 8863 8864 for (i = 0 ; i < GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE ; i++) 8865 if (intr_cause & BIT(i)) { 8866 gaudi2_print_event(hdev, event_type, true, 8867 "err cause: %s", gaudi2_dma_core_interrupts_cause[i]); 8868 error_count++; 8869 } 8870 8871 hl_check_for_glbl_errors(hdev); 8872 8873 return error_count; 8874 } 8875 8876 static void gaudi2_print_pcie_mstr_rr_mstr_if_razwi_info(struct hl_device *hdev, u64 *event_mask) 8877 { 8878 u32 mstr_if_base_addr = mmPCIE_MSTR_RR_MSTR_IF_RR_SHRD_HBW_BASE, razwi_happened_addr; 8879 8880 razwi_happened_addr = mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HAPPENED; 8881 if (RREG32(razwi_happened_addr)) { 8882 gaudi2_razwi_rr_hbw_shared_printf_info(hdev, mstr_if_base_addr, true, "PCIE", 8883 GAUDI2_ENGINE_ID_PCIE, event_mask); 8884 WREG32(razwi_happened_addr, 0x1); 8885 } 8886 8887 razwi_happened_addr = mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HAPPENED; 8888 if (RREG32(razwi_happened_addr)) { 8889 gaudi2_razwi_rr_hbw_shared_printf_info(hdev, mstr_if_base_addr, false, "PCIE", 8890 GAUDI2_ENGINE_ID_PCIE, event_mask); 8891 WREG32(razwi_happened_addr, 0x1); 8892 } 8893 8894 razwi_happened_addr = mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_HAPPENED; 8895 if (RREG32(razwi_happened_addr)) { 8896 gaudi2_razwi_rr_lbw_shared_printf_info(hdev, mstr_if_base_addr, true, "PCIE", 8897 GAUDI2_ENGINE_ID_PCIE, event_mask); 8898 WREG32(razwi_happened_addr, 0x1); 8899 } 8900 8901 razwi_happened_addr = mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_HAPPENED; 8902 if (RREG32(razwi_happened_addr)) { 8903 gaudi2_razwi_rr_lbw_shared_printf_info(hdev, mstr_if_base_addr, false, "PCIE", 8904 GAUDI2_ENGINE_ID_PCIE, event_mask); 8905 WREG32(razwi_happened_addr, 0x1); 8906 } 8907 } 8908 8909 static int gaudi2_print_pcie_addr_dec_info(struct hl_device *hdev, u16 event_type, 8910 u64 intr_cause_data, u64 *event_mask) 8911 { 8912 u32 error_count = 0; 8913 int i; 8914 8915 gaudi2_print_event(hdev, event_type, true, 8916 "intr_cause_data: %#llx", intr_cause_data); 8917 8918 for (i = 0 ; i < GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE ; i++) { 8919 if (!(intr_cause_data & BIT_ULL(i))) 8920 continue; 8921 8922 gaudi2_print_event(hdev, event_type, true, 8923 "err cause: %s", gaudi2_pcie_addr_dec_error_cause[i]); 8924 error_count++; 8925 8926 /* 8927 * Always check for LBW and HBW additional info as the indication itself is 8928 * sometimes missing 8929 */ 8930 } 8931 8932 hl_check_for_glbl_errors(hdev); 8933 gaudi2_print_pcie_mstr_rr_mstr_if_razwi_info(hdev, event_mask); 8934 8935 return error_count; 8936 } 8937 8938 static int gaudi2_handle_pif_fatal(struct hl_device *hdev, u16 event_type, 8939 u64 intr_cause_data) 8940 8941 { 8942 u32 error_count = 0; 8943 int i; 8944 8945 for (i = 0 ; i < GAUDI2_NUM_OF_PMMU_FATAL_ERR_CAUSE ; i++) { 8946 if (intr_cause_data & BIT_ULL(i)) { 8947 gaudi2_print_event(hdev, event_type, true, 8948 "err cause: %s", gaudi2_pmmu_fatal_interrupts_cause[i]); 8949 error_count++; 8950 } 8951 } 8952 8953 return error_count; 8954 } 8955 8956 static int gaudi2_handle_hif_fatal(struct hl_device *hdev, u16 event_type, u64 intr_cause_data) 8957 { 8958 u32 error_count = 0; 8959 int i; 8960 8961 for (i = 0 ; i < GAUDI2_NUM_OF_HIF_FATAL_ERR_CAUSE ; i++) { 8962 if (intr_cause_data & BIT_ULL(i)) { 8963 gaudi2_print_event(hdev, event_type, true, 8964 "err cause: %s", gaudi2_hif_fatal_interrupts_cause[i]); 8965 error_count++; 8966 } 8967 } 8968 8969 return error_count; 8970 } 8971 8972 static void gaudi2_handle_page_error(struct hl_device *hdev, u64 mmu_base, bool is_pmmu, 8973 u64 *event_mask) 8974 { 8975 u32 valid, val; 8976 u64 addr; 8977 8978 valid = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID)); 8979 8980 if (!(valid & DCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID_PAGE_ERR_VALID_ENTRY_MASK)) 8981 return; 8982 8983 val = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_PAGE_ERROR_CAPTURE)); 8984 addr = val & DCORE0_HMMU0_MMU_PAGE_ERROR_CAPTURE_VA_63_32_MASK; 8985 addr <<= 32; 8986 addr |= RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_PAGE_ERROR_CAPTURE_VA)); 8987 8988 if (is_pmmu) { 8989 dev_err_ratelimited(hdev->dev, "PMMU page fault on va 0x%llx\n", addr); 8990 } else { 8991 8992 addr = gaudi2_mmu_descramble_addr(hdev, addr); 8993 addr &= HW_UNSCRAMBLED_BITS_MASK; 8994 dev_err_ratelimited(hdev->dev, "HMMU page fault on va range 0x%llx - 0x%llx\n", 8995 addr, addr + ~HW_UNSCRAMBLED_BITS_MASK); 8996 } 8997 8998 hl_handle_page_fault(hdev, addr, 0, is_pmmu, event_mask); 8999 9000 WREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID), 0); 9001 } 9002 9003 static void gaudi2_handle_access_error(struct hl_device *hdev, u64 mmu_base, bool is_pmmu) 9004 { 9005 u32 valid, val; 9006 u64 addr; 9007 9008 valid = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID)); 9009 9010 if (!(valid & DCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID_ACCESS_ERR_VALID_ENTRY_MASK)) 9011 return; 9012 9013 val = RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_ERROR_CAPTURE)); 9014 addr = val & DCORE0_HMMU0_MMU_ACCESS_ERROR_CAPTURE_VA_63_32_MASK; 9015 addr <<= 32; 9016 addr |= RREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_ERROR_CAPTURE_VA)); 9017 9018 if (!is_pmmu) 9019 addr = gaudi2_mmu_descramble_addr(hdev, addr); 9020 9021 dev_err_ratelimited(hdev->dev, "%s access error on va 0x%llx\n", 9022 is_pmmu ? "PMMU" : "HMMU", addr); 9023 WREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_ACCESS_PAGE_ERROR_VALID), 0); 9024 } 9025 9026 static int gaudi2_handle_mmu_spi_sei_generic(struct hl_device *hdev, u16 event_type, 9027 u64 mmu_base, bool is_pmmu, u64 *event_mask) 9028 { 9029 u32 spi_sei_cause, interrupt_clr = 0x0, error_count = 0; 9030 int i; 9031 9032 spi_sei_cause = RREG32(mmu_base + MMU_SPI_SEI_CAUSE_OFFSET); 9033 9034 for (i = 0 ; i < GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE ; i++) { 9035 if (spi_sei_cause & BIT(i)) { 9036 gaudi2_print_event(hdev, event_type, true, 9037 "err cause: %s", gaudi2_mmu_spi_sei[i].cause); 9038 9039 if (i == 0) 9040 gaudi2_handle_page_error(hdev, mmu_base, is_pmmu, event_mask); 9041 else if (i == 1) 9042 gaudi2_handle_access_error(hdev, mmu_base, is_pmmu); 9043 9044 if (gaudi2_mmu_spi_sei[i].clear_bit >= 0) 9045 interrupt_clr |= BIT(gaudi2_mmu_spi_sei[i].clear_bit); 9046 9047 error_count++; 9048 } 9049 } 9050 9051 /* Clear cause */ 9052 WREG32_AND(mmu_base + MMU_SPI_SEI_CAUSE_OFFSET, ~spi_sei_cause); 9053 9054 /* Clear interrupt */ 9055 WREG32(mmu_base + MMU_INTERRUPT_CLR_OFFSET, interrupt_clr); 9056 9057 return error_count; 9058 } 9059 9060 static int gaudi2_handle_sm_err(struct hl_device *hdev, u16 event_type, u8 sm_index) 9061 { 9062 u32 sei_cause_addr, sei_cause_val, sei_cause_cause, sei_cause_log, 9063 cq_intr_addr, cq_intr_val, cq_intr_queue_index, error_count = 0; 9064 int i; 9065 9066 sei_cause_addr = mmDCORE0_SYNC_MNGR_GLBL_SM_SEI_CAUSE + DCORE_OFFSET * sm_index; 9067 cq_intr_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_INTR + DCORE_OFFSET * sm_index; 9068 9069 sei_cause_val = RREG32(sei_cause_addr); 9070 sei_cause_cause = FIELD_GET(DCORE0_SYNC_MNGR_GLBL_SM_SEI_CAUSE_CAUSE_MASK, sei_cause_val); 9071 cq_intr_val = RREG32(cq_intr_addr); 9072 9073 /* SEI interrupt */ 9074 if (sei_cause_cause) { 9075 /* There are corresponding SEI_CAUSE_log bits for every SEI_CAUSE_cause bit */ 9076 sei_cause_log = FIELD_GET(DCORE0_SYNC_MNGR_GLBL_SM_SEI_CAUSE_LOG_MASK, 9077 sei_cause_val); 9078 9079 for (i = 0 ; i < GAUDI2_NUM_OF_SM_SEI_ERR_CAUSE ; i++) { 9080 if (!(sei_cause_cause & BIT(i))) 9081 continue; 9082 9083 gaudi2_print_event(hdev, event_type, true, 9084 "err cause: %s. %s: 0x%X", 9085 gaudi2_sm_sei_cause[i].cause_name, 9086 gaudi2_sm_sei_cause[i].log_name, 9087 sei_cause_log); 9088 error_count++; 9089 break; 9090 } 9091 9092 /* Clear SM_SEI_CAUSE */ 9093 WREG32(sei_cause_addr, 0); 9094 } 9095 9096 /* CQ interrupt */ 9097 if (cq_intr_val & DCORE0_SYNC_MNGR_GLBL_CQ_INTR_CQ_SEC_INTR_MASK) { 9098 cq_intr_queue_index = 9099 FIELD_GET(DCORE0_SYNC_MNGR_GLBL_CQ_INTR_CQ_INTR_QUEUE_INDEX_MASK, 9100 cq_intr_val); 9101 9102 dev_err_ratelimited(hdev->dev, "SM%u err. err cause: CQ_INTR. queue index: %u\n", 9103 sm_index, cq_intr_queue_index); 9104 error_count++; 9105 9106 /* Clear CQ_INTR */ 9107 WREG32(cq_intr_addr, 0); 9108 } 9109 9110 hl_check_for_glbl_errors(hdev); 9111 9112 return error_count; 9113 } 9114 9115 static u64 get_hmmu_base(u16 event_type) 9116 { 9117 u8 dcore, index_in_dcore; 9118 9119 switch (event_type) { 9120 case GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP: 9121 case GAUDI2_EVENT_HMMU0_SPI_BASE ... GAUDI2_EVENT_HMMU0_SECURITY_ERROR: 9122 dcore = 0; 9123 index_in_dcore = 0; 9124 break; 9125 case GAUDI2_EVENT_HMMU_1_AXI_ERR_RSP: 9126 case GAUDI2_EVENT_HMMU1_SPI_BASE ... GAUDI2_EVENT_HMMU1_SECURITY_ERROR: 9127 dcore = 1; 9128 index_in_dcore = 0; 9129 break; 9130 case GAUDI2_EVENT_HMMU_2_AXI_ERR_RSP: 9131 case GAUDI2_EVENT_HMMU2_SPI_BASE ... GAUDI2_EVENT_HMMU2_SECURITY_ERROR: 9132 dcore = 0; 9133 index_in_dcore = 1; 9134 break; 9135 case GAUDI2_EVENT_HMMU_3_AXI_ERR_RSP: 9136 case GAUDI2_EVENT_HMMU3_SPI_BASE ... GAUDI2_EVENT_HMMU3_SECURITY_ERROR: 9137 dcore = 1; 9138 index_in_dcore = 1; 9139 break; 9140 case GAUDI2_EVENT_HMMU_4_AXI_ERR_RSP: 9141 case GAUDI2_EVENT_HMMU4_SPI_BASE ... GAUDI2_EVENT_HMMU4_SECURITY_ERROR: 9142 dcore = 3; 9143 index_in_dcore = 2; 9144 break; 9145 case GAUDI2_EVENT_HMMU_5_AXI_ERR_RSP: 9146 case GAUDI2_EVENT_HMMU5_SPI_BASE ... GAUDI2_EVENT_HMMU5_SECURITY_ERROR: 9147 dcore = 2; 9148 index_in_dcore = 2; 9149 break; 9150 case GAUDI2_EVENT_HMMU_6_AXI_ERR_RSP: 9151 case GAUDI2_EVENT_HMMU6_SPI_BASE ... GAUDI2_EVENT_HMMU6_SECURITY_ERROR: 9152 dcore = 3; 9153 index_in_dcore = 3; 9154 break; 9155 case GAUDI2_EVENT_HMMU_7_AXI_ERR_RSP: 9156 case GAUDI2_EVENT_HMMU7_SPI_BASE ... GAUDI2_EVENT_HMMU7_SECURITY_ERROR: 9157 dcore = 2; 9158 index_in_dcore = 3; 9159 break; 9160 case GAUDI2_EVENT_HMMU_8_AXI_ERR_RSP: 9161 case GAUDI2_EVENT_HMMU8_SPI_BASE ... GAUDI2_EVENT_HMMU8_SECURITY_ERROR: 9162 dcore = 0; 9163 index_in_dcore = 2; 9164 break; 9165 case GAUDI2_EVENT_HMMU_9_AXI_ERR_RSP: 9166 case GAUDI2_EVENT_HMMU9_SPI_BASE ... GAUDI2_EVENT_HMMU9_SECURITY_ERROR: 9167 dcore = 1; 9168 index_in_dcore = 2; 9169 break; 9170 case GAUDI2_EVENT_HMMU_10_AXI_ERR_RSP: 9171 case GAUDI2_EVENT_HMMU10_SPI_BASE ... GAUDI2_EVENT_HMMU10_SECURITY_ERROR: 9172 dcore = 0; 9173 index_in_dcore = 3; 9174 break; 9175 case GAUDI2_EVENT_HMMU_11_AXI_ERR_RSP: 9176 case GAUDI2_EVENT_HMMU11_SPI_BASE ... GAUDI2_EVENT_HMMU11_SECURITY_ERROR: 9177 dcore = 1; 9178 index_in_dcore = 3; 9179 break; 9180 case GAUDI2_EVENT_HMMU_12_AXI_ERR_RSP: 9181 case GAUDI2_EVENT_HMMU12_SPI_BASE ... GAUDI2_EVENT_HMMU12_SECURITY_ERROR: 9182 dcore = 3; 9183 index_in_dcore = 0; 9184 break; 9185 case GAUDI2_EVENT_HMMU_13_AXI_ERR_RSP: 9186 case GAUDI2_EVENT_HMMU13_SPI_BASE ... GAUDI2_EVENT_HMMU13_SECURITY_ERROR: 9187 dcore = 2; 9188 index_in_dcore = 0; 9189 break; 9190 case GAUDI2_EVENT_HMMU_14_AXI_ERR_RSP: 9191 case GAUDI2_EVENT_HMMU14_SPI_BASE ... GAUDI2_EVENT_HMMU14_SECURITY_ERROR: 9192 dcore = 3; 9193 index_in_dcore = 1; 9194 break; 9195 case GAUDI2_EVENT_HMMU_15_AXI_ERR_RSP: 9196 case GAUDI2_EVENT_HMMU15_SPI_BASE ... GAUDI2_EVENT_HMMU15_SECURITY_ERROR: 9197 dcore = 2; 9198 index_in_dcore = 1; 9199 break; 9200 default: 9201 return ULONG_MAX; 9202 } 9203 9204 return mmDCORE0_HMMU0_MMU_BASE + dcore * DCORE_OFFSET + index_in_dcore * DCORE_HMMU_OFFSET; 9205 } 9206 9207 static int gaudi2_handle_mmu_spi_sei_err(struct hl_device *hdev, u16 event_type, u64 *event_mask) 9208 { 9209 bool is_pmmu = false; 9210 u32 error_count = 0; 9211 u64 mmu_base; 9212 9213 switch (event_type) { 9214 case GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP ... GAUDI2_EVENT_HMMU_12_AXI_ERR_RSP: 9215 case GAUDI2_EVENT_HMMU0_SPI_BASE ... GAUDI2_EVENT_HMMU12_SECURITY_ERROR: 9216 mmu_base = get_hmmu_base(event_type); 9217 break; 9218 9219 case GAUDI2_EVENT_PMMU0_PAGE_FAULT_WR_PERM ... GAUDI2_EVENT_PMMU0_SECURITY_ERROR: 9220 case GAUDI2_EVENT_PMMU_AXI_ERR_RSP_0: 9221 is_pmmu = true; 9222 mmu_base = mmPMMU_HBW_MMU_BASE; 9223 break; 9224 default: 9225 return 0; 9226 } 9227 9228 if (mmu_base == ULONG_MAX) 9229 return 0; 9230 9231 error_count = gaudi2_handle_mmu_spi_sei_generic(hdev, event_type, mmu_base, 9232 is_pmmu, event_mask); 9233 hl_check_for_glbl_errors(hdev); 9234 9235 return error_count; 9236 } 9237 9238 9239 /* returns true if hard reset is required (ECC DERR or Read parity), false otherwise (ECC SERR) */ 9240 static bool gaudi2_hbm_sei_handle_read_err(struct hl_device *hdev, 9241 struct hl_eq_hbm_sei_read_err_intr_info *rd_err_data, u32 err_cnt) 9242 { 9243 u32 addr, beat, beat_shift; 9244 bool rc = false; 9245 9246 dev_err_ratelimited(hdev->dev, 9247 "READ ERROR count: ECC SERR: %d, ECC DERR: %d, RD_PARITY: %d\n", 9248 FIELD_GET(HBM_ECC_SERR_CNTR_MASK, err_cnt), 9249 FIELD_GET(HBM_ECC_DERR_CNTR_MASK, err_cnt), 9250 FIELD_GET(HBM_RD_PARITY_CNTR_MASK, err_cnt)); 9251 9252 addr = le32_to_cpu(rd_err_data->dbg_rd_err_addr.rd_addr_val); 9253 dev_err_ratelimited(hdev->dev, 9254 "READ ERROR address: sid(%u), bg(%u), ba(%u), col(%u), row(%u)\n", 9255 FIELD_GET(HBM_RD_ADDR_SID_MASK, addr), 9256 FIELD_GET(HBM_RD_ADDR_BG_MASK, addr), 9257 FIELD_GET(HBM_RD_ADDR_BA_MASK, addr), 9258 FIELD_GET(HBM_RD_ADDR_COL_MASK, addr), 9259 FIELD_GET(HBM_RD_ADDR_ROW_MASK, addr)); 9260 9261 /* For each beat (RDQS edge), look for possible errors and print relevant info */ 9262 for (beat = 0 ; beat < 4 ; beat++) { 9263 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & 9264 (HBM_RD_ERR_SERR_BEAT0_MASK << beat)) 9265 dev_err_ratelimited(hdev->dev, "Beat%d ECC SERR: DM: %#x, Syndrome: %#x\n", 9266 beat, 9267 le32_to_cpu(rd_err_data->dbg_rd_err_dm), 9268 le32_to_cpu(rd_err_data->dbg_rd_err_syndrome)); 9269 9270 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & 9271 (HBM_RD_ERR_DERR_BEAT0_MASK << beat)) { 9272 dev_err_ratelimited(hdev->dev, "Beat%d ECC DERR: DM: %#x, Syndrome: %#x\n", 9273 beat, 9274 le32_to_cpu(rd_err_data->dbg_rd_err_dm), 9275 le32_to_cpu(rd_err_data->dbg_rd_err_syndrome)); 9276 rc |= true; 9277 } 9278 9279 beat_shift = beat * HBM_RD_ERR_BEAT_SHIFT; 9280 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & 9281 (HBM_RD_ERR_PAR_ERR_BEAT0_MASK << beat_shift)) { 9282 dev_err_ratelimited(hdev->dev, 9283 "Beat%d read PARITY: DM: %#x, PAR data: %#x\n", 9284 beat, 9285 le32_to_cpu(rd_err_data->dbg_rd_err_dm), 9286 (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & 9287 (HBM_RD_ERR_PAR_DATA_BEAT0_MASK << beat_shift)) >> 9288 (HBM_RD_ERR_PAR_DATA_BEAT0_SHIFT + beat_shift)); 9289 rc |= true; 9290 } 9291 9292 dev_err_ratelimited(hdev->dev, "Beat%d DQ data:\n", beat); 9293 dev_err_ratelimited(hdev->dev, "\t0x%08x\n", 9294 le32_to_cpu(rd_err_data->dbg_rd_err_data[beat * 2])); 9295 dev_err_ratelimited(hdev->dev, "\t0x%08x\n", 9296 le32_to_cpu(rd_err_data->dbg_rd_err_data[beat * 2 + 1])); 9297 } 9298 9299 return rc; 9300 } 9301 9302 static void gaudi2_hbm_sei_print_wr_par_info(struct hl_device *hdev, 9303 struct hl_eq_hbm_sei_wr_par_intr_info *wr_par_err_data, u32 err_cnt) 9304 { 9305 struct hbm_sei_wr_cmd_address *wr_cmd_addr = wr_par_err_data->dbg_last_wr_cmds; 9306 u32 i, curr_addr, derr = wr_par_err_data->dbg_derr; 9307 9308 dev_err_ratelimited(hdev->dev, "WRITE PARITY ERROR count: %d\n", err_cnt); 9309 9310 dev_err_ratelimited(hdev->dev, "CK-0 DERR: 0x%02x, CK-1 DERR: 0x%02x\n", 9311 derr & 0x3, derr & 0xc); 9312 9313 /* JIRA H6-3286 - the following prints may not be valid */ 9314 dev_err_ratelimited(hdev->dev, "Last latched write commands addresses:\n"); 9315 for (i = 0 ; i < HBM_WR_PAR_CMD_LIFO_LEN ; i++) { 9316 curr_addr = le32_to_cpu(wr_cmd_addr[i].dbg_wr_cmd_addr); 9317 dev_err_ratelimited(hdev->dev, 9318 "\twrite cmd[%u]: Address: SID(%u) BG(%u) BA(%u) COL(%u).\n", 9319 i, 9320 FIELD_GET(WR_PAR_LAST_CMD_SID_MASK, curr_addr), 9321 FIELD_GET(WR_PAR_LAST_CMD_BG_MASK, curr_addr), 9322 FIELD_GET(WR_PAR_LAST_CMD_BA_MASK, curr_addr), 9323 FIELD_GET(WR_PAR_LAST_CMD_COL_MASK, curr_addr)); 9324 } 9325 } 9326 9327 static void gaudi2_hbm_sei_print_ca_par_info(struct hl_device *hdev, 9328 struct hl_eq_hbm_sei_ca_par_intr_info *ca_par_err_data, u32 err_cnt) 9329 { 9330 __le32 *col_cmd = ca_par_err_data->dbg_col; 9331 __le16 *row_cmd = ca_par_err_data->dbg_row; 9332 u32 i; 9333 9334 dev_err_ratelimited(hdev->dev, "CA ERROR count: %d\n", err_cnt); 9335 9336 dev_err_ratelimited(hdev->dev, "Last latched C&R bus commands:\n"); 9337 for (i = 0 ; i < HBM_CA_ERR_CMD_LIFO_LEN ; i++) 9338 dev_err_ratelimited(hdev->dev, "cmd%u: ROW(0x%04x) COL(0x%05x)\n", i, 9339 le16_to_cpu(row_cmd[i]) & (u16)GENMASK(13, 0), 9340 le32_to_cpu(col_cmd[i]) & (u32)GENMASK(17, 0)); 9341 } 9342 9343 /* Returns true if hard reset is needed or false otherwise */ 9344 static bool gaudi2_handle_hbm_mc_sei_err(struct hl_device *hdev, u16 event_type, 9345 struct hl_eq_hbm_sei_data *sei_data) 9346 { 9347 bool require_hard_reset = false; 9348 u32 hbm_id, mc_id, cause_idx; 9349 9350 hbm_id = (event_type - GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE) / 4; 9351 mc_id = ((event_type - GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE) / 2) % 2; 9352 9353 cause_idx = sei_data->hdr.sei_cause; 9354 if (cause_idx > GAUDI2_NUM_OF_HBM_SEI_CAUSE - 1) { 9355 gaudi2_print_event(hdev, event_type, true, 9356 "err cause: %s", 9357 "Invalid HBM SEI event cause (%d) provided by FW", cause_idx); 9358 return true; 9359 } 9360 9361 gaudi2_print_event(hdev, event_type, !sei_data->hdr.is_critical, 9362 "System %s Error Interrupt - HBM(%u) MC(%u) MC_CH(%u) MC_PC(%u). Error cause: %s", 9363 sei_data->hdr.is_critical ? "Critical" : "Non-critical", 9364 hbm_id, mc_id, sei_data->hdr.mc_channel, sei_data->hdr.mc_pseudo_channel, 9365 hbm_mc_sei_cause[cause_idx]); 9366 9367 /* Print error-specific info */ 9368 switch (cause_idx) { 9369 case HBM_SEI_CATTRIP: 9370 require_hard_reset = true; 9371 break; 9372 9373 case HBM_SEI_CMD_PARITY_EVEN: 9374 gaudi2_hbm_sei_print_ca_par_info(hdev, &sei_data->ca_parity_even_info, 9375 le32_to_cpu(sei_data->hdr.cnt)); 9376 require_hard_reset = true; 9377 break; 9378 9379 case HBM_SEI_CMD_PARITY_ODD: 9380 gaudi2_hbm_sei_print_ca_par_info(hdev, &sei_data->ca_parity_odd_info, 9381 le32_to_cpu(sei_data->hdr.cnt)); 9382 require_hard_reset = true; 9383 break; 9384 9385 case HBM_SEI_WRITE_DATA_PARITY_ERR: 9386 gaudi2_hbm_sei_print_wr_par_info(hdev, &sei_data->wr_parity_info, 9387 le32_to_cpu(sei_data->hdr.cnt)); 9388 require_hard_reset = true; 9389 break; 9390 9391 case HBM_SEI_READ_ERR: 9392 /* Unlike other SEI events, read error requires further processing of the 9393 * raw data in order to determine the root cause. 9394 */ 9395 require_hard_reset = gaudi2_hbm_sei_handle_read_err(hdev, 9396 &sei_data->read_err_info, 9397 le32_to_cpu(sei_data->hdr.cnt)); 9398 break; 9399 9400 default: 9401 break; 9402 } 9403 9404 require_hard_reset |= !!sei_data->hdr.is_critical; 9405 9406 return require_hard_reset; 9407 } 9408 9409 static int gaudi2_handle_hbm_cattrip(struct hl_device *hdev, u16 event_type, 9410 u64 intr_cause_data) 9411 { 9412 if (intr_cause_data) { 9413 gaudi2_print_event(hdev, event_type, true, 9414 "temperature error cause: %#llx", intr_cause_data); 9415 return 1; 9416 } 9417 9418 return 0; 9419 } 9420 9421 static int gaudi2_handle_hbm_mc_spi(struct hl_device *hdev, u64 intr_cause_data) 9422 { 9423 u32 i, error_count = 0; 9424 9425 for (i = 0 ; i < GAUDI2_NUM_OF_HBM_MC_SPI_CAUSE ; i++) 9426 if (intr_cause_data & hbm_mc_spi[i].mask) { 9427 dev_dbg(hdev->dev, "HBM spi event: notification cause(%s)\n", 9428 hbm_mc_spi[i].cause); 9429 error_count++; 9430 } 9431 9432 return error_count; 9433 } 9434 9435 static void gaudi2_print_clk_change_info(struct hl_device *hdev, u16 event_type, u64 *event_mask) 9436 { 9437 ktime_t zero_time = ktime_set(0, 0); 9438 9439 mutex_lock(&hdev->clk_throttling.lock); 9440 9441 switch (event_type) { 9442 case GAUDI2_EVENT_CPU_FIX_POWER_ENV_S: 9443 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_POWER; 9444 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_POWER; 9445 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].start = ktime_get(); 9446 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = zero_time; 9447 dev_dbg_ratelimited(hdev->dev, "Clock throttling due to power consumption\n"); 9448 break; 9449 9450 case GAUDI2_EVENT_CPU_FIX_POWER_ENV_E: 9451 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_POWER; 9452 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = ktime_get(); 9453 dev_dbg_ratelimited(hdev->dev, "Power envelop is safe, back to optimal clock\n"); 9454 break; 9455 9456 case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_S: 9457 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_THERMAL; 9458 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_THERMAL; 9459 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].start = ktime_get(); 9460 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = zero_time; 9461 *event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9462 dev_info_ratelimited(hdev->dev, "Clock throttling due to overheating\n"); 9463 break; 9464 9465 case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_E: 9466 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_THERMAL; 9467 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = ktime_get(); 9468 *event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9469 dev_info_ratelimited(hdev->dev, "Thermal envelop is safe, back to optimal clock\n"); 9470 break; 9471 9472 default: 9473 dev_err(hdev->dev, "Received invalid clock change event %d\n", event_type); 9474 break; 9475 } 9476 9477 mutex_unlock(&hdev->clk_throttling.lock); 9478 } 9479 9480 static void gaudi2_print_out_of_sync_info(struct hl_device *hdev, u16 event_type, 9481 struct cpucp_pkt_sync_err *sync_err) 9482 { 9483 struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ]; 9484 9485 gaudi2_print_event(hdev, event_type, false, 9486 "FW: pi=%u, ci=%u, LKD: pi=%u, ci=%d", 9487 le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), 9488 q->pi, atomic_read(&q->ci)); 9489 } 9490 9491 static int gaudi2_handle_pcie_p2p_msix(struct hl_device *hdev, u16 event_type) 9492 { 9493 u32 p2p_intr, msix_gw_intr, error_count = 0; 9494 9495 p2p_intr = RREG32(mmPCIE_WRAP_P2P_INTR); 9496 msix_gw_intr = RREG32(mmPCIE_WRAP_MSIX_GW_INTR); 9497 9498 if (p2p_intr) { 9499 gaudi2_print_event(hdev, event_type, true, 9500 "pcie p2p transaction terminated due to security, req_id(0x%x)", 9501 RREG32(mmPCIE_WRAP_P2P_REQ_ID)); 9502 9503 WREG32(mmPCIE_WRAP_P2P_INTR, 0x1); 9504 error_count++; 9505 } 9506 9507 if (msix_gw_intr) { 9508 gaudi2_print_event(hdev, event_type, true, 9509 "pcie msi-x gen denied due to vector num check failure, vec(0x%X)", 9510 RREG32(mmPCIE_WRAP_MSIX_GW_VEC)); 9511 9512 WREG32(mmPCIE_WRAP_MSIX_GW_INTR, 0x1); 9513 error_count++; 9514 } 9515 9516 return error_count; 9517 } 9518 9519 static int gaudi2_handle_pcie_drain(struct hl_device *hdev, 9520 struct hl_eq_pcie_drain_ind_data *drain_data) 9521 { 9522 u64 lbw_rd, lbw_wr, hbw_rd, hbw_wr, cause, error_count = 0; 9523 9524 cause = le64_to_cpu(drain_data->intr_cause.intr_cause_data); 9525 lbw_rd = le64_to_cpu(drain_data->drain_rd_addr_lbw); 9526 lbw_wr = le64_to_cpu(drain_data->drain_wr_addr_lbw); 9527 hbw_rd = le64_to_cpu(drain_data->drain_rd_addr_hbw); 9528 hbw_wr = le64_to_cpu(drain_data->drain_wr_addr_hbw); 9529 9530 if (cause & BIT_ULL(0)) { 9531 dev_err_ratelimited(hdev->dev, 9532 "PCIE AXI drain LBW completed, read_err %u, write_err %u\n", 9533 !!lbw_rd, !!lbw_wr); 9534 error_count++; 9535 } 9536 9537 if (cause & BIT_ULL(1)) { 9538 dev_err_ratelimited(hdev->dev, 9539 "PCIE AXI drain HBW completed, raddr %#llx, waddr %#llx\n", 9540 hbw_rd, hbw_wr); 9541 error_count++; 9542 } 9543 9544 return error_count; 9545 } 9546 9547 static int gaudi2_handle_psoc_drain(struct hl_device *hdev, u64 intr_cause_data) 9548 { 9549 u32 error_count = 0; 9550 int i; 9551 9552 for (i = 0 ; i < GAUDI2_NUM_OF_AXI_DRAIN_ERR_CAUSE ; i++) { 9553 if (intr_cause_data & BIT_ULL(i)) { 9554 dev_err_ratelimited(hdev->dev, "PSOC %s completed\n", 9555 gaudi2_psoc_axi_drain_interrupts_cause[i]); 9556 error_count++; 9557 } 9558 } 9559 9560 hl_check_for_glbl_errors(hdev); 9561 9562 return error_count; 9563 } 9564 9565 static void gaudi2_print_cpu_pkt_failure_info(struct hl_device *hdev, u16 event_type, 9566 struct cpucp_pkt_sync_err *sync_err) 9567 { 9568 struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ]; 9569 9570 gaudi2_print_event(hdev, event_type, false, 9571 "FW reported sanity check failure, FW: pi=%u, ci=%u, LKD: pi=%u, ci=%d", 9572 le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), q->pi, atomic_read(&q->ci)); 9573 } 9574 9575 static int hl_arc_event_handle(struct hl_device *hdev, u16 event_type, 9576 struct hl_eq_engine_arc_intr_data *data) 9577 { 9578 struct hl_engine_arc_dccm_queue_full_irq *q; 9579 u32 intr_type, engine_id; 9580 u64 payload; 9581 9582 intr_type = le32_to_cpu(data->intr_type); 9583 engine_id = le32_to_cpu(data->engine_id); 9584 payload = le64_to_cpu(data->payload); 9585 9586 switch (intr_type) { 9587 case ENGINE_ARC_DCCM_QUEUE_FULL_IRQ: 9588 q = (struct hl_engine_arc_dccm_queue_full_irq *) &payload; 9589 9590 gaudi2_print_event(hdev, event_type, true, 9591 "ARC DCCM Full event: EngId: %u, Intr_type: %u, Qidx: %u", 9592 engine_id, intr_type, q->queue_index); 9593 return 1; 9594 default: 9595 gaudi2_print_event(hdev, event_type, true, "Unknown ARC event type"); 9596 return 0; 9597 } 9598 } 9599 9600 static u16 event_id_to_engine_id(struct hl_device *hdev, u16 event_type) 9601 { 9602 enum gaudi2_block_types type = GAUDI2_BLOCK_TYPE_MAX; 9603 u16 index; 9604 9605 switch (event_type) { 9606 case GAUDI2_EVENT_TPC0_AXI_ERR_RSP ... GAUDI2_EVENT_TPC24_AXI_ERR_RSP: 9607 index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP; 9608 type = GAUDI2_BLOCK_TYPE_TPC; 9609 break; 9610 case GAUDI2_EVENT_TPC0_QM ... GAUDI2_EVENT_TPC24_QM: 9611 index = event_type - GAUDI2_EVENT_TPC0_QM; 9612 type = GAUDI2_BLOCK_TYPE_TPC; 9613 break; 9614 case GAUDI2_EVENT_MME0_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE: 9615 case GAUDI2_EVENT_MME0_SPI_BASE ... GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID: 9616 case GAUDI2_EVENT_MME0_QM: 9617 index = 0; 9618 type = GAUDI2_BLOCK_TYPE_MME; 9619 break; 9620 case GAUDI2_EVENT_MME1_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE: 9621 case GAUDI2_EVENT_MME1_SPI_BASE ... GAUDI2_EVENT_MME1_WAP_SOURCE_RESULT_INVALID: 9622 case GAUDI2_EVENT_MME1_QM: 9623 index = 1; 9624 type = GAUDI2_BLOCK_TYPE_MME; 9625 break; 9626 case GAUDI2_EVENT_MME2_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME2_CTRL_AXI_ERROR_RESPONSE: 9627 case GAUDI2_EVENT_MME2_SPI_BASE ... GAUDI2_EVENT_MME2_WAP_SOURCE_RESULT_INVALID: 9628 case GAUDI2_EVENT_MME2_QM: 9629 index = 2; 9630 type = GAUDI2_BLOCK_TYPE_MME; 9631 break; 9632 case GAUDI2_EVENT_MME3_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME3_CTRL_AXI_ERROR_RESPONSE: 9633 case GAUDI2_EVENT_MME3_SPI_BASE ... GAUDI2_EVENT_MME3_WAP_SOURCE_RESULT_INVALID: 9634 case GAUDI2_EVENT_MME3_QM: 9635 index = 3; 9636 type = GAUDI2_BLOCK_TYPE_MME; 9637 break; 9638 case GAUDI2_EVENT_KDMA_CH0_AXI_ERR_RSP: 9639 case GAUDI2_EVENT_KDMA_BM_SPMU: 9640 case GAUDI2_EVENT_KDMA0_CORE: 9641 return GAUDI2_ENGINE_ID_KDMA; 9642 case GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP: 9643 case GAUDI2_EVENT_PDMA0_CORE: 9644 case GAUDI2_EVENT_PDMA0_BM_SPMU: 9645 case GAUDI2_EVENT_PDMA0_QM: 9646 return GAUDI2_ENGINE_ID_PDMA_0; 9647 case GAUDI2_EVENT_PDMA_CH1_AXI_ERR_RSP: 9648 case GAUDI2_EVENT_PDMA1_CORE: 9649 case GAUDI2_EVENT_PDMA1_BM_SPMU: 9650 case GAUDI2_EVENT_PDMA1_QM: 9651 return GAUDI2_ENGINE_ID_PDMA_1; 9652 case GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE ... GAUDI2_EVENT_DEC9_AXI_ERR_RSPONSE: 9653 index = event_type - GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE; 9654 type = GAUDI2_BLOCK_TYPE_DEC; 9655 break; 9656 case GAUDI2_EVENT_DEC0_SPI ... GAUDI2_EVENT_DEC9_BMON_SPMU: 9657 index = (event_type - GAUDI2_EVENT_DEC0_SPI) >> 1; 9658 type = GAUDI2_BLOCK_TYPE_DEC; 9659 break; 9660 case GAUDI2_EVENT_NIC0_AXI_ERROR_RESPONSE ... GAUDI2_EVENT_NIC11_AXI_ERROR_RESPONSE: 9661 index = event_type - GAUDI2_EVENT_NIC0_AXI_ERROR_RESPONSE; 9662 return GAUDI2_ENGINE_ID_NIC0_0 + (index * 2); 9663 case GAUDI2_EVENT_NIC0_QM0 ... GAUDI2_EVENT_NIC11_QM1: 9664 index = event_type - GAUDI2_EVENT_NIC0_QM0; 9665 return GAUDI2_ENGINE_ID_NIC0_0 + index; 9666 case GAUDI2_EVENT_NIC0_BMON_SPMU ... GAUDI2_EVENT_NIC11_SW_ERROR: 9667 index = event_type - GAUDI2_EVENT_NIC0_BMON_SPMU; 9668 return GAUDI2_ENGINE_ID_NIC0_0 + (index * 2); 9669 case GAUDI2_EVENT_TPC0_BMON_SPMU ... GAUDI2_EVENT_TPC24_KERNEL_ERR: 9670 index = (event_type - GAUDI2_EVENT_TPC0_BMON_SPMU) >> 1; 9671 type = GAUDI2_BLOCK_TYPE_TPC; 9672 break; 9673 case GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE: 9674 case GAUDI2_EVENT_ROTATOR0_BMON_SPMU: 9675 case GAUDI2_EVENT_ROTATOR0_ROT0_QM: 9676 return GAUDI2_ENGINE_ID_ROT_0; 9677 case GAUDI2_EVENT_ROTATOR1_AXI_ERROR_RESPONSE: 9678 case GAUDI2_EVENT_ROTATOR1_BMON_SPMU: 9679 case GAUDI2_EVENT_ROTATOR1_ROT1_QM: 9680 return GAUDI2_ENGINE_ID_ROT_1; 9681 case GAUDI2_EVENT_HDMA0_BM_SPMU: 9682 case GAUDI2_EVENT_HDMA0_QM: 9683 case GAUDI2_EVENT_HDMA0_CORE: 9684 return GAUDI2_DCORE0_ENGINE_ID_EDMA_0; 9685 case GAUDI2_EVENT_HDMA1_BM_SPMU: 9686 case GAUDI2_EVENT_HDMA1_QM: 9687 case GAUDI2_EVENT_HDMA1_CORE: 9688 return GAUDI2_DCORE0_ENGINE_ID_EDMA_1; 9689 case GAUDI2_EVENT_HDMA2_BM_SPMU: 9690 case GAUDI2_EVENT_HDMA2_QM: 9691 case GAUDI2_EVENT_HDMA2_CORE: 9692 return GAUDI2_DCORE1_ENGINE_ID_EDMA_0; 9693 case GAUDI2_EVENT_HDMA3_BM_SPMU: 9694 case GAUDI2_EVENT_HDMA3_QM: 9695 case GAUDI2_EVENT_HDMA3_CORE: 9696 return GAUDI2_DCORE1_ENGINE_ID_EDMA_1; 9697 case GAUDI2_EVENT_HDMA4_BM_SPMU: 9698 case GAUDI2_EVENT_HDMA4_QM: 9699 case GAUDI2_EVENT_HDMA4_CORE: 9700 return GAUDI2_DCORE2_ENGINE_ID_EDMA_0; 9701 case GAUDI2_EVENT_HDMA5_BM_SPMU: 9702 case GAUDI2_EVENT_HDMA5_QM: 9703 case GAUDI2_EVENT_HDMA5_CORE: 9704 return GAUDI2_DCORE2_ENGINE_ID_EDMA_1; 9705 case GAUDI2_EVENT_HDMA6_BM_SPMU: 9706 case GAUDI2_EVENT_HDMA6_QM: 9707 case GAUDI2_EVENT_HDMA6_CORE: 9708 return GAUDI2_DCORE3_ENGINE_ID_EDMA_0; 9709 case GAUDI2_EVENT_HDMA7_BM_SPMU: 9710 case GAUDI2_EVENT_HDMA7_QM: 9711 case GAUDI2_EVENT_HDMA7_CORE: 9712 return GAUDI2_DCORE3_ENGINE_ID_EDMA_1; 9713 default: 9714 break; 9715 } 9716 9717 switch (type) { 9718 case GAUDI2_BLOCK_TYPE_TPC: 9719 switch (index) { 9720 case TPC_ID_DCORE0_TPC0 ... TPC_ID_DCORE0_TPC5: 9721 return GAUDI2_DCORE0_ENGINE_ID_TPC_0 + index; 9722 case TPC_ID_DCORE1_TPC0 ... TPC_ID_DCORE1_TPC5: 9723 return GAUDI2_DCORE1_ENGINE_ID_TPC_0 + index - TPC_ID_DCORE1_TPC0; 9724 case TPC_ID_DCORE2_TPC0 ... TPC_ID_DCORE2_TPC5: 9725 return GAUDI2_DCORE2_ENGINE_ID_TPC_0 + index - TPC_ID_DCORE2_TPC0; 9726 case TPC_ID_DCORE3_TPC0 ... TPC_ID_DCORE3_TPC5: 9727 return GAUDI2_DCORE3_ENGINE_ID_TPC_0 + index - TPC_ID_DCORE3_TPC0; 9728 default: 9729 break; 9730 } 9731 break; 9732 case GAUDI2_BLOCK_TYPE_MME: 9733 switch (index) { 9734 case MME_ID_DCORE0: return GAUDI2_DCORE0_ENGINE_ID_MME; 9735 case MME_ID_DCORE1: return GAUDI2_DCORE1_ENGINE_ID_MME; 9736 case MME_ID_DCORE2: return GAUDI2_DCORE2_ENGINE_ID_MME; 9737 case MME_ID_DCORE3: return GAUDI2_DCORE3_ENGINE_ID_MME; 9738 default: 9739 break; 9740 } 9741 break; 9742 case GAUDI2_BLOCK_TYPE_DEC: 9743 switch (index) { 9744 case DEC_ID_DCORE0_DEC0: return GAUDI2_DCORE0_ENGINE_ID_DEC_0; 9745 case DEC_ID_DCORE0_DEC1: return GAUDI2_DCORE0_ENGINE_ID_DEC_1; 9746 case DEC_ID_DCORE1_DEC0: return GAUDI2_DCORE1_ENGINE_ID_DEC_0; 9747 case DEC_ID_DCORE1_DEC1: return GAUDI2_DCORE1_ENGINE_ID_DEC_1; 9748 case DEC_ID_DCORE2_DEC0: return GAUDI2_DCORE2_ENGINE_ID_DEC_0; 9749 case DEC_ID_DCORE2_DEC1: return GAUDI2_DCORE2_ENGINE_ID_DEC_1; 9750 case DEC_ID_DCORE3_DEC0: return GAUDI2_DCORE3_ENGINE_ID_DEC_0; 9751 case DEC_ID_DCORE3_DEC1: return GAUDI2_DCORE3_ENGINE_ID_DEC_1; 9752 case DEC_ID_PCIE_VDEC0: return GAUDI2_PCIE_ENGINE_ID_DEC_0; 9753 case DEC_ID_PCIE_VDEC1: return GAUDI2_PCIE_ENGINE_ID_DEC_1; 9754 default: 9755 break; 9756 } 9757 break; 9758 default: 9759 break; 9760 } 9761 9762 return U16_MAX; 9763 } 9764 9765 static void hl_eq_heartbeat_event_handle(struct hl_device *hdev) 9766 { 9767 hdev->eq_heartbeat_received = true; 9768 } 9769 9770 static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry) 9771 { 9772 struct gaudi2_device *gaudi2 = hdev->asic_specific; 9773 bool reset_required = false, is_critical = false; 9774 u32 index, ctl, reset_flags = 0, error_count = 0; 9775 u64 event_mask = 0; 9776 u16 event_type; 9777 9778 ctl = le32_to_cpu(eq_entry->hdr.ctl); 9779 event_type = ((ctl & EQ_CTL_EVENT_TYPE_MASK) >> EQ_CTL_EVENT_TYPE_SHIFT); 9780 9781 if (event_type >= GAUDI2_EVENT_SIZE) { 9782 dev_err(hdev->dev, "Event type %u exceeds maximum of %u", 9783 event_type, GAUDI2_EVENT_SIZE - 1); 9784 return; 9785 } 9786 9787 gaudi2->events_stat[event_type]++; 9788 gaudi2->events_stat_aggregate[event_type]++; 9789 9790 switch (event_type) { 9791 case GAUDI2_EVENT_PCIE_CORE_SERR ... GAUDI2_EVENT_ARC0_ECC_DERR: 9792 fallthrough; 9793 case GAUDI2_EVENT_ROTATOR0_SERR ... GAUDI2_EVENT_ROTATOR1_DERR: 9794 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9795 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9796 reset_required = gaudi2_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data); 9797 is_critical = eq_entry->ecc_data.is_critical; 9798 error_count++; 9799 break; 9800 9801 case GAUDI2_EVENT_TPC0_QM ... GAUDI2_EVENT_PDMA1_QM: 9802 fallthrough; 9803 case GAUDI2_EVENT_ROTATOR0_ROT0_QM ... GAUDI2_EVENT_ROTATOR1_ROT1_QM: 9804 fallthrough; 9805 case GAUDI2_EVENT_NIC0_QM0 ... GAUDI2_EVENT_NIC11_QM1: 9806 error_count = gaudi2_handle_qman_err(hdev, event_type, &event_mask); 9807 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9808 break; 9809 9810 case GAUDI2_EVENT_ARC_AXI_ERROR_RESPONSE_0: 9811 error_count = gaudi2_handle_arc_farm_sei_err(hdev, event_type, &event_mask); 9812 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9813 break; 9814 9815 case GAUDI2_EVENT_CPU_AXI_ERR_RSP: 9816 error_count = gaudi2_handle_cpu_sei_err(hdev, event_type); 9817 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9818 event_mask |= HL_NOTIFIER_EVENT_CRITICL_FW_ERR; 9819 break; 9820 9821 case GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP: 9822 case GAUDI2_EVENT_PDMA_CH1_AXI_ERR_RSP: 9823 error_count = gaudi2_handle_qm_sei_err(hdev, event_type, true, &event_mask); 9824 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9825 break; 9826 9827 case GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE: 9828 case GAUDI2_EVENT_ROTATOR1_AXI_ERROR_RESPONSE: 9829 index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE; 9830 error_count = gaudi2_handle_rot_err(hdev, index, event_type, 9831 &eq_entry->razwi_with_intr_cause, &event_mask); 9832 error_count += gaudi2_handle_qm_sei_err(hdev, event_type, false, &event_mask); 9833 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9834 break; 9835 9836 case GAUDI2_EVENT_TPC0_AXI_ERR_RSP ... GAUDI2_EVENT_TPC24_AXI_ERR_RSP: 9837 index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP; 9838 error_count = gaudi2_tpc_ack_interrupts(hdev, index, event_type, 9839 &eq_entry->razwi_with_intr_cause, &event_mask); 9840 error_count += gaudi2_handle_qm_sei_err(hdev, event_type, false, &event_mask); 9841 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9842 break; 9843 9844 case GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE ... GAUDI2_EVENT_DEC9_AXI_ERR_RSPONSE: 9845 index = event_type - GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE; 9846 error_count = gaudi2_handle_dec_err(hdev, index, event_type, &event_mask); 9847 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9848 break; 9849 9850 case GAUDI2_EVENT_TPC0_KERNEL_ERR: 9851 case GAUDI2_EVENT_TPC1_KERNEL_ERR: 9852 case GAUDI2_EVENT_TPC2_KERNEL_ERR: 9853 case GAUDI2_EVENT_TPC3_KERNEL_ERR: 9854 case GAUDI2_EVENT_TPC4_KERNEL_ERR: 9855 case GAUDI2_EVENT_TPC5_KERNEL_ERR: 9856 case GAUDI2_EVENT_TPC6_KERNEL_ERR: 9857 case GAUDI2_EVENT_TPC7_KERNEL_ERR: 9858 case GAUDI2_EVENT_TPC8_KERNEL_ERR: 9859 case GAUDI2_EVENT_TPC9_KERNEL_ERR: 9860 case GAUDI2_EVENT_TPC10_KERNEL_ERR: 9861 case GAUDI2_EVENT_TPC11_KERNEL_ERR: 9862 case GAUDI2_EVENT_TPC12_KERNEL_ERR: 9863 case GAUDI2_EVENT_TPC13_KERNEL_ERR: 9864 case GAUDI2_EVENT_TPC14_KERNEL_ERR: 9865 case GAUDI2_EVENT_TPC15_KERNEL_ERR: 9866 case GAUDI2_EVENT_TPC16_KERNEL_ERR: 9867 case GAUDI2_EVENT_TPC17_KERNEL_ERR: 9868 case GAUDI2_EVENT_TPC18_KERNEL_ERR: 9869 case GAUDI2_EVENT_TPC19_KERNEL_ERR: 9870 case GAUDI2_EVENT_TPC20_KERNEL_ERR: 9871 case GAUDI2_EVENT_TPC21_KERNEL_ERR: 9872 case GAUDI2_EVENT_TPC22_KERNEL_ERR: 9873 case GAUDI2_EVENT_TPC23_KERNEL_ERR: 9874 case GAUDI2_EVENT_TPC24_KERNEL_ERR: 9875 index = (event_type - GAUDI2_EVENT_TPC0_KERNEL_ERR) / 9876 (GAUDI2_EVENT_TPC1_KERNEL_ERR - GAUDI2_EVENT_TPC0_KERNEL_ERR); 9877 error_count = gaudi2_tpc_ack_interrupts(hdev, index, event_type, 9878 &eq_entry->razwi_with_intr_cause, &event_mask); 9879 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9880 break; 9881 9882 case GAUDI2_EVENT_DEC0_SPI: 9883 case GAUDI2_EVENT_DEC1_SPI: 9884 case GAUDI2_EVENT_DEC2_SPI: 9885 case GAUDI2_EVENT_DEC3_SPI: 9886 case GAUDI2_EVENT_DEC4_SPI: 9887 case GAUDI2_EVENT_DEC5_SPI: 9888 case GAUDI2_EVENT_DEC6_SPI: 9889 case GAUDI2_EVENT_DEC7_SPI: 9890 case GAUDI2_EVENT_DEC8_SPI: 9891 case GAUDI2_EVENT_DEC9_SPI: 9892 index = (event_type - GAUDI2_EVENT_DEC0_SPI) / 9893 (GAUDI2_EVENT_DEC1_SPI - GAUDI2_EVENT_DEC0_SPI); 9894 error_count = gaudi2_handle_dec_err(hdev, index, event_type, &event_mask); 9895 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9896 break; 9897 9898 case GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE: 9899 case GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE: 9900 case GAUDI2_EVENT_MME2_CTRL_AXI_ERROR_RESPONSE: 9901 case GAUDI2_EVENT_MME3_CTRL_AXI_ERROR_RESPONSE: 9902 index = (event_type - GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE) / 9903 (GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE - 9904 GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE); 9905 error_count = gaudi2_handle_mme_err(hdev, index, event_type, &event_mask); 9906 error_count += gaudi2_handle_qm_sei_err(hdev, event_type, false, &event_mask); 9907 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9908 break; 9909 9910 case GAUDI2_EVENT_MME0_QMAN_SW_ERROR: 9911 case GAUDI2_EVENT_MME1_QMAN_SW_ERROR: 9912 case GAUDI2_EVENT_MME2_QMAN_SW_ERROR: 9913 case GAUDI2_EVENT_MME3_QMAN_SW_ERROR: 9914 index = (event_type - GAUDI2_EVENT_MME0_QMAN_SW_ERROR) / 9915 (GAUDI2_EVENT_MME1_QMAN_SW_ERROR - 9916 GAUDI2_EVENT_MME0_QMAN_SW_ERROR); 9917 error_count = gaudi2_handle_mme_err(hdev, index, event_type, &event_mask); 9918 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9919 break; 9920 9921 case GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID: 9922 case GAUDI2_EVENT_MME1_WAP_SOURCE_RESULT_INVALID: 9923 case GAUDI2_EVENT_MME2_WAP_SOURCE_RESULT_INVALID: 9924 case GAUDI2_EVENT_MME3_WAP_SOURCE_RESULT_INVALID: 9925 index = (event_type - GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID) / 9926 (GAUDI2_EVENT_MME1_WAP_SOURCE_RESULT_INVALID - 9927 GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID); 9928 error_count = gaudi2_handle_mme_wap_err(hdev, index, event_type, &event_mask); 9929 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9930 break; 9931 9932 case GAUDI2_EVENT_KDMA_CH0_AXI_ERR_RSP: 9933 case GAUDI2_EVENT_KDMA0_CORE: 9934 error_count = gaudi2_handle_kdma_core_event(hdev, event_type, 9935 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 9936 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9937 break; 9938 9939 case GAUDI2_EVENT_HDMA2_CORE ... GAUDI2_EVENT_HDMA5_CORE: 9940 error_count = gaudi2_handle_dma_core_event(hdev, event_type, 9941 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 9942 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9943 break; 9944 9945 case GAUDI2_EVENT_PDMA0_CORE ... GAUDI2_EVENT_PDMA1_CORE: 9946 error_count = gaudi2_handle_dma_core_event(hdev, event_type, 9947 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 9948 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9949 break; 9950 9951 case GAUDI2_EVENT_PCIE_ADDR_DEC_ERR: 9952 error_count = gaudi2_print_pcie_addr_dec_info(hdev, event_type, 9953 le64_to_cpu(eq_entry->intr_cause.intr_cause_data), &event_mask); 9954 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9955 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9956 break; 9957 9958 case GAUDI2_EVENT_HMMU0_PAGE_FAULT_OR_WR_PERM ... GAUDI2_EVENT_HMMU12_SECURITY_ERROR: 9959 case GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP ... GAUDI2_EVENT_HMMU_12_AXI_ERR_RSP: 9960 case GAUDI2_EVENT_PMMU0_PAGE_FAULT_WR_PERM ... GAUDI2_EVENT_PMMU0_SECURITY_ERROR: 9961 case GAUDI2_EVENT_PMMU_AXI_ERR_RSP_0: 9962 error_count = gaudi2_handle_mmu_spi_sei_err(hdev, event_type, &event_mask); 9963 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9964 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9965 break; 9966 9967 case GAUDI2_EVENT_HIF0_FATAL ... GAUDI2_EVENT_HIF12_FATAL: 9968 error_count = gaudi2_handle_hif_fatal(hdev, event_type, 9969 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 9970 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9971 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9972 break; 9973 9974 case GAUDI2_EVENT_PMMU_FATAL_0: 9975 error_count = gaudi2_handle_pif_fatal(hdev, event_type, 9976 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 9977 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9978 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9979 break; 9980 9981 case GAUDI2_EVENT_PSOC63_RAZWI_OR_PID_MIN_MAX_INTERRUPT: 9982 error_count = gaudi2_ack_psoc_razwi_event_handler(hdev, &event_mask); 9983 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 9984 break; 9985 9986 case GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE ... GAUDI2_EVENT_HBM5_MC1_SEI_NON_SEVERE: 9987 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9988 if (gaudi2_handle_hbm_mc_sei_err(hdev, event_type, &eq_entry->sei_data)) { 9989 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 9990 reset_required = true; 9991 } 9992 error_count++; 9993 break; 9994 9995 case GAUDI2_EVENT_HBM_CATTRIP_0 ... GAUDI2_EVENT_HBM_CATTRIP_5: 9996 error_count = gaudi2_handle_hbm_cattrip(hdev, event_type, 9997 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 9998 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 9999 break; 10000 10001 case GAUDI2_EVENT_HBM0_MC0_SPI ... GAUDI2_EVENT_HBM5_MC1_SPI: 10002 error_count = gaudi2_handle_hbm_mc_spi(hdev, 10003 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 10004 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10005 break; 10006 10007 case GAUDI2_EVENT_PCIE_DRAIN_COMPLETE: 10008 error_count = gaudi2_handle_pcie_drain(hdev, &eq_entry->pcie_drain_ind_data); 10009 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10010 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10011 if (hl_is_fw_sw_ver_equal_or_greater(hdev, 1, 13)) 10012 is_critical = true; 10013 break; 10014 10015 case GAUDI2_EVENT_PSOC59_RPM_ERROR_OR_DRAIN: 10016 error_count = gaudi2_handle_psoc_drain(hdev, 10017 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); 10018 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10019 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10020 break; 10021 10022 case GAUDI2_EVENT_CPU_AXI_ECC: 10023 error_count = GAUDI2_NA_EVENT_CAUSE; 10024 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10025 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10026 break; 10027 case GAUDI2_EVENT_CPU_L2_RAM_ECC: 10028 error_count = GAUDI2_NA_EVENT_CAUSE; 10029 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10030 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10031 break; 10032 case GAUDI2_EVENT_MME0_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME0_SBTE4_AXI_ERR_RSP: 10033 case GAUDI2_EVENT_MME1_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME1_SBTE4_AXI_ERR_RSP: 10034 case GAUDI2_EVENT_MME2_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME2_SBTE4_AXI_ERR_RSP: 10035 case GAUDI2_EVENT_MME3_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME3_SBTE4_AXI_ERR_RSP: 10036 error_count = gaudi2_handle_mme_sbte_err(hdev, event_type); 10037 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10038 break; 10039 case GAUDI2_EVENT_VM0_ALARM_A ... GAUDI2_EVENT_VM3_ALARM_B: 10040 error_count = GAUDI2_NA_EVENT_CAUSE; 10041 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10042 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10043 break; 10044 case GAUDI2_EVENT_PSOC_AXI_ERR_RSP: 10045 error_count = GAUDI2_NA_EVENT_CAUSE; 10046 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10047 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10048 break; 10049 case GAUDI2_EVENT_PSOC_PRSTN_FALL: 10050 error_count = GAUDI2_NA_EVENT_CAUSE; 10051 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10052 break; 10053 case GAUDI2_EVENT_PCIE_APB_TIMEOUT: 10054 error_count = GAUDI2_NA_EVENT_CAUSE; 10055 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10056 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10057 break; 10058 case GAUDI2_EVENT_PCIE_FATAL_ERR: 10059 error_count = GAUDI2_NA_EVENT_CAUSE; 10060 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10061 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10062 break; 10063 case GAUDI2_EVENT_TPC0_BMON_SPMU: 10064 case GAUDI2_EVENT_TPC1_BMON_SPMU: 10065 case GAUDI2_EVENT_TPC2_BMON_SPMU: 10066 case GAUDI2_EVENT_TPC3_BMON_SPMU: 10067 case GAUDI2_EVENT_TPC4_BMON_SPMU: 10068 case GAUDI2_EVENT_TPC5_BMON_SPMU: 10069 case GAUDI2_EVENT_TPC6_BMON_SPMU: 10070 case GAUDI2_EVENT_TPC7_BMON_SPMU: 10071 case GAUDI2_EVENT_TPC8_BMON_SPMU: 10072 case GAUDI2_EVENT_TPC9_BMON_SPMU: 10073 case GAUDI2_EVENT_TPC10_BMON_SPMU: 10074 case GAUDI2_EVENT_TPC11_BMON_SPMU: 10075 case GAUDI2_EVENT_TPC12_BMON_SPMU: 10076 case GAUDI2_EVENT_TPC13_BMON_SPMU: 10077 case GAUDI2_EVENT_TPC14_BMON_SPMU: 10078 case GAUDI2_EVENT_TPC15_BMON_SPMU: 10079 case GAUDI2_EVENT_TPC16_BMON_SPMU: 10080 case GAUDI2_EVENT_TPC17_BMON_SPMU: 10081 case GAUDI2_EVENT_TPC18_BMON_SPMU: 10082 case GAUDI2_EVENT_TPC19_BMON_SPMU: 10083 case GAUDI2_EVENT_TPC20_BMON_SPMU: 10084 case GAUDI2_EVENT_TPC21_BMON_SPMU: 10085 case GAUDI2_EVENT_TPC22_BMON_SPMU: 10086 case GAUDI2_EVENT_TPC23_BMON_SPMU: 10087 case GAUDI2_EVENT_TPC24_BMON_SPMU: 10088 case GAUDI2_EVENT_MME0_CTRL_BMON_SPMU: 10089 case GAUDI2_EVENT_MME0_SBTE_BMON_SPMU: 10090 case GAUDI2_EVENT_MME0_WAP_BMON_SPMU: 10091 case GAUDI2_EVENT_MME1_CTRL_BMON_SPMU: 10092 case GAUDI2_EVENT_MME1_SBTE_BMON_SPMU: 10093 case GAUDI2_EVENT_MME1_WAP_BMON_SPMU: 10094 case GAUDI2_EVENT_MME2_CTRL_BMON_SPMU: 10095 case GAUDI2_EVENT_MME2_SBTE_BMON_SPMU: 10096 case GAUDI2_EVENT_MME2_WAP_BMON_SPMU: 10097 case GAUDI2_EVENT_MME3_CTRL_BMON_SPMU: 10098 case GAUDI2_EVENT_MME3_SBTE_BMON_SPMU: 10099 case GAUDI2_EVENT_MME3_WAP_BMON_SPMU: 10100 case GAUDI2_EVENT_HDMA2_BM_SPMU ... GAUDI2_EVENT_PDMA1_BM_SPMU: 10101 fallthrough; 10102 case GAUDI2_EVENT_DEC0_BMON_SPMU: 10103 case GAUDI2_EVENT_DEC1_BMON_SPMU: 10104 case GAUDI2_EVENT_DEC2_BMON_SPMU: 10105 case GAUDI2_EVENT_DEC3_BMON_SPMU: 10106 case GAUDI2_EVENT_DEC4_BMON_SPMU: 10107 case GAUDI2_EVENT_DEC5_BMON_SPMU: 10108 case GAUDI2_EVENT_DEC6_BMON_SPMU: 10109 case GAUDI2_EVENT_DEC7_BMON_SPMU: 10110 case GAUDI2_EVENT_DEC8_BMON_SPMU: 10111 case GAUDI2_EVENT_DEC9_BMON_SPMU: 10112 case GAUDI2_EVENT_ROTATOR0_BMON_SPMU ... GAUDI2_EVENT_SM3_BMON_SPMU: 10113 error_count = GAUDI2_NA_EVENT_CAUSE; 10114 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10115 break; 10116 10117 case GAUDI2_EVENT_CPU_FIX_POWER_ENV_S: 10118 case GAUDI2_EVENT_CPU_FIX_POWER_ENV_E: 10119 case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_S: 10120 case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_E: 10121 gaudi2_print_clk_change_info(hdev, event_type, &event_mask); 10122 error_count = GAUDI2_NA_EVENT_CAUSE; 10123 break; 10124 10125 case GAUDI2_EVENT_CPU_PKT_QUEUE_OUT_SYNC: 10126 gaudi2_print_out_of_sync_info(hdev, event_type, &eq_entry->pkt_sync_err); 10127 error_count = GAUDI2_NA_EVENT_CAUSE; 10128 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10129 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10130 break; 10131 10132 case GAUDI2_EVENT_PCIE_FLR_REQUESTED: 10133 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10134 error_count = GAUDI2_NA_EVENT_CAUSE; 10135 /* Do nothing- FW will handle it */ 10136 break; 10137 10138 case GAUDI2_EVENT_PCIE_P2P_MSIX: 10139 error_count = gaudi2_handle_pcie_p2p_msix(hdev, event_type); 10140 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10141 break; 10142 10143 case GAUDI2_EVENT_SM0_AXI_ERROR_RESPONSE ... GAUDI2_EVENT_SM3_AXI_ERROR_RESPONSE: 10144 index = event_type - GAUDI2_EVENT_SM0_AXI_ERROR_RESPONSE; 10145 error_count = gaudi2_handle_sm_err(hdev, event_type, index); 10146 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10147 break; 10148 10149 case GAUDI2_EVENT_PSOC_MME_PLL_LOCK_ERR ... GAUDI2_EVENT_DCORE2_HBM_PLL_LOCK_ERR: 10150 error_count = GAUDI2_NA_EVENT_CAUSE; 10151 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10152 break; 10153 10154 case GAUDI2_EVENT_CPU_CPLD_SHUTDOWN_CAUSE: 10155 dev_info(hdev->dev, "CPLD shutdown cause, reset reason: 0x%llx\n", 10156 le64_to_cpu(eq_entry->data[0])); 10157 error_count = GAUDI2_NA_EVENT_CAUSE; 10158 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10159 break; 10160 case GAUDI2_EVENT_CPU_CPLD_SHUTDOWN_EVENT: 10161 dev_err(hdev->dev, "CPLD shutdown event, reset reason: 0x%llx\n", 10162 le64_to_cpu(eq_entry->data[0])); 10163 error_count = GAUDI2_NA_EVENT_CAUSE; 10164 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10165 break; 10166 10167 case GAUDI2_EVENT_CPU_PKT_SANITY_FAILED: 10168 gaudi2_print_cpu_pkt_failure_info(hdev, event_type, &eq_entry->pkt_sync_err); 10169 error_count = GAUDI2_NA_EVENT_CAUSE; 10170 reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; 10171 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10172 break; 10173 10174 case GAUDI2_EVENT_ARC_DCCM_FULL: 10175 error_count = hl_arc_event_handle(hdev, event_type, &eq_entry->arc_data); 10176 event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; 10177 break; 10178 10179 case GAUDI2_EVENT_CPU_FP32_NOT_SUPPORTED: 10180 case GAUDI2_EVENT_CPU_DEV_RESET_REQ: 10181 event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; 10182 error_count = GAUDI2_NA_EVENT_CAUSE; 10183 is_critical = true; 10184 break; 10185 10186 case GAUDI2_EVENT_ARC_PWR_BRK_ENTRY: 10187 case GAUDI2_EVENT_ARC_PWR_BRK_EXT: 10188 case GAUDI2_EVENT_ARC_PWR_RD_MODE0: 10189 case GAUDI2_EVENT_ARC_PWR_RD_MODE1: 10190 case GAUDI2_EVENT_ARC_PWR_RD_MODE2: 10191 case GAUDI2_EVENT_ARC_PWR_RD_MODE3: 10192 error_count = GAUDI2_NA_EVENT_CAUSE; 10193 dev_info_ratelimited(hdev->dev, "%s event received\n", 10194 gaudi2_irq_map_table[event_type].name); 10195 break; 10196 10197 case GAUDI2_EVENT_ARC_EQ_HEARTBEAT: 10198 hl_eq_heartbeat_event_handle(hdev); 10199 error_count = GAUDI2_NA_EVENT_CAUSE; 10200 break; 10201 default: 10202 if (gaudi2_irq_map_table[event_type].valid) { 10203 dev_err_ratelimited(hdev->dev, "Cannot find handler for event %d\n", 10204 event_type); 10205 error_count = GAUDI2_NA_EVENT_CAUSE; 10206 } 10207 } 10208 10209 if (event_mask & HL_NOTIFIER_EVENT_USER_ENGINE_ERR) 10210 hl_capture_engine_err(hdev, event_id_to_engine_id(hdev, event_type), error_count); 10211 10212 /* Make sure to dump an error in case no error cause was printed so far. 10213 * Note that although we have counted the errors, we use this number as 10214 * a boolean. 10215 */ 10216 if (error_count == GAUDI2_NA_EVENT_CAUSE && !is_info_event(event_type)) 10217 gaudi2_print_event(hdev, event_type, true, "%d", event_type); 10218 else if (error_count == 0) 10219 gaudi2_print_event(hdev, event_type, true, 10220 "No error cause for H/W event %u", event_type); 10221 10222 if ((gaudi2_irq_map_table[event_type].reset != EVENT_RESET_TYPE_NONE) || 10223 reset_required) { 10224 if (reset_required || 10225 (gaudi2_irq_map_table[event_type].reset == EVENT_RESET_TYPE_HARD)) 10226 reset_flags |= HL_DRV_RESET_HARD; 10227 10228 if (hdev->hard_reset_on_fw_events || 10229 (hdev->asic_prop.fw_security_enabled && is_critical)) 10230 goto reset_device; 10231 } 10232 10233 /* Send unmask irq only for interrupts not classified as MSG */ 10234 if (!gaudi2_irq_map_table[event_type].msg) 10235 hl_fw_unmask_irq(hdev, event_type); 10236 10237 if (event_mask) 10238 hl_notifier_event_send_all(hdev, event_mask); 10239 10240 return; 10241 10242 reset_device: 10243 if (hdev->asic_prop.fw_security_enabled && is_critical) { 10244 reset_flags |= HL_DRV_RESET_BYPASS_REQ_TO_FW; 10245 event_mask |= HL_NOTIFIER_EVENT_DEVICE_UNAVAILABLE; 10246 } else { 10247 reset_flags |= HL_DRV_RESET_DELAY; 10248 } 10249 /* escalate general hw errors to critical/fatal error */ 10250 if (event_mask & HL_NOTIFIER_EVENT_GENERAL_HW_ERR) 10251 hl_handle_critical_hw_err(hdev, event_type, &event_mask); 10252 10253 event_mask |= HL_NOTIFIER_EVENT_DEVICE_RESET; 10254 hl_device_cond_reset(hdev, reset_flags, event_mask); 10255 } 10256 10257 static int gaudi2_memset_memory_chunk_using_edma_qm(struct hl_device *hdev, 10258 struct packet_lin_dma *lin_dma_pkt, dma_addr_t pkt_dma_addr, 10259 u32 hw_queue_id, u32 size, u64 addr, u32 val) 10260 { 10261 u32 ctl, pkt_size; 10262 int rc = 0; 10263 10264 ctl = FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_LIN_DMA); 10265 ctl |= FIELD_PREP(GAUDI2_PKT_LIN_DMA_CTL_MEMSET_MASK, 1); 10266 ctl |= FIELD_PREP(GAUDI2_PKT_LIN_DMA_CTL_WRCOMP_MASK, 1); 10267 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 1); 10268 10269 lin_dma_pkt->ctl = cpu_to_le32(ctl); 10270 lin_dma_pkt->src_addr = cpu_to_le64(val); 10271 lin_dma_pkt->dst_addr = cpu_to_le64(addr); 10272 lin_dma_pkt->tsize = cpu_to_le32(size); 10273 10274 pkt_size = sizeof(struct packet_lin_dma); 10275 10276 rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, pkt_size, pkt_dma_addr); 10277 if (rc) 10278 dev_err(hdev->dev, "Failed to send lin dma packet to H/W queue %d\n", 10279 hw_queue_id); 10280 10281 return rc; 10282 } 10283 10284 static int gaudi2_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, u64 val) 10285 { 10286 u32 edma_queues_id[] = {GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0, 10287 GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0, 10288 GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0, 10289 GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0}; 10290 u32 chunk_size, dcore, edma_idx, sob_offset, sob_addr, comp_val, 10291 old_mmubp, mmubp, num_of_pkts, busy, pkt_size; 10292 u64 comp_addr, cur_addr = addr, end_addr = addr + size; 10293 struct asic_fixed_properties *prop = &hdev->asic_prop; 10294 void *lin_dma_pkts_arr; 10295 dma_addr_t pkt_dma_addr; 10296 int rc = 0, dma_num = 0; 10297 10298 if (prop->edma_enabled_mask == 0) { 10299 dev_info(hdev->dev, "non of the EDMA engines is enabled - skip dram scrubbing\n"); 10300 return -EIO; 10301 } 10302 10303 sob_offset = hdev->asic_prop.first_available_user_sob[0] * 4; 10304 sob_addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset; 10305 comp_addr = CFG_BASE + sob_addr; 10306 comp_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1) | 10307 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 1); 10308 mmubp = FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_WR_MASK, 1) | 10309 FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_RD_MASK, 1); 10310 10311 /* Calculate how many lin dma pkts we'll need */ 10312 num_of_pkts = div64_u64(round_up(size, SZ_2G), SZ_2G); 10313 pkt_size = sizeof(struct packet_lin_dma); 10314 10315 lin_dma_pkts_arr = hl_asic_dma_alloc_coherent(hdev, pkt_size * num_of_pkts, 10316 &pkt_dma_addr, GFP_KERNEL); 10317 if (!lin_dma_pkts_arr) 10318 return -ENOMEM; 10319 10320 /* 10321 * set mmu bypass for the scrubbing - all ddmas are configured the same so save 10322 * only the first one to restore later 10323 * also set the sob addr for all edma cores for completion. 10324 * set QM as trusted to allow it to access physical address with MMU bp. 10325 */ 10326 old_mmubp = RREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP); 10327 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 10328 for (edma_idx = 0 ; edma_idx < NUM_OF_EDMA_PER_DCORE ; edma_idx++) { 10329 u32 edma_offset = dcore * DCORE_OFFSET + edma_idx * DCORE_EDMA_OFFSET; 10330 u32 edma_bit = dcore * NUM_OF_EDMA_PER_DCORE + edma_idx; 10331 10332 if (!(prop->edma_enabled_mask & BIT(edma_bit))) 10333 continue; 10334 10335 WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP + 10336 edma_offset, mmubp); 10337 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_LO + edma_offset, 10338 lower_32_bits(comp_addr)); 10339 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_HI + edma_offset, 10340 upper_32_bits(comp_addr)); 10341 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_WDATA + edma_offset, 10342 comp_val); 10343 gaudi2_qman_set_test_mode(hdev, 10344 edma_queues_id[dcore] + 4 * edma_idx, true); 10345 } 10346 } 10347 10348 WREG32(sob_addr, 0); 10349 10350 while (cur_addr < end_addr) { 10351 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 10352 for (edma_idx = 0 ; edma_idx < NUM_OF_EDMA_PER_DCORE ; edma_idx++) { 10353 u32 edma_bit = dcore * NUM_OF_EDMA_PER_DCORE + edma_idx; 10354 10355 if (!(prop->edma_enabled_mask & BIT(edma_bit))) 10356 continue; 10357 10358 chunk_size = min_t(u64, SZ_2G, end_addr - cur_addr); 10359 10360 rc = gaudi2_memset_memory_chunk_using_edma_qm(hdev, 10361 (struct packet_lin_dma *)lin_dma_pkts_arr + dma_num, 10362 pkt_dma_addr + dma_num * pkt_size, 10363 edma_queues_id[dcore] + edma_idx * 4, 10364 chunk_size, cur_addr, val); 10365 if (rc) 10366 goto end; 10367 10368 dma_num++; 10369 cur_addr += chunk_size; 10370 if (cur_addr == end_addr) 10371 break; 10372 } 10373 } 10374 } 10375 10376 rc = hl_poll_timeout(hdev, sob_addr, busy, (busy == dma_num), 1000, 1000000); 10377 if (rc) { 10378 dev_err(hdev->dev, "DMA Timeout during HBM scrubbing\n"); 10379 goto end; 10380 } 10381 end: 10382 for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) { 10383 for (edma_idx = 0 ; edma_idx < NUM_OF_EDMA_PER_DCORE ; edma_idx++) { 10384 u32 edma_offset = dcore * DCORE_OFFSET + edma_idx * DCORE_EDMA_OFFSET; 10385 u32 edma_bit = dcore * NUM_OF_EDMA_PER_DCORE + edma_idx; 10386 10387 if (!(prop->edma_enabled_mask & BIT(edma_bit))) 10388 continue; 10389 10390 WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP + edma_offset, old_mmubp); 10391 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_LO + edma_offset, 0); 10392 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_HI + edma_offset, 0); 10393 WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_WDATA + edma_offset, 0); 10394 gaudi2_qman_set_test_mode(hdev, 10395 edma_queues_id[dcore] + 4 * edma_idx, false); 10396 } 10397 } 10398 10399 WREG32(sob_addr, 0); 10400 hl_asic_dma_free_coherent(hdev, pkt_size * num_of_pkts, lin_dma_pkts_arr, pkt_dma_addr); 10401 10402 return rc; 10403 } 10404 10405 static int gaudi2_scrub_device_dram(struct hl_device *hdev, u64 val) 10406 { 10407 int rc; 10408 struct asic_fixed_properties *prop = &hdev->asic_prop; 10409 u64 size = prop->dram_end_address - prop->dram_user_base_address; 10410 10411 rc = gaudi2_memset_device_memory(hdev, prop->dram_user_base_address, size, val); 10412 10413 if (rc) 10414 dev_err(hdev->dev, "Failed to scrub dram, address: 0x%llx size: %llu\n", 10415 prop->dram_user_base_address, size); 10416 return rc; 10417 } 10418 10419 static int gaudi2_scrub_device_mem(struct hl_device *hdev) 10420 { 10421 int rc; 10422 struct asic_fixed_properties *prop = &hdev->asic_prop; 10423 u64 val = hdev->memory_scrub_val; 10424 u64 addr, size; 10425 10426 if (!hdev->memory_scrub) 10427 return 0; 10428 10429 /* scrub SRAM */ 10430 addr = prop->sram_user_base_address; 10431 size = hdev->pldm ? 0x10000 : (prop->sram_size - SRAM_USER_BASE_OFFSET); 10432 dev_dbg(hdev->dev, "Scrubbing SRAM: 0x%09llx - 0x%09llx, val: 0x%llx\n", 10433 addr, addr + size, val); 10434 rc = gaudi2_memset_device_memory(hdev, addr, size, val); 10435 if (rc) { 10436 dev_err(hdev->dev, "scrubbing SRAM failed (%d)\n", rc); 10437 return rc; 10438 } 10439 10440 /* scrub DRAM */ 10441 rc = gaudi2_scrub_device_dram(hdev, val); 10442 if (rc) { 10443 dev_err(hdev->dev, "scrubbing DRAM failed (%d)\n", rc); 10444 return rc; 10445 } 10446 return 0; 10447 } 10448 10449 static void gaudi2_restore_user_sm_registers(struct hl_device *hdev) 10450 { 10451 u64 addr, mon_sts_addr, mon_cfg_addr, cq_lbw_l_addr, cq_lbw_h_addr, 10452 cq_lbw_data_addr, cq_base_l_addr, cq_base_h_addr, cq_size_addr; 10453 u32 val, size, offset; 10454 int dcore_id; 10455 10456 offset = hdev->asic_prop.first_available_cq[0] * 4; 10457 cq_lbw_l_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0 + offset; 10458 cq_lbw_h_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 + offset; 10459 cq_lbw_data_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_DATA_0 + offset; 10460 cq_base_l_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_L_0 + offset; 10461 cq_base_h_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_H_0 + offset; 10462 cq_size_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_SIZE_LOG2_0 + offset; 10463 size = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 - 10464 (mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0 + offset); 10465 10466 /* memset dcore0 CQ registers */ 10467 gaudi2_memset_device_lbw(hdev, cq_lbw_l_addr, size, 0); 10468 gaudi2_memset_device_lbw(hdev, cq_lbw_h_addr, size, 0); 10469 gaudi2_memset_device_lbw(hdev, cq_lbw_data_addr, size, 0); 10470 gaudi2_memset_device_lbw(hdev, cq_base_l_addr, size, 0); 10471 gaudi2_memset_device_lbw(hdev, cq_base_h_addr, size, 0); 10472 gaudi2_memset_device_lbw(hdev, cq_size_addr, size, 0); 10473 10474 cq_lbw_l_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0 + DCORE_OFFSET; 10475 cq_lbw_h_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 + DCORE_OFFSET; 10476 cq_lbw_data_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_DATA_0 + DCORE_OFFSET; 10477 cq_base_l_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_L_0 + DCORE_OFFSET; 10478 cq_base_h_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_BASE_ADDR_H_0 + DCORE_OFFSET; 10479 cq_size_addr = mmDCORE0_SYNC_MNGR_GLBL_CQ_SIZE_LOG2_0 + DCORE_OFFSET; 10480 size = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 - mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0; 10481 10482 for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) { 10483 gaudi2_memset_device_lbw(hdev, cq_lbw_l_addr, size, 0); 10484 gaudi2_memset_device_lbw(hdev, cq_lbw_h_addr, size, 0); 10485 gaudi2_memset_device_lbw(hdev, cq_lbw_data_addr, size, 0); 10486 gaudi2_memset_device_lbw(hdev, cq_base_l_addr, size, 0); 10487 gaudi2_memset_device_lbw(hdev, cq_base_h_addr, size, 0); 10488 gaudi2_memset_device_lbw(hdev, cq_size_addr, size, 0); 10489 10490 cq_lbw_l_addr += DCORE_OFFSET; 10491 cq_lbw_h_addr += DCORE_OFFSET; 10492 cq_lbw_data_addr += DCORE_OFFSET; 10493 cq_base_l_addr += DCORE_OFFSET; 10494 cq_base_h_addr += DCORE_OFFSET; 10495 cq_size_addr += DCORE_OFFSET; 10496 } 10497 10498 offset = hdev->asic_prop.first_available_user_mon[0] * 4; 10499 addr = mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + offset; 10500 val = 1 << DCORE0_SYNC_MNGR_OBJS_MON_STATUS_PROT_SHIFT; 10501 size = mmDCORE0_SYNC_MNGR_OBJS_SM_SEC_0 - (mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + offset); 10502 10503 /* memset dcore0 monitors */ 10504 gaudi2_memset_device_lbw(hdev, addr, size, val); 10505 10506 addr = mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + offset; 10507 gaudi2_memset_device_lbw(hdev, addr, size, 0); 10508 10509 mon_sts_addr = mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + DCORE_OFFSET; 10510 mon_cfg_addr = mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + DCORE_OFFSET; 10511 size = mmDCORE0_SYNC_MNGR_OBJS_SM_SEC_0 - mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0; 10512 10513 for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) { 10514 gaudi2_memset_device_lbw(hdev, mon_sts_addr, size, val); 10515 gaudi2_memset_device_lbw(hdev, mon_cfg_addr, size, 0); 10516 mon_sts_addr += DCORE_OFFSET; 10517 mon_cfg_addr += DCORE_OFFSET; 10518 } 10519 10520 offset = hdev->asic_prop.first_available_user_sob[0] * 4; 10521 addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + offset; 10522 val = 0; 10523 size = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 - 10524 (mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + offset); 10525 10526 /* memset dcore0 sobs */ 10527 gaudi2_memset_device_lbw(hdev, addr, size, val); 10528 10529 addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + DCORE_OFFSET; 10530 size = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 - mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0; 10531 10532 for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) { 10533 gaudi2_memset_device_lbw(hdev, addr, size, val); 10534 addr += DCORE_OFFSET; 10535 } 10536 10537 /* Flush all WREG to prevent race */ 10538 val = RREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + offset); 10539 } 10540 10541 static void gaudi2_restore_user_qm_registers(struct hl_device *hdev) 10542 { 10543 u32 reg_base, hw_queue_id; 10544 10545 for (hw_queue_id = GAUDI2_QUEUE_ID_PDMA_0_0 ; hw_queue_id <= GAUDI2_QUEUE_ID_ROT_1_0; 10546 hw_queue_id += NUM_OF_PQ_PER_QMAN) { 10547 if (!gaudi2_is_queue_enabled(hdev, hw_queue_id)) 10548 continue; 10549 10550 gaudi2_clear_qm_fence_counters_common(hdev, hw_queue_id, false); 10551 10552 reg_base = gaudi2_qm_blocks_bases[hw_queue_id]; 10553 WREG32(reg_base + QM_ARB_CFG_0_OFFSET, 0); 10554 } 10555 10556 /* Flush all WREG to prevent race */ 10557 RREG32(mmPDMA0_QM_ARB_CFG_0); 10558 } 10559 10560 static void gaudi2_restore_nic_qm_registers(struct hl_device *hdev) 10561 { 10562 u32 reg_base, hw_queue_id; 10563 10564 for (hw_queue_id = GAUDI2_QUEUE_ID_NIC_0_0 ; hw_queue_id <= GAUDI2_QUEUE_ID_NIC_23_3; 10565 hw_queue_id += NUM_OF_PQ_PER_QMAN) { 10566 if (!gaudi2_is_queue_enabled(hdev, hw_queue_id)) 10567 continue; 10568 10569 gaudi2_clear_qm_fence_counters_common(hdev, hw_queue_id, false); 10570 10571 reg_base = gaudi2_qm_blocks_bases[hw_queue_id]; 10572 WREG32(reg_base + QM_ARB_CFG_0_OFFSET, 0); 10573 } 10574 10575 /* Flush all WREG to prevent race */ 10576 RREG32(mmPDMA0_QM_ARB_CFG_0); 10577 } 10578 10579 static int gaudi2_context_switch(struct hl_device *hdev, u32 asid) 10580 { 10581 return 0; 10582 } 10583 10584 static void gaudi2_restore_phase_topology(struct hl_device *hdev) 10585 { 10586 } 10587 10588 static void gaudi2_init_block_instances(struct hl_device *hdev, u32 block_idx, 10589 struct dup_block_ctx *cfg_ctx) 10590 { 10591 u64 block_base = cfg_ctx->base + block_idx * cfg_ctx->block_off; 10592 u8 seq; 10593 int i; 10594 10595 for (i = 0 ; i < cfg_ctx->instances ; i++) { 10596 seq = block_idx * cfg_ctx->instances + i; 10597 10598 /* skip disabled instance */ 10599 if (!(cfg_ctx->enabled_mask & BIT_ULL(seq))) 10600 continue; 10601 10602 cfg_ctx->instance_cfg_fn(hdev, block_base + i * cfg_ctx->instance_off, 10603 cfg_ctx->data); 10604 } 10605 } 10606 10607 static void gaudi2_init_blocks_with_mask(struct hl_device *hdev, struct dup_block_ctx *cfg_ctx, 10608 u64 mask) 10609 { 10610 int i; 10611 10612 cfg_ctx->enabled_mask = mask; 10613 10614 for (i = 0 ; i < cfg_ctx->blocks ; i++) 10615 gaudi2_init_block_instances(hdev, i, cfg_ctx); 10616 } 10617 10618 void gaudi2_init_blocks(struct hl_device *hdev, struct dup_block_ctx *cfg_ctx) 10619 { 10620 gaudi2_init_blocks_with_mask(hdev, cfg_ctx, U64_MAX); 10621 } 10622 10623 static int gaudi2_debugfs_read_dma(struct hl_device *hdev, u64 addr, u32 size, void *blob_addr) 10624 { 10625 void *host_mem_virtual_addr; 10626 dma_addr_t host_mem_dma_addr; 10627 u64 reserved_va_base; 10628 u32 pos, size_left, size_to_dma; 10629 struct hl_ctx *ctx; 10630 int rc = 0; 10631 10632 /* Fetch the ctx */ 10633 ctx = hl_get_compute_ctx(hdev); 10634 if (!ctx) { 10635 dev_err(hdev->dev, "No ctx available\n"); 10636 return -EINVAL; 10637 } 10638 10639 /* Allocate buffers for read and for poll */ 10640 host_mem_virtual_addr = hl_asic_dma_alloc_coherent(hdev, SZ_2M, &host_mem_dma_addr, 10641 GFP_KERNEL | __GFP_ZERO); 10642 if (host_mem_virtual_addr == NULL) { 10643 dev_err(hdev->dev, "Failed to allocate memory for KDMA read\n"); 10644 rc = -ENOMEM; 10645 goto put_ctx; 10646 } 10647 10648 /* Reserve VM region on asic side */ 10649 reserved_va_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST, SZ_2M, 10650 HL_MMU_VA_ALIGNMENT_NOT_NEEDED); 10651 if (!reserved_va_base) { 10652 dev_err(hdev->dev, "Failed to reserve vmem on asic\n"); 10653 rc = -ENOMEM; 10654 goto free_data_buffer; 10655 } 10656 10657 /* Create mapping on asic side */ 10658 mutex_lock(&hdev->mmu_lock); 10659 10660 rc = hl_mmu_map_contiguous(ctx, reserved_va_base, host_mem_dma_addr, SZ_2M); 10661 if (rc) { 10662 dev_err(hdev->dev, "Failed to create mapping on asic mmu\n"); 10663 goto unreserve_va; 10664 } 10665 10666 rc = hl_mmu_invalidate_cache_range(hdev, false, 10667 MMU_OP_USERPTR | MMU_OP_SKIP_LOW_CACHE_INV, 10668 ctx->asid, reserved_va_base, SZ_2M); 10669 if (rc) { 10670 hl_mmu_unmap_contiguous(ctx, reserved_va_base, SZ_2M); 10671 goto unreserve_va; 10672 } 10673 10674 mutex_unlock(&hdev->mmu_lock); 10675 10676 /* Enable MMU on KDMA */ 10677 gaudi2_kdma_set_mmbp_asid(hdev, false, ctx->asid); 10678 10679 pos = 0; 10680 size_left = size; 10681 size_to_dma = SZ_2M; 10682 10683 while (size_left > 0) { 10684 if (size_left < SZ_2M) 10685 size_to_dma = size_left; 10686 10687 rc = gaudi2_send_job_to_kdma(hdev, addr, reserved_va_base, size_to_dma, false); 10688 if (rc) 10689 break; 10690 10691 memcpy(blob_addr + pos, host_mem_virtual_addr, size_to_dma); 10692 10693 if (size_left <= SZ_2M) 10694 break; 10695 10696 pos += SZ_2M; 10697 addr += SZ_2M; 10698 size_left -= SZ_2M; 10699 } 10700 10701 gaudi2_kdma_set_mmbp_asid(hdev, true, HL_KERNEL_ASID_ID); 10702 10703 mutex_lock(&hdev->mmu_lock); 10704 10705 rc = hl_mmu_unmap_contiguous(ctx, reserved_va_base, SZ_2M); 10706 if (rc) 10707 goto unreserve_va; 10708 10709 rc = hl_mmu_invalidate_cache_range(hdev, false, MMU_OP_USERPTR, 10710 ctx->asid, reserved_va_base, SZ_2M); 10711 10712 unreserve_va: 10713 mutex_unlock(&hdev->mmu_lock); 10714 hl_unreserve_va_block(hdev, ctx, reserved_va_base, SZ_2M); 10715 free_data_buffer: 10716 hl_asic_dma_free_coherent(hdev, SZ_2M, host_mem_virtual_addr, host_mem_dma_addr); 10717 put_ctx: 10718 hl_ctx_put(ctx); 10719 10720 return rc; 10721 } 10722 10723 static int gaudi2_internal_cb_pool_init(struct hl_device *hdev, struct hl_ctx *ctx) 10724 { 10725 struct gaudi2_device *gaudi2 = hdev->asic_specific; 10726 int min_alloc_order, rc; 10727 10728 if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) 10729 return 0; 10730 10731 hdev->internal_cb_pool_virt_addr = hl_asic_dma_alloc_coherent(hdev, 10732 HOST_SPACE_INTERNAL_CB_SZ, 10733 &hdev->internal_cb_pool_dma_addr, 10734 GFP_KERNEL | __GFP_ZERO); 10735 10736 if (!hdev->internal_cb_pool_virt_addr) 10737 return -ENOMEM; 10738 10739 min_alloc_order = ilog2(min(gaudi2_get_signal_cb_size(hdev), 10740 gaudi2_get_wait_cb_size(hdev))); 10741 10742 hdev->internal_cb_pool = gen_pool_create(min_alloc_order, -1); 10743 if (!hdev->internal_cb_pool) { 10744 dev_err(hdev->dev, "Failed to create internal CB pool\n"); 10745 rc = -ENOMEM; 10746 goto free_internal_cb_pool; 10747 } 10748 10749 rc = gen_pool_add(hdev->internal_cb_pool, (uintptr_t) hdev->internal_cb_pool_virt_addr, 10750 HOST_SPACE_INTERNAL_CB_SZ, -1); 10751 if (rc) { 10752 dev_err(hdev->dev, "Failed to add memory to internal CB pool\n"); 10753 rc = -EFAULT; 10754 goto destroy_internal_cb_pool; 10755 } 10756 10757 hdev->internal_cb_va_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST, 10758 HOST_SPACE_INTERNAL_CB_SZ, HL_MMU_VA_ALIGNMENT_NOT_NEEDED); 10759 10760 if (!hdev->internal_cb_va_base) { 10761 rc = -ENOMEM; 10762 goto destroy_internal_cb_pool; 10763 } 10764 10765 mutex_lock(&hdev->mmu_lock); 10766 10767 rc = hl_mmu_map_contiguous(ctx, hdev->internal_cb_va_base, hdev->internal_cb_pool_dma_addr, 10768 HOST_SPACE_INTERNAL_CB_SZ); 10769 if (rc) 10770 goto unreserve_internal_cb_pool; 10771 10772 rc = hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR); 10773 if (rc) 10774 goto unmap_internal_cb_pool; 10775 10776 mutex_unlock(&hdev->mmu_lock); 10777 10778 return 0; 10779 10780 unmap_internal_cb_pool: 10781 hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); 10782 unreserve_internal_cb_pool: 10783 mutex_unlock(&hdev->mmu_lock); 10784 hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); 10785 destroy_internal_cb_pool: 10786 gen_pool_destroy(hdev->internal_cb_pool); 10787 free_internal_cb_pool: 10788 hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr, 10789 hdev->internal_cb_pool_dma_addr); 10790 10791 return rc; 10792 } 10793 10794 static void gaudi2_internal_cb_pool_fini(struct hl_device *hdev, struct hl_ctx *ctx) 10795 { 10796 struct gaudi2_device *gaudi2 = hdev->asic_specific; 10797 10798 if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) 10799 return; 10800 10801 mutex_lock(&hdev->mmu_lock); 10802 hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); 10803 hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); 10804 hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR); 10805 mutex_unlock(&hdev->mmu_lock); 10806 10807 gen_pool_destroy(hdev->internal_cb_pool); 10808 10809 hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr, 10810 hdev->internal_cb_pool_dma_addr); 10811 } 10812 10813 static void gaudi2_restore_user_registers(struct hl_device *hdev) 10814 { 10815 gaudi2_restore_user_sm_registers(hdev); 10816 gaudi2_restore_user_qm_registers(hdev); 10817 } 10818 10819 static int gaudi2_map_virtual_msix_doorbell_memory(struct hl_ctx *ctx) 10820 { 10821 struct hl_device *hdev = ctx->hdev; 10822 struct asic_fixed_properties *prop = &hdev->asic_prop; 10823 struct gaudi2_device *gaudi2 = hdev->asic_specific; 10824 int rc; 10825 10826 rc = hl_mmu_map_page(ctx, RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START, 10827 gaudi2->virt_msix_db_dma_addr, prop->pmmu.page_size, true); 10828 if (rc) 10829 dev_err(hdev->dev, "Failed to map VA %#llx for virtual MSI-X doorbell memory\n", 10830 RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START); 10831 10832 return rc; 10833 } 10834 10835 static void gaudi2_unmap_virtual_msix_doorbell_memory(struct hl_ctx *ctx) 10836 { 10837 struct hl_device *hdev = ctx->hdev; 10838 struct asic_fixed_properties *prop = &hdev->asic_prop; 10839 int rc; 10840 10841 rc = hl_mmu_unmap_page(ctx, RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START, 10842 prop->pmmu.page_size, true); 10843 if (rc) 10844 dev_err(hdev->dev, "Failed to unmap VA %#llx of virtual MSI-X doorbell memory\n", 10845 RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START); 10846 } 10847 10848 static int gaudi2_ctx_init(struct hl_ctx *ctx) 10849 { 10850 int rc; 10851 10852 if (ctx->asid == HL_KERNEL_ASID_ID) 10853 return 0; 10854 10855 rc = gaudi2_mmu_prepare(ctx->hdev, ctx->asid); 10856 if (rc) 10857 return rc; 10858 10859 /* No need to clear user registers if the device has just 10860 * performed reset, we restore only nic qm registers 10861 */ 10862 if (ctx->hdev->reset_upon_device_release) 10863 gaudi2_restore_nic_qm_registers(ctx->hdev); 10864 else 10865 gaudi2_restore_user_registers(ctx->hdev); 10866 10867 rc = gaudi2_internal_cb_pool_init(ctx->hdev, ctx); 10868 if (rc) 10869 return rc; 10870 10871 rc = gaudi2_map_virtual_msix_doorbell_memory(ctx); 10872 if (rc) 10873 gaudi2_internal_cb_pool_fini(ctx->hdev, ctx); 10874 10875 return rc; 10876 } 10877 10878 static void gaudi2_ctx_fini(struct hl_ctx *ctx) 10879 { 10880 if (ctx->asid == HL_KERNEL_ASID_ID) 10881 return; 10882 10883 gaudi2_internal_cb_pool_fini(ctx->hdev, ctx); 10884 10885 gaudi2_unmap_virtual_msix_doorbell_memory(ctx); 10886 } 10887 10888 static int gaudi2_pre_schedule_cs(struct hl_cs *cs) 10889 { 10890 struct hl_device *hdev = cs->ctx->hdev; 10891 int index = cs->sequence & (hdev->asic_prop.max_pending_cs - 1); 10892 u32 mon_payload, sob_id, mon_id; 10893 10894 if (!cs_needs_completion(cs)) 10895 return 0; 10896 10897 /* 10898 * First 64 SOB/MON are reserved for driver for QMAN auto completion 10899 * mechanism. Each SOB/MON pair are used for a pending CS with the same 10900 * cyclic index. The SOB value is increased when each of the CS jobs is 10901 * completed. When the SOB reaches the number of CS jobs, the monitor 10902 * generates MSI-X interrupt. 10903 */ 10904 10905 sob_id = mon_id = index; 10906 mon_payload = (1 << CQ_ENTRY_SHADOW_INDEX_VALID_SHIFT) | 10907 (1 << CQ_ENTRY_READY_SHIFT) | index; 10908 10909 gaudi2_arm_cq_monitor(hdev, sob_id, mon_id, GAUDI2_RESERVED_CQ_CS_COMPLETION, mon_payload, 10910 cs->jobs_cnt); 10911 10912 return 0; 10913 } 10914 10915 static u32 gaudi2_get_queue_id_for_cq(struct hl_device *hdev, u32 cq_idx) 10916 { 10917 return HL_INVALID_QUEUE; 10918 } 10919 10920 static u32 gaudi2_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id, u32 size, bool eb) 10921 { 10922 struct hl_cb *cb = data; 10923 struct packet_msg_short *pkt; 10924 u32 value, ctl, pkt_size = sizeof(*pkt); 10925 10926 pkt = (struct packet_msg_short *) (uintptr_t) (cb->kernel_address + size); 10927 memset(pkt, 0, pkt_size); 10928 10929 /* Inc by 1, Mode ADD */ 10930 value = FIELD_PREP(GAUDI2_PKT_SHORT_VAL_SOB_SYNC_VAL_MASK, 1); 10931 value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_SOB_MOD_MASK, 1); 10932 10933 ctl = FIELD_PREP(GAUDI2_PKT_SHORT_CTL_ADDR_MASK, sob_id * 4); 10934 ctl |= FIELD_PREP(GAUDI2_PKT_SHORT_CTL_BASE_MASK, 1); /* SOB base */ 10935 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); 10936 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, eb); 10937 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1); 10938 10939 pkt->value = cpu_to_le32(value); 10940 pkt->ctl = cpu_to_le32(ctl); 10941 10942 return size + pkt_size; 10943 } 10944 10945 static u32 gaudi2_add_mon_msg_short(struct packet_msg_short *pkt, u32 value, u16 addr) 10946 { 10947 u32 ctl, pkt_size = sizeof(*pkt); 10948 10949 memset(pkt, 0, pkt_size); 10950 10951 ctl = FIELD_PREP(GAUDI2_PKT_SHORT_CTL_ADDR_MASK, addr); 10952 ctl |= FIELD_PREP(GAUDI2_PKT_SHORT_CTL_BASE_MASK, 0); /* MON base */ 10953 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); 10954 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 0); 10955 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 0); 10956 10957 pkt->value = cpu_to_le32(value); 10958 pkt->ctl = cpu_to_le32(ctl); 10959 10960 return pkt_size; 10961 } 10962 10963 static u32 gaudi2_add_arm_monitor_pkt(struct hl_device *hdev, struct packet_msg_short *pkt, 10964 u16 sob_base, u8 sob_mask, u16 sob_val, u16 addr) 10965 { 10966 u32 ctl, value, pkt_size = sizeof(*pkt); 10967 u8 mask; 10968 10969 if (hl_gen_sob_mask(sob_base, sob_mask, &mask)) { 10970 dev_err(hdev->dev, "sob_base %u (mask %#x) is not valid\n", sob_base, sob_mask); 10971 return 0; 10972 } 10973 10974 memset(pkt, 0, pkt_size); 10975 10976 value = FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_SYNC_GID_MASK, sob_base / 8); 10977 value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_SYNC_VAL_MASK, sob_val); 10978 value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_MODE_MASK, 0); /* GREATER OR EQUAL*/ 10979 value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_MON_MASK_MASK, mask); 10980 10981 ctl = FIELD_PREP(GAUDI2_PKT_SHORT_CTL_ADDR_MASK, addr); 10982 ctl |= FIELD_PREP(GAUDI2_PKT_SHORT_CTL_BASE_MASK, 0); /* MON base */ 10983 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); 10984 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 0); 10985 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1); 10986 10987 pkt->value = cpu_to_le32(value); 10988 pkt->ctl = cpu_to_le32(ctl); 10989 10990 return pkt_size; 10991 } 10992 10993 static u32 gaudi2_add_fence_pkt(struct packet_fence *pkt) 10994 { 10995 u32 ctl, cfg, pkt_size = sizeof(*pkt); 10996 10997 memset(pkt, 0, pkt_size); 10998 10999 cfg = FIELD_PREP(GAUDI2_PKT_FENCE_CFG_DEC_VAL_MASK, 1); 11000 cfg |= FIELD_PREP(GAUDI2_PKT_FENCE_CFG_TARGET_VAL_MASK, 1); 11001 cfg |= FIELD_PREP(GAUDI2_PKT_FENCE_CFG_ID_MASK, 2); 11002 11003 ctl = FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_FENCE); 11004 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 0); 11005 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1); 11006 11007 pkt->cfg = cpu_to_le32(cfg); 11008 pkt->ctl = cpu_to_le32(ctl); 11009 11010 return pkt_size; 11011 } 11012 11013 static u32 gaudi2_gen_wait_cb(struct hl_device *hdev, struct hl_gen_wait_properties *prop) 11014 { 11015 struct hl_cb *cb = prop->data; 11016 void *buf = (void *) (uintptr_t) (cb->kernel_address); 11017 11018 u64 monitor_base, fence_addr = 0; 11019 u32 stream_index, size = prop->size; 11020 u16 msg_addr_offset; 11021 11022 stream_index = prop->q_idx % 4; 11023 fence_addr = CFG_BASE + gaudi2_qm_blocks_bases[prop->q_idx] + 11024 QM_FENCE2_OFFSET + stream_index * 4; 11025 11026 /* 11027 * monitor_base should be the content of the base0 address registers, 11028 * so it will be added to the msg short offsets 11029 */ 11030 monitor_base = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0; 11031 11032 /* First monitor config packet: low address of the sync */ 11033 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + prop->mon_id * 4) - 11034 monitor_base; 11035 11036 size += gaudi2_add_mon_msg_short(buf + size, (u32) fence_addr, msg_addr_offset); 11037 11038 /* Second monitor config packet: high address of the sync */ 11039 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + prop->mon_id * 4) - 11040 monitor_base; 11041 11042 size += gaudi2_add_mon_msg_short(buf + size, (u32) (fence_addr >> 32), msg_addr_offset); 11043 11044 /* 11045 * Third monitor config packet: the payload, i.e. what to write when the 11046 * sync triggers 11047 */ 11048 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + prop->mon_id * 4) - 11049 monitor_base; 11050 11051 size += gaudi2_add_mon_msg_short(buf + size, 1, msg_addr_offset); 11052 11053 /* Fourth monitor config packet: bind the monitor to a sync object */ 11054 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + prop->mon_id * 4) - monitor_base; 11055 11056 size += gaudi2_add_arm_monitor_pkt(hdev, buf + size, prop->sob_base, prop->sob_mask, 11057 prop->sob_val, msg_addr_offset); 11058 11059 /* Fence packet */ 11060 size += gaudi2_add_fence_pkt(buf + size); 11061 11062 return size; 11063 } 11064 11065 static void gaudi2_reset_sob(struct hl_device *hdev, void *data) 11066 { 11067 struct hl_hw_sob *hw_sob = data; 11068 11069 dev_dbg(hdev->dev, "reset SOB, q_idx: %d, sob_id: %d\n", hw_sob->q_idx, hw_sob->sob_id); 11070 11071 WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + hw_sob->sob_id * 4, 0); 11072 11073 kref_init(&hw_sob->kref); 11074 } 11075 11076 static void gaudi2_reset_sob_group(struct hl_device *hdev, u16 sob_group) 11077 { 11078 } 11079 11080 static u64 gaudi2_get_device_time(struct hl_device *hdev) 11081 { 11082 u64 device_time = ((u64) RREG32(mmPSOC_TIMESTAMP_CNTCVU)) << 32; 11083 11084 return device_time | RREG32(mmPSOC_TIMESTAMP_CNTCVL); 11085 } 11086 11087 static int gaudi2_collective_wait_init_cs(struct hl_cs *cs) 11088 { 11089 return 0; 11090 } 11091 11092 static int gaudi2_collective_wait_create_jobs(struct hl_device *hdev, struct hl_ctx *ctx, 11093 struct hl_cs *cs, u32 wait_queue_id, 11094 u32 collective_engine_id, u32 encaps_signal_offset) 11095 { 11096 return -EINVAL; 11097 } 11098 11099 /* 11100 * hl_mmu_scramble - converts a dram (non power of 2) page-size aligned address 11101 * to DMMU page-size address (64MB) before mapping it in 11102 * the MMU. 11103 * The operation is performed on both the virtual and physical addresses. 11104 * for device with 6 HBMs the scramble is: 11105 * (addr[47:0] / 48M) * 64M + addr % 48M + addr[63:48] 11106 * 11107 * Example: 11108 * ============================================================================= 11109 * Allocated DRAM Reserved VA scrambled VA for MMU mapping Scrambled PA 11110 * Phys address in MMU last 11111 * HOP 11112 * ============================================================================= 11113 * PA1 0x3000000 VA1 0x9C000000 SVA1= (VA1/48M)*64M 0xD0000000 <- PA1/48M 0x1 11114 * PA2 0x9000000 VA2 0x9F000000 SVA2= (VA2/48M)*64M 0xD4000000 <- PA2/48M 0x3 11115 * ============================================================================= 11116 */ 11117 static u64 gaudi2_mmu_scramble_addr(struct hl_device *hdev, u64 raw_addr) 11118 { 11119 struct asic_fixed_properties *prop = &hdev->asic_prop; 11120 u32 divisor, mod_va; 11121 u64 div_va; 11122 11123 /* accept any address in the DRAM address space */ 11124 if (hl_mem_area_inside_range(raw_addr, sizeof(raw_addr), DRAM_PHYS_BASE, 11125 VA_HBM_SPACE_END)) { 11126 11127 divisor = prop->num_functional_hbms * GAUDI2_HBM_MMU_SCRM_MEM_SIZE; 11128 div_va = div_u64_rem(raw_addr & GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK, divisor, &mod_va); 11129 return (raw_addr & ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK) | 11130 (div_va << GAUDI2_HBM_MMU_SCRM_DIV_SHIFT) | 11131 (mod_va << GAUDI2_HBM_MMU_SCRM_MOD_SHIFT); 11132 } 11133 11134 return raw_addr; 11135 } 11136 11137 static u64 gaudi2_mmu_descramble_addr(struct hl_device *hdev, u64 scrambled_addr) 11138 { 11139 struct asic_fixed_properties *prop = &hdev->asic_prop; 11140 u32 divisor, mod_va; 11141 u64 div_va; 11142 11143 /* accept any address in the DRAM address space */ 11144 if (hl_mem_area_inside_range(scrambled_addr, sizeof(scrambled_addr), DRAM_PHYS_BASE, 11145 VA_HBM_SPACE_END)) { 11146 11147 divisor = prop->num_functional_hbms * GAUDI2_HBM_MMU_SCRM_MEM_SIZE; 11148 div_va = div_u64_rem(scrambled_addr & GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK, 11149 PAGE_SIZE_64MB, &mod_va); 11150 11151 return ((scrambled_addr & ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK) + 11152 (div_va * divisor + mod_va)); 11153 } 11154 11155 return scrambled_addr; 11156 } 11157 11158 static u32 gaudi2_get_dec_base_addr(struct hl_device *hdev, u32 core_id) 11159 { 11160 u32 base = 0, dcore_id, dec_id; 11161 11162 if (core_id >= NUMBER_OF_DEC) { 11163 dev_err(hdev->dev, "Unexpected core number %d for DEC\n", core_id); 11164 goto out; 11165 } 11166 11167 if (core_id < 8) { 11168 dcore_id = core_id / NUM_OF_DEC_PER_DCORE; 11169 dec_id = core_id % NUM_OF_DEC_PER_DCORE; 11170 11171 base = mmDCORE0_DEC0_CMD_BASE + dcore_id * DCORE_OFFSET + 11172 dec_id * DCORE_VDEC_OFFSET; 11173 } else { 11174 /* PCIe Shared Decoder */ 11175 base = mmPCIE_DEC0_CMD_BASE + ((core_id % 8) * PCIE_VDEC_OFFSET); 11176 } 11177 out: 11178 return base; 11179 } 11180 11181 static int gaudi2_get_hw_block_id(struct hl_device *hdev, u64 block_addr, 11182 u32 *block_size, u32 *block_id) 11183 { 11184 struct gaudi2_device *gaudi2 = hdev->asic_specific; 11185 int i; 11186 11187 for (i = 0 ; i < NUM_USER_MAPPED_BLOCKS ; i++) { 11188 if (block_addr == CFG_BASE + gaudi2->mapped_blocks[i].address) { 11189 *block_id = i; 11190 if (block_size) 11191 *block_size = gaudi2->mapped_blocks[i].size; 11192 return 0; 11193 } 11194 } 11195 11196 dev_err(hdev->dev, "Invalid block address %#llx", block_addr); 11197 11198 return -EINVAL; 11199 } 11200 11201 static int gaudi2_block_mmap(struct hl_device *hdev, struct vm_area_struct *vma, 11202 u32 block_id, u32 block_size) 11203 { 11204 struct gaudi2_device *gaudi2 = hdev->asic_specific; 11205 u64 offset_in_bar; 11206 u64 address; 11207 int rc; 11208 11209 if (block_id >= NUM_USER_MAPPED_BLOCKS) { 11210 dev_err(hdev->dev, "Invalid block id %u", block_id); 11211 return -EINVAL; 11212 } 11213 11214 /* we allow mapping only an entire block */ 11215 if (block_size != gaudi2->mapped_blocks[block_id].size) { 11216 dev_err(hdev->dev, "Invalid block size %u", block_size); 11217 return -EINVAL; 11218 } 11219 11220 offset_in_bar = CFG_BASE + gaudi2->mapped_blocks[block_id].address - STM_FLASH_BASE_ADDR; 11221 11222 address = pci_resource_start(hdev->pdev, SRAM_CFG_BAR_ID) + offset_in_bar; 11223 11224 vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | 11225 VM_DONTCOPY | VM_NORESERVE); 11226 11227 rc = remap_pfn_range(vma, vma->vm_start, address >> PAGE_SHIFT, 11228 block_size, vma->vm_page_prot); 11229 if (rc) 11230 dev_err(hdev->dev, "remap_pfn_range error %d", rc); 11231 11232 return rc; 11233 } 11234 11235 static void gaudi2_enable_events_from_fw(struct hl_device *hdev) 11236 { 11237 struct gaudi2_device *gaudi2 = hdev->asic_specific; 11238 11239 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; 11240 u32 irq_handler_offset = le32_to_cpu(dyn_regs->gic_host_ints_irq); 11241 11242 if (gaudi2->hw_cap_initialized & HW_CAP_CPU_Q) 11243 WREG32(irq_handler_offset, 11244 gaudi2_irq_map_table[GAUDI2_EVENT_CPU_INTS_REGISTER].cpu_id); 11245 } 11246 11247 static int gaudi2_get_mmu_base(struct hl_device *hdev, u64 mmu_id, u32 *mmu_base) 11248 { 11249 switch (mmu_id) { 11250 case HW_CAP_DCORE0_DMMU0: 11251 *mmu_base = mmDCORE0_HMMU0_MMU_BASE; 11252 break; 11253 case HW_CAP_DCORE0_DMMU1: 11254 *mmu_base = mmDCORE0_HMMU1_MMU_BASE; 11255 break; 11256 case HW_CAP_DCORE0_DMMU2: 11257 *mmu_base = mmDCORE0_HMMU2_MMU_BASE; 11258 break; 11259 case HW_CAP_DCORE0_DMMU3: 11260 *mmu_base = mmDCORE0_HMMU3_MMU_BASE; 11261 break; 11262 case HW_CAP_DCORE1_DMMU0: 11263 *mmu_base = mmDCORE1_HMMU0_MMU_BASE; 11264 break; 11265 case HW_CAP_DCORE1_DMMU1: 11266 *mmu_base = mmDCORE1_HMMU1_MMU_BASE; 11267 break; 11268 case HW_CAP_DCORE1_DMMU2: 11269 *mmu_base = mmDCORE1_HMMU2_MMU_BASE; 11270 break; 11271 case HW_CAP_DCORE1_DMMU3: 11272 *mmu_base = mmDCORE1_HMMU3_MMU_BASE; 11273 break; 11274 case HW_CAP_DCORE2_DMMU0: 11275 *mmu_base = mmDCORE2_HMMU0_MMU_BASE; 11276 break; 11277 case HW_CAP_DCORE2_DMMU1: 11278 *mmu_base = mmDCORE2_HMMU1_MMU_BASE; 11279 break; 11280 case HW_CAP_DCORE2_DMMU2: 11281 *mmu_base = mmDCORE2_HMMU2_MMU_BASE; 11282 break; 11283 case HW_CAP_DCORE2_DMMU3: 11284 *mmu_base = mmDCORE2_HMMU3_MMU_BASE; 11285 break; 11286 case HW_CAP_DCORE3_DMMU0: 11287 *mmu_base = mmDCORE3_HMMU0_MMU_BASE; 11288 break; 11289 case HW_CAP_DCORE3_DMMU1: 11290 *mmu_base = mmDCORE3_HMMU1_MMU_BASE; 11291 break; 11292 case HW_CAP_DCORE3_DMMU2: 11293 *mmu_base = mmDCORE3_HMMU2_MMU_BASE; 11294 break; 11295 case HW_CAP_DCORE3_DMMU3: 11296 *mmu_base = mmDCORE3_HMMU3_MMU_BASE; 11297 break; 11298 case HW_CAP_PMMU: 11299 *mmu_base = mmPMMU_HBW_MMU_BASE; 11300 break; 11301 default: 11302 return -EINVAL; 11303 } 11304 11305 return 0; 11306 } 11307 11308 static void gaudi2_ack_mmu_error(struct hl_device *hdev, u64 mmu_id) 11309 { 11310 bool is_pmmu = (mmu_id == HW_CAP_PMMU); 11311 struct gaudi2_device *gaudi2 = hdev->asic_specific; 11312 u32 mmu_base; 11313 11314 if (!(gaudi2->hw_cap_initialized & mmu_id)) 11315 return; 11316 11317 if (gaudi2_get_mmu_base(hdev, mmu_id, &mmu_base)) 11318 return; 11319 11320 gaudi2_handle_page_error(hdev, mmu_base, is_pmmu, NULL); 11321 gaudi2_handle_access_error(hdev, mmu_base, is_pmmu); 11322 } 11323 11324 static int gaudi2_ack_mmu_page_fault_or_access_error(struct hl_device *hdev, u64 mmu_cap_mask) 11325 { 11326 u32 i, mmu_id, num_of_hmmus = NUM_OF_HMMU_PER_DCORE * NUM_OF_DCORES; 11327 11328 /* check all HMMUs */ 11329 for (i = 0 ; i < num_of_hmmus ; i++) { 11330 mmu_id = HW_CAP_DCORE0_DMMU0 << i; 11331 11332 if (mmu_cap_mask & mmu_id) 11333 gaudi2_ack_mmu_error(hdev, mmu_id); 11334 } 11335 11336 /* check PMMU */ 11337 if (mmu_cap_mask & HW_CAP_PMMU) 11338 gaudi2_ack_mmu_error(hdev, HW_CAP_PMMU); 11339 11340 return 0; 11341 } 11342 11343 static void gaudi2_get_msi_info(__le32 *table) 11344 { 11345 table[CPUCP_EVENT_QUEUE_MSI_TYPE] = cpu_to_le32(GAUDI2_EVENT_QUEUE_MSIX_IDX); 11346 table[CPUCP_EVENT_QUEUE_ERR_MSI_TYPE] = cpu_to_le32(GAUDI2_IRQ_NUM_EQ_ERROR); 11347 } 11348 11349 static int gaudi2_map_pll_idx_to_fw_idx(u32 pll_idx) 11350 { 11351 switch (pll_idx) { 11352 case HL_GAUDI2_CPU_PLL: return CPU_PLL; 11353 case HL_GAUDI2_PCI_PLL: return PCI_PLL; 11354 case HL_GAUDI2_NIC_PLL: return NIC_PLL; 11355 case HL_GAUDI2_DMA_PLL: return DMA_PLL; 11356 case HL_GAUDI2_MESH_PLL: return MESH_PLL; 11357 case HL_GAUDI2_MME_PLL: return MME_PLL; 11358 case HL_GAUDI2_TPC_PLL: return TPC_PLL; 11359 case HL_GAUDI2_IF_PLL: return IF_PLL; 11360 case HL_GAUDI2_SRAM_PLL: return SRAM_PLL; 11361 case HL_GAUDI2_HBM_PLL: return HBM_PLL; 11362 case HL_GAUDI2_VID_PLL: return VID_PLL; 11363 case HL_GAUDI2_MSS_PLL: return MSS_PLL; 11364 default: return -EINVAL; 11365 } 11366 } 11367 11368 static int gaudi2_gen_sync_to_engine_map(struct hl_device *hdev, struct hl_sync_to_engine_map *map) 11369 { 11370 /* Not implemented */ 11371 return 0; 11372 } 11373 11374 static int gaudi2_monitor_valid(struct hl_mon_state_dump *mon) 11375 { 11376 /* Not implemented */ 11377 return 0; 11378 } 11379 11380 static int gaudi2_print_single_monitor(char **buf, size_t *size, size_t *offset, 11381 struct hl_device *hdev, struct hl_mon_state_dump *mon) 11382 { 11383 /* Not implemented */ 11384 return 0; 11385 } 11386 11387 11388 static int gaudi2_print_fences_single_engine(struct hl_device *hdev, u64 base_offset, 11389 u64 status_base_offset, enum hl_sync_engine_type engine_type, 11390 u32 engine_id, char **buf, size_t *size, size_t *offset) 11391 { 11392 /* Not implemented */ 11393 return 0; 11394 } 11395 11396 11397 static struct hl_state_dump_specs_funcs gaudi2_state_dump_funcs = { 11398 .monitor_valid = gaudi2_monitor_valid, 11399 .print_single_monitor = gaudi2_print_single_monitor, 11400 .gen_sync_to_engine_map = gaudi2_gen_sync_to_engine_map, 11401 .print_fences_single_engine = gaudi2_print_fences_single_engine, 11402 }; 11403 11404 static void gaudi2_state_dump_init(struct hl_device *hdev) 11405 { 11406 /* Not implemented */ 11407 hdev->state_dump_specs.props = gaudi2_state_dump_specs_props; 11408 hdev->state_dump_specs.funcs = gaudi2_state_dump_funcs; 11409 } 11410 11411 static u32 gaudi2_get_sob_addr(struct hl_device *hdev, u32 sob_id) 11412 { 11413 return 0; 11414 } 11415 11416 static u32 *gaudi2_get_stream_master_qid_arr(void) 11417 { 11418 return NULL; 11419 } 11420 11421 static void gaudi2_add_device_attr(struct hl_device *hdev, struct attribute_group *dev_clk_attr_grp, 11422 struct attribute_group *dev_vrm_attr_grp) 11423 { 11424 hl_sysfs_add_dev_clk_attr(hdev, dev_clk_attr_grp); 11425 hl_sysfs_add_dev_vrm_attr(hdev, dev_vrm_attr_grp); 11426 } 11427 11428 static int gaudi2_mmu_get_real_page_size(struct hl_device *hdev, struct hl_mmu_properties *mmu_prop, 11429 u32 page_size, u32 *real_page_size, bool is_dram_addr) 11430 { 11431 struct asic_fixed_properties *prop = &hdev->asic_prop; 11432 11433 /* for host pages the page size must be */ 11434 if (!is_dram_addr) { 11435 if (page_size % mmu_prop->page_size) 11436 goto page_size_err; 11437 11438 *real_page_size = mmu_prop->page_size; 11439 return 0; 11440 } 11441 11442 if ((page_size % prop->dram_page_size) || (prop->dram_page_size > mmu_prop->page_size)) 11443 goto page_size_err; 11444 11445 /* 11446 * MMU page size is different from DRAM page size (more precisely, DMMU page is greater 11447 * than DRAM page size). 11448 * for this reason work with the DRAM page size and let the MMU scrambling routine handle 11449 * this mismatch when calculating the address to place in the MMU page table. 11450 * (in that case also make sure that the dram_page_size is not greater than the 11451 * mmu page size) 11452 */ 11453 *real_page_size = prop->dram_page_size; 11454 11455 return 0; 11456 11457 page_size_err: 11458 dev_err(hdev->dev, "page size of %u is not %uKB aligned, can't map\n", 11459 page_size, mmu_prop->page_size >> 10); 11460 return -EFAULT; 11461 } 11462 11463 static int gaudi2_get_monitor_dump(struct hl_device *hdev, void *data) 11464 { 11465 return -EOPNOTSUPP; 11466 } 11467 11468 int gaudi2_send_device_activity(struct hl_device *hdev, bool open) 11469 { 11470 struct gaudi2_device *gaudi2 = hdev->asic_specific; 11471 11472 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) 11473 return 0; 11474 11475 return hl_fw_send_device_activity(hdev, open); 11476 } 11477 11478 static const struct hl_asic_funcs gaudi2_funcs = { 11479 .early_init = gaudi2_early_init, 11480 .early_fini = gaudi2_early_fini, 11481 .late_init = gaudi2_late_init, 11482 .late_fini = gaudi2_late_fini, 11483 .sw_init = gaudi2_sw_init, 11484 .sw_fini = gaudi2_sw_fini, 11485 .hw_init = gaudi2_hw_init, 11486 .hw_fini = gaudi2_hw_fini, 11487 .halt_engines = gaudi2_halt_engines, 11488 .suspend = gaudi2_suspend, 11489 .resume = gaudi2_resume, 11490 .mmap = gaudi2_mmap, 11491 .ring_doorbell = gaudi2_ring_doorbell, 11492 .pqe_write = gaudi2_pqe_write, 11493 .asic_dma_alloc_coherent = gaudi2_dma_alloc_coherent, 11494 .asic_dma_free_coherent = gaudi2_dma_free_coherent, 11495 .scrub_device_mem = gaudi2_scrub_device_mem, 11496 .scrub_device_dram = gaudi2_scrub_device_dram, 11497 .get_int_queue_base = NULL, 11498 .test_queues = gaudi2_test_queues, 11499 .asic_dma_pool_zalloc = gaudi2_dma_pool_zalloc, 11500 .asic_dma_pool_free = gaudi2_dma_pool_free, 11501 .cpu_accessible_dma_pool_alloc = gaudi2_cpu_accessible_dma_pool_alloc, 11502 .cpu_accessible_dma_pool_free = gaudi2_cpu_accessible_dma_pool_free, 11503 .dma_unmap_sgtable = hl_asic_dma_unmap_sgtable, 11504 .cs_parser = gaudi2_cs_parser, 11505 .dma_map_sgtable = hl_asic_dma_map_sgtable, 11506 .add_end_of_cb_packets = NULL, 11507 .update_eq_ci = gaudi2_update_eq_ci, 11508 .context_switch = gaudi2_context_switch, 11509 .restore_phase_topology = gaudi2_restore_phase_topology, 11510 .debugfs_read_dma = gaudi2_debugfs_read_dma, 11511 .add_device_attr = gaudi2_add_device_attr, 11512 .handle_eqe = gaudi2_handle_eqe, 11513 .get_events_stat = gaudi2_get_events_stat, 11514 .read_pte = NULL, 11515 .write_pte = NULL, 11516 .mmu_invalidate_cache = gaudi2_mmu_invalidate_cache, 11517 .mmu_invalidate_cache_range = gaudi2_mmu_invalidate_cache_range, 11518 .mmu_prefetch_cache_range = NULL, 11519 .send_heartbeat = gaudi2_send_heartbeat, 11520 .debug_coresight = gaudi2_debug_coresight, 11521 .is_device_idle = gaudi2_is_device_idle, 11522 .compute_reset_late_init = gaudi2_compute_reset_late_init, 11523 .hw_queues_lock = gaudi2_hw_queues_lock, 11524 .hw_queues_unlock = gaudi2_hw_queues_unlock, 11525 .get_pci_id = gaudi2_get_pci_id, 11526 .get_eeprom_data = gaudi2_get_eeprom_data, 11527 .get_monitor_dump = gaudi2_get_monitor_dump, 11528 .send_cpu_message = gaudi2_send_cpu_message, 11529 .pci_bars_map = gaudi2_pci_bars_map, 11530 .init_iatu = gaudi2_init_iatu, 11531 .rreg = hl_rreg, 11532 .wreg = hl_wreg, 11533 .halt_coresight = gaudi2_halt_coresight, 11534 .ctx_init = gaudi2_ctx_init, 11535 .ctx_fini = gaudi2_ctx_fini, 11536 .pre_schedule_cs = gaudi2_pre_schedule_cs, 11537 .get_queue_id_for_cq = gaudi2_get_queue_id_for_cq, 11538 .load_firmware_to_device = NULL, 11539 .load_boot_fit_to_device = NULL, 11540 .get_signal_cb_size = gaudi2_get_signal_cb_size, 11541 .get_wait_cb_size = gaudi2_get_wait_cb_size, 11542 .gen_signal_cb = gaudi2_gen_signal_cb, 11543 .gen_wait_cb = gaudi2_gen_wait_cb, 11544 .reset_sob = gaudi2_reset_sob, 11545 .reset_sob_group = gaudi2_reset_sob_group, 11546 .get_device_time = gaudi2_get_device_time, 11547 .pb_print_security_errors = gaudi2_pb_print_security_errors, 11548 .collective_wait_init_cs = gaudi2_collective_wait_init_cs, 11549 .collective_wait_create_jobs = gaudi2_collective_wait_create_jobs, 11550 .get_dec_base_addr = gaudi2_get_dec_base_addr, 11551 .scramble_addr = gaudi2_mmu_scramble_addr, 11552 .descramble_addr = gaudi2_mmu_descramble_addr, 11553 .ack_protection_bits_errors = gaudi2_ack_protection_bits_errors, 11554 .get_hw_block_id = gaudi2_get_hw_block_id, 11555 .hw_block_mmap = gaudi2_block_mmap, 11556 .enable_events_from_fw = gaudi2_enable_events_from_fw, 11557 .ack_mmu_errors = gaudi2_ack_mmu_page_fault_or_access_error, 11558 .get_msi_info = gaudi2_get_msi_info, 11559 .map_pll_idx_to_fw_idx = gaudi2_map_pll_idx_to_fw_idx, 11560 .init_firmware_preload_params = gaudi2_init_firmware_preload_params, 11561 .init_firmware_loader = gaudi2_init_firmware_loader, 11562 .init_cpu_scrambler_dram = gaudi2_init_scrambler_hbm, 11563 .state_dump_init = gaudi2_state_dump_init, 11564 .get_sob_addr = &gaudi2_get_sob_addr, 11565 .set_pci_memory_regions = gaudi2_set_pci_memory_regions, 11566 .get_stream_master_qid_arr = gaudi2_get_stream_master_qid_arr, 11567 .check_if_razwi_happened = gaudi2_check_if_razwi_happened, 11568 .mmu_get_real_page_size = gaudi2_mmu_get_real_page_size, 11569 .access_dev_mem = hl_access_dev_mem, 11570 .set_dram_bar_base = gaudi2_set_hbm_bar_base, 11571 .set_engine_cores = gaudi2_set_engine_cores, 11572 .set_engines = gaudi2_set_engines, 11573 .send_device_activity = gaudi2_send_device_activity, 11574 .set_dram_properties = gaudi2_set_dram_properties, 11575 .set_binning_masks = gaudi2_set_binning_masks, 11576 }; 11577 11578 void gaudi2_set_asic_funcs(struct hl_device *hdev) 11579 { 11580 hdev->asic_funcs = &gaudi2_funcs; 11581 } 11582