1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 // FIXME XXX 25 #define NVOC_KERNEL_GRAPHICS_MANAGER_H_PRIVATE_ACCESS_ALLOWED 26 27 #include "kernel/gpu/fifo/kernel_fifo.h" 28 #include "kernel/gpu/fifo/kernel_channel_group_api.h" 29 #include "kernel/gpu/fifo/kernel_channel_group.h" 30 #include "kernel/gpu/ce/kernel_ce_shared.h" 31 #include "kernel/gpu/mig_mgr/kernel_mig_manager.h" 32 #include "kernel/gpu/bus/kern_bus.h" 33 34 #include "vgpu/vgpu_events.h" 35 36 #include "published/ampere/ga100/dev_ram.h" 37 #include "published/ampere/ga100/dev_ctrl.h" 38 39 40 NV_STATUS 41 kfifoEngineInfoXlate_GA100 42 ( 43 OBJGPU *pGpu, 44 KernelFifo *pKernelFifo, 45 ENGINE_INFO_TYPE inType, 46 NvU32 inVal, 47 ENGINE_INFO_TYPE outType, 48 NvU32 *pOutVal 49 ) 50 { 51 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 52 53 // We no longer store ENGINE_INFO_TYPE_INTR on Ampere+ (bug 24110055) 54 if (inType == ENGINE_INFO_TYPE_INTR || outType == ENGINE_INFO_TYPE_INTR) 55 { 56 return NV_ERR_NOT_SUPPORTED; 57 } 58 59 // 60 // We need extra logic for translation when SMC is enabled and input or output is 61 // MMU_FAULT_ID because device Info cannot translate MMU_FAULT_ID to/from any type for GR > 0 62 // 63 if (IS_MIG_IN_USE(pGpu)) 64 { 65 NvU32 baseGrFaultId; 66 NV_ASSERT_OK_OR_RETURN(kfifoEngineInfoXlate_GV100(pGpu, pKernelFifo, 67 ENGINE_INFO_TYPE_ENG_DESC, ENG_GR(0), 68 ENGINE_INFO_TYPE_MMU_FAULT_ID, &baseGrFaultId)); 69 70 if (inType == ENGINE_INFO_TYPE_MMU_FAULT_ID) 71 { 72 NvU32 subctxId, grIdx; 73 NvU32 maxSubctx = kfifoGetMaxSubcontext_HAL(pGpu, pKernelFifo, NV_FALSE); 74 75 // check if input fault ID corresponds to GR 76 if ((inVal >= baseGrFaultId) && (inVal < (baseGrFaultId + maxSubctx))) 77 { 78 subctxId = inVal - baseGrFaultId; 79 NV_ASSERT_OK_OR_RETURN(kgrmgrGetGrIdxForVeid(pGpu, pKernelGraphicsManager, subctxId, &grIdx)); 80 inVal = RM_ENGINE_TYPE_GR(grIdx); 81 inType = ENGINE_INFO_TYPE_RM_ENGINE_TYPE; 82 } 83 } 84 85 if (outType == ENGINE_INFO_TYPE_MMU_FAULT_ID) 86 { 87 NvU32 grIdx, startSubctxId; 88 NV_STATUS status; 89 RM_ENGINE_TYPE rmEngineType; 90 91 status = kfifoEngineInfoXlate_GV100(pGpu, pKernelFifo, inType, inVal, 92 ENGINE_INFO_TYPE_RM_ENGINE_TYPE, (NvU32 *)&rmEngineType); 93 if (status != NV_OK) 94 return status; 95 96 // check if rmEngineType corresponding to input is GR 97 if (RM_ENGINE_TYPE_IS_GR(rmEngineType)) 98 { 99 grIdx = RM_ENGINE_TYPE_GR_IDX(rmEngineType); 100 NV_ASSERT_OK_OR_RETURN(kgrmgrGetVeidBaseForGrIdx(pGpu, pKernelGraphicsManager, grIdx, &startSubctxId)); 101 *pOutVal = baseGrFaultId + startSubctxId; 102 return NV_OK; 103 } 104 } 105 } 106 107 return kfifoEngineInfoXlate_GV100(pGpu, pKernelFifo, inType, inVal, outType, pOutVal); 108 } 109 110 111 /*! 112 * @brief Get the local maximum number of subctx allowed in this TSG 113 * 114 * @param pGpu 115 * @param pKernelFifo 116 * @param[in] pKernelChannelGroup 117 * @param[in] bLegacyMode Is TSG in legacy mode. 118 */ 119 NvU32 120 kfifoChannelGroupGetLocalMaxSubcontext_GA100 121 ( 122 OBJGPU *pGpu, 123 KernelFifo *pKernelFifo, 124 KernelChannelGroup *pKernelChannelGroup, 125 NvBool bLegacyMode 126 ) 127 { 128 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 129 130 NV_ASSERT_OR_RETURN(pKernelChannelGroup != NULL, NV_ERR_INVALID_ARGUMENT); 131 132 if (IS_MIG_IN_USE(pGpu) && !bLegacyMode && 133 RM_ENGINE_TYPE_IS_GR(pKernelChannelGroup->engineType)) 134 { 135 NvU32 grIdx = RM_ENGINE_TYPE_GR_IDX(pKernelChannelGroup->engineType); 136 return nvPopCount64(pKernelGraphicsManager->grIdxVeidMask[grIdx]); 137 } 138 139 // In SMC-Legacy mode, revert to pre-Ampere behavior 140 return kfifoChannelGroupGetLocalMaxSubcontext_GM107(pGpu, pKernelFifo, 141 pKernelChannelGroup, 142 bLegacyMode); 143 } 144 145 /*! 146 * @brief Update the usermode doorbell register with work submit token to notify 147 * host that work is available on this channel. 148 * 149 * @param[in] pGpu 150 * @param[in] pKernelFifo 151 * @param[in] workSubmitToken Token to update the doorbell with 152 * @param[in] runlistId Runlist ID 153 */ 154 NV_STATUS 155 kfifoUpdateUsermodeDoorbell_GA100 156 ( 157 OBJGPU *pGpu, 158 KernelFifo *pKernelFifo, 159 NvU32 workSubmitToken, 160 NvU32 runlistId 161 ) 162 { 163 // 164 // Updating the usermode doorbell is different for CPU vs. GSP. 165 // 166 if (!RMCFG_FEATURE_PLATFORM_GSP) 167 { 168 return kfifoUpdateUsermodeDoorbell_TU102(pGpu, pKernelFifo, workSubmitToken, runlistId); 169 } 170 else 171 { 172 return kfifoUpdateInternalDoorbellForUsermode_HAL(pGpu, pKernelFifo, workSubmitToken, runlistId); 173 } 174 175 return NV_OK; 176 } 177 178 /*! 179 * @brief Construct the worksubmit token. Caller cannot make assumption about this handle. 180 * 181 * @param[in] pGpu 182 * @param[in] pKernelFifo 183 * @param[in] pKernelChannel 184 * @param[out] pGeneratedToken Store the generated token 185 * @param[in] bUsedForHost Used on Host RM 186 * 187 */ 188 NV_STATUS 189 kfifoGenerateWorkSubmitToken_GA100 190 ( 191 OBJGPU *pGpu, 192 KernelFifo *pKernelFifo, 193 KernelChannel *pKernelChannel, 194 NvU32 *pGeneratedToken, 195 NvBool bUsedForHost 196 ) 197 { 198 NvU32 chId; 199 NvU32 gfId; 200 NvU32 runlistId; 201 NvU32 val = 0; 202 203 NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_CHANNEL); 204 205 NV_ASSERT_OR_RETURN(pGeneratedToken != NULL, NV_ERR_INVALID_OBJECT); 206 NV_ASSERT_OR_RETURN((pKernelChannel->pKernelChannelGroupApi != NULL) && 207 (pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup != NULL), 208 NV_ERR_INVALID_STATE); 209 210 chId = pKernelChannel->ChID; 211 212 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &gfId)); 213 214 if (!RMCFG_FEATURE_PLATFORM_GSP || (IS_VGPU_GSP_PLUGIN_OFFLOAD_ENABLED(pGpu) && IS_GFID_VF(gfId))) 215 { 216 217 // TODO: Remove check on Ampere. Bug 200606706. 218 if (!bUsedForHost && IS_GFID_VF(gfId)) 219 { 220 NvU32 vChId; 221 222 NV_ASSERT_OK_OR_RETURN(kfifoGetVChIdForSChId_HAL(pGpu, pKernelFifo, 223 chId, gfId, 224 kchannelGetEngineType(pKernelChannel), 225 &vChId)); 226 chId = vChId; 227 } 228 229 // TODO: Remove, on Ampere channels should be set to a valid runlist before allocation. Bug 200606706. 230 if (!kchannelIsRunlistSet(pGpu, pKernelChannel)) 231 { 232 NV_PRINTF(LEVEL_NOTICE, 233 "FAILED Channel 0x%x is not assigned to runlist yet\n", 234 chId); 235 return NV_ERR_INVALID_STATE; 236 } 237 238 runlistId = kchannelGetRunlistId(pKernelChannel); 239 240 // Here we construct token to be a concatenation of runlist id and channel id 241 val = FLD_SET_DRF_NUM(_CTRL, _VF_DOORBELL, _RUNLIST_ID, runlistId, val); 242 val = FLD_SET_DRF_NUM(_CTRL, _VF_DOORBELL, _VECTOR, chId, val); 243 244 NV_PRINTF(LEVEL_INFO, 245 "Generated workSubmitToken 0x%x for channel 0x%x runlist 0x%x\n", 246 val, chId, runlistId); 247 } 248 else // RMCFG_FEATURE_PLATFORM_GSP 249 { 250 NV_ASSERT_OK_OR_RETURN(kfifoGenerateInternalWorkSubmitToken_HAL(pGpu, pKernelFifo, pKernelChannel)); 251 } 252 253 *pGeneratedToken = val; 254 255 return NV_OK; 256 } 257 258 /** 259 * @brief Get the runlist base shift amount 260 * 261 * @param pKernelFifo 262 * 263 * @return shift amount 264 */ 265 NvU32 266 kfifoRunlistGetBaseShift_GA100 267 ( 268 KernelFifo *pKernelFifo 269 ) 270 { 271 return NV_RAMRL_ENTRY_BASE_SHIFT; 272 } 273 274 /*! 275 * Special function to be used early when the CHID_MGRs aren't and cannot be 276 * constructed in all cases. Do not use otherwise 277 */ 278 NvU32 279 kfifoGetMaxCeChannelGroups_GA100 280 ( 281 OBJGPU *pGpu, 282 KernelFifo *pKernelFifo 283 ) 284 { 285 ENGDESCRIPTOR eng = 0; 286 NvU32 deviceIndex; 287 const ENGINE_INFO *pEngineInfo = kfifoGetEngineInfo(pKernelFifo); 288 NvU32 maxCeChannels = 0; 289 290 // If called before kfifoConstructEngineList has executed 291 if (pEngineInfo == NULL) 292 return 0; 293 294 // 295 // We can't use chidMgr here as this gets called before chidMgr is completely initialized 296 // Use device info table instead 297 // 298 for (deviceIndex = 0; deviceIndex < pEngineInfo->engineInfoListSize; deviceIndex++) 299 { 300 eng = pEngineInfo->engineInfoList[deviceIndex].engineData[ENGINE_INFO_TYPE_ENG_DESC]; 301 302 // All GR CE use the same pool as GR 303 if ((eng == ENG_GR(0)) || 304 (IS_CE(eng) && 305 (!ceIsCeGrce(pGpu, pEngineInfo->engineInfoList[deviceIndex].engineData[ENGINE_INFO_TYPE_RM_ENGINE_TYPE])))) 306 { 307 maxCeChannels += kfifoRunlistQueryNumChannels_HAL(pGpu, pKernelFifo, 0); 308 } 309 } 310 311 // override max channels if we can run out of BAR2 page tables 312 if (kbusIsBug2751296LimitBar2PtSize(GPU_GET_KERNEL_BUS(pGpu))) 313 { 314 // 2k for GR CE and 2k for the rest 315 maxCeChannels = 4096; 316 } 317 318 return maxCeChannels; 319 } 320