1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 #include "gpu/falcon/kernel_falcon.h" 24 #include "gpu/sec2/kernel_sec2.h" 25 #include "gpu/gsp/kernel_gsp.h" 26 27 #include "gpu/fifo/kernel_fifo.h" 28 #include "gpu/fifo/kernel_channel.h" 29 #include "gpu/fifo/kernel_channel_group.h" 30 #include "gpu/fifo/kernel_channel_group_api.h" 31 #include "gpu/intr/intr.h" 32 #include "gpu/subdevice/subdevice.h" 33 #include "gpu/mem_mgr/mem_desc.h" 34 #include "mem_mgr/gpu_vaspace.h" 35 #include "mem_mgr/ctx_buf_pool.h" 36 #include "rmapi/rmapi.h" 37 38 39 void kflcnConfigureEngine_IMPL(OBJGPU *pGpu, KernelFalcon *pKernelFalcon, KernelFalconEngineConfig *pFalconConfig) 40 { 41 pKernelFalcon->registerBase = pFalconConfig->registerBase; 42 pKernelFalcon->riscvRegisterBase = pFalconConfig->riscvRegisterBase; 43 pKernelFalcon->fbifBase = pFalconConfig->fbifBase; 44 pKernelFalcon->bBootFromHs = pFalconConfig->bBootFromHs; 45 pKernelFalcon->pmcEnableMask = pFalconConfig->pmcEnableMask; 46 pKernelFalcon->bIsPmcDeviceEngine = pFalconConfig->bIsPmcDeviceEngine; 47 pKernelFalcon->physEngDesc = pFalconConfig->physEngDesc; 48 pKernelFalcon->ctxAttr = pFalconConfig->ctxAttr; 49 pKernelFalcon->ctxBufferSize = pFalconConfig->ctxBufferSize; 50 pKernelFalcon->addrSpaceList = pFalconConfig->addrSpaceList; 51 52 NV_PRINTF(LEVEL_INFO, "for physEngDesc 0x%x\n", pKernelFalcon->physEngDesc); 53 } 54 55 KernelFalcon *kflcnGetKernelFalconForEngine_IMPL(OBJGPU *pGpu, ENGDESCRIPTOR physEngDesc) 56 { 57 // 58 // Check for any special objects that are instantiated as GPU children. 59 // Otherwise, OBJGPU keeps track of all falcons as reported by GSP 60 // 61 switch (physEngDesc) 62 { 63 // this list is mirrored in subdeviceCtrlCmdInternalGetConstructedFalconInfo_IMPL 64 case ENG_SEC2: return staticCast(GPU_GET_KERNEL_SEC2(pGpu), KernelFalcon); 65 case ENG_GSP: return staticCast(GPU_GET_KERNEL_GSP(pGpu), KernelFalcon); 66 default: 67 return staticCast(gpuGetGenericKernelFalconForEngine(pGpu, physEngDesc), KernelFalcon); 68 } 69 } 70 71 72 static NvBool _kflcnNeedToAllocContext(OBJGPU *pGpu, KernelChannel *pKernelChannel) 73 { 74 NvU32 gfid = kchannelGetGfid(pKernelChannel); 75 76 // 77 // In case of vGPU, when client allocated ctx buffer feature enabled, vGPU guest 78 // RM will alloc all FLCN context buffers for VF channels. 79 // But, for PF channels (IS_GFID_PF(gfid) is TRUE), host RM needs to allocate the 80 // FLCN buffers. 81 // 82 if (!gpuIsClientRmAllocatedCtxBufferEnabled(pGpu) || IS_GFID_VF(gfid)) 83 return NV_FALSE; 84 85 return NV_TRUE; 86 } 87 88 static NV_STATUS _kflcnAllocAndMapCtxBuffer 89 ( 90 OBJGPU *pGpu, 91 KernelFalcon *pKernelFalcon, 92 KernelChannel *pKernelChannel 93 ) 94 { 95 MEMORY_DESCRIPTOR *pCtxMemDesc = NULL; 96 CTX_BUF_POOL_INFO *pCtxBufPool = NULL; 97 KernelChannelGroup *pKernelChannelGroup = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup; 98 OBJGVASPACE *pGVAS = dynamicCast(pKernelChannel->pVAS, OBJGVASPACE); 99 NvU8 *pInstMem; 100 NV_STATUS status = NV_OK; 101 NvU64 flags = MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE; 102 103 if (kchannelIsCtxBufferAllocSkipped(pKernelChannel)) 104 return NV_OK; 105 106 kchangrpGetEngineContextMemDesc(pGpu, pKernelChannelGroup, &pCtxMemDesc); 107 if (pCtxMemDesc != NULL) 108 { 109 NV_PRINTF(LEVEL_ERROR, "This channel already has a falcon engine instance on engine %d:%d\n", 110 ENGDESC_FIELD(pKernelFalcon->physEngDesc, _CLASS), 111 ENGDESC_FIELD(pKernelFalcon->physEngDesc, _INST)); 112 return NV_OK; 113 } 114 115 if (ctxBufPoolIsSupported(pGpu) && pKernelChannelGroup->pCtxBufPool != NULL) 116 { 117 flags |= MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL; 118 pCtxBufPool = pKernelChannelGroup->pCtxBufPool; 119 } 120 121 // 122 // Setup an engine context and initialize. 123 // 124 NV_ASSERT_OK_OR_RETURN(memdescCreate(&pCtxMemDesc, pGpu, 125 pKernelFalcon->ctxBufferSize, 126 FLCN_BLK_ALIGNMENT, 127 NV_TRUE, 128 ADDR_UNKNOWN, 129 pKernelFalcon->ctxAttr, 130 flags)); 131 NV_ASSERT_OK_OR_GOTO(status, 132 memdescSetCtxBufPool(pCtxMemDesc, pCtxBufPool), 133 done); 134 NV_ASSERT_OK_OR_GOTO(status, 135 memdescAllocList(pCtxMemDesc, memdescU32ToAddrSpaceList(pKernelFalcon->addrSpaceList)), 136 done); 137 138 pInstMem = memdescMapInternal(pGpu, pCtxMemDesc, 0); 139 if (pInstMem != NULL) 140 { 141 // Clear the engine context buffer 142 NvU32 i; 143 for (i = 0; i < pKernelFalcon->ctxBufferSize; i += 4) 144 { 145 MEM_WR32(pInstMem + i, 0); 146 } 147 memdescUnmapInternal(pGpu, pCtxMemDesc, 0); 148 } 149 else 150 { 151 status = NV_ERR_INSUFFICIENT_RESOURCES; 152 goto done; 153 } 154 155 NV_ASSERT_OK_OR_GOTO(status, 156 kchannelSetEngineContextMemDesc(pGpu, pKernelChannel, pKernelFalcon->physEngDesc, pCtxMemDesc), 157 done); 158 159 if (!gvaspaceIsExternallyOwned(pGVAS)) 160 { 161 NV_ASSERT_OK_OR_GOTO(status, 162 kchannelMapEngineCtxBuf(pGpu, pKernelChannel, pKernelFalcon->physEngDesc), 163 done); 164 } 165 166 done: 167 if (status != NV_OK) 168 { 169 memdescFree(pCtxMemDesc); 170 memdescDestroy(pCtxMemDesc); 171 } 172 173 return status; 174 } 175 176 static NV_STATUS _kflcnPromoteContext 177 ( 178 OBJGPU *pGpu, 179 KernelFalcon *pKernelFalcon, 180 KernelChannel *pKernelChannel 181 ) 182 { 183 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); 184 RsClient *pClient = RES_GET_CLIENT(pKernelChannel); 185 Subdevice *pSubdevice; 186 NvU64 addr; 187 RM_ENGINE_TYPE rmEngineType; 188 ENGINE_CTX_DESCRIPTOR *pEngCtx; 189 NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS rmCtrlParams = {0}; 190 191 NV_ASSERT_OK_OR_RETURN(subdeviceGetByGpu(pClient, pGpu, &pSubdevice)); 192 NV_ASSERT_OR_RETURN(gpumgrGetSubDeviceInstanceFromGpu(pGpu) == 0, NV_ERR_INVALID_STATE); 193 194 pEngCtx = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->ppEngCtxDesc[0]; 195 NV_ASSERT_OR_RETURN(pEngCtx != NULL, NV_ERR_INVALID_ARGUMENT); 196 NV_ASSERT_OK_OR_RETURN(vaListFindVa(&pEngCtx->vaList, pKernelChannel->pVAS, &addr)); 197 198 NV_ASSERT_OK_OR_RETURN(kfifoEngineInfoXlate_HAL(pGpu, GPU_GET_KERNEL_FIFO(pGpu), 199 ENGINE_INFO_TYPE_ENG_DESC, pKernelFalcon->physEngDesc, 200 ENGINE_INFO_TYPE_RM_ENGINE_TYPE, (NvU32 *)&rmEngineType)); 201 202 rmCtrlParams.hClient = pClient->hClient; 203 rmCtrlParams.hObject = RES_GET_HANDLE(pKernelChannel); 204 rmCtrlParams.hChanClient = pClient->hClient; 205 rmCtrlParams.virtAddress = addr; 206 rmCtrlParams.size = pKernelFalcon->ctxBufferSize; 207 rmCtrlParams.engineType = gpuGetNv2080EngineType(rmEngineType); 208 rmCtrlParams.ChID = pKernelChannel->ChID; 209 210 NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi, pClient->hClient, RES_GET_HANDLE(pSubdevice), 211 NV2080_CTRL_CMD_GPU_PROMOTE_CTX, &rmCtrlParams, sizeof(rmCtrlParams))); 212 213 return NV_OK; 214 } 215 216 217 NV_STATUS kflcnAllocContext_IMPL 218 ( 219 OBJGPU *pGpu, 220 KernelFalcon *pKernelFalcon, 221 KernelChannel *pKernelChannel, 222 NvU32 classNum 223 ) 224 { 225 NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_CHANNEL); 226 227 if (!_kflcnNeedToAllocContext(pGpu, pKernelChannel)) 228 return NV_OK; 229 230 NV_ASSERT_OR_RETURN(gpuIsClassSupported(pGpu, classNum), NV_ERR_INVALID_OBJECT); 231 232 NV_ASSERT_OK_OR_RETURN(_kflcnAllocAndMapCtxBuffer(pGpu, pKernelFalcon, pKernelChannel)); 233 234 return _kflcnPromoteContext(pGpu, pKernelFalcon, pKernelChannel); 235 } 236 237 NV_STATUS kflcnFreeContext_IMPL 238 ( 239 OBJGPU *pGpu, 240 KernelFalcon *pKernelFalcon, 241 KernelChannel *pKernelChannel, 242 NvU32 classNum 243 ) 244 { 245 MEMORY_DESCRIPTOR *pCtxMemDesc = NULL; 246 NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_CHANNEL); 247 248 if (!_kflcnNeedToAllocContext(pGpu, pKernelChannel)) 249 return NV_OK; 250 251 if (kchannelIsCtxBufferAllocSkipped(pKernelChannel)) 252 return NV_OK; 253 254 kchangrpGetEngineContextMemDesc(pGpu, 255 pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup, 256 &pCtxMemDesc); 257 258 if (pCtxMemDesc == NULL) 259 { 260 NV_PRINTF(LEVEL_WARNING, 261 "The channel 0x%x does not have a falcon engine instance for engDesc=0x%x\n", 262 kchannelGetDebugTag(pKernelChannel), pKernelFalcon->physEngDesc); 263 return NV_OK; 264 } 265 266 kchannelUnmapEngineCtxBuf(pGpu, pKernelChannel, pKernelFalcon->physEngDesc); 267 kchannelSetEngineContextMemDesc(pGpu, pKernelChannel, pKernelFalcon->physEngDesc, NULL); 268 memdescFree(pCtxMemDesc); 269 memdescDestroy(pCtxMemDesc); 270 271 return NV_OK; 272 } 273 274 NV_STATUS gkflcnConstruct_IMPL 275 ( 276 GenericKernelFalcon *pGenericKernelFalcon, 277 OBJGPU *pGpu, 278 KernelFalconEngineConfig *pFalconConfig 279 ) 280 { 281 KernelFalcon *pKernelFalcon = staticCast(pGenericKernelFalcon, KernelFalcon); 282 if (pFalconConfig != NULL) 283 { 284 kflcnConfigureEngine(pGpu, pKernelFalcon, pFalconConfig); 285 } 286 return NV_OK; 287 } 288 289 NV_STATUS gkflcnResetHw_IMPL(OBJGPU *pGpu, GenericKernelFalcon *pGenKernFlcn) 290 { 291 NV_ASSERT_FAILED("This should only be called on full KernelFalcon implementations"); 292 return NV_ERR_NOT_SUPPORTED; 293 } 294 295 void gkflcnRegisterIntrService_IMPL(OBJGPU *pGpu, GenericKernelFalcon *pGenericKernelFalcon, IntrServiceRecord pRecords[MC_ENGINE_IDX_MAX]) 296 { 297 KernelFalcon *pKernelFalcon = staticCast(pGenericKernelFalcon, KernelFalcon); 298 NV_ASSERT_OR_RETURN_VOID(pKernelFalcon); 299 300 NV_PRINTF(LEVEL_INFO, "physEngDesc 0x%x\n", pKernelFalcon->physEngDesc); 301 302 if (!IS_NVDEC(pKernelFalcon->physEngDesc) && 303 pKernelFalcon->physEngDesc != ENG_OFA && 304 !IS_NVJPEG(pKernelFalcon->physEngDesc) && 305 !IS_MSENC(pKernelFalcon->physEngDesc)) 306 return; 307 308 // Register to handle nonstalling interrupts of the corresponding physical falcon in kernel rm 309 if (pKernelFalcon->physEngDesc != ENG_INVALID) 310 { 311 NvU32 mcIdx = MC_ENGINE_IDX_NULL; 312 313 NV_STATUS status = kfifoEngineInfoXlate_HAL(pGpu, GPU_GET_KERNEL_FIFO(pGpu), 314 ENGINE_INFO_TYPE_ENG_DESC, pKernelFalcon->physEngDesc, 315 ENGINE_INFO_TYPE_MC, &mcIdx); 316 317 NV_ASSERT_OR_RETURN_VOID(status == NV_OK); 318 319 NV_PRINTF(LEVEL_INFO, "Registering 0x%x/0x%x to handle nonstall intr\n", pKernelFalcon->physEngDesc, mcIdx); 320 321 NV_ASSERT(pRecords[mcIdx].pNotificationService == NULL); 322 pRecords[mcIdx].bFifoWaiveNotify = NV_FALSE; 323 pRecords[mcIdx].pNotificationService = staticCast(pGenericKernelFalcon, IntrService); 324 } 325 } 326 327 NV_STATUS gkflcnServiceNotificationInterrupt_IMPL(OBJGPU *pGpu, GenericKernelFalcon *pGenericKernelFalcon, IntrServiceServiceNotificationInterruptArguments *pParams) 328 { 329 NvU32 idxMc = pParams->engineIdx; 330 RM_ENGINE_TYPE rmEngineType = RM_ENGINE_TYPE_NULL; 331 332 NV_PRINTF(LEVEL_INFO, "nonstall intr for MC 0x%x\n", idxMc); 333 334 if (MC_ENGINE_IDX_NVDECn(0) <= idxMc && 335 idxMc < MC_ENGINE_IDX_NVDECn(RM_ENGINE_TYPE_NVDEC_SIZE)) 336 { 337 NvU32 nvdecIdx = idxMc - MC_ENGINE_IDX_NVDECn(0); 338 rmEngineType = RM_ENGINE_TYPE_NVDEC(nvdecIdx); 339 } 340 else if (idxMc == MC_ENGINE_IDX_OFA0) 341 { 342 rmEngineType = RM_ENGINE_TYPE_OFA; 343 } 344 else if (MC_ENGINE_IDX_NVJPEGn(0) <= idxMc && 345 idxMc < MC_ENGINE_IDX_NVJPEGn(RM_ENGINE_TYPE_NVJPEG_SIZE)) 346 { 347 NvU32 nvjpgIdx = idxMc - MC_ENGINE_IDX_NVJPEGn(0); 348 rmEngineType = RM_ENGINE_TYPE_NVJPEG(nvjpgIdx); 349 } 350 else if (MC_ENGINE_IDX_MSENCn(0) <= idxMc && 351 idxMc < MC_ENGINE_IDX_MSENCn(RM_ENGINE_TYPE_NVENC_SIZE)) 352 { 353 NvU32 msencIdx = idxMc - MC_ENGINE_IDX_MSENCn(0); 354 rmEngineType = RM_ENGINE_TYPE_NVENC(msencIdx); 355 } 356 357 NV_ASSERT_OR_RETURN(rmEngineType != RM_ENGINE_TYPE_NULL, NV_ERR_INVALID_STATE); 358 359 // Wake up channels waiting on this event 360 engineNonStallIntrNotify(pGpu, rmEngineType); 361 362 return NV_OK; 363 } 364