1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #define NVOC_KERNEL_CHANNEL_H_PRIVATE_ACCESS_ALLOWED 25 26 #include "kernel/gpu/fifo/kernel_channel.h" 27 28 #include "kernel/core/locks.h" 29 #include "gpu/subdevice/subdevice.h" 30 #include "kernel/diagnostics/gpu_acct.h" 31 #include "kernel/gpu/conf_compute/conf_compute.h" 32 #include "kernel/gpu/device/device.h" 33 #include "kernel/gpu/fifo/kernel_ctxshare.h" 34 #include "kernel/gpu/fifo/kernel_channel_group.h" 35 #include "kernel/gpu/gr/kernel_graphics.h" 36 #include "kernel/gpu/mem_mgr/context_dma.h" 37 #include "kernel/gpu/mem_mgr/heap.h" 38 #include "kernel/gpu/mem_mgr/mem_mgr.h" 39 #include "kernel/gpu/mig_mgr/kernel_mig_manager.h" 40 #include "kernel/gpu/rc/kernel_rc.h" 41 #include "kernel/mem_mgr/ctx_buf_pool.h" 42 #include "kernel/mem_mgr/gpu_vaspace.h" 43 #include "kernel/rmapi/event.h" 44 #include "kernel/rmapi/rmapi.h" 45 #include "kernel/rmapi/rs_utils.h" 46 #include "kernel/virtualization/hypervisor/hypervisor.h" 47 #include "gpu/bus/kern_bus.h" 48 #include "gpu/mem_mgr/virt_mem_allocator.h" 49 #include "objtmr.h" 50 #include "platform/sli/sli.h" 51 52 #include "class/cl0090.h" // KERNEL_GRAPHICS_CONTEXT 53 #include "class/cl906fsw.h" // GF100_GPFIFO 54 #include "class/cla06c.h" // KEPLER_CHANNEL_GROUP_A 55 #include "class/cla06f.h" // KEPLER_CHANNEL_GPFIFO_A 56 #include "class/cla06fsw.h" // KEPLER_CHANNEL_GPFIFO_A 57 #include "class/cla16f.h" // KEPLER_CHANNEL_GPFIFO_B 58 #include "class/cla16fsw.h" // KEPLER_CHANNEL_GPFIFO_B 59 #include "class/clb06f.h" // MAXWELL_CHANNEL_GPFIFO_A 60 #include "class/clb06fsw.h" // MAXWELL_CHANNEL_GPFIFO_A 61 #include "class/clc06f.h" // PASCAL_CHANNEL_GPFIFO_A 62 #include "class/clc06fsw.h" // PASCAL_CHANNEL_GPFIFO_A 63 #include "class/clc36f.h" // VOLTA_CHANNEL_GPFIFO_A 64 #include "class/clc36fsw.h" // VOLTA_CHANNEL_GPFIFO_A 65 #include "class/clc46f.h" // TURING_CHANNEL_GPFIFO_A 66 #include "class/clc46fsw.h" // TURING_CHANNEL_GPFIFO_A 67 #include "class/clc56f.h" // AMPERE_CHANNEL_GPFIFO_A 68 #include "class/clc56fsw.h" // AMPERE_CHANNEL_GPFIFO_A 69 #include "class/clc572.h" // PHYSICAL_CHANNEL_GPFIFO 70 #include "class/clc86f.h" // HOPPER_CHANNEL_GPFIFO_A 71 #include "class/clc86fsw.h" // HOPPER_CHANNEL_GPFIFO_A 72 73 #include "ctrl/ctrl906f.h" 74 #include "ctrl/ctrlc46f.h" 75 #include "ctrl/ctrlc86f.h" 76 77 #include "Nvcm.h" 78 #include "libraries/resserv/resserv.h" 79 #include "libraries/resserv/rs_client.h" 80 #include "libraries/resserv/rs_resource.h" 81 #include "libraries/resserv/rs_server.h" 82 #include "nvRmReg.h" 83 #include "nvstatuscodes.h" 84 #include "vgpu/rpc.h" 85 86 // Instmem static functions 87 static NV_STATUS _kchannelAllocHalData(OBJGPU *pGpu, KernelChannel *pKernelChannel); 88 static void _kchannelFreeHalData(OBJGPU *pGpu, KernelChannel *pKernelChannel); 89 static NV_STATUS _kchannelAllocOrDescribeInstMem( 90 KernelChannel *pKernelChannel, 91 NV_CHANNEL_ALLOC_PARAMS *pChannelGpfifoParams); 92 static NV_STATUS _kchannelDescribeMemDescsFromParams( 93 OBJGPU *pGpu, 94 KernelChannel *pKernelChannel, 95 NV_CHANNEL_ALLOC_PARAMS *pChannelGpfifoParams); 96 static NV_STATUS _kchannelDescribeMemDescsHeavySriov(OBJGPU *pGpu, KernelChannel *pKernelChannel); 97 static NV_STATUS _kchannelSendChannelAllocRpc( 98 KernelChannel *pKernelChannel, 99 NV_CHANNEL_ALLOC_PARAMS *pChannelGpfifoParams, 100 KernelChannelGroup *pKernelChannelGroup, 101 NvBool bFullSriov); 102 103 static NV_STATUS _kchannelSetupNotifyActions(KernelChannel *pKernelChannel, 104 NvU32 classNum); 105 static void _kchannelCleanupNotifyActions(KernelChannel *pKernelChannel); 106 static NV_STATUS _kchannelNotifyOfChid(OBJGPU *pGpu, KernelChannel *pKernelChannel, RsClient *pRsClient); 107 static NV_STATUS _kchannelGetUserMemDesc(OBJGPU *pGpu, KernelChannel *pKernelChannel, PMEMORY_DESCRIPTOR *ppMemDesc); 108 static void _kchannelUpdateFifoMapping(KernelChannel *pKernelChannel, 109 OBJGPU *pGpu, 110 NvBool bKernel, 111 NvP64 cpuAddress, 112 NvP64 priv, 113 NvU64 cpuMapLength, 114 NvU32 flags, 115 NvHandle hSubdevice, 116 RsCpuMapping *pMapping); 117 static NvNotification* 118 _kchannelGetKeyRotationNotifier(KernelChannel *pKernelChannel); 119 120 /*! 121 * @brief Construct a new KernelChannel, which also creates a Channel. 122 * 123 * @param[in,out] pCallContext The call context 124 * @param[in,out] pParams Params for the *_CHANNEL_GPFIFO class 125 * object being created 126 * 127 * @returns NV_OK on success, specific error code on failure. 128 */ 129 NV_STATUS 130 kchannelConstruct_IMPL 131 ( 132 KernelChannel *pKernelChannel, 133 CALL_CONTEXT *pCallContext, 134 RS_RES_ALLOC_PARAMS_INTERNAL *pParams 135 ) 136 { 137 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannel); 138 OBJSYS *pSys = SYS_GET_INSTANCE(); 139 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 140 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 141 RsClient *pRsClient = pCallContext->pClient; 142 RmClient *pRmClient = NULL; 143 RsResourceRef *pResourceRef = pCallContext->pResourceRef; 144 RsResourceRef *pKernelCtxShareRef = NULL; 145 NV_STATUS status; 146 RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL); 147 NvHandle hClient = pRsClient->hClient; 148 NvHandle hParent = pResourceRef->pParentRef->hResource; 149 NV_CHANNEL_ALLOC_PARAMS *pChannelGpfifoParams = pParams->pAllocParams; 150 RsResourceRef *pChanGrpRef = NULL; 151 KernelChannelGroupApi *pKernelChannelGroupApi = NULL; 152 NvHandle hKernelCtxShare = pChannelGpfifoParams->hContextShare; 153 NvBool bTsgAllocated = NV_FALSE; 154 NvHandle hChanGrp = NV01_NULL_OBJECT; 155 RsResourceRef *pDeviceRef = NULL; 156 RsResourceRef *pVASpaceRef = NULL; 157 KernelGraphicsContext *pKernelGraphicsContext = NULL; 158 NvBool bMIGInUse; 159 KernelChannelGroup *pKernelChannelGroup = NULL; 160 NvU32 chID = ~0; 161 NvU32 flags = pChannelGpfifoParams->flags; 162 RM_ENGINE_TYPE globalRmEngineType = RM_ENGINE_TYPE_NULL; 163 NvU32 verifFlags2 = 0; 164 NvBool bChidAllocated = NV_FALSE; 165 NvBool bLockAcquired = NV_FALSE; 166 NvBool bNotifyActionsSetup = NV_FALSE; 167 CTX_BUF_POOL_INFO *pChannelBufPool = NULL; 168 CTX_BUF_INFO bufInfo = {0}; 169 NvBool bRpcAllocated = NV_FALSE; 170 NvBool bFullSriov = IS_VIRTUAL_WITH_SRIOV(pGpu) && !gpuIsWarBug200577889SriovHeavyEnabled(pGpu); 171 NvBool bAddedToGroup = NV_FALSE; 172 NvU32 callingContextGfid; 173 Device *pDevice; 174 175 // We only support physical channels. 176 NV_ASSERT_OR_RETURN(FLD_TEST_DRF(OS04, _FLAGS, _CHANNEL_TYPE, _PHYSICAL, flags), 177 NV_ERR_NOT_SUPPORTED); 178 179 pKernelChannel->refCount = 1; 180 pKernelChannel->bIsContextBound = NV_FALSE; 181 pKernelChannel->nextObjectClassID = 0; 182 pKernelChannel->subctxId = 0; 183 pKernelChannel->bSkipCtxBufferAlloc = FLD_TEST_DRF(OS04, _FLAGS, 184 _SKIP_CTXBUFFER_ALLOC, _TRUE, flags); 185 pKernelChannel->cid = portAtomicIncrementU32(&pSys->currentCid); 186 pKernelChannel->runqueue = DRF_VAL(OS04, _FLAGS, _GROUP_CHANNEL_RUNQUEUE, flags); 187 pKernelChannel->engineType = RM_ENGINE_TYPE_NULL; 188 pChannelGpfifoParams->cid = pKernelChannel->cid; 189 NV_ASSERT_OK_OR_GOTO(status, refFindAncestorOfType(pResourceRef, classId(Device), &pDeviceRef), cleanup); 190 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &callingContextGfid)); 191 192 pDevice = dynamicCast(pDeviceRef->pResource, Device); 193 194 // Internal fields must be cleared when RMAPI call is from client 195 if (!hypervisorIsVgxHyper() || IS_GSP_CLIENT(pGpu)) 196 pChannelGpfifoParams->hPhysChannelGroup = NV01_NULL_OBJECT; 197 pChannelGpfifoParams->internalFlags = 0; 198 portMemSet(&pChannelGpfifoParams->errorNotifierMem, 0, 199 sizeof pChannelGpfifoParams->errorNotifierMem); 200 portMemSet(&pChannelGpfifoParams->eccErrorNotifierMem, 0, 201 sizeof pChannelGpfifoParams->eccErrorNotifierMem); 202 pChannelGpfifoParams->ProcessID = 0; 203 pChannelGpfifoParams->SubProcessID = 0; 204 portMemSet(pChannelGpfifoParams->encryptIv, 0, sizeof(pChannelGpfifoParams->encryptIv)); 205 portMemSet(pChannelGpfifoParams->decryptIv, 0, sizeof(pChannelGpfifoParams->decryptIv)); 206 portMemSet(pChannelGpfifoParams->hmacNonce, 0, sizeof(pChannelGpfifoParams->hmacNonce)); 207 208 pRmClient = dynamicCast(pRsClient, RmClient); 209 if (pRmClient == NULL) 210 { 211 return NV_ERR_OBJECT_NOT_FOUND; 212 } 213 pKernelChannel->pUserInfo = pRmClient->pUserInfo; 214 215 // 216 // GSP-RM needs privilegeLevel passed in as an alloc param because it cannot 217 // check pRmClient for kernel/admin. 218 // Other platforms check pRmClient to determine privilegeLevel. 219 // 220 if (RMCFG_FEATURE_PLATFORM_GSP) 221 { 222 // Guest-RM clients can allocate a privileged channel to perform 223 // actions such as updating page tables in physical mode or scrubbing. 224 // Security for these channels is enforced by VMMU and IOMMU 225 if (gpuIsSriovEnabled(pGpu) && IS_GFID_VF(callingContextGfid) && 226 FLD_TEST_DRF(OS04, _FLAGS, _PRIVILEGED_CHANNEL, _TRUE, flags)) 227 { 228 pKernelChannel->privilegeLevel = NV_KERNELCHANNEL_ALLOC_INTERNALFLAGS_PRIVILEGE_ADMIN; 229 } 230 else 231 { 232 pKernelChannel->privilegeLevel = 233 DRF_VAL(_KERNELCHANNEL, _ALLOC_INTERNALFLAGS, _PRIVILEGE, pChannelGpfifoParams->internalFlags); 234 } 235 236 // In GSP, all vGPU channel's will simply consider GFID as the processID 237 if (IS_VGPU_GSP_PLUGIN_OFFLOAD_ENABLED(pGpu) && IS_GFID_VF(callingContextGfid)) 238 { 239 pKernelChannel->ProcessID = callingContextGfid; 240 } 241 else 242 { 243 pKernelChannel->ProcessID = pChannelGpfifoParams->ProcessID; 244 } 245 246 pKernelChannel->SubProcessID = pChannelGpfifoParams->SubProcessID; 247 } 248 else 249 { 250 RS_PRIV_LEVEL privLevel = pCallContext->secInfo.privLevel; 251 if (privLevel >= RS_PRIV_LEVEL_KERNEL) 252 { 253 pKernelChannel->privilegeLevel = NV_KERNELCHANNEL_ALLOC_INTERNALFLAGS_PRIVILEGE_KERNEL; 254 pChannelGpfifoParams->flags = FLD_SET_DRF(OS04, _FLAGS, _PRIVILEGED_CHANNEL, _TRUE, pChannelGpfifoParams->flags); 255 } 256 else if (rmclientIsAdmin(pRmClient, privLevel) || hypervisorCheckForObjectAccess(hClient)) 257 { 258 pKernelChannel->privilegeLevel = NV_KERNELCHANNEL_ALLOC_INTERNALFLAGS_PRIVILEGE_ADMIN; 259 pChannelGpfifoParams->flags = FLD_SET_DRF(OS04, _FLAGS, _PRIVILEGED_CHANNEL, _TRUE, pChannelGpfifoParams->flags); 260 } 261 else 262 { 263 pKernelChannel->privilegeLevel = NV_KERNELCHANNEL_ALLOC_INTERNALFLAGS_PRIVILEGE_USER; 264 } 265 266 pKernelChannel->ProcessID = pRmClient->ProcID; 267 pKernelChannel->SubProcessID = pRmClient->SubProcessID; 268 } 269 270 // Context share and vaspace handles can't be active at the same time. 271 if ((hKernelCtxShare != NV01_NULL_OBJECT) && (pChannelGpfifoParams->hVASpace != NV01_NULL_OBJECT)) 272 { 273 NV_PRINTF(LEVEL_ERROR, 274 "Both context share and vaspace handles can't be valid at the same time\n"); 275 return NV_ERR_INVALID_ARGUMENT; 276 } 277 278 bMIGInUse = IS_MIG_IN_USE(pGpu); 279 280 // 281 // The scrubber is allocated by Kernel RM in offload mode, and is disabled 282 // completely on GSP, so it is not possible for GSP to determine whether 283 // this allocation should be allowed or not. CPU RM can and should properly 284 // check this. 285 // 286 if (IS_MIG_ENABLED(pGpu) && !RMCFG_FEATURE_PLATFORM_GSP && !bMIGInUse) 287 { 288 NvBool bTopLevelScrubberEnabled = NV_FALSE; 289 NvBool bTopLevelScrubberConstructed = NV_FALSE; 290 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 291 292 if (memmgrIsPmaInitialized(pMemoryManager)) 293 { 294 Heap *pHeap = GPU_GET_HEAP(pGpu); 295 NvU32 pmaConfigs = PMA_QUERY_SCRUB_ENABLED | PMA_QUERY_SCRUB_VALID; 296 NV_ASSERT_OK(pmaQueryConfigs(&pHeap->pmaObject, &pmaConfigs)); 297 bTopLevelScrubberEnabled = (pmaConfigs & PMA_QUERY_SCRUB_ENABLED) != 0x0; 298 bTopLevelScrubberConstructed = (pmaConfigs & PMA_QUERY_SCRUB_VALID) != 0x0; 299 } 300 301 // 302 // Exception: Top level scrubber must be initialized before 303 // GPU instances can be created, and therefore must be allowed to 304 // create a CE context if the scrubber is supported. 305 // 306 307 if (!bTopLevelScrubberEnabled || bTopLevelScrubberConstructed || 308 !kchannelCheckIsKernel(pKernelChannel)) 309 { 310 NV_PRINTF(LEVEL_ERROR, 311 "Channel allocation not allowed when MIG is enabled without GPU instancing\n"); 312 return NV_ERR_INVALID_STATE; 313 } 314 } 315 316 // Find the TSG, or create the TSG if we need to wrap it 317 status = clientGetResourceRefByType(pRsClient, hParent, 318 classId(KernelChannelGroupApi), 319 &pChanGrpRef); 320 if (status != NV_OK) 321 { 322 NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS tsgParams = { 0 }; 323 324 // Context share can only be used with a TSG channel 325 if (hKernelCtxShare != NV01_NULL_OBJECT) 326 { 327 NV_PRINTF(LEVEL_ERROR, 328 "Non-TSG channels can't use context share\n"); 329 status = NV_ERR_INVALID_ARGUMENT; 330 goto cleanup; 331 } 332 333 tsgParams.hVASpace = pChannelGpfifoParams->hVASpace; 334 tsgParams.engineType = pChannelGpfifoParams->engineType; 335 // vGpu plugin context flag should only be set if context is plugin 336 if (gpuIsSriovEnabled(pGpu)) 337 { 338 tsgParams.bIsCallingContextVgpuPlugin = FLD_TEST_DRF(OS04, _FLAGS, _CHANNEL_VGPU_PLUGIN_CONTEXT, _TRUE, pChannelGpfifoParams->flags); 339 } 340 // 341 // Internally allocate a TSG to wrap this channel. There is no point 342 // in mirroring this allocation in the host, as the channel is 343 // already mirrored. 344 // 345 status = pRmApi->AllocWithSecInfo(pRmApi, 346 hClient, 347 hParent, 348 &pChannelGpfifoParams->hPhysChannelGroup, 349 KEPLER_CHANNEL_GROUP_A, 350 NV_PTR_TO_NvP64(&tsgParams), 351 sizeof(tsgParams), 352 RMAPI_ALLOC_FLAGS_SKIP_RPC, 353 NvP64_NULL, 354 &pRmApi->defaultSecInfo); 355 356 NV_ASSERT_OR_GOTO(status == NV_OK, cleanup); 357 bTsgAllocated = NV_TRUE; 358 hChanGrp = pChannelGpfifoParams->hPhysChannelGroup; 359 360 status = clientGetResourceRefByType(pRsClient, hChanGrp, 361 classId(KernelChannelGroupApi), 362 &pChanGrpRef); 363 NV_ASSERT_OR_GOTO(status == NV_OK, cleanup); 364 365 pKernelChannelGroupApi = dynamicCast(pChanGrpRef->pResource, 366 KernelChannelGroupApi); 367 pKernelChannelGroup = pKernelChannelGroupApi->pKernelChannelGroup; 368 pKernelChannelGroup->bAllocatedByRm = NV_TRUE; 369 } 370 else 371 { 372 hChanGrp = hParent; 373 pKernelChannelGroupApi = dynamicCast(pChanGrpRef->pResource, 374 KernelChannelGroupApi); 375 if (pKernelChannelGroupApi == NULL || 376 pKernelChannelGroupApi->pKernelChannelGroup == NULL) 377 { 378 NV_PRINTF(LEVEL_ERROR, "Invalid KernelChannelGroup* for channel 0x%x\n", 379 pResourceRef->hResource); 380 status = NV_ERR_INVALID_POINTER; 381 NV_ASSERT_OR_GOTO(status == NV_OK, cleanup); 382 } 383 pKernelChannelGroup = pKernelChannelGroupApi->pKernelChannelGroup; 384 385 // TSG channel should specify a context share object, rather than vaspace directly 386 if (pChannelGpfifoParams->hVASpace != NV01_NULL_OBJECT) 387 { 388 NV_PRINTF(LEVEL_ERROR, 389 "TSG channels can't use an explicit vaspace\n"); 390 status = NV_ERR_INVALID_ARGUMENT; 391 NV_ASSERT_OR_GOTO(status == NV_OK, cleanup); 392 } 393 } 394 pKernelChannel->pKernelChannelGroupApi = pKernelChannelGroupApi; 395 396 NV_ASSERT_OR_RETURN(pKernelChannelGroupApi != NULL, NV_ERR_INVALID_STATE); 397 NV_ASSERT_OR_RETURN(pKernelChannelGroup != NULL, NV_ERR_INVALID_STATE); 398 399 // 400 // Reserve memory for channel instance block from PMA 401 // into a pool tied to channel's parent TSG. 402 // RM will later allocate memory for instance block from this pool. 403 // 404 pChannelBufPool = pKernelChannelGroup->pChannelBufPool; 405 if (pChannelBufPool != NULL) 406 { 407 NvBool bIsScrubSkipped; 408 NvBool bRequestScrubSkip = FLD_TEST_DRF(OS04, _FLAGS, _CHANNEL_SKIP_SCRUBBER, _TRUE, pChannelGpfifoParams->flags); 409 410 if (bRequestScrubSkip) 411 { 412 if (!kchannelCheckIsKernel(pKernelChannel)) 413 { 414 status = NV_ERR_INVALID_ARGUMENT; 415 NV_PRINTF(LEVEL_ERROR, "Only kernel priv clients can skip scrubber\n"); 416 goto cleanup; 417 } 418 419 // 420 // If this is first channel in the TSG then setup ctx buf pool to skip scrubbing. 421 // For subsequent channels, setting should match with ctx buf pool's state. 422 // 423 if (pKernelChannelGroup->chanCount == 0) 424 { 425 ctxBufPoolSetScrubSkip(pChannelBufPool, NV_TRUE); 426 NV_PRINTF(LEVEL_INFO, "Skipping scrubber for all allocations on this context\n"); 427 } 428 } 429 430 bIsScrubSkipped = ctxBufPoolIsScrubSkipped(pChannelBufPool); 431 if (bIsScrubSkipped ^ bRequestScrubSkip) 432 { 433 status = NV_ERR_INVALID_ARGUMENT; 434 NV_PRINTF(LEVEL_ERROR, "Mismatch between channel and parent TSG's policy on skipping scrubber\n"); 435 NV_PRINTF(LEVEL_ERROR, "scrubbing %s skipped for TSG and %s for channel\n", (bIsScrubSkipped ? "is" : "is not"), 436 (bRequestScrubSkip ? "is" : "is not")); 437 goto cleanup; 438 } 439 NV_ASSERT_OK_OR_GOTO(status, 440 kfifoGetInstMemInfo_HAL(pKernelFifo, &bufInfo.size, &bufInfo.align, NULL, NULL, NULL), 441 cleanup); 442 bufInfo.attr = RM_ATTR_PAGE_SIZE_DEFAULT; 443 NV_ASSERT_OK_OR_GOTO(status, ctxBufPoolReserve(pGpu, pChannelBufPool, &bufInfo, 1), cleanup); 444 } 445 else 446 { 447 NV_PRINTF(LEVEL_INFO, "Not using ctx buf pool\n"); 448 } 449 450 //-------------------------------------------------------------------------- 451 // we acquire the GPU lock below. 452 // From here down do not return early, use goto cleanup 453 //-------------------------------------------------------------------------- 454 455 NV_ASSERT_OK_OR_GOTO(status, 456 rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_FIFO), 457 cleanup); 458 bLockAcquired = NV_TRUE; 459 460 // 461 // Initialize the notification indices used for different notifications 462 // 463 pKernelChannel->notifyIndex[NV_CHANNELGPFIFO_NOTIFICATION_TYPE_ERROR] 464 = NV_CHANNELGPFIFO_NOTIFICATION_TYPE_ERROR; 465 pKernelChannel->notifyIndex[NV_CHANNELGPFIFO_NOTIFICATION_TYPE_WORK_SUBMIT_TOKEN] 466 = NV_CHANNELGPFIFO_NOTIFICATION_TYPE_WORK_SUBMIT_TOKEN; 467 468 // Bake channel group error handlers into the channel 469 pKernelChannel->hErrorContext = pChannelGpfifoParams->hObjectError; 470 pKernelChannel->hEccErrorContext = pChannelGpfifoParams->hObjectEccError; 471 472 if (pKernelChannel->hErrorContext == NV01_NULL_OBJECT) 473 { 474 pKernelChannel->hErrorContext = ( 475 pKernelChannel->pKernelChannelGroupApi->hErrorContext); 476 } 477 if (pKernelChannel->hEccErrorContext == NV01_NULL_OBJECT) 478 { 479 pKernelChannel->hEccErrorContext = ( 480 pKernelChannel->pKernelChannelGroupApi->hEccErrorContext); 481 } 482 483 if (pKernelChannel->hErrorContext == NV01_NULL_OBJECT) 484 { 485 pKernelChannel->errorContextType = ERROR_NOTIFIER_TYPE_NONE; 486 } 487 else if (!RMCFG_FEATURE_PLATFORM_GSP) 488 { 489 NV_ASSERT_OK(kchannelGetNotifierInfo(pGpu, pDevice, 490 pKernelChannel->hErrorContext, 491 &pKernelChannel->pErrContextMemDesc, 492 &pKernelChannel->errorContextType, 493 &pKernelChannel->errorContextOffset)); 494 NV_ASSERT(pKernelChannel->errorContextType != 495 ERROR_NOTIFIER_TYPE_NONE); 496 } 497 if (pKernelChannel->hEccErrorContext == NV01_NULL_OBJECT) 498 { 499 pKernelChannel->eccErrorContextType = ERROR_NOTIFIER_TYPE_NONE; 500 } 501 else if (!RMCFG_FEATURE_PLATFORM_GSP) 502 { 503 NV_ASSERT_OK(kchannelGetNotifierInfo(pGpu, pDevice, 504 pKernelChannel->hEccErrorContext, 505 &pKernelChannel->pEccErrContextMemDesc, 506 &pKernelChannel->eccErrorContextType, 507 &pKernelChannel->eccErrorContextOffset)); 508 NV_ASSERT(pKernelChannel->eccErrorContextType != 509 ERROR_NOTIFIER_TYPE_NONE); 510 } 511 512 if (IS_GSP_CLIENT(pGpu) || bFullSriov) 513 { 514 if (pKernelChannel->hErrorContext != NV01_NULL_OBJECT) 515 { 516 pChannelGpfifoParams->errorNotifierMem.base = ( 517 memdescGetPhysAddr(pKernelChannel->pErrContextMemDesc, 518 AT_GPU, 0) + 519 pKernelChannel->errorContextOffset); 520 pChannelGpfifoParams->errorNotifierMem.size = ( 521 pKernelChannel->pErrContextMemDesc->Size - 522 pKernelChannel->errorContextOffset); 523 pChannelGpfifoParams->errorNotifierMem.addressSpace = 524 memdescGetAddressSpace(pKernelChannel->pErrContextMemDesc); 525 pChannelGpfifoParams->errorNotifierMem.cacheAttrib = 526 memdescGetCpuCacheAttrib(pKernelChannel->pErrContextMemDesc); 527 528 } 529 if (pKernelChannel->hEccErrorContext != NV01_NULL_OBJECT) 530 { 531 pChannelGpfifoParams->eccErrorNotifierMem.base = ( 532 memdescGetPhysAddr(pKernelChannel->pEccErrContextMemDesc, 533 AT_GPU, 0) + 534 pKernelChannel->eccErrorContextOffset); 535 pChannelGpfifoParams->eccErrorNotifierMem.size = ( 536 pKernelChannel->pEccErrContextMemDesc->Size - 537 pKernelChannel->eccErrorContextOffset); 538 pChannelGpfifoParams->eccErrorNotifierMem.addressSpace = 539 memdescGetAddressSpace(pKernelChannel->pEccErrContextMemDesc); 540 pChannelGpfifoParams->eccErrorNotifierMem.cacheAttrib = 541 memdescGetCpuCacheAttrib(pKernelChannel->pEccErrContextMemDesc); 542 } 543 544 pChannelGpfifoParams->internalFlags = FLD_SET_DRF_NUM( 545 _KERNELCHANNEL_ALLOC, _INTERNALFLAGS, _ERROR_NOTIFIER_TYPE, 546 pKernelChannel->errorContextType, 547 pChannelGpfifoParams->internalFlags); 548 pChannelGpfifoParams->internalFlags = FLD_SET_DRF_NUM( 549 _KERNELCHANNEL_ALLOC, _INTERNALFLAGS, _ECC_ERROR_NOTIFIER_TYPE, 550 pKernelChannel->eccErrorContextType, 551 pChannelGpfifoParams->internalFlags); 552 } 553 554 // 555 // The error context types should be set on all RM configurations 556 // (GSP/baremetal/CPU-GSP client) 557 // 558 NV_ASSERT(pKernelChannel->errorContextType != ERROR_NOTIFIER_TYPE_UNKNOWN); 559 NV_ASSERT(pKernelChannel->eccErrorContextType != 560 ERROR_NOTIFIER_TYPE_UNKNOWN); 561 562 563 if ((pKernelChannelGroup->chanCount != 0) && 564 (( pKernelChannelGroup->bLegacyMode && (hKernelCtxShare != NV01_NULL_OBJECT)) || 565 (!pKernelChannelGroup->bLegacyMode && (hKernelCtxShare == NV01_NULL_OBJECT)))) 566 { 567 // 568 // Check if this channnel allocation specifying (or not) a user 569 // allocated context share matches with previous channel allocations (if 570 // any) in this group specifiying (or not) a user allocated context 571 // share. 572 // 573 // A channel group cannot have a mix of channels with some of them 574 // specifying a user allocated context share and some having RM 575 // allocated context share. 576 // 577 NV_PRINTF(LEVEL_NOTICE, 578 "All channels in a channel group must specify a CONTEXT_SHARE if any one of them specifies it\n"); 579 status = NV_ERR_INVALID_ARGUMENT; 580 goto cleanup; 581 } 582 583 // Get KernelCtxShare (supplied or legacy) 584 if (hKernelCtxShare != NV01_NULL_OBJECT) 585 { 586 // Get object pointers from supplied hKernelCtxShare. 587 NV_ASSERT_OK_OR_GOTO(status, 588 clientGetResourceRefByType(pRsClient, 589 hKernelCtxShare, 590 classId(KernelCtxShareApi), 591 &pKernelCtxShareRef), 592 cleanup); 593 594 // 595 // If hKernelCtxShare is nonzero, the ChannelGroup is not internal 596 // either, so it should have the same parent as hParent. 597 // 598 NV_ASSERT_TRUE_OR_GOTO(status, 599 pKernelCtxShareRef->pParentRef != NULL && 600 pKernelCtxShareRef->pParentRef->hResource == hParent, 601 NV_ERR_INVALID_OBJECT_PARENT, 602 cleanup); 603 } 604 else 605 { 606 NvU32 subctxFlag; 607 NvHandle hLegacyKernelCtxShare; 608 609 if (!pKernelChannelGroup->bLegacyMode) 610 { 611 // 612 // Set this ChannelGroup to legacy mode and get the KernelCtxShare 613 // from it. 614 // 615 NV_ASSERT_OK_OR_GOTO(status, 616 kchangrpapiSetLegacyMode(pKernelChannelGroupApi, 617 pGpu, pKernelFifo, hClient), 618 cleanup); 619 } 620 621 subctxFlag = DRF_VAL(OS04, _FLAGS, _GROUP_CHANNEL_THREAD, flags); 622 hLegacyKernelCtxShare = (subctxFlag == 623 NV_CTXSHARE_ALLOCATION_FLAGS_SUBCONTEXT_SYNC) ? 624 pKernelChannelGroupApi->hLegacykCtxShareSync : 625 pKernelChannelGroupApi->hLegacykCtxShareAsync; 626 627 NV_ASSERT_OK_OR_GOTO(status, 628 clientGetResourceRefByType(pRsClient, 629 hLegacyKernelCtxShare, 630 classId(KernelCtxShareApi), 631 &pKernelCtxShareRef), 632 cleanup); 633 } 634 635 pKernelChannel->pKernelCtxShareApi = dynamicCast( 636 pKernelCtxShareRef->pResource, 637 KernelCtxShareApi); 638 NV_ASSERT_TRUE_OR_GOTO(status, 639 pKernelChannel->pKernelCtxShareApi != NULL, 640 NV_ERR_INVALID_OBJECT, 641 cleanup); 642 NV_ASSERT_TRUE_OR_GOTO(status, 643 pKernelChannel->pKernelCtxShareApi->pShareData != 644 NULL, 645 NV_ERR_INVALID_OBJECT, 646 cleanup); 647 pKernelChannel->pVAS = pKernelChannel->pKernelCtxShareApi->pShareData->pVAS; 648 NV_ASSERT_TRUE_OR_GOTO(status, 649 pKernelChannel->pVAS != NULL, 650 NV_ERR_INVALID_OBJECT, 651 cleanup); 652 653 if (kfifoIsPerRunlistChramSupportedInHw(pKernelFifo)) 654 { 655 // TSG should always have a valid engine Id. 656 NV_ASSERT_TRUE_OR_GOTO(status, 657 RM_ENGINE_TYPE_IS_VALID(pKernelChannelGroup->engineType), 658 NV_ERR_INVALID_STATE, 659 cleanup); 660 661 if (NV2080_ENGINE_TYPE_IS_VALID(pChannelGpfifoParams->engineType)) 662 { 663 globalRmEngineType = gpuGetRmEngineType(pChannelGpfifoParams->engineType); 664 // Convert it to global engine id if MIG is enabled 665 if (bMIGInUse) 666 { 667 MIG_INSTANCE_REF ref; 668 669 NV_CHECK_OK_OR_GOTO( 670 status, 671 LEVEL_ERROR, 672 kmigmgrGetInstanceRefFromDevice(pGpu, pKernelMIGManager, 673 pDevice, &ref), 674 cleanup); 675 676 NV_CHECK_OK_OR_GOTO( 677 status, 678 LEVEL_ERROR, 679 kmigmgrGetLocalToGlobalEngineType(pGpu, pKernelMIGManager, ref, 680 globalRmEngineType, 681 &globalRmEngineType), 682 cleanup); 683 } 684 685 // Throw an error if TSG engine Id does NOT match with channel engine Id 686 if (globalRmEngineType != pKernelChannelGroup->engineType) 687 { 688 NV_PRINTF(LEVEL_ERROR, 689 "Engine type of channel = 0x%x (0x%x) not compatible with engine type of TSG = 0x%x (0x%x)\n", 690 gpuGetNv2080EngineType(pChannelGpfifoParams->engineType), 691 pChannelGpfifoParams->engineType, 692 gpuGetNv2080EngineType(pKernelChannelGroup->engineType), 693 pKernelChannelGroup->engineType); 694 695 status = NV_ERR_INVALID_ARGUMENT; 696 goto cleanup; 697 } 698 } 699 700 // Assign the engine type from the parent TSG 701 pKernelChannel->engineType = pKernelChannelGroup->engineType; 702 } 703 704 // Determine initial runlist ID (based on engine type if provided or inherited from TSG) 705 pKernelChannel->runlistId = kfifoGetDefaultRunlist_HAL(pGpu, pKernelFifo, pKernelChannel->engineType); 706 707 pKernelChannel->bCCSecureChannel = FLD_TEST_DRF(OS04, _FLAGS, _CC_SECURE, _TRUE, flags); 708 if (pKernelChannel->bCCSecureChannel) 709 { 710 ConfidentialCompute* pConfCompute = GPU_GET_CONF_COMPUTE(pGpu); 711 712 // return early if gpu is not ready to accept work 713 if (pConfCompute && kchannelCheckIsUserMode(pKernelChannel) 714 && !confComputeAcceptClientRequest(pGpu, pConfCompute)) 715 { 716 return NV_ERR_NOT_READY; 717 } 718 719 if (pConfCompute->getProperty(pConfCompute, PDB_PROP_CONFCOMPUTE_KEY_ROTATION_SUPPORTED)) 720 { 721 KEY_ROTATION_STATUS state; 722 NvU32 h2dKey; 723 NV_ASSERT_OK_OR_GOTO(status, 724 confComputeGetKeyPairByChannel(pGpu, pConfCompute, pKernelChannel, &h2dKey, NULL), 725 cleanup); 726 NV_ASSERT_OK_OR_GOTO(status, 727 confComputeGetKeyRotationStatus(pConfCompute, h2dKey, &state), 728 cleanup); 729 if (state != KEY_ROTATION_STATUS_IDLE) 730 { 731 status = NV_ERR_KEY_ROTATION_IN_PROGRESS; 732 goto cleanup; 733 } 734 } 735 status = kchannelRetrieveKmb_HAL(pGpu, pKernelChannel, ROTATE_IV_ALL_VALID, 736 NV_TRUE, &pKernelChannel->clientKmb); 737 NV_ASSERT_OR_GOTO(status == NV_OK, cleanup); 738 739 portMemCopy(pChannelGpfifoParams->encryptIv, 740 sizeof(pChannelGpfifoParams->encryptIv), 741 pKernelChannel->clientKmb.encryptBundle.iv, 742 sizeof(pKernelChannel->clientKmb.encryptBundle.iv)); 743 744 portMemCopy(pChannelGpfifoParams->decryptIv, 745 sizeof(pChannelGpfifoParams->decryptIv), 746 pKernelChannel->clientKmb.decryptBundle.iv, 747 sizeof(pKernelChannel->clientKmb.decryptBundle.iv)); 748 749 portMemCopy(pChannelGpfifoParams->hmacNonce, 750 sizeof(pChannelGpfifoParams->hmacNonce), 751 pKernelChannel->clientKmb.hmacBundle.nonce, 752 sizeof(pKernelChannel->clientKmb.hmacBundle.nonce)); 753 754 } 755 756 // Set TLS state and BAR0 window if we are working with Gr 757 if (bMIGInUse && RM_ENGINE_TYPE_IS_GR(pKernelChannel->engineType)) 758 { 759 NV_ASSERT_OK(kmigmgrGetInstanceRefFromDevice(pGpu, pKernelMIGManager, 760 pDevice, &pKernelChannel->partitionRef)); 761 } 762 763 // Allocate the ChId (except legacy VGPU which allocates ChID on the host) 764 if (!IS_VIRTUAL_WITHOUT_SRIOV(pGpu)) 765 { 766 status = kchannelAllocHwID_HAL(pGpu, pKernelChannel, hClient, 767 flags, verifFlags2, chID); 768 769 if (status != NV_OK) 770 { 771 NV_PRINTF(LEVEL_ERROR, "Error in Allocating channel id 0x%x for hClient 0x%x hKernelChannel 0x%x \n", 772 chID, hClient, pResourceRef->hResource); 773 DBG_BREAKPOINT(); 774 goto cleanup; 775 776 } 777 778 chID = pKernelChannel->ChID; 779 bChidAllocated = NV_TRUE; 780 } 781 782 // 783 // RPC alloc the channel in legacy VGPU / Heavy SRIOV so that instmem details can be gotten from it 784 // 785 if (IS_VIRTUAL(pGpu) && (!bFullSriov)) 786 { 787 NV_ASSERT_OK_OR_GOTO(status, 788 _kchannelSendChannelAllocRpc(pKernelChannel, 789 pChannelGpfifoParams, 790 pKernelChannelGroup, 791 bFullSriov), 792 cleanup); 793 bRpcAllocated = NV_TRUE; 794 } 795 796 // Legacy VGPU: allocate chid that the host provided 797 if (IS_VIRTUAL_WITHOUT_SRIOV(pGpu)) 798 { 799 chID = pKernelChannel->ChID; 800 801 status = kchannelAllocHwID_HAL(pGpu, pKernelChannel, hClient, 802 flags, verifFlags2, chID); 803 804 if (status != NV_OK) 805 { 806 NV_PRINTF(LEVEL_ERROR, "Error in Allocating channel id 0x%x for hClient 0x%x hKernelChannel 0x%x \n", 807 chID, hClient, pResourceRef->hResource); 808 chID = ~0; 809 DBG_BREAKPOINT(); 810 goto cleanup; 811 } 812 813 bChidAllocated = NV_TRUE; 814 } 815 816 // 817 // Do instmem setup here 818 // (Requires the channel to be created on the host if legacy VGPU / Heavy SRIOV. 819 // Does not require a Channel object.) 820 // 821 NV_ASSERT_OK_OR_GOTO(status, 822 _kchannelAllocOrDescribeInstMem(pKernelChannel, pChannelGpfifoParams), 823 cleanup); 824 825 // Join the channel group here 826 NV_ASSERT_OK_OR_GOTO(status, 827 kchangrpAddChannel(pGpu, pKernelChannelGroup, pKernelChannel), 828 cleanup); 829 bAddedToGroup = NV_TRUE; 830 831 // Assign to the same runlistId as the KernelChannelGroup if it's already determined 832 if (pKernelChannelGroup->bRunlistAssigned) 833 { 834 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 835 { 836 NV_ASSERT_OK_OR_ELSE(status, 837 kfifoRunlistSetId_HAL(pGpu, 838 GPU_GET_KERNEL_FIFO(pGpu), 839 pKernelChannel, 840 pKernelChannelGroup->runlistId), 841 SLI_LOOP_GOTO(cleanup)); 842 } 843 SLI_LOOP_END 844 } 845 846 // Allocate the physical channel 847 NV_ASSERT_OK_OR_GOTO(status, 848 kchannelAllocChannel_HAL(pKernelChannel, pChannelGpfifoParams), 849 cleanup); 850 851 // Set up pNotifyActions 852 _kchannelSetupNotifyActions(pKernelChannel, pResourceRef->externalClassId); 853 bNotifyActionsSetup = NV_TRUE; 854 855 // Initialize the userd length 856 if (!pKernelChannel->bClientAllocatedUserD) 857 { 858 NvU64 temp_offset; 859 860 kchannelGetUserdInfo_HAL(pGpu, 861 pKernelChannel, 862 NULL, 863 &temp_offset, 864 &pKernelChannel->userdLength); 865 } 866 else 867 { 868 kfifoGetUserdSizeAlign_HAL(pKernelFifo, (NvU32*)&pKernelChannel->userdLength, NULL); 869 } 870 871 // Set GPU accounting 872 if (RMCFG_MODULE_GPUACCT && 873 pGpu->getProperty(pGpu, PDB_PROP_GPU_ACCOUNTING_ON)) 874 { 875 GpuAccounting *pGpuAcct = SYS_GET_GPUACCT(SYS_GET_INSTANCE()); 876 877 gpuacctSetProcType(pGpuAcct, 878 pGpu->gpuInstance, 879 pRmClient->ProcID, 880 pRmClient->SubProcessID, 881 NV_GPUACCT_PROC_TYPE_GPU); 882 } 883 884 // 885 // RPC to allocate the channel on GSPFW/host. 886 // (Requires a Channel object but only for hPhysChannel.) 887 // 888 if (IS_GSP_CLIENT(pGpu) || bFullSriov) 889 { 890 NV_ASSERT_OK_OR_GOTO(status, 891 _kchannelSendChannelAllocRpc(pKernelChannel, 892 pChannelGpfifoParams, 893 pKernelChannelGroup, 894 bFullSriov), 895 cleanup); 896 bRpcAllocated = NV_TRUE; 897 } 898 899 if (kfifoIsPerRunlistChramEnabled(pKernelFifo) || 900 (gpuIsCCorApmFeatureEnabled(pGpu) || bMIGInUse)) 901 { 902 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 903 { 904 KernelFifo *pTempKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 905 // 906 // If we have a separate channel RAM for each runlist then we need to set 907 // runlistId as we already picked a chID from channel RAM based on this runlistId. 908 // This will also ensure runlistId is not overridden later to a different value 909 // 910 NV_ASSERT_OK_OR_GOTO(status, 911 kfifoRunlistSetId_HAL(pGpu, pTempKernelFifo, pKernelChannel, pKernelChannel->runlistId), 912 cleanup); 913 } 914 SLI_LOOP_END; 915 } 916 917 // 918 // If we alloced this group, we want to free KernelChannel first, 919 // so we should set KernelChannel as its dependent. 920 // 921 if (bTsgAllocated) 922 { 923 NV_ASSERT_OK_OR_GOTO(status, refAddDependant(pChanGrpRef, pResourceRef), cleanup); 924 } 925 926 // We depend on VASpace if it was provided 927 if (pChannelGpfifoParams->hVASpace != NV01_NULL_OBJECT) 928 { 929 NV_ASSERT_OK_OR_GOTO(status, 930 clientGetResourceRef(pRsClient, 931 pChannelGpfifoParams->hVASpace, 932 &pVASpaceRef), 933 cleanup); 934 NV_ASSERT_TRUE_OR_GOTO(status, 935 pVASpaceRef != NULL, 936 NV_ERR_INVALID_OBJECT, 937 cleanup); 938 NV_ASSERT_OK_OR_GOTO(status, 939 refAddDependant(pVASpaceRef, pResourceRef), 940 cleanup); 941 } 942 943 // 944 // If KernelCtxShare was provided, we depend on it (and if we created it then we 945 // also want KernelChannel to be freed first.) 946 // 947 if (pKernelChannel->pKernelCtxShareApi != NULL) 948 { 949 NV_ASSERT_OK_OR_GOTO( 950 status, 951 refAddDependant(RES_GET_REF(pKernelChannel->pKernelCtxShareApi), pResourceRef), 952 cleanup); 953 } 954 955 pKernelChannel->hKernelGraphicsContext = pKernelChannelGroupApi->hKernelGraphicsContext; 956 if (pKernelChannel->hKernelGraphicsContext != NV01_NULL_OBJECT) 957 { 958 NV_ASSERT_OK_OR_GOTO(status, 959 kgrctxFromKernelChannel(pKernelChannel, &pKernelGraphicsContext), 960 cleanup); 961 962 NV_ASSERT_OK_OR_GOTO(status, 963 refAddDependant(RES_GET_REF(pKernelGraphicsContext), pResourceRef), 964 cleanup); 965 } 966 967 if (pChannelGpfifoParams->hObjectError != 0) 968 { 969 NV_ASSERT_OK_OR_GOTO( 970 status, 971 _kchannelNotifyOfChid(pGpu, pKernelChannel, pRsClient), 972 cleanup); 973 } 974 975 // Cache the hVASpace for this channel in the KernelChannel object 976 pKernelChannel->hVASpace = pKernelChannel->pKernelCtxShareApi->hVASpace; 977 978 ConfidentialCompute *pConfCompute = GPU_GET_CONF_COMPUTE(pGpu); 979 if ((pConfCompute != NULL) && 980 (pConfCompute->getProperty(pConfCompute, PDB_PROP_CONFCOMPUTE_CC_FEATURE_ENABLED)) && 981 (pConfCompute->getProperty(pConfCompute, PDB_PROP_CONFCOMPUTE_KEY_ROTATION_SUPPORTED)) && 982 (pKernelChannel->bCCSecureChannel)) 983 { 984 if (!FLD_TEST_DRF(OS04, _FLAGS, _CHANNEL_SKIP_SCRUBBER, _TRUE, pChannelGpfifoParams->flags) && 985 pConfCompute->getProperty(pConfCompute, PDB_PROP_CONFCOMPUTE_DUMMY_KEY_ROTATION_ENABLED)) 986 { 987 // 988 // If conf compute feature is enabled AND 989 // If key rotation is supported AND 990 // If key rotation callbacks are not enabled yet AND 991 // If this is a secure channel being created AND 992 // If its not the scrubber channel then increment refcount 993 // 994 pConfCompute->keyRotationChannelRefCount++; 995 } 996 997 // Create persistent mapping to key rotation notifier 998 NV_ASSERT_OK_OR_GOTO( 999 status, 1000 kchannelSetKeyRotationNotifier_HAL(pGpu, pKernelChannel, NV_TRUE), 1001 cleanup); 1002 } 1003 1004 cleanup: 1005 if (bLockAcquired) 1006 rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); 1007 1008 // These fields are only needed internally; clear them here 1009 pChannelGpfifoParams->hPhysChannelGroup = 0; 1010 pChannelGpfifoParams->internalFlags = 0; 1011 portMemSet(&pChannelGpfifoParams->errorNotifierMem, 0, 1012 sizeof pChannelGpfifoParams->errorNotifierMem); 1013 portMemSet(&pChannelGpfifoParams->eccErrorNotifierMem, 0, 1014 sizeof pChannelGpfifoParams->eccErrorNotifierMem); 1015 pChannelGpfifoParams->ProcessID = 0; 1016 pChannelGpfifoParams->SubProcessID = 0; 1017 portMemSet(pChannelGpfifoParams->encryptIv, 0, sizeof(pChannelGpfifoParams->encryptIv)); 1018 portMemSet(pChannelGpfifoParams->decryptIv, 0, sizeof(pChannelGpfifoParams->decryptIv)); 1019 portMemSet(pChannelGpfifoParams->hmacNonce, 0, sizeof(pChannelGpfifoParams->hmacNonce)); 1020 1021 // Free the allocated resources if there was an error 1022 if (status != NV_OK) 1023 { 1024 if (bNotifyActionsSetup) 1025 { 1026 _kchannelCleanupNotifyActions(pKernelChannel); 1027 } 1028 1029 // Remove any dependencies we may have added; we don't want our destructor called when freeing anything below 1030 if (pKernelGraphicsContext != NULL) 1031 { 1032 refRemoveDependant(RES_GET_REF(pKernelGraphicsContext), pResourceRef); 1033 } 1034 if (pKernelChannel->pKernelCtxShareApi != NULL) 1035 { 1036 refRemoveDependant(RES_GET_REF(pKernelChannel->pKernelCtxShareApi), pResourceRef); 1037 } 1038 if (pVASpaceRef != NULL) 1039 { 1040 refRemoveDependant(pVASpaceRef, pResourceRef); 1041 } 1042 if (bTsgAllocated) 1043 { 1044 refRemoveDependant(pChanGrpRef, pResourceRef); 1045 } 1046 1047 if (bAddedToGroup) 1048 { 1049 kchangrpRemoveChannel(pGpu, pKernelChannelGroup, pKernelChannel); 1050 } 1051 1052 if (RMCFG_FEATURE_PLATFORM_GSP) 1053 { 1054 // Free memdescs created during construct on GSP path. 1055 memdescFree(pKernelChannel->pErrContextMemDesc); 1056 memdescDestroy(pKernelChannel->pErrContextMemDesc); 1057 memdescFree(pKernelChannel->pEccErrContextMemDesc); 1058 memdescDestroy(pKernelChannel->pEccErrContextMemDesc); 1059 } 1060 pKernelChannel->pErrContextMemDesc = NULL; 1061 pKernelChannel->pEccErrContextMemDesc = NULL; 1062 1063 if (bRpcAllocated) 1064 { 1065 NV_RM_RPC_FREE_ON_ERROR(pGpu, hClient, hParent, RES_GET_HANDLE(pKernelChannel)); 1066 } 1067 1068 _kchannelFreeHalData(pGpu, pKernelChannel); 1069 1070 if (pChannelBufPool != NULL) 1071 { 1072 ctxBufPoolRelease(pChannelBufPool); 1073 } 1074 1075 if (bTsgAllocated) 1076 { 1077 pRmApi->Free(pRmApi, hClient, hChanGrp); 1078 } 1079 1080 if (bChidAllocated) 1081 { 1082 kchannelFreeHwID_HAL(pGpu, pKernelChannel); 1083 } 1084 } 1085 1086 return status; 1087 } 1088 1089 void 1090 kchannelDestruct_IMPL 1091 ( 1092 KernelChannel *pKernelChannel 1093 ) 1094 { 1095 CALL_CONTEXT *pCallContext; 1096 RS_RES_FREE_PARAMS_INTERNAL *pParams; 1097 NvHandle hClient; 1098 RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL); 1099 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannel); 1100 NV_STATUS status = NV_OK; 1101 KernelChannelGroup *pKernelChannelGroup = NULL; 1102 1103 NV_ASSERT(pKernelChannel->pKernelChannelGroupApi != NULL); 1104 pKernelChannelGroup = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup; 1105 NV_ASSERT(pKernelChannelGroup != NULL); 1106 1107 resGetFreeParams(staticCast(pKernelChannel, RsResource), &pCallContext, &pParams); 1108 hClient = pCallContext->pClient->hClient; 1109 1110 ConfidentialCompute *pConfCompute = GPU_GET_CONF_COMPUTE(pGpu); 1111 if ((pConfCompute != NULL) && 1112 (pConfCompute->getProperty(pConfCompute, PDB_PROP_CONFCOMPUTE_CC_FEATURE_ENABLED)) && 1113 (pConfCompute->getProperty(pConfCompute, PDB_PROP_CONFCOMPUTE_KEY_ROTATION_SUPPORTED)) && 1114 (pKernelChannel->bCCSecureChannel)) 1115 { 1116 if (pConfCompute->getProperty(pConfCompute, PDB_PROP_CONFCOMPUTE_DUMMY_KEY_ROTATION_ENABLED)) 1117 { 1118 if (pConfCompute->keyRotationChannelRefCount > 0) 1119 { 1120 pConfCompute->keyRotationChannelRefCount--; 1121 } 1122 if (pConfCompute->keyRotationChannelRefCount == 0) 1123 { 1124 pConfCompute->keyRotationCallbackCount = 0; 1125 } 1126 } 1127 NV_ASSERT_OK(confComputeUpdateFreedChannelStats(pGpu, pConfCompute, pKernelChannel)); 1128 NV_ASSERT_OK(kchannelSetEncryptionStatsBuffer_HAL(pGpu, pKernelChannel, NV_FALSE)); 1129 NV_ASSERT_OK(kchannelSetKeyRotationNotifier_HAL(pGpu, pKernelChannel, NV_FALSE)); 1130 } 1131 1132 if (RMCFG_FEATURE_PLATFORM_GSP) 1133 { 1134 // Free memdescs created during construct on GSP path. 1135 memdescFree(pKernelChannel->pErrContextMemDesc); 1136 memdescDestroy(pKernelChannel->pErrContextMemDesc); 1137 memdescFree(pKernelChannel->pEccErrContextMemDesc); 1138 memdescDestroy(pKernelChannel->pEccErrContextMemDesc); 1139 } 1140 pKernelChannel->pErrContextMemDesc = NULL; 1141 pKernelChannel->pEccErrContextMemDesc = NULL; 1142 1143 // GSP and vGPU support 1144 if ((IS_GSP_CLIENT(pGpu) || IS_VIRTUAL(pGpu))) 1145 { 1146 // 1147 // GSP: 1148 // 1149 // Method buffer is allocated by CPU-RM during TSG construct 1150 // but mapped to invisible BAR2 in GSP during channel construct 1151 // During Free, first the BAR2 mapping must be unmapped in GSP 1152 // and then freeing of method buffer should be done on CPU. 1153 // This RPC call is especially required for the internal channel case 1154 // where channelDestruct calls free for its TSG 1155 // 1156 NV_RM_RPC_FREE(pGpu, 1157 hClient, 1158 RES_GET_PARENT_HANDLE(pKernelChannel), 1159 RES_GET_HANDLE(pKernelChannel), 1160 status); 1161 } 1162 1163 { 1164 KernelGraphicsContext *pKernelGraphicsContext; 1165 1166 // Perform GR ctx cleanup tasks on channel destruction 1167 if ((kgrctxFromKernelChannel(pKernelChannel, &pKernelGraphicsContext) == NV_OK) && 1168 kgrctxIsValid(pGpu, pKernelGraphicsContext, pKernelChannel)) 1169 { 1170 shrkgrctxDetach(pGpu, kgrctxGetShared(pGpu, pKernelGraphicsContext), pKernelGraphicsContext, pKernelChannel); 1171 } 1172 } 1173 1174 _kchannelCleanupNotifyActions(pKernelChannel); 1175 1176 _kchannelFreeHalData(pGpu, pKernelChannel); 1177 1178 NV_ASSERT(pKernelChannel->pKernelChannelGroupApi != NULL); 1179 1180 pKernelChannelGroup = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup; 1181 1182 NV_ASSERT(pKernelChannelGroup != NULL); 1183 1184 // remove channel from the group 1185 kchangrpRemoveChannel(pGpu, pKernelChannelGroup, pKernelChannel); 1186 1187 // Free the Ctx Buf pool 1188 if (pKernelChannelGroup->pChannelBufPool != NULL) 1189 { 1190 ctxBufPoolRelease(pKernelChannelGroup->pChannelBufPool); 1191 } 1192 1193 // Free the channel group, if we alloced it 1194 if (pKernelChannelGroup->bAllocatedByRm) 1195 { 1196 pRmApi->Free(pRmApi, hClient, 1197 RES_GET_HANDLE(pKernelChannel->pKernelChannelGroupApi)); 1198 pKernelChannelGroup = NULL; 1199 pKernelChannel->pKernelChannelGroupApi = NULL; 1200 } 1201 1202 kchannelFreeHwID_HAL(pGpu, pKernelChannel); 1203 kchannelFreeMmuExceptionInfo(pKernelChannel); 1204 1205 NV_ASSERT(pKernelChannel->refCount == 1); 1206 } 1207 1208 NV_STATUS 1209 kchannelMap_IMPL 1210 ( 1211 KernelChannel *pKernelChannel, 1212 CALL_CONTEXT *pCallContext, 1213 RS_CPU_MAP_PARAMS *pParams, 1214 RsCpuMapping *pCpuMapping 1215 ) 1216 { 1217 OBJGPU *pGpu; 1218 NV_STATUS rmStatus; 1219 RsClient *pRsClient = pCallContext->pClient; 1220 RmClient *pRmClient = dynamicCast(pRsClient, RmClient); 1221 GpuResource *pGpuResource; 1222 1223 NV_ASSERT_OR_RETURN(!pKernelChannel->bClientAllocatedUserD, NV_ERR_INVALID_REQUEST); 1224 1225 rmStatus = gpuresGetByDeviceOrSubdeviceHandle(pRsClient, 1226 pCpuMapping->pContextRef->hResource, 1227 &pGpuResource); 1228 if (rmStatus != NV_OK) 1229 return rmStatus; 1230 1231 pGpu = GPU_RES_GET_GPU(pGpuResource); 1232 GPU_RES_SET_THREAD_BC_STATE(pGpuResource); 1233 1234 // If the flags are fifo default then offset/length passed in 1235 if (DRF_VAL(OS33, _FLAGS, _FIFO_MAPPING, pCpuMapping->flags) == NVOS33_FLAGS_FIFO_MAPPING_DEFAULT) 1236 { 1237 // Validate the offset and limit passed in. 1238 if (pCpuMapping->offset >= pKernelChannel->userdLength) 1239 return NV_ERR_INVALID_BASE; 1240 if (pCpuMapping->length == 0) 1241 return NV_ERR_INVALID_LIMIT; 1242 if (pCpuMapping->offset + pCpuMapping->length > pKernelChannel->userdLength) 1243 return NV_ERR_INVALID_LIMIT; 1244 } 1245 else 1246 { 1247 pCpuMapping->offset = 0x0; 1248 pCpuMapping->length = pKernelChannel->userdLength; 1249 } 1250 1251 rmStatus = kchannelMapUserD(pGpu, pKernelChannel, 1252 rmclientGetCachedPrivilege(pRmClient), 1253 pCpuMapping->offset, 1254 pCpuMapping->pPrivate->protect, 1255 &pCpuMapping->pLinearAddress, 1256 &(pCpuMapping->pPrivate->pPriv)); 1257 1258 if (rmStatus != NV_OK) 1259 return rmStatus; 1260 1261 // Save off the mapping 1262 _kchannelUpdateFifoMapping(pKernelChannel, 1263 pGpu, 1264 (pRsClient->type == CLIENT_TYPE_KERNEL), 1265 pCpuMapping->pLinearAddress, 1266 pCpuMapping->pPrivate->pPriv, 1267 pCpuMapping->length, 1268 pCpuMapping->flags, 1269 pCpuMapping->pContextRef->hResource, 1270 pCpuMapping); 1271 1272 return NV_OK; 1273 } 1274 1275 NV_STATUS 1276 kchannelUnmap_IMPL 1277 ( 1278 KernelChannel *pKernelChannel, 1279 CALL_CONTEXT *pCallContext, 1280 RsCpuMapping *pCpuMapping 1281 ) 1282 { 1283 OBJGPU *pGpu; 1284 RsClient *pRsClient = pCallContext->pClient; 1285 RmClient *pRmClient = dynamicCast(pRsClient, RmClient); 1286 1287 if (pKernelChannel->bClientAllocatedUserD) 1288 { 1289 DBG_BREAKPOINT(); 1290 return NV_ERR_INVALID_REQUEST; 1291 } 1292 1293 pGpu = pCpuMapping->pPrivate->pGpu; 1294 1295 kchannelUnmapUserD(pGpu, 1296 pKernelChannel, 1297 rmclientGetCachedPrivilege(pRmClient), 1298 &pCpuMapping->pLinearAddress, 1299 &pCpuMapping->pPrivate->pPriv); 1300 1301 return NV_OK; 1302 } 1303 1304 NV_STATUS 1305 kchannelGetMapAddrSpace_IMPL 1306 ( 1307 KernelChannel *pKernelChannel, 1308 CALL_CONTEXT *pCallContext, 1309 NvU32 mapFlags, 1310 NV_ADDRESS_SPACE *pAddrSpace 1311 ) 1312 { 1313 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannel); 1314 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 1315 NvU32 userdAperture; 1316 NvU32 userdAttribute; 1317 1318 NV_ASSERT_OK_OR_RETURN(kfifoGetUserdLocation_HAL(pKernelFifo, 1319 &userdAperture, 1320 &userdAttribute)); 1321 if (pAddrSpace) 1322 *pAddrSpace = userdAperture; 1323 1324 return NV_OK; 1325 } 1326 1327 NV_STATUS 1328 kchannelGetMemInterMapParams_IMPL 1329 ( 1330 KernelChannel *pKernelChannel, 1331 RMRES_MEM_INTER_MAP_PARAMS *pParams 1332 ) 1333 { 1334 OBJGPU *pGpu = pParams->pGpu; 1335 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 1336 MEMORY_DESCRIPTOR *pSrcMemDesc = NULL; 1337 NV_STATUS status; 1338 1339 if (pParams->bSubdeviceHandleProvided) 1340 { 1341 NV_PRINTF(LEVEL_ERROR, "Unicast DMA mappings of USERD not supported.\n"); 1342 return NV_ERR_NOT_SUPPORTED; 1343 } 1344 1345 if (!kfifoIsUserdMapDmaSupported(pKernelFifo)) 1346 return NV_ERR_INVALID_OBJECT_HANDLE; 1347 1348 status = _kchannelGetUserMemDesc(pGpu, pKernelChannel, &pSrcMemDesc); 1349 if (status != NV_OK) 1350 return status; 1351 1352 pParams->pSrcMemDesc = pSrcMemDesc; 1353 pParams->pSrcGpu = pSrcMemDesc->pGpu; 1354 1355 return NV_OK; 1356 } 1357 1358 NV_STATUS 1359 kchannelCheckMemInterUnmap_IMPL 1360 ( 1361 KernelChannel *pKernelChannel, 1362 NvBool bSubdeviceHandleProvided 1363 ) 1364 { 1365 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannel); 1366 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 1367 1368 if (bSubdeviceHandleProvided) 1369 { 1370 NV_PRINTF(LEVEL_ERROR, "Unicast DMA mappings of channels not supported.\n"); 1371 return NV_ERR_NOT_SUPPORTED; 1372 } 1373 1374 1375 if (!kfifoIsUserdMapDmaSupported(pKernelFifo)) 1376 return NV_ERR_INVALID_OBJECT_HANDLE; 1377 1378 return NV_OK; 1379 } 1380 1381 /** 1382 * @brief Creates an iterator to iterate all channels in a given scope. 1383 * 1384 * Iterates over all channels under a given scope. For a device it will loop 1385 * through all channels that are descendants of the device (including children 1386 * of channel groups). For a channel group it will only iterate over the 1387 * channels within that group. Ordering is ensured for channel group. 1388 * All channels within a channel group will be iterated together before moving to 1389 * another channel group or channel. 1390 * 1391 * @param[in] pClient 1392 * @param[in] pScopeRef The resource that defines the scope of iteration 1393 */ 1394 RS_ORDERED_ITERATOR 1395 kchannelGetIter 1396 ( 1397 RsClient *pClient, 1398 RsResourceRef *pScopeRef 1399 ) 1400 { 1401 return clientRefOrderedIter(pClient, pScopeRef, classId(KernelChannel), NV_TRUE); 1402 } 1403 1404 /** 1405 * @brief Given a client, parent, and KernelChannel handle retrieves the 1406 * KernelChannel object 1407 * 1408 * @param[in] hClient 1409 * @param[in] hParent Device or Channel Group parent 1410 * @param[in] hKernelChannel 1411 * @param[out] ppKernelChannel Valid iff NV_OK is returned. 1412 * 1413 * @return NV_OK if successful, appropriate error otherwise 1414 */ 1415 NV_STATUS 1416 CliGetKernelChannelWithDevice 1417 ( 1418 RsClient *pClient, 1419 NvHandle hParent, 1420 NvHandle hKernelChannel, 1421 KernelChannel **ppKernelChannel 1422 ) 1423 { 1424 RsResourceRef *pParentRef; 1425 RsResourceRef *pResourceRef; 1426 KernelChannel *pKernelChannel; 1427 1428 if (ppKernelChannel == NULL) 1429 return NV_ERR_INVALID_ARGUMENT; 1430 1431 *ppKernelChannel = NULL; 1432 1433 NV_ASSERT_OK_OR_RETURN(clientGetResourceRef(pClient, hKernelChannel, &pResourceRef)); 1434 1435 pKernelChannel = dynamicCast(pResourceRef->pResource, KernelChannel); 1436 NV_CHECK_OR_RETURN(LEVEL_INFO, pKernelChannel != NULL, NV_ERR_OBJECT_NOT_FOUND); 1437 1438 pParentRef = pResourceRef->pParentRef; 1439 NV_CHECK_OR_RETURN(LEVEL_INFO, pParentRef != NULL, NV_ERR_OBJECT_NOT_FOUND); 1440 1441 // 1442 // Check that the parent matches requested handle. Parent handle can be a 1443 // device or a ChannelGroup. The first case can match either, the second 1444 // matches a Device when the parent is a ChannelGroup. 1445 // 1446 NV_CHECK_OR_RETURN(LEVEL_INFO, (pParentRef->hResource == hParent) || 1447 (RES_GET_HANDLE(GPU_RES_GET_DEVICE(pKernelChannel)) == hParent), 1448 NV_ERR_OBJECT_NOT_FOUND); 1449 1450 *ppKernelChannel = pKernelChannel; 1451 return NV_OK; 1452 } // end of CliGetKernelChannelWithDevice() 1453 1454 1455 /** 1456 * @brief Given a classNum this routine returns various sdk specific values for 1457 * that class. 1458 * 1459 * @param[in] classNum 1460 * @param[out] pClassInfo 1461 */ 1462 void 1463 CliGetChannelClassInfo 1464 ( 1465 NvU32 classNum, 1466 CLI_CHANNEL_CLASS_INFO *pClassInfo 1467 ) 1468 { 1469 switch (classNum) 1470 { 1471 case GF100_CHANNEL_GPFIFO: 1472 { 1473 pClassInfo->notifiersMaxCount = NV906F_NOTIFIERS_MAXCOUNT; 1474 pClassInfo->eventActionDisable = NV906F_CTRL_EVENT_SET_NOTIFICATION_ACTION_DISABLE; 1475 pClassInfo->eventActionSingle = NV906F_CTRL_EVENT_SET_NOTIFICATION_ACTION_SINGLE; 1476 pClassInfo->eventActionRepeat = NV906F_CTRL_EVENT_SET_NOTIFICATION_ACTION_REPEAT; 1477 pClassInfo->rcNotifierIndex = NV906F_NOTIFIERS_RC; 1478 pClassInfo->classType = CHANNEL_CLASS_TYPE_GPFIFO; 1479 break; 1480 } 1481 case KEPLER_CHANNEL_GPFIFO_A: 1482 { 1483 pClassInfo->notifiersMaxCount = NVA06F_NOTIFIERS_MAXCOUNT; 1484 pClassInfo->eventActionDisable = NVA06F_CTRL_EVENT_SET_NOTIFICATION_ACTION_DISABLE; 1485 pClassInfo->eventActionSingle = NVA06F_CTRL_EVENT_SET_NOTIFICATION_ACTION_SINGLE; 1486 pClassInfo->eventActionRepeat = NVA06F_CTRL_EVENT_SET_NOTIFICATION_ACTION_REPEAT; 1487 pClassInfo->rcNotifierIndex = NVA06F_NOTIFIERS_RC; 1488 pClassInfo->classType = CHANNEL_CLASS_TYPE_GPFIFO; 1489 break; 1490 } 1491 case KEPLER_CHANNEL_GPFIFO_B: 1492 { 1493 pClassInfo->notifiersMaxCount = NVA16F_NOTIFIERS_MAXCOUNT; 1494 pClassInfo->eventActionDisable = NVA16F_CTRL_EVENT_SET_NOTIFICATION_ACTION_DISABLE; 1495 pClassInfo->eventActionSingle = NVA16F_CTRL_EVENT_SET_NOTIFICATION_ACTION_SINGLE; 1496 pClassInfo->eventActionRepeat = NVA16F_CTRL_EVENT_SET_NOTIFICATION_ACTION_REPEAT; 1497 pClassInfo->rcNotifierIndex = NVA16F_NOTIFIERS_RC; 1498 pClassInfo->classType = CHANNEL_CLASS_TYPE_GPFIFO; 1499 break; 1500 } 1501 case MAXWELL_CHANNEL_GPFIFO_A: 1502 { 1503 pClassInfo->notifiersMaxCount = NVB06F_NOTIFIERS_MAXCOUNT; 1504 pClassInfo->eventActionDisable = NVB06F_CTRL_EVENT_SET_NOTIFICATION_ACTION_DISABLE; 1505 pClassInfo->eventActionSingle = NVB06F_CTRL_EVENT_SET_NOTIFICATION_ACTION_SINGLE; 1506 pClassInfo->eventActionRepeat = NVB06F_CTRL_EVENT_SET_NOTIFICATION_ACTION_REPEAT; 1507 pClassInfo->rcNotifierIndex = NVB06F_NOTIFIERS_RC; 1508 pClassInfo->classType = CHANNEL_CLASS_TYPE_GPFIFO; 1509 break; 1510 } 1511 case PASCAL_CHANNEL_GPFIFO_A: 1512 { 1513 pClassInfo->notifiersMaxCount = NVC06F_NOTIFIERS_MAXCOUNT; 1514 pClassInfo->eventActionDisable = NVC06F_CTRL_EVENT_SET_NOTIFICATION_ACTION_DISABLE; 1515 pClassInfo->eventActionSingle = NVC06F_CTRL_EVENT_SET_NOTIFICATION_ACTION_SINGLE; 1516 pClassInfo->eventActionRepeat = NVC06F_CTRL_EVENT_SET_NOTIFICATION_ACTION_REPEAT; 1517 pClassInfo->rcNotifierIndex = NVC06F_NOTIFIERS_RC; 1518 pClassInfo->classType = CHANNEL_CLASS_TYPE_GPFIFO; 1519 break; 1520 } 1521 case VOLTA_CHANNEL_GPFIFO_A: 1522 { 1523 pClassInfo->notifiersMaxCount = NVC36F_NOTIFIERS_MAXCOUNT; 1524 pClassInfo->eventActionDisable = NVC36F_CTRL_EVENT_SET_NOTIFICATION_ACTION_DISABLE; 1525 pClassInfo->eventActionSingle = NVC36F_CTRL_EVENT_SET_NOTIFICATION_ACTION_SINGLE; 1526 pClassInfo->eventActionRepeat = NVC36F_CTRL_EVENT_SET_NOTIFICATION_ACTION_REPEAT; 1527 pClassInfo->rcNotifierIndex = NVC36F_NOTIFIERS_RC; 1528 pClassInfo->classType = CHANNEL_CLASS_TYPE_GPFIFO; 1529 break; 1530 } 1531 case TURING_CHANNEL_GPFIFO_A: 1532 { 1533 pClassInfo->notifiersMaxCount = NVC46F_NOTIFIERS_MAXCOUNT; 1534 pClassInfo->eventActionDisable = NVC46F_CTRL_EVENT_SET_NOTIFICATION_ACTION_DISABLE; 1535 pClassInfo->eventActionSingle = NVC46F_CTRL_EVENT_SET_NOTIFICATION_ACTION_SINGLE; 1536 pClassInfo->eventActionRepeat = NVC46F_CTRL_EVENT_SET_NOTIFICATION_ACTION_REPEAT; 1537 pClassInfo->rcNotifierIndex = NVC46F_NOTIFIERS_RC; 1538 pClassInfo->classType = CHANNEL_CLASS_TYPE_GPFIFO; 1539 break; 1540 } 1541 case AMPERE_CHANNEL_GPFIFO_A: 1542 { 1543 pClassInfo->notifiersMaxCount = NVC56F_NOTIFIERS_MAXCOUNT; 1544 pClassInfo->eventActionDisable = NVC56F_CTRL_EVENT_SET_NOTIFICATION_ACTION_DISABLE; 1545 pClassInfo->eventActionSingle = NVC56F_CTRL_EVENT_SET_NOTIFICATION_ACTION_SINGLE; 1546 pClassInfo->eventActionRepeat = NVC56F_CTRL_EVENT_SET_NOTIFICATION_ACTION_REPEAT; 1547 pClassInfo->rcNotifierIndex = NVC56F_NOTIFIERS_RC; 1548 pClassInfo->classType = CHANNEL_CLASS_TYPE_GPFIFO; 1549 break; 1550 } 1551 case HOPPER_CHANNEL_GPFIFO_A: 1552 { 1553 pClassInfo->notifiersMaxCount = NVC86F_NOTIFIERS_MAXCOUNT; 1554 pClassInfo->eventActionDisable = NVC86F_CTRL_EVENT_SET_NOTIFICATION_ACTION_DISABLE; 1555 pClassInfo->eventActionSingle = NVC86F_CTRL_EVENT_SET_NOTIFICATION_ACTION_SINGLE; 1556 pClassInfo->eventActionRepeat = NVC86F_CTRL_EVENT_SET_NOTIFICATION_ACTION_REPEAT; 1557 pClassInfo->rcNotifierIndex = NVC86F_NOTIFIERS_RC; 1558 pClassInfo->classType = CHANNEL_CLASS_TYPE_GPFIFO; 1559 break; 1560 } 1561 1562 // 1563 // Does not make sense. Call with the class type from the client not the 1564 // internal type 1565 // 1566 case PHYSICAL_CHANNEL_GPFIFO: 1567 NV_PRINTF(LEVEL_ERROR, 1568 "Invalid class for CliGetChannelClassInfo\n"); 1569 1570 default: 1571 { 1572 pClassInfo->notifiersMaxCount = 0; 1573 pClassInfo->eventActionDisable = 0; 1574 pClassInfo->eventActionSingle = 0; 1575 pClassInfo->eventActionRepeat = 0; 1576 pClassInfo->rcNotifierIndex = 0; 1577 pClassInfo->classType = CHANNEL_CLASS_TYPE_DMA; 1578 break; 1579 } 1580 } 1581 } 1582 1583 1584 /** 1585 * @brief Returns the next KernelChannel from the iterator. 1586 * 1587 * Iterates over runlist IDs and ChIDs and returns the next KernelChannel found 1588 * on the heap, if any. 1589 * 1590 * (error guaranteed if pointer is NULL; non-NULL pointer guaranteed if NV_OK) 1591 * 1592 * @param[in] pGpu 1593 * @param[in] pIt the channel iterator 1594 * @param[out] ppKernelChannel returns a KernelChannel * 1595 * 1596 * @return NV_OK if the returned pointer is valid or error 1597 */ 1598 NV_STATUS kchannelGetNextKernelChannel 1599 ( 1600 OBJGPU *pGpu, 1601 CHANNEL_ITERATOR *pIt, 1602 KernelChannel **ppKernelChannel 1603 ) 1604 { 1605 KernelChannel *pKernelChannel; 1606 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 1607 1608 if (ppKernelChannel == NULL) 1609 return NV_ERR_INVALID_ARGUMENT; 1610 1611 *ppKernelChannel = NULL; 1612 1613 while (pIt->runlistId < pIt->numRunlists) 1614 { 1615 CHID_MGR *pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, pIt->runlistId); 1616 1617 if (pChidMgr == NULL) 1618 { 1619 pIt->runlistId++; 1620 continue; 1621 } 1622 1623 pIt->numChannels = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr); 1624 while (pIt->physicalChannelID < pIt->numChannels) 1625 { 1626 pKernelChannel = kfifoChidMgrGetKernelChannel(pGpu, pKernelFifo, 1627 pChidMgr, pIt->physicalChannelID); 1628 pIt->physicalChannelID++; 1629 1630 // 1631 // This iterator can be used during an interrupt, when a KernelChannel may 1632 // be in the process of being destroyed. Don't return it if so. 1633 // 1634 if (pKernelChannel == NULL) 1635 continue; 1636 if (!kchannelIsValid_HAL(pKernelChannel)) 1637 continue; 1638 1639 *ppKernelChannel = pKernelChannel; 1640 return NV_OK; 1641 } 1642 1643 pIt->runlistId++; 1644 // Reset channel index to 0 for next runlist 1645 pIt->physicalChannelID = 0; 1646 } 1647 1648 return NV_ERR_OBJECT_NOT_FOUND; 1649 } 1650 1651 /** 1652 * @brief Finds the corresponding KernelChannel given client object and channel handle 1653 * 1654 * Looks in client object store for the channel handle. Scales with total 1655 * number of registered objects in the client, not just the number of channels. 1656 * 1657 * @param[in] pClient 1658 * @param[in] hKernelChannel a KernelChannel Channel handle 1659 * @param[out] ppKernelChannel 1660 * 1661 * @return NV_STATUS 1662 */ 1663 NV_STATUS 1664 CliGetKernelChannel 1665 ( 1666 RsClient *pClient, 1667 NvHandle hKernelChannel, 1668 KernelChannel **ppKernelChannel 1669 ) 1670 { 1671 NV_STATUS status; 1672 RsResourceRef *pResourceRef; 1673 1674 *ppKernelChannel = NULL; 1675 1676 status = clientGetResourceRef(pClient, hKernelChannel, &pResourceRef); 1677 if (status != NV_OK) 1678 { 1679 return status; 1680 } 1681 1682 *ppKernelChannel = dynamicCast(pResourceRef->pResource, KernelChannel); 1683 NV_CHECK_OR_RETURN(LEVEL_INFO, 1684 *ppKernelChannel != NULL, 1685 NV_ERR_INVALID_CHANNEL); 1686 return NV_OK; 1687 } 1688 1689 /*! 1690 * @brief Notify client that channel is stopped. 1691 * 1692 * @param[in] pKernelChannnel 1693 */ 1694 NV_STATUS 1695 kchannelNotifyRc_IMPL 1696 ( 1697 KernelChannel *pKernelChannel 1698 ) 1699 { 1700 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannel); 1701 RM_ENGINE_TYPE rmEngineType = RM_ENGINE_TYPE_NULL; 1702 NV_STATUS rmStatus = NV_OK; 1703 1704 if (IS_GFID_VF(kchannelGetGfid(pKernelChannel))) 1705 { 1706 NV_PRINTF(LEVEL_INFO, "Notification for channel 0x%x stop is already performed on guest-RM\n", 1707 kchannelGetDebugTag(pKernelChannel)); 1708 return NV_OK; 1709 } 1710 1711 if (pKernelChannel->hErrorContext == NV01_NULL_OBJECT && 1712 pKernelChannel->hEccErrorContext == NV01_NULL_OBJECT) 1713 { 1714 NV_PRINTF(LEVEL_WARNING, "Channel 0x%x has no notifier set\n", 1715 kchannelGetDebugTag(pKernelChannel)); 1716 return NV_OK; 1717 } 1718 1719 if (RM_ENGINE_TYPE_IS_VALID(kchannelGetEngineType(pKernelChannel))) 1720 { 1721 rmEngineType = kchannelGetEngineType(pKernelChannel); 1722 } 1723 rmStatus = krcErrorSetNotifier(pGpu, GPU_GET_KERNEL_RC(pGpu), 1724 pKernelChannel, 1725 ROBUST_CHANNEL_PREEMPTIVE_REMOVAL, 1726 rmEngineType, 1727 RC_NOTIFIER_SCOPE_CHANNEL); 1728 if (rmStatus != NV_OK) 1729 { 1730 NV_PRINTF(LEVEL_ERROR, 1731 "Failed to set error notifier for channel 0x%x with error 0x%x.\n", 1732 kchannelGetDebugTag(pKernelChannel), rmStatus); 1733 } 1734 return rmStatus; 1735 } 1736 1737 /** 1738 * @brief Sends event corresponding to index to notify clients 1739 * 1740 * @param[in] pKernelChannel 1741 * @param[in] notifyIndex 1742 * @param[in] pNotifyParams 1743 * @parms[in] notifyParamsSize 1744 */ 1745 void kchannelNotifyEvent_IMPL 1746 ( 1747 KernelChannel *pKernelChannel, 1748 NvU32 notifyIndex, 1749 NvU32 info32, 1750 NvU16 info16, 1751 void *pNotifyParams, 1752 NvU32 notifyParamsSize 1753 ) 1754 { 1755 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannel); 1756 ContextDma *pContextDma; 1757 EVENTNOTIFICATION *pEventNotification; 1758 CLI_CHANNEL_CLASS_INFO classInfo; 1759 1760 CliGetChannelClassInfo(RES_GET_EXT_CLASS_ID(pKernelChannel), &classInfo); 1761 1762 // validate notifyIndex 1763 NV_CHECK_OR_RETURN_VOID(LEVEL_INFO, notifyIndex < classInfo.notifiersMaxCount); 1764 1765 // handle notification if client wants it 1766 if (pKernelChannel->pNotifyActions[notifyIndex] != classInfo.eventActionDisable) 1767 { 1768 // get notifier context dma for the channel 1769 if (ctxdmaGetByHandle(RES_GET_CLIENT(pKernelChannel), 1770 pKernelChannel->hErrorContext, 1771 &pContextDma) == NV_OK) 1772 { 1773 // make sure it's big enough 1774 if (pContextDma->Limit >= 1775 ((classInfo.notifiersMaxCount * sizeof (NvNotification)) - 1)) 1776 { 1777 // finally, write out the notifier 1778 notifyFillNotifierArray(pGpu, pContextDma, 1779 0x0, 0x0, 0x0, 1780 notifyIndex); 1781 } 1782 } 1783 } 1784 1785 // handle event if client wants it 1786 pEventNotification = inotifyGetNotificationList(staticCast(pKernelChannel, INotifier)); 1787 if (pEventNotification != NULL) 1788 { 1789 NV_PRINTF(LEVEL_INFO, "Posting event on channel = 0x%x with info16 = 0x%x\n", 1790 kchannelGetDebugTag(pKernelChannel), (NvU32)info16); 1791 // ping any events on the list of type notifyIndex 1792 osEventNotificationWithInfo(pGpu, pEventNotification, notifyIndex, info32, info16, 1793 pNotifyParams, notifyParamsSize); 1794 } 1795 else 1796 { 1797 NV_PRINTF(LEVEL_INFO, "No event on channel = 0x%x\n", kchannelGetDebugTag(pKernelChannel)); 1798 } 1799 1800 // reset if single shot notify action 1801 if (pKernelChannel->pNotifyActions[notifyIndex] == classInfo.eventActionSingle) 1802 pKernelChannel->pNotifyActions[notifyIndex] = classInfo.eventActionDisable; 1803 1804 return; 1805 } 1806 1807 /** 1808 * @brief Writes notifier memory at given index with given info 1809 * 1810 * @param[in] pKernelChannel 1811 * @param[in] notifyIndex 1812 * @param[in] info32 1813 * @param[in] info16 1814 * @parms[in] notifierStatus 1815 */ 1816 NV_STATUS kchannelUpdateNotifierMem_IMPL 1817 ( 1818 KernelChannel *pKernelChannel, 1819 NvU32 notifyIndex, 1820 NvU32 info32, 1821 NvU16 info16, 1822 NvU32 notifierStatus 1823 ) 1824 { 1825 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannel); 1826 MEMORY_DESCRIPTOR *pNotifierMemDesc = pKernelChannel->pErrContextMemDesc; 1827 NV_ADDRESS_SPACE addressSpace; 1828 OBJTMR *pTmr = GPU_GET_TIMER(pGpu); 1829 NvU64 time; 1830 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 1831 KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu); 1832 TRANSFER_SURFACE surf = {0}; 1833 NvNotification *pNotifier = NULL; 1834 NvBool bMemEndTransfer = NV_FALSE; 1835 1836 if (pNotifierMemDesc == NULL) 1837 return NV_OK; 1838 1839 addressSpace = memdescGetAddressSpace(pNotifierMemDesc); 1840 if (RMCFG_FEATURE_PLATFORM_GSP) 1841 NV_ASSERT_OR_RETURN(addressSpace == ADDR_FBMEM, NV_ERR_INVALID_STATE); 1842 1843 // 1844 // If clients did not allocate enough memory for the doorbell 1845 // notifier, return NV_OK so as not to regress older clients 1846 // 1847 NV_CHECK_OR_RETURN(LEVEL_INFO, memdescGetSize(pNotifierMemDesc) >= (notifyIndex + 1) * sizeof(NvNotification), NV_OK); 1848 1849 // 1850 // we rely on persistent mapping for key rotation notifier 1851 // since this may be called in top half and mappings are not allowed 1852 // in contexts that can't sleep on KVM or similar HCC systems. 1853 // 1854 ConfidentialCompute *pConfCompute = GPU_GET_CONF_COMPUTE(pGpu); 1855 if ((pConfCompute != NULL) && 1856 (pConfCompute->getProperty(pConfCompute, PDB_PROP_CONFCOMPUTE_KEY_ROTATION_SUPPORTED)) && 1857 (notifyIndex == NV_CHANNELGPFIFO_NOTIFICATION_TYPE_KEY_ROTATION_STATUS)) 1858 { 1859 pNotifier = _kchannelGetKeyRotationNotifier(pKernelChannel); 1860 NV_ASSERT_OR_RETURN(pNotifier != NULL, NV_ERR_INVALID_STATE); 1861 bMemEndTransfer = NV_FALSE; 1862 } 1863 else 1864 { 1865 pNotifier = (NvNotification *)memdescGetKernelMapping(pNotifierMemDesc); 1866 if (pNotifier == NULL) 1867 { 1868 surf.pMemDesc = pNotifierMemDesc; 1869 surf.offset = notifyIndex * sizeof(NvNotification); 1870 1871 pNotifier = 1872 (NvNotification *) memmgrMemBeginTransfer(pMemoryManager, &surf, 1873 sizeof(NvNotification), 1874 TRANSFER_FLAGS_SHADOW_ALLOC); 1875 NV_ASSERT_OR_RETURN(pNotifier != NULL, NV_ERR_INVALID_STATE); 1876 bMemEndTransfer = NV_TRUE; 1877 } 1878 else 1879 { 1880 // 1881 // If a CPU pointer has been passed by caller ensure that the notifier 1882 // is in sysmem or in case it in vidmem, BAR access to the same is not 1883 // blocked (for HCC) 1884 // 1885 NV_ASSERT_OR_RETURN( 1886 memdescGetAddressSpace(pNotifierMemDesc) == ADDR_SYSMEM || 1887 !kbusIsBarAccessBlocked(pKernelBus), NV_ERR_INVALID_ARGUMENT); 1888 pNotifier = &pNotifier[notifyIndex]; 1889 } 1890 } 1891 1892 tmrGetCurrentTime(pTmr, &time); 1893 1894 notifyFillNvNotification(pGpu, pNotifier, info32, info16, 1895 notifierStatus, NV_TRUE, time); 1896 1897 if (bMemEndTransfer) 1898 { 1899 memmgrMemEndTransfer(pMemoryManager, &surf, sizeof(NvNotification), 0); 1900 } 1901 return NV_OK; 1902 } 1903 1904 /*! 1905 * @brief Stop channel and notify client 1906 * 1907 * @param[in] pKernelChannnel 1908 * @param[in] pStopChannelParams 1909 */ 1910 NV_STATUS 1911 kchannelCtrlCmdStopChannel_IMPL 1912 ( 1913 KernelChannel *pKernelChannel, 1914 NVA06F_CTRL_STOP_CHANNEL_PARAMS *pStopChannelParams 1915 ) 1916 { 1917 NV_STATUS rmStatus = NV_OK; 1918 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannel); 1919 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 1920 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams; 1921 1922 if (IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu)) 1923 { 1924 NV_RM_RPC_CONTROL(pGpu, 1925 pRmCtrlParams->hClient, 1926 RES_GET_HANDLE(pKernelChannel), 1927 pRmCtrlParams->cmd, 1928 pRmCtrlParams->pParams, 1929 pRmCtrlParams->paramsSize, 1930 rmStatus); 1931 if (rmStatus != NV_OK) 1932 return rmStatus; 1933 } 1934 else 1935 { 1936 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, 1937 kchannelFwdToInternalCtrl_HAL(pGpu, 1938 pKernelChannel, 1939 NVA06F_CTRL_CMD_INTERNAL_STOP_CHANNEL, 1940 pRmCtrlParams)); 1941 } 1942 1943 NV_ASSERT_OK_OR_RETURN(kchannelNotifyRc_HAL(pKernelChannel)); 1944 1945 return NV_OK; 1946 } 1947 1948 /*! 1949 * @brief Helper to get type and memdesc of a channel notifier (memory/ctxdma) 1950 */ 1951 NV_STATUS 1952 kchannelGetNotifierInfo 1953 ( 1954 OBJGPU *pGpu, 1955 Device *pDevice, 1956 NvHandle hErrorContext, 1957 MEMORY_DESCRIPTOR **ppMemDesc, 1958 ErrorNotifierType *pNotifierType, 1959 NvU64 *pOffset 1960 ) 1961 { 1962 RsClient *pRsClient = RES_GET_CLIENT(pDevice); 1963 NvHandle hDevice = RES_GET_HANDLE(pDevice); 1964 ContextDma *pContextDma = NULL; 1965 Memory *pMemory = NULL; 1966 1967 NV_ASSERT_OR_RETURN(ppMemDesc != NULL, NV_ERR_INVALID_PARAMETER); 1968 NV_ASSERT_OR_RETURN(pNotifierType != NULL, NV_ERR_INVALID_PARAMETER); 1969 1970 *ppMemDesc = NULL; 1971 *pNotifierType = ERROR_NOTIFIER_TYPE_UNKNOWN; 1972 *pOffset = 0; 1973 1974 if (hErrorContext == NV01_NULL_OBJECT) 1975 { 1976 *pNotifierType = ERROR_NOTIFIER_TYPE_NONE; 1977 return NV_OK; 1978 } 1979 1980 if (memGetByHandleAndDevice(pRsClient, hErrorContext, hDevice, &pMemory) == 1981 NV_OK) 1982 { 1983 if (memdescGetAddressSpace(pMemory->pMemDesc) == ADDR_VIRTUAL) 1984 { 1985 // 1986 // GPUVA case: Get the underlying DMA mapping in this case. In GSP 1987 // client mode + SLI, GSP won't be able to write to notifiers on 1988 // other GPUs. 1989 // 1990 NvU64 offset; 1991 NvU32 subdeviceInstance; 1992 NvU64 notifyGpuVA = memdescGetPhysAddr(pMemory->pMemDesc, 1993 AT_GPU_VA, 0); 1994 CLI_DMA_MAPPING_INFO *pDmaMappingInfo; 1995 NvBool bFound; 1996 1997 bFound = CliGetDmaMappingInfo( 1998 pRsClient, 1999 RES_GET_HANDLE(pDevice), 2000 RES_GET_HANDLE(pMemory), 2001 notifyGpuVA, 2002 gpumgrGetDeviceGpuMask(pGpu->deviceInstance), 2003 &pDmaMappingInfo); 2004 2005 if (!bFound) 2006 { 2007 NV_PRINTF(LEVEL_ERROR, 2008 "Cannot find DMA mapping for GPU_VA notifier\n"); 2009 return NV_ERR_INVALID_STATE; 2010 } 2011 2012 offset = notifyGpuVA - pDmaMappingInfo->DmaOffset; 2013 if (offset + sizeof(NOTIFICATION) > pDmaMappingInfo->pMemDesc->Size) 2014 { 2015 NV_PRINTF(LEVEL_ERROR, 2016 "Notifier does not fit within DMA mapping for GPU_VA\n"); 2017 return NV_ERR_INVALID_STATE; 2018 } 2019 2020 subdeviceInstance = gpumgrGetSubDeviceInstanceFromGpu( 2021 gpumgrGetParentGPU(pGpu)); 2022 SLI_LOOP_START(SLI_LOOP_FLAGS_NONE) 2023 if (IsSLIEnabled(pGpu) && IS_GSP_CLIENT(pGpu)) 2024 { 2025 NV_PRINTF(LEVEL_ERROR, "GSP does not support SLI\n"); 2026 return NV_ERR_NOT_SUPPORTED; 2027 } 2028 SLI_LOOP_END 2029 2030 if (!pDmaMappingInfo->KernelVAddr[subdeviceInstance]) 2031 { 2032 NV_PRINTF(LEVEL_ERROR, 2033 "Kernel VA addr mapping not present for notifier\n"); 2034 return NV_ERR_INVALID_STATE; 2035 } 2036 *ppMemDesc = pDmaMappingInfo->pMemDesc; 2037 // The notifier format here is struct NOTIFICATION, same as ctxdma 2038 *pNotifierType = ERROR_NOTIFIER_TYPE_CTXDMA; 2039 *pOffset = offset; 2040 } 2041 else 2042 { 2043 *ppMemDesc = pMemory->pMemDesc; 2044 *pNotifierType = ERROR_NOTIFIER_TYPE_MEMORY; 2045 } 2046 return NV_OK; 2047 } 2048 2049 if (ctxdmaGetByHandle(pRsClient, hErrorContext, &pContextDma) == NV_OK) 2050 { 2051 *ppMemDesc = pContextDma->pMemDesc; 2052 *pNotifierType = ERROR_NOTIFIER_TYPE_CTXDMA; 2053 return NV_OK; 2054 } 2055 2056 return NV_ERR_OBJECT_NOT_FOUND; 2057 } 2058 2059 /*! 2060 * @brief Check if the client that owns this channel is in user mode. 2061 * 2062 * This replaces using call context for privilege checking, 2063 * and is callable from both CPU and GSP. 2064 * 2065 * @param[in] pGpu 2066 * @param[in] pKernelChannel 2067 * 2068 * @returns NV_TRUE if owned by user mode or NV_FALSE. 2069 */ 2070 NvBool 2071 kchannelCheckIsUserMode_IMPL 2072 ( 2073 KernelChannel *pKernelChannel 2074 ) 2075 { 2076 return (pKernelChannel->privilegeLevel == NV_KERNELCHANNEL_ALLOC_INTERNALFLAGS_PRIVILEGE_USER) || 2077 (pKernelChannel->privilegeLevel == NV_KERNELCHANNEL_ALLOC_INTERNALFLAGS_PRIVILEGE_ADMIN); 2078 } 2079 2080 /*! 2081 * @brief Check if the client that owns this channel is kernel. 2082 * 2083 * This replaces using call context for privilege checking, 2084 * and is callable from both CPU and GSP. 2085 * 2086 * @param[in] pGpu 2087 * @param[in] pKernelChannel 2088 * 2089 * @returns NV_TRUE if owned by kernel or NV_FALSE. 2090 */ 2091 NvBool 2092 kchannelCheckIsKernel_IMPL 2093 ( 2094 KernelChannel *pKernelChannel 2095 ) 2096 { 2097 return pKernelChannel->privilegeLevel == NV_KERNELCHANNEL_ALLOC_INTERNALFLAGS_PRIVILEGE_KERNEL; 2098 } 2099 2100 /*! 2101 * @brief Check if the client that owns this channel is admin. 2102 * 2103 * This replaces using call context for admin privilege checking, 2104 * but is callable from both CPU and GSP. 2105 * 2106 * @param[in] pGpu 2107 * @param[in] pKernelChannel 2108 * 2109 * @returns NV_TRUE if owned by admin or NV_FALSE. 2110 */ 2111 NvBool 2112 kchannelCheckIsAdmin_IMPL 2113 ( 2114 KernelChannel *pKernelChannel 2115 ) 2116 { 2117 return (pKernelChannel->privilegeLevel == NV_KERNELCHANNEL_ALLOC_INTERNALFLAGS_PRIVILEGE_KERNEL) || 2118 (pKernelChannel->privilegeLevel == NV_KERNELCHANNEL_ALLOC_INTERNALFLAGS_PRIVILEGE_ADMIN); 2119 } 2120 2121 2122 /*! 2123 * @brief Check if the channel is bound to its resources. 2124 * 2125 * This is to make sure channel went through the UVM registration step before it can be scheduled. 2126 * This applies only to UVM owned channels. 2127 * 2128 * @param[in] pKernelChannel 2129 * @param[in] pGVAS 2130 * 2131 * @returns NV_TRUE if bound. 2132 */ 2133 NvBool 2134 kchannelIsSchedulable_IMPL 2135 ( 2136 OBJGPU *pGpu, 2137 KernelChannel *pKernelChannel 2138 ) 2139 { 2140 OBJGVASPACE *pGVAS = NULL; 2141 NvU32 engineDesc = 0; 2142 NvU32 gfId; 2143 2144 gfId = kchannelGetGfid(pKernelChannel); 2145 if (IS_GFID_VF(gfId)) 2146 { 2147 NV_PRINTF(LEVEL_INFO, "Check for channel schedulability for channel 0x%x is already performed on guest-RM\n", 2148 kchannelGetDebugTag(pKernelChannel)); 2149 return NV_TRUE; 2150 } 2151 2152 pGVAS = dynamicCast(pKernelChannel->pVAS, OBJGVASPACE); 2153 2154 // 2155 // It should be an error to have allocated and attempt to schedule a 2156 // channel without having allocated a GVAS. We ignore this check on 2157 // AMODEL, which has its own dummy AVAS. 2158 // 2159 NV_ASSERT_OR_RETURN(pGVAS != NULL || IS_MODS_AMODEL(pGpu), NV_FALSE); 2160 2161 NV_ASSERT_OR_RETURN(kchannelGetEngine_HAL(pGpu, pKernelChannel, &engineDesc) == NV_OK, NV_FALSE); 2162 2163 if (pGVAS != NULL && gvaspaceIsExternallyOwned(pGVAS) && IS_GR(engineDesc) && !pKernelChannel->bIsContextBound) 2164 { 2165 NV_PRINTF(LEVEL_ERROR, 2166 "Cannot schedule externally-owned channel with unbound allocations :0x%x!\n", 2167 kchannelGetDebugTag(pKernelChannel)); 2168 return NV_FALSE; 2169 } 2170 return NV_TRUE; 2171 } 2172 2173 // Alloc pFifoHalData 2174 static NV_STATUS 2175 _kchannelAllocHalData 2176 ( 2177 OBJGPU *pGpu, 2178 KernelChannel *pKernelChannel 2179 ) 2180 { 2181 portMemSet(pKernelChannel->pFifoHalData, 0, sizeof(pKernelChannel->pFifoHalData)); 2182 2183 // Alloc 1 page of instmem per GPU instance 2184 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 2185 2186 pKernelChannel->pFifoHalData[gpumgrGetSubDeviceInstanceFromGpu(pGpu)] = portMemAllocNonPaged(sizeof(FIFO_INSTANCE_BLOCK)); 2187 2188 NV_ASSERT_OR_ELSE(pKernelChannel->pFifoHalData[gpumgrGetSubDeviceInstanceFromGpu(pGpu)] != NULL, 2189 SLI_LOOP_GOTO(failed)); 2190 2191 portMemSet(pKernelChannel->pFifoHalData[gpumgrGetSubDeviceInstanceFromGpu(pGpu)], 0, sizeof(FIFO_INSTANCE_BLOCK)); 2192 2193 SLI_LOOP_END 2194 2195 return NV_OK; 2196 2197 failed: 2198 DBG_BREAKPOINT(); 2199 _kchannelFreeHalData(pGpu, pKernelChannel); 2200 return NV_ERR_NO_MEMORY; 2201 } 2202 2203 // Free memdescs and pFifoHalData, if any 2204 static void 2205 _kchannelFreeHalData 2206 ( 2207 OBJGPU *pGpu, 2208 KernelChannel *pKernelChannel 2209 ) 2210 { 2211 // Unmap / delete memdescs 2212 kchannelDestroyMem_HAL(pGpu, pKernelChannel); 2213 2214 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 2215 kchannelDestroyUserdMemDesc(pGpu, pKernelChannel); 2216 2217 // Free pFifoHalData 2218 portMemFree(pKernelChannel->pFifoHalData[gpumgrGetSubDeviceInstanceFromGpu(pGpu)]); 2219 pKernelChannel->pFifoHalData[gpumgrGetSubDeviceInstanceFromGpu(pGpu)] = NULL; 2220 SLI_LOOP_END 2221 } 2222 2223 // Returns the proper VerifFlags for kchannelAllocMem 2224 static NvU32 2225 _kchannelgetVerifFlags 2226 ( 2227 OBJGPU *pGpu, 2228 NV_CHANNEL_ALLOC_PARAMS *pChannelGpfifoParams 2229 ) 2230 { 2231 NvU32 verifFlags = 0; 2232 2233 return verifFlags; 2234 } 2235 2236 // Allocate and describe instance memory 2237 static NV_STATUS 2238 _kchannelAllocOrDescribeInstMem 2239 ( 2240 KernelChannel *pKernelChannel, 2241 NV_CHANNEL_ALLOC_PARAMS *pChannelGpfifoParams 2242 ) 2243 { 2244 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannel); 2245 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 2246 KernelChannelGroupApi *pKernelChannelGroupApi = pKernelChannel->pKernelChannelGroupApi; 2247 KernelChannelGroup *pKernelChannelGroup = pKernelChannelGroupApi->pKernelChannelGroup; 2248 NvU32 gfid = pKernelChannelGroup->gfid; 2249 NV_STATUS status; 2250 NvHandle hClient = RES_GET_CLIENT_HANDLE(pKernelChannel); 2251 2252 // Alloc pFifoHalData 2253 NV_ASSERT_OK_OR_RETURN(_kchannelAllocHalData(pGpu, pKernelChannel)); 2254 2255 // 2256 // GSP RM and host RM on full SRIOV setup will not be aware of the client allocated userd handles, 2257 // translate the handle on client GSP. GSP RM or host RM on full SRIOV setup will get the translated 2258 // addresses which it will later memdescribe. 2259 // 2260 // However it is still client allocated userd from GSP RM or host RM on full SRIOV setup 2261 // perspective so set the flag accordingly. 2262 // 2263 if (!RMCFG_FEATURE_PLATFORM_GSP && 2264 !(IS_GFID_VF(gfid) && !gpuIsWarBug200577889SriovHeavyEnabled(pGpu))) 2265 { 2266 pKernelChannel->bClientAllocatedUserD = NV_FALSE; 2267 NV_ASSERT_OK_OR_GOTO(status, 2268 kchannelCreateUserdMemDescBc_HAL(pGpu, pKernelChannel, hClient, 2269 pChannelGpfifoParams->hUserdMemory, 2270 pChannelGpfifoParams->userdOffset), 2271 failed); 2272 } 2273 else 2274 { 2275 pKernelChannel->bClientAllocatedUserD = NV_TRUE; 2276 } 2277 2278 // Alloc/describe instmem memdescs depending on platform 2279 if (IS_VIRTUAL_WITH_SRIOV(pGpu) && gpuIsWarBug200577889SriovHeavyEnabled(pGpu)) 2280 { 2281 // On Heavy SRIOV, describe memdescs using RPC 2282 NV_ASSERT_OK_OR_GOTO(status, 2283 _kchannelDescribeMemDescsHeavySriov(pGpu, pKernelChannel), 2284 failed); 2285 } 2286 else if (RMCFG_FEATURE_PLATFORM_GSP || 2287 (IS_GFID_VF(gfid) && !gpuIsWarBug200577889SriovHeavyEnabled(pGpu))) 2288 { 2289 // On GSPFW or non-heavy SRIOV, describe memdescs from params 2290 NV_ASSERT_OK_OR_GOTO(status, 2291 _kchannelDescribeMemDescsFromParams(pGpu, pKernelChannel, pChannelGpfifoParams), 2292 failed); 2293 } 2294 else if (!IS_VIRTUAL_WITHOUT_SRIOV(pGpu)) 2295 { 2296 // On baremetal, GSP client, or SRIOV host, alloc mem 2297 NV_ASSERT_OK_OR_GOTO(status, 2298 kchannelAllocMem_HAL(pGpu, 2299 pKernelChannel, 2300 pChannelGpfifoParams->flags, 2301 _kchannelgetVerifFlags(pGpu, pChannelGpfifoParams)), 2302 failed); 2303 } 2304 2305 // Setup USERD 2306 if (IS_VIRTUAL(pGpu)) 2307 { 2308 PMEMORY_DESCRIPTOR pUserdSubDeviceMemDesc = 2309 pKernelChannel->pUserdSubDeviceMemDesc[gpumgrGetSubDeviceInstanceFromGpu(pGpu)]; 2310 NvBool bFullSriov = IS_VIRTUAL_WITH_SRIOV(pGpu) && 2311 !gpuIsWarBug200577889SriovHeavyEnabled(pGpu); 2312 2313 // Clear Userd if it is in FB for SRIOV environment without BUG 200577889 or if in SYSMEM 2314 if (pUserdSubDeviceMemDesc != NULL && 2315 ((memdescGetAddressSpace(pUserdSubDeviceMemDesc) == ADDR_SYSMEM) 2316 || ((memdescGetAddressSpace(pUserdSubDeviceMemDesc) == ADDR_FBMEM) && bFullSriov))) 2317 { 2318 kfifoSetupUserD_HAL(pGpu, pKernelFifo, pUserdSubDeviceMemDesc); 2319 } 2320 } 2321 return NV_OK; 2322 2323 failed: 2324 _kchannelFreeHalData(pGpu, pKernelChannel); 2325 return status; 2326 } 2327 2328 /** 2329 * @brief Create and describe channel instance memory ramfc and userd memdescs 2330 * Done using info in pChanGpfifoParams 2331 * 2332 * @param pGpu : OBJGPU pointer 2333 * @param pKernelChannel : KernelChannel pointer 2334 * @param pChanGpfifoParams : Pointer to channel allocation params 2335 */ 2336 static NV_STATUS 2337 _kchannelDescribeMemDescsFromParams 2338 ( 2339 OBJGPU *pGpu, 2340 KernelChannel *pKernelChannel, 2341 NV_CHANNEL_ALLOC_PARAMS *pChannelGpfifoParams 2342 ) 2343 { 2344 NV_STATUS status = NV_OK; 2345 FIFO_INSTANCE_BLOCK *pInstanceBlock = NULL; 2346 NvU32 subDevInst; 2347 NvU32 gfid = GPU_GFID_PF; 2348 NvU32 runqueue; 2349 KernelChannelGroupApi *pKernelChannelGroupApi = 2350 pKernelChannel->pKernelChannelGroupApi; 2351 2352 NV_ASSERT_OR_RETURN((pKernelChannelGroupApi != NULL), NV_ERR_INVALID_STATE); 2353 gfid = pKernelChannelGroupApi->pKernelChannelGroup->gfid; 2354 2355 NV_ASSERT_OR_RETURN(RMCFG_FEATURE_PLATFORM_GSP || 2356 (IS_GFID_VF(gfid) && !gpuIsWarBug200577889SriovHeavyEnabled(pGpu)), 2357 NV_ERR_INVALID_STATE); 2358 2359 NV_ASSERT_OR_RETURN((pChannelGpfifoParams != NULL), NV_ERR_INVALID_ARGUMENT); 2360 2361 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 2362 2363 subDevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu); 2364 2365 pInstanceBlock = (FIFO_INSTANCE_BLOCK*) pKernelChannel->pFifoHalData[subDevInst]; 2366 2367 // Create memory descriptor for the instance memory 2368 status = memdescCreate(&pInstanceBlock->pInstanceBlockDesc, pGpu, 2369 pChannelGpfifoParams->instanceMem.size, 1 , NV_TRUE, 2370 pChannelGpfifoParams->instanceMem.addressSpace, 2371 pChannelGpfifoParams->instanceMem.cacheAttrib, 2372 MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE); 2373 2374 if (status != NV_OK) 2375 { 2376 NV_PRINTF(LEVEL_ERROR, 2377 "Unable to allocate instance memory descriptor!\n"); 2378 SLI_LOOP_RETURN(status); 2379 } 2380 2381 memdescDescribe(pInstanceBlock->pInstanceBlockDesc, pChannelGpfifoParams->instanceMem.addressSpace, 2382 pChannelGpfifoParams->instanceMem.base, pChannelGpfifoParams->instanceMem.size); 2383 2384 2385 // Create memory descriptor for the ramfc 2386 status = memdescCreate(&pInstanceBlock->pRamfcDesc, pGpu, 2387 pChannelGpfifoParams->ramfcMem.size, 1 , NV_TRUE, 2388 pChannelGpfifoParams->ramfcMem.addressSpace, 2389 pChannelGpfifoParams->ramfcMem.cacheAttrib, 2390 MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE); 2391 2392 if (status != NV_OK) 2393 { 2394 NV_PRINTF(LEVEL_ERROR, 2395 "Unable to allocate instance memory descriptor!\n"); 2396 SLI_LOOP_RETURN(status); 2397 } 2398 2399 memdescDescribe(pInstanceBlock->pRamfcDesc, pChannelGpfifoParams->ramfcMem.addressSpace, 2400 pChannelGpfifoParams->ramfcMem.base, pChannelGpfifoParams->ramfcMem.size); 2401 2402 // Create userd memory descriptor 2403 status = memdescCreate(&pKernelChannel->pUserdSubDeviceMemDesc[subDevInst], pGpu, 2404 pChannelGpfifoParams->userdMem.size, 1 , NV_TRUE, 2405 pChannelGpfifoParams->userdMem.addressSpace, 2406 pChannelGpfifoParams->userdMem.cacheAttrib, 2407 MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE); 2408 2409 if (status != NV_OK) 2410 { 2411 NV_PRINTF(LEVEL_ERROR, 2412 "Unable to allocate instance memory descriptor!\n"); 2413 SLI_LOOP_RETURN(status); 2414 } 2415 2416 memdescDescribe(pKernelChannel->pUserdSubDeviceMemDesc[subDevInst], 2417 pChannelGpfifoParams->userdMem.addressSpace, 2418 pChannelGpfifoParams->userdMem.base, pChannelGpfifoParams->userdMem.size); 2419 2420 if (IS_GFID_VF(gfid) && !gpuIsWarBug200577889SriovHeavyEnabled(pGpu)) 2421 { 2422 /* 2423 * For full SRIOV, guest RM allocates and sends istance, ramfc and userd memory. 2424 * Set MEMDESC_FLAGS_GUEST_ALLOCATED flag in memory descriptor 2425 */ 2426 memdescSetFlag(pInstanceBlock->pInstanceBlockDesc, MEMDESC_FLAGS_GUEST_ALLOCATED, NV_TRUE); 2427 memdescSetFlag(pInstanceBlock->pRamfcDesc, MEMDESC_FLAGS_GUEST_ALLOCATED, NV_TRUE); 2428 memdescSetFlag(pKernelChannel->pUserdSubDeviceMemDesc[subDevInst], MEMDESC_FLAGS_GUEST_ALLOCATED, NV_TRUE); 2429 } 2430 2431 // Create method buffer memory descriptor 2432 runqueue = DRF_VAL(OS04, _FLAGS, _GROUP_CHANNEL_RUNQUEUE, pChannelGpfifoParams->flags); 2433 if (IS_GFID_VF(gfid) && !gpuIsWarBug200577889SriovHeavyEnabled(pGpu)) 2434 { 2435 pKernelChannelGroupApi->pKernelChannelGroup->pMthdBuffers[runqueue] 2436 .bar2Addr = pChannelGpfifoParams->mthdbufMem.base; 2437 } 2438 else if (pKernelChannelGroupApi->pKernelChannelGroup 2439 ->pMthdBuffers[runqueue].pMemDesc == NULL) 2440 { 2441 NV_ASSERT(pChannelGpfifoParams->mthdbufMem.size > 0); 2442 NV_ASSERT(pChannelGpfifoParams->mthdbufMem.base != 0); 2443 status = memdescCreate(&pKernelChannelGroupApi->pKernelChannelGroup 2444 ->pMthdBuffers[runqueue].pMemDesc, 2445 pGpu, 2446 pChannelGpfifoParams->mthdbufMem.size, 2447 1, 2448 NV_TRUE, 2449 pChannelGpfifoParams->mthdbufMem.addressSpace, 2450 pChannelGpfifoParams->mthdbufMem.cacheAttrib, 2451 MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE); 2452 2453 if (status != NV_OK) 2454 { 2455 NV_PRINTF(LEVEL_ERROR, 2456 "Unable to allocate instance memory descriptor!\n"); 2457 SLI_LOOP_RETURN(status); 2458 } 2459 memdescDescribe(pKernelChannelGroupApi->pKernelChannelGroup 2460 ->pMthdBuffers[runqueue].pMemDesc, 2461 pChannelGpfifoParams->mthdbufMem.addressSpace, 2462 pChannelGpfifoParams->mthdbufMem.base, 2463 pChannelGpfifoParams->mthdbufMem.size); 2464 } 2465 2466 NV_PRINTF(LEVEL_INFO, 2467 "hChannel 0x%x hClient 0x%x, Class ID 0x%x " 2468 "Instance Block @ 0x%llx (%s %x) " 2469 "USERD @ 0x%llx " 2470 "for subdevice %d\n", 2471 RES_GET_HANDLE(pKernelChannel), RES_GET_CLIENT_HANDLE(pKernelChannel), RES_GET_EXT_CLASS_ID(pKernelChannel), 2472 memdescGetPhysAddr(pInstanceBlock->pInstanceBlockDesc, AT_GPU, 0), 2473 memdescGetApertureString(memdescGetAddressSpace(pInstanceBlock->pInstanceBlockDesc)), 2474 (NvU32)(memdescGetAddressSpace(pInstanceBlock->pInstanceBlockDesc)), 2475 (pKernelChannel->pUserdSubDeviceMemDesc[subDevInst] == NULL) ? 0x0LL : 2476 memdescGetPhysAddr(pKernelChannel->pUserdSubDeviceMemDesc[subDevInst], AT_GPU, 0LL), subDevInst); 2477 2478 SLI_LOOP_END 2479 2480 return status; 2481 } 2482 2483 /** 2484 * @brief Create and describe channel instance memory ramfc and userd memdescs 2485 * Done using RPC for Heavy SRIOV guest 2486 * 2487 * @param pGpu : OBJGPU pointer 2488 * @param pKernelChannel : KernelChannel pointer 2489 */ 2490 static NV_STATUS 2491 _kchannelDescribeMemDescsHeavySriov 2492 ( 2493 OBJGPU *pGpu, 2494 KernelChannel *pKernelChannel 2495 ) 2496 { 2497 NV_STATUS status = NV_OK; 2498 FIFO_INSTANCE_BLOCK *pInstanceBlock = NULL; 2499 NvU32 subDevInst; 2500 Subdevice *pSubDevice; 2501 NvHandle hSubDevice = 0; 2502 NvU32 apert = ADDR_UNKNOWN; 2503 NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_INFO_PARAMS memInfoParams; 2504 Device *pDevice = GPU_RES_GET_DEVICE(pKernelChannel); 2505 2506 NV_ASSERT_OR_RETURN(IS_VIRTUAL_WITH_SRIOV(pGpu) && gpuIsWarBug200577889SriovHeavyEnabled(pGpu), 2507 NV_ERR_INVALID_STATE); 2508 2509 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 2510 2511 subDevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu); 2512 2513 pInstanceBlock = (FIFO_INSTANCE_BLOCK*) pKernelChannel->pFifoHalData[subDevInst]; 2514 2515 // 2516 // In SRIOV enabled systems, MMU fault interrupts for guest contexts are received and handled in guests. 2517 // Inorder to correctly find the faulting channel, faulting instance address has be compared with list of allocated channels. 2518 // But since contexts are currently allocated in host during channelConstruct, we need 2519 // context info from host and save it locally for the above channel lookup to pass. This piece of code uses GET_CHANNEL_MEM_INFO 2520 // to fetch the info and update pFifoHalData with the relevant details. 2521 // 2522 2523 portMemSet(&memInfoParams, 0, sizeof(NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_INFO_PARAMS)); 2524 memInfoParams.hChannel = RES_GET_HANDLE(pKernelChannel); 2525 2526 status = subdeviceGetByInstance(RES_GET_CLIENT(pKernelChannel), 2527 RES_GET_HANDLE(pDevice), 2528 subDevInst, 2529 &pSubDevice); 2530 if (status != NV_OK) 2531 { 2532 NV_PRINTF(LEVEL_ERROR, "Unable to get subdevice object.\n"); 2533 DBG_BREAKPOINT(); 2534 SLI_LOOP_RETURN(status); 2535 } 2536 2537 GPU_RES_SET_THREAD_BC_STATE(pSubDevice); 2538 2539 hSubDevice = RES_GET_HANDLE(pSubDevice); 2540 2541 NV_RM_RPC_CONTROL(pGpu, 2542 RES_GET_CLIENT_HANDLE(pKernelChannel), 2543 hSubDevice, 2544 NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_INFO, 2545 &memInfoParams, 2546 sizeof(NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_INFO_PARAMS), 2547 status); 2548 if (status != NV_OK) 2549 { 2550 NV_PRINTF(LEVEL_ERROR, 2551 "RM Control call to fetch channel meminfo failed, hKernelChannel 0x%x\n", 2552 RES_GET_HANDLE(pKernelChannel)); 2553 DBG_BREAKPOINT(); 2554 SLI_LOOP_RETURN(status); 2555 } 2556 2557 // Find the aperture 2558 if (memInfoParams.chMemInfo.inst.aperture == NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_APERTURE_VIDMEM) 2559 { 2560 apert = ADDR_FBMEM; 2561 } 2562 else if ((memInfoParams.chMemInfo.inst.aperture == NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_APERTURE_SYSMEM_COH) || 2563 (memInfoParams.chMemInfo.inst.aperture == NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_APERTURE_SYSMEM_NCOH)) 2564 { 2565 apert = ADDR_SYSMEM; 2566 } 2567 else 2568 { 2569 NV_PRINTF(LEVEL_ERROR, 2570 "Unknown aperture, hClient 0x%x, hKernelChannel 0x%x\n", 2571 RES_GET_CLIENT_HANDLE(pKernelChannel), 2572 RES_GET_HANDLE(pKernelChannel)); 2573 status = NV_ERR_INVALID_ARGUMENT; 2574 DBG_BREAKPOINT(); 2575 SLI_LOOP_RETURN(status); 2576 } 2577 2578 status = memdescCreate(&pInstanceBlock->pInstanceBlockDesc, pGpu, 2579 memInfoParams.chMemInfo.inst.size, 1 , NV_TRUE, 2580 apert, NV_MEMORY_UNCACHED, MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE); 2581 2582 if (status != NV_OK) 2583 { 2584 NV_PRINTF(LEVEL_ERROR, 2585 "Unable to allocate instance memory descriptor!\n"); 2586 SLI_LOOP_RETURN(status); 2587 } 2588 2589 memdescDescribe(pInstanceBlock->pInstanceBlockDesc, apert, memInfoParams.chMemInfo.inst.base, memInfoParams.chMemInfo.inst.size); 2590 2591 NV_PRINTF(LEVEL_INFO, 2592 "hChannel 0x%x hClient 0x%x, Class ID 0x%x " 2593 "Instance Block @ 0x%llx (%s %x) " 2594 "USERD @ 0x%llx " 2595 "for subdevice %d\n", 2596 RES_GET_HANDLE(pKernelChannel), RES_GET_CLIENT_HANDLE(pKernelChannel), RES_GET_EXT_CLASS_ID(pKernelChannel), 2597 memdescGetPhysAddr(pInstanceBlock->pInstanceBlockDesc, AT_GPU, 0), 2598 memdescGetApertureString(memdescGetAddressSpace(pInstanceBlock->pInstanceBlockDesc)), 2599 (NvU32)(memdescGetAddressSpace(pInstanceBlock->pInstanceBlockDesc)), 2600 (pKernelChannel->pUserdSubDeviceMemDesc[subDevInst] == NULL) ? 0x0LL : 2601 memdescGetPhysAddr(pKernelChannel->pUserdSubDeviceMemDesc[subDevInst], AT_GPU, 0LL), subDevInst); 2602 2603 SLI_LOOP_END 2604 2605 return status; 2606 } 2607 2608 static NV_STATUS 2609 _kchannelSendChannelAllocRpc 2610 ( 2611 KernelChannel *pKernelChannel, 2612 NV_CHANNEL_ALLOC_PARAMS *pChannelGpfifoParams, 2613 KernelChannelGroup *pKernelChannelGroup, 2614 NvBool bFullSriov 2615 ) 2616 { 2617 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannel); 2618 NV_CHANNEL_ALLOC_PARAMS *pRpcParams; 2619 NV_STATUS status = NV_OK; 2620 2621 pRpcParams = portMemAllocNonPaged(sizeof(*pRpcParams)); 2622 NV_ASSERT_OR_RETURN(pRpcParams != NULL, NV_ERR_NO_MEMORY); 2623 portMemSet(pRpcParams, 0, sizeof(*pRpcParams)); 2624 2625 pRpcParams->hObjectError = pChannelGpfifoParams->hObjectError; 2626 pRpcParams->hObjectBuffer = 0; 2627 pRpcParams->gpFifoOffset = pChannelGpfifoParams->gpFifoOffset; 2628 pRpcParams->gpFifoEntries = pChannelGpfifoParams->gpFifoEntries; 2629 pRpcParams->flags = pChannelGpfifoParams->flags; 2630 pRpcParams->hContextShare = pChannelGpfifoParams->hContextShare; 2631 pRpcParams->hVASpace = pChannelGpfifoParams->hVASpace; 2632 pRpcParams->engineType = pChannelGpfifoParams->engineType; 2633 pRpcParams->subDeviceId = pChannelGpfifoParams->subDeviceId; 2634 pRpcParams->hObjectEccError = pChannelGpfifoParams->hObjectEccError; 2635 pRpcParams->hPhysChannelGroup = pChannelGpfifoParams->hPhysChannelGroup; 2636 pRpcParams->internalFlags = pChannelGpfifoParams->internalFlags; 2637 2638 portMemCopy((void*)pRpcParams->hUserdMemory, 2639 sizeof(NvHandle) * NV2080_MAX_SUBDEVICES, 2640 (const void*)pChannelGpfifoParams->hUserdMemory, 2641 sizeof(NvHandle) * NV2080_MAX_SUBDEVICES); 2642 2643 portMemCopy((void*)pRpcParams->userdOffset, 2644 sizeof(NvU64) * NV2080_MAX_SUBDEVICES, 2645 (const void*)pChannelGpfifoParams->userdOffset, 2646 sizeof(NvU64) * NV2080_MAX_SUBDEVICES); 2647 2648 if (pKernelChannel->bCCSecureChannel) 2649 { 2650 portMemCopy((void*)pRpcParams->encryptIv, 2651 sizeof(pRpcParams->encryptIv), 2652 (const void*)pChannelGpfifoParams->encryptIv, 2653 sizeof(pChannelGpfifoParams->encryptIv)); 2654 2655 portMemCopy((void*)pRpcParams->decryptIv, 2656 sizeof(pRpcParams->decryptIv), 2657 (const void*)pChannelGpfifoParams->decryptIv, 2658 sizeof(pChannelGpfifoParams->decryptIv)); 2659 2660 portMemCopy((void*)pRpcParams->hmacNonce, 2661 sizeof(pRpcParams->hmacNonce), 2662 (const void*)pChannelGpfifoParams->hmacNonce, 2663 sizeof(pChannelGpfifoParams->hmacNonce)); 2664 } 2665 2666 // 2667 // These fields are only filled out for GSP client or full SRIOV 2668 // i.e. the guest independently allocs ChID and instmem 2669 // 2670 if (IS_GSP_CLIENT(pGpu) || bFullSriov) 2671 { 2672 NvU32 subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu); 2673 FIFO_INSTANCE_BLOCK *pInstanceBlock = pKernelChannel->pFifoHalData[subdevInst]; 2674 NvU32 runqueue = DRF_VAL(OS04, _FLAGS, _GROUP_CHANNEL_RUNQUEUE, pChannelGpfifoParams->flags); 2675 2676 NV_ASSERT_TRUE_OR_GOTO(status, 2677 pInstanceBlock != NULL, 2678 NV_ERR_INVALID_STATE, 2679 cleanup); 2680 2681 portMemCopy(&pRpcParams->errorNotifierMem, 2682 sizeof pRpcParams->errorNotifierMem, 2683 &(pChannelGpfifoParams->errorNotifierMem), 2684 sizeof pChannelGpfifoParams->errorNotifierMem); 2685 portMemCopy(&pRpcParams->eccErrorNotifierMem, 2686 sizeof pRpcParams->eccErrorNotifierMem, 2687 &(pChannelGpfifoParams->eccErrorNotifierMem), 2688 sizeof pChannelGpfifoParams->eccErrorNotifierMem); 2689 2690 // Fill the instance block 2691 if (pInstanceBlock) 2692 { 2693 pRpcParams->instanceMem.base = 2694 memdescGetPhysAddr(pInstanceBlock->pInstanceBlockDesc, AT_GPU, 0); 2695 pRpcParams->instanceMem.size = pInstanceBlock->pInstanceBlockDesc->Size; 2696 pRpcParams->instanceMem.addressSpace = 2697 memdescGetAddressSpace(pInstanceBlock->pInstanceBlockDesc); 2698 pRpcParams->instanceMem.cacheAttrib = 2699 memdescGetCpuCacheAttrib(pInstanceBlock->pInstanceBlockDesc); 2700 2701 pRpcParams->ramfcMem.base = 2702 memdescGetPhysAddr(pInstanceBlock->pRamfcDesc, AT_GPU, 0); 2703 pRpcParams->ramfcMem.size = pInstanceBlock->pRamfcDesc->Size; 2704 pRpcParams->ramfcMem.addressSpace = 2705 memdescGetAddressSpace(pInstanceBlock->pRamfcDesc); 2706 pRpcParams->ramfcMem.cacheAttrib = 2707 memdescGetCpuCacheAttrib(pInstanceBlock->pRamfcDesc); 2708 } 2709 2710 // Fill the userd memory descriptor 2711 if (pKernelChannel->pUserdSubDeviceMemDesc[subdevInst]) 2712 { 2713 pRpcParams->userdMem.base = 2714 memdescGetPhysAddr(pKernelChannel->pUserdSubDeviceMemDesc[subdevInst], AT_GPU, 0); 2715 pRpcParams->userdMem.size = pKernelChannel->pUserdSubDeviceMemDesc[subdevInst]->Size; 2716 pRpcParams->userdMem.addressSpace = 2717 memdescGetAddressSpace(pKernelChannel->pUserdSubDeviceMemDesc[subdevInst]); 2718 pRpcParams->userdMem.cacheAttrib = 2719 memdescGetCpuCacheAttrib(pKernelChannel->pUserdSubDeviceMemDesc[subdevInst]); 2720 } 2721 2722 // Fill the method buffer memory descriptor 2723 if (pKernelChannelGroup->pMthdBuffers != NULL && 2724 pKernelChannelGroup->pMthdBuffers[runqueue].pMemDesc != NULL) 2725 { 2726 if (bFullSriov) 2727 { 2728 pRpcParams->mthdbufMem.base = 2729 pKernelChannelGroup->pMthdBuffers[runqueue].bar2Addr; 2730 pRpcParams->mthdbufMem.size = 2731 pKernelChannelGroup->pMthdBuffers[runqueue].pMemDesc->Size; 2732 pRpcParams->mthdbufMem.addressSpace = ADDR_VIRTUAL; 2733 pRpcParams->mthdbufMem.cacheAttrib = 0; 2734 } 2735 else 2736 { 2737 pRpcParams->mthdbufMem.base = memdescGetPhysAddr( 2738 pKernelChannelGroup->pMthdBuffers[runqueue].pMemDesc, 2739 AT_GPU, 0); 2740 pRpcParams->mthdbufMem.size = 2741 pKernelChannelGroup->pMthdBuffers[runqueue].pMemDesc->Size; 2742 pRpcParams->mthdbufMem.addressSpace = memdescGetAddressSpace( 2743 pKernelChannelGroup->pMthdBuffers[runqueue].pMemDesc); 2744 pRpcParams->mthdbufMem.cacheAttrib = memdescGetCpuCacheAttrib( 2745 pKernelChannelGroup->pMthdBuffers[runqueue].pMemDesc); 2746 } 2747 } 2748 2749 if (IS_GSP_CLIENT(pGpu)) 2750 { 2751 // 2752 // Setting these param flags will make the Physical RMAPI use our 2753 // ChID (which is already decided) 2754 // 2755 2756 NvU32 numChannelsPerUserd = NVBIT(DRF_SIZE(NVOS04_FLAGS_CHANNEL_USERD_INDEX_VALUE)); // 1<<3 -> 4K / 512B 2757 2758 pRpcParams->flags = FLD_SET_DRF(OS04, _FLAGS, 2759 _CHANNEL_USERD_INDEX_FIXED, _FALSE, pRpcParams->flags); 2760 pRpcParams->flags = FLD_SET_DRF(OS04, _FLAGS, 2761 _CHANNEL_USERD_INDEX_PAGE_FIXED, _TRUE, pRpcParams->flags); 2762 pRpcParams->flags = FLD_SET_DRF_NUM(OS04, _FLAGS, 2763 _CHANNEL_USERD_INDEX_VALUE, pKernelChannel->ChID % numChannelsPerUserd, pRpcParams->flags); 2764 pRpcParams->flags = FLD_SET_DRF_NUM(OS04, _FLAGS, 2765 _CHANNEL_USERD_INDEX_PAGE_VALUE, pKernelChannel->ChID / numChannelsPerUserd, pRpcParams->flags); 2766 2767 // GSP client needs to pass in privilege level as an alloc param since GSP-RM cannot check this 2768 pRpcParams->internalFlags = 2769 FLD_SET_DRF_NUM(_KERNELCHANNEL, _ALLOC_INTERNALFLAGS, _PRIVILEGE, 2770 pKernelChannel->privilegeLevel, pRpcParams->internalFlags); 2771 pRpcParams->ProcessID = pKernelChannel->ProcessID; 2772 pRpcParams->SubProcessID= pKernelChannel->SubProcessID; 2773 } 2774 } 2775 2776 NV_RM_RPC_ALLOC_CHANNEL(pGpu, 2777 RES_GET_CLIENT_HANDLE(pKernelChannel), 2778 RES_GET_PARENT_HANDLE(pKernelChannel), 2779 RES_GET_HANDLE(pKernelChannel), 2780 RES_GET_EXT_CLASS_ID(pKernelChannel), 2781 pRpcParams, 2782 &pKernelChannel->ChID, 2783 status); 2784 NV_ASSERT_OK_OR_GOTO(status, status, cleanup); 2785 2786 NV_PRINTF(LEVEL_INFO, 2787 "Alloc Channel chid %d, hClient:0x%x, hParent:0x%x, hObject:0x%x, hClass:0x%x\n", 2788 pKernelChannel->ChID, 2789 RES_GET_CLIENT_HANDLE(pKernelChannel), 2790 RES_GET_PARENT_HANDLE(pKernelChannel), 2791 RES_GET_HANDLE(pKernelChannel), 2792 RES_GET_EXT_CLASS_ID(pKernelChannel)); 2793 2794 cleanup: 2795 portMemFree(pRpcParams); 2796 2797 return status; 2798 } 2799 2800 /*! 2801 * @brief Bind a single channel to a runlist 2802 * 2803 * This is a helper function for kchannelCtrlCmdBind and kchangrpapiCtrlCmdBind 2804 */ 2805 NV_STATUS kchannelBindToRunlist_IMPL 2806 ( 2807 KernelChannel *pKernelChannel, 2808 RM_ENGINE_TYPE localRmEngineType, 2809 ENGDESCRIPTOR engineDesc 2810 ) 2811 { 2812 OBJGPU *pGpu; 2813 NV_STATUS status = NV_OK; 2814 2815 NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_ARGUMENT); 2816 pGpu = GPU_RES_GET_GPU(pKernelChannel); 2817 2818 // copied from setRunlistIdByEngineType 2819 if ((engineDesc == ENG_SW) || (engineDesc == ENG_BUS)) 2820 { 2821 return NV_OK; 2822 } 2823 2824 // 2825 // vGPU: 2826 // 2827 // Since vGPU does all real hardware management in the 2828 // host, if we are in guest OS (where IS_VIRTUAL(pGpu) is true), 2829 // do an RPC to the host to do the hardware update. 2830 // 2831 if (IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu)) 2832 { 2833 NVA06F_CTRL_BIND_PARAMS params; 2834 2835 params.engineType = gpuGetNv2080EngineType(localRmEngineType); 2836 2837 NV_RM_RPC_CONTROL(pGpu, 2838 RES_GET_CLIENT_HANDLE(pKernelChannel), 2839 RES_GET_HANDLE(pKernelChannel), 2840 NVA06F_CTRL_CMD_BIND, 2841 ¶ms, 2842 sizeof(params), 2843 status); 2844 2845 NV_ASSERT_OR_RETURN(status == NV_OK, status); 2846 } 2847 2848 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY) 2849 2850 status = kfifoRunlistSetIdByEngine_HAL(pGpu, GPU_GET_KERNEL_FIFO(pGpu), 2851 pKernelChannel, engineDesc); 2852 2853 if (status != NV_OK) 2854 { 2855 NV_PRINTF(LEVEL_ERROR, 2856 "Failed to set RunlistID 0x%08x for channel 0x%08x\n", 2857 engineDesc, kchannelGetDebugTag(pKernelChannel)); 2858 SLI_LOOP_BREAK; 2859 } 2860 2861 SLI_LOOP_END; 2862 2863 return status; 2864 } 2865 2866 // 2867 // channelCtrlCmdEventSetNotification 2868 // 2869 // This command handles set notification operations for all tesla, 2870 // fermi, kepler, and maxwell based gpfifo classes: 2871 // 2872 // NV50_DISPLAY (Class: NV5070) 2873 // GF100_CHANNEL_GPFIFO (Class: NV906F) 2874 // KEPLER_CHANNEL_GPFIFO_A (Class: NVA06F) 2875 // KEPLER_CHANNEL_GPFIFO_B (Class: NVA16F) 2876 // KEPLER_CHANNEL_GPFIFO_C (Class: NVA26F) 2877 // MAXWELL_CHANNEL_GPFIFO_A (Class: NVB06F) 2878 // PASCAL_CHANNEL_GPFIFO_A (Class: NVC06F) 2879 // 2880 NV_STATUS 2881 kchannelCtrlCmdEventSetNotification_IMPL 2882 ( 2883 KernelChannel *pKernelChannel, 2884 NV906F_CTRL_EVENT_SET_NOTIFICATION_PARAMS *pSetEventParams 2885 ) 2886 { 2887 CLI_CHANNEL_CLASS_INFO classInfo; 2888 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 2889 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams; 2890 2891 // NV01_EVENT must have been plugged into this subdevice 2892 if (inotifyGetNotificationList(staticCast(pKernelChannel, INotifier)) == NULL) 2893 { 2894 NV_PRINTF(LEVEL_INFO, "cmd 0x%x: no event list\n", pRmCtrlParams->cmd); 2895 return NV_ERR_INVALID_STATE; 2896 } 2897 2898 // get channel class-specific properties 2899 CliGetChannelClassInfo(REF_VAL(NVXXXX_CTRL_CMD_CLASS, pRmCtrlParams->cmd), 2900 &classInfo); 2901 2902 if (pSetEventParams->event >= classInfo.notifiersMaxCount) 2903 { 2904 NV_PRINTF(LEVEL_INFO, "bad event 0x%x\n", pSetEventParams->event); 2905 return NV_ERR_INVALID_ARGUMENT; 2906 } 2907 2908 if ((pSetEventParams->action == classInfo.eventActionSingle) || 2909 (pSetEventParams->action == classInfo.eventActionRepeat)) 2910 { 2911 // must be in disabled state to transition to an active state 2912 if (pKernelChannel->pNotifyActions[pSetEventParams->event] != classInfo.eventActionDisable) 2913 { 2914 return NV_ERR_INVALID_STATE; 2915 } 2916 2917 pKernelChannel->pNotifyActions[pSetEventParams->event] = pSetEventParams->action; 2918 } 2919 else if (pSetEventParams->action == classInfo.eventActionDisable) 2920 { 2921 pKernelChannel->pNotifyActions[pSetEventParams->event] = pSetEventParams->action; 2922 } 2923 else 2924 { 2925 return NV_ERR_INVALID_ARGUMENT; 2926 } 2927 2928 return NV_OK; 2929 } 2930 2931 NV_STATUS 2932 kchannelCtrlCmdGetClassEngineid_IMPL 2933 ( 2934 KernelChannel *pKernelChannel, 2935 NV906F_CTRL_GET_CLASS_ENGINEID_PARAMS *pParams 2936 ) 2937 { 2938 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannel); 2939 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 2940 NV_STATUS status = NV_OK; 2941 RM_ENGINE_TYPE rmEngineType; 2942 2943 // 2944 // MODS uses hObject 0 to figure out if this call is supported or not. 2945 // In SRIOV VF scenario, plugin asserts if host returns an error code 2946 // for a control call. Adding a temporary work around till MODS submits 2947 // a proper fix. 2948 // 2949 if (pParams->hObject == NV01_NULL_OBJECT) 2950 { 2951 return NV_ERR_OBJECT_NOT_FOUND; 2952 } 2953 2954 NV_CHECK_OR_RETURN(LEVEL_ERROR, 2955 pParams->hObject != RES_GET_CLIENT_HANDLE(pKernelChannel), 2956 NV_ERR_INVALID_ARGUMENT); 2957 2958 if (IS_VIRTUAL_WITHOUT_SRIOV(pGpu) || 2959 (IS_VIRTUAL_WITH_SRIOV(pGpu) && gpuIsWarBug200577889SriovHeavyEnabled(pGpu))) 2960 { 2961 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 2962 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams; 2963 2964 NV_RM_RPC_CONTROL(pGpu, 2965 pRmCtrlParams->hClient, 2966 RES_GET_HANDLE(pKernelChannel), 2967 pRmCtrlParams->cmd, 2968 pRmCtrlParams->pParams, 2969 pRmCtrlParams->paramsSize, 2970 status); 2971 return status; 2972 } 2973 2974 NV_ASSERT_OK_OR_RETURN( 2975 kchannelGetClassEngineID_HAL(pGpu, pKernelChannel, pParams->hObject, 2976 &pParams->classEngineID, 2977 &pParams->classID, 2978 &rmEngineType)); 2979 2980 pParams->engineID = gpuGetNv2080EngineType(rmEngineType); 2981 2982 if (IS_MIG_IN_USE(pGpu) && 2983 kmigmgrIsEnginePartitionable(pGpu, pKernelMIGManager, rmEngineType)) 2984 { 2985 MIG_INSTANCE_REF ref; 2986 RM_ENGINE_TYPE localRmEngineType; 2987 2988 NV_ASSERT_OK_OR_RETURN( 2989 kmigmgrGetInstanceRefFromDevice(pGpu, pKernelMIGManager, 2990 GPU_RES_GET_DEVICE(pKernelChannel), 2991 &ref)); 2992 2993 NV_ASSERT_OK_OR_RETURN( 2994 kmigmgrGetGlobalToLocalEngineType(pGpu, pKernelMIGManager, ref, 2995 rmEngineType, 2996 &localRmEngineType)); 2997 2998 NV_PRINTF(LEVEL_INFO, "Overriding global engine type 0x%x to local engine type 0x%x (0x%x) due to MIG\n", 2999 pParams->engineID, gpuGetNv2080EngineType(localRmEngineType), localRmEngineType); 3000 3001 pParams->engineID = gpuGetNv2080EngineType(localRmEngineType); 3002 } 3003 3004 return status; 3005 } 3006 3007 NV_STATUS 3008 kchannelCtrlCmdResetIsolatedChannel_IMPL 3009 ( 3010 KernelChannel *pKernelChannel, 3011 NV506F_CTRL_CMD_RESET_ISOLATED_CHANNEL_PARAMS *pResetIsolatedChannelParams 3012 ) 3013 { 3014 NV_STATUS status = NV_OK; 3015 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannel); 3016 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); 3017 3018 3019 // This ctrl sets bIsRcPending in the KernelChannel object. Because Kernel-RM is 3020 // the source of truth on this, it's important that this ctrl is called from CPU-RM 3021 NV_ASSERT_OR_RETURN(!RMCFG_FEATURE_PLATFORM_GSP, NV_ERR_INVALID_OPERATION); 3022 3023 // Call internal RMCTRL on physical-RM, kchannelFwdToInternalCtrl() is not 3024 // used because no conversion from KernelChannel to Channel is required 3025 status = pRmApi->Control(pRmApi, 3026 resservGetTlsCallContext()->pControlParams->hClient, 3027 RES_GET_HANDLE(pKernelChannel), 3028 NV506F_CTRL_CMD_INTERNAL_RESET_ISOLATED_CHANNEL, 3029 pResetIsolatedChannelParams, 3030 sizeof(NV506F_CTRL_CMD_INTERNAL_RESET_ISOLATED_CHANNEL_PARAMS)); 3031 3032 // If physical RM successfully reset the isolated channel, 3033 // mark that the RC is no longer pending 3034 if (status == NV_OK) 3035 pKernelChannel->bIsRcPending[gpumgrGetSubDeviceInstanceFromGpu(pGpu)] = NV_FALSE; 3036 3037 return status; 3038 } 3039 3040 // This ctrl accesses bIsRcPending in the KernelChannel object to populate 3041 // information required by physical RM. Because Kernel-RM is the source of 3042 // truth on this, it's important that this ctrl be called originally from CPU-RM. 3043 NV_STATUS 3044 kchannelCtrlCmdResetChannel_IMPL 3045 ( 3046 KernelChannel *pKernelChannel, 3047 NV906F_CTRL_CMD_RESET_CHANNEL_PARAMS *pResetChannelParams 3048 ) 3049 { 3050 NV_STATUS status = NV_OK; 3051 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannel); 3052 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 3053 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams; 3054 3055 if (!(pRmCtrlParams->bInternal || 3056 pResetChannelParams->resetReason < 3057 NV906F_CTRL_CMD_RESET_CHANNEL_REASON_ENUM_MAX)) 3058 { 3059 return NV_ERR_INVALID_PARAMETER; 3060 } 3061 3062 // Send physical RM info on if an RC is pending 3063 pResetChannelParams->bIsRcPending = 3064 pKernelChannel->bIsRcPending[gpumgrGetSubDeviceInstanceFromGpu(pGpu)]; 3065 3066 // 3067 // All real hardware management is done in the host. 3068 // Do an RPC to the host to do the hardware update and return. 3069 // 3070 if (IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu)) 3071 { 3072 NV_RM_RPC_CONTROL(pGpu, 3073 pRmCtrlParams->hClient, 3074 RES_GET_HANDLE(pKernelChannel), 3075 NV906F_CTRL_CMD_RESET_CHANNEL, 3076 pResetChannelParams, 3077 pRmCtrlParams->paramsSize, 3078 status); 3079 return status; 3080 } 3081 3082 // 3083 // Do an internal control call to do channel reset 3084 // on Host (Physical) RM 3085 // 3086 return kchannelFwdToInternalCtrl_HAL(pGpu, 3087 pKernelChannel, 3088 NVA06F_CTRL_CMD_INTERNAL_RESET_CHANNEL, 3089 pRmCtrlParams); 3090 } 3091 3092 // 3093 // channelCtrlCmdEventSetTrigger 3094 // 3095 // This command handles set trigger operations for all kepler and maxwell based 3096 // gpfifo classes: 3097 // 3098 // KEPLER_CHANNEL_GPFIFO_A (Class: NVA06F) 3099 // KEPLER_CHANNEL_GPFIFO_B (Class: NVA16F) 3100 // KEPLER_CHANNEL_GPFIFO_C (Class: NVA26F) 3101 // MAXWELL_CHANNEL_GPFIFO_A (Class: NVB06F) 3102 // PASCAL_CHANNEL_GPFIFO_A (Class: NVC06F) 3103 // 3104 NV_STATUS 3105 kchannelCtrlCmdEventSetTrigger_IMPL 3106 ( 3107 KernelChannel *pKernelChannel 3108 ) 3109 { 3110 kchannelNotifyEvent(pKernelChannel, NVA06F_NOTIFIERS_SW, 0, 0, NULL, 0); 3111 3112 return NV_OK; 3113 } 3114 3115 NV_STATUS 3116 kchannelCtrlCmdGpFifoSchedule_IMPL 3117 ( 3118 KernelChannel *pKernelChannel, 3119 NVA06F_CTRL_GPFIFO_SCHEDULE_PARAMS *pSchedParams 3120 ) 3121 { 3122 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannel); 3123 NV_STATUS rmStatus = NV_OK; 3124 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 3125 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams; 3126 3127 // 3128 // Bug 1737765: Prevent Externally Owned Channels from running unless bound 3129 // It is possible for clients to allocate and schedule channels while 3130 // skipping the UVM registration step which binds the appropriate 3131 // allocations in RM. We need to fail channel scheduling if the channels 3132 // have not been registered with UVM. 3133 // This check is performed on baremetal, CPU-RM and guest-RM 3134 // 3135 NV_ASSERT_OR_RETURN(kchannelIsSchedulable_HAL(pGpu, pKernelChannel), NV_ERR_INVALID_STATE); 3136 3137 // 3138 // If this was a host-only channel we'll have never set the runlist id, so 3139 // force it here to ensure it is immutable now that the channel is scheduled. 3140 // 3141 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY) 3142 kchannelSetRunlistSet(pGpu, pKernelChannel, NV_TRUE); 3143 SLI_LOOP_END 3144 3145 3146 // 3147 // All real hardware management is done in the host. 3148 // Do an RPC to the host to do the hardware update and return. 3149 // 3150 if (IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu)) 3151 { 3152 NV_RM_RPC_CONTROL(pGpu, 3153 RES_GET_CLIENT_HANDLE(pKernelChannel), 3154 RES_GET_HANDLE(pKernelChannel), 3155 NVA06F_CTRL_CMD_GPFIFO_SCHEDULE, 3156 pRmCtrlParams->pParams, 3157 pRmCtrlParams->paramsSize, 3158 rmStatus); 3159 3160 return rmStatus; 3161 } 3162 3163 // 3164 // Do an internal control call to do channel reset 3165 // on Host (Physical) RM 3166 // 3167 return kchannelFwdToInternalCtrl_HAL(pGpu, 3168 pKernelChannel, 3169 NVA06F_CTRL_CMD_INTERNAL_GPFIFO_SCHEDULE, 3170 pRmCtrlParams); 3171 } 3172 3173 NV_STATUS 3174 kchannelCtrlCmdGetEngineCtxSize_IMPL 3175 ( 3176 KernelChannel *pKernelChannel, 3177 NVB06F_CTRL_GET_ENGINE_CTX_SIZE_PARAMS *pCtxSizeParams 3178 ) 3179 { 3180 return NV_ERR_NOT_SUPPORTED; 3181 } 3182 3183 NV_STATUS 3184 kchannelCtrlCmdSetErrorNotifier_IMPL 3185 ( 3186 KernelChannel *pKernelChannel, 3187 NVA06F_CTRL_SET_ERROR_NOTIFIER_PARAMS *pSetErrorNotifierParams 3188 ) 3189 { 3190 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannel); 3191 RC_NOTIFIER_SCOPE scope; 3192 NV_STATUS rmStatus = NV_OK; 3193 3194 NV_PRINTF(LEVEL_INFO, 3195 "calling setErrorNotifier on channel: 0x%x, broadcast to TSG: %s\n", 3196 kchannelGetDebugTag(pKernelChannel), 3197 pSetErrorNotifierParams->bNotifyEachChannelInTSG ? "true" : "false"); 3198 3199 scope = pSetErrorNotifierParams->bNotifyEachChannelInTSG ? 3200 RC_NOTIFIER_SCOPE_TSG : 3201 RC_NOTIFIER_SCOPE_CHANNEL; 3202 3203 rmStatus = krcErrorSetNotifier(pGpu, GPU_GET_KERNEL_RC(pGpu), 3204 pKernelChannel, 3205 ROBUST_CHANNEL_GR_EXCEPTION, 3206 kchannelGetEngineType(pKernelChannel), 3207 scope); 3208 return rmStatus; 3209 } 3210 3211 NV_STATUS 3212 kchannelCtrlCmdBind_IMPL 3213 ( 3214 KernelChannel *pKernelChannel, 3215 NVA06F_CTRL_BIND_PARAMS *pParams 3216 ) 3217 { 3218 RM_ENGINE_TYPE globalRmEngineType; 3219 RM_ENGINE_TYPE localRmEngineType; 3220 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannel); 3221 NvBool bMIGInUse = IS_MIG_IN_USE(pGpu); 3222 NV_STATUS rmStatus = NV_OK; 3223 ENGDESCRIPTOR engineDesc; 3224 3225 if (!pParams) 3226 return NV_ERR_INVALID_ARGUMENT; 3227 3228 // Check if channel belongs to TSG that is not internal RM TSG 3229 if (!pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->bAllocatedByRm) 3230 { 3231 // This may be valid request if we added new channel to TSG that is 3232 // already running. In that case we just have to check that it uses 3233 // the same runlist as whole TSG. 3234 // We do that in fifoRunlistSetId() 3235 NV_PRINTF(LEVEL_INFO, 3236 "Bind requested for channel %d belonging to TSG %d.\n", 3237 kchannelGetDebugTag(pKernelChannel), 3238 pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->grpID); 3239 } 3240 3241 localRmEngineType = globalRmEngineType = gpuGetRmEngineType(pParams->engineType); 3242 3243 if (bMIGInUse) 3244 { 3245 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 3246 MIG_INSTANCE_REF ref; 3247 3248 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3249 kmigmgrGetInstanceRefFromDevice(pGpu, pKernelMIGManager, 3250 GPU_RES_GET_DEVICE(pKernelChannel), 3251 &ref)); 3252 3253 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3254 kmigmgrGetLocalToGlobalEngineType(pGpu, pKernelMIGManager, ref, localRmEngineType, 3255 &globalRmEngineType)); 3256 3257 } 3258 3259 NV_PRINTF(LEVEL_INFO, "Binding Channel %d to Engine %d\n", 3260 kchannelGetDebugTag(pKernelChannel), globalRmEngineType); 3261 3262 // Translate globalRmEngineType -> enginedesc 3263 NV_ASSERT_OK_OR_CAPTURE_FIRST_ERROR(rmStatus, 3264 gpuXlateClientEngineIdToEngDesc(pGpu, globalRmEngineType, &engineDesc)); 3265 3266 if (rmStatus == NV_OK) 3267 { 3268 NV_ASSERT_OK_OR_CAPTURE_FIRST_ERROR(rmStatus, 3269 kchannelBindToRunlist(pKernelChannel, localRmEngineType, engineDesc)); 3270 } 3271 3272 return rmStatus; 3273 } 3274 3275 NV_STATUS 3276 kchannelCtrlCmdSetInterleaveLevel_IMPL 3277 ( 3278 KernelChannel *pKernelChannel, 3279 NVA06F_CTRL_INTERLEAVE_LEVEL_PARAMS *pParams 3280 ) 3281 { 3282 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannel); 3283 NV_STATUS status = NV_OK; 3284 3285 if (IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu)) 3286 { 3287 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 3288 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams; 3289 3290 NV_RM_RPC_CONTROL(pGpu, 3291 RES_GET_CLIENT_HANDLE(pKernelChannel), 3292 RES_GET_HANDLE(pKernelChannel), 3293 pRmCtrlParams->cmd, 3294 pRmCtrlParams->pParams, 3295 pRmCtrlParams->paramsSize, 3296 status); 3297 NV_CHECK_OR_RETURN(LEVEL_INFO, status == NV_OK, NV_ERR_NOT_SUPPORTED); 3298 } 3299 3300 status = kchangrpSetInterleaveLevel(pGpu, pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup, pParams->channelInterleaveLevel); 3301 3302 return status; 3303 } 3304 3305 NV_STATUS 3306 kchannelCtrlCmdGetInterleaveLevel_IMPL 3307 ( 3308 KernelChannel *pKernelChannel, 3309 NVA06F_CTRL_INTERLEAVE_LEVEL_PARAMS *pParams 3310 ) 3311 { 3312 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannel); 3313 3314 pParams->channelInterleaveLevel = 3315 pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->pInterleaveLevel[gpumgrGetSubDeviceInstanceFromGpu(pGpu)]; 3316 3317 return NV_OK; 3318 } 3319 3320 NV_STATUS 3321 kchannelCtrlCmdGpfifoGetWorkSubmitToken_IMPL 3322 ( 3323 KernelChannel *pKernelChannel, 3324 NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN_PARAMS *pTokenParams 3325 ) 3326 { 3327 NV_STATUS rmStatus = NV_OK; 3328 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannel); 3329 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 3330 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 3331 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams; 3332 NvBool bIsMIGEnabled = IS_MIG_ENABLED(pGpu); 3333 3334 NvBool bIsModsVgpu = NV_FALSE; 3335 3336 NvBool bIsVgpuRpcNeeded = (bIsModsVgpu || (IS_VIRTUAL(pGpu) && 3337 !(IS_VIRTUAL_WITH_SRIOV(pGpu) && !bIsMIGEnabled && 3338 kfifoIsPerRunlistChramEnabled(pKernelFifo)))) && 3339 (!pKernelFifo->bGuestGenenratesWorkSubmitToken); 3340 // 3341 // vGPU: 3342 // If required call into the host to get the worksubmit token. 3343 // 3344 if (bIsVgpuRpcNeeded) 3345 { 3346 NV_RM_RPC_CONTROL(pGpu, 3347 pRmCtrlParams->hClient, 3348 RES_GET_HANDLE(pKernelChannel), 3349 pRmCtrlParams->cmd, 3350 pRmCtrlParams->pParams, 3351 pRmCtrlParams->paramsSize, 3352 rmStatus); 3353 // 3354 // All done if error or for non-MODS vGPU guest (host did notification in RPC). 3355 // GSP FW is not able to perform the notification, nor is MODS vGPU host, 3356 // so it still needs to be handled by the client/guest outside the RPC. 3357 // 3358 if (rmStatus != NV_OK) 3359 { 3360 return rmStatus; 3361 } 3362 3363 if (IS_VIRTUAL(pGpu)) 3364 { 3365 return rmStatus; 3366 } 3367 } 3368 3369 if (!bIsModsVgpu || pKernelFifo->bGuestGenenratesWorkSubmitToken) 3370 { 3371 NV_ASSERT_OR_RETURN(pKernelChannel->pKernelChannelGroupApi != NULL, NV_ERR_INVALID_STATE); 3372 NV_ASSERT_OR_RETURN(pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup != NULL, NV_ERR_INVALID_STATE); 3373 rmStatus = kfifoGenerateWorkSubmitToken_HAL(pGpu, pKernelFifo, pKernelChannel, 3374 &pTokenParams->workSubmitToken, 3375 pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->bIsCallingContextVgpuPlugin); 3376 NV_CHECK_OR_RETURN(LEVEL_INFO, rmStatus == NV_OK, rmStatus); 3377 } 3378 3379 rmStatus = kchannelNotifyWorkSubmitToken(pGpu, pKernelChannel, pTokenParams->workSubmitToken); 3380 return rmStatus; 3381 } 3382 3383 NV_STATUS 3384 kchannelCtrlCmdGpfifoSetWorkSubmitTokenNotifIndex_IMPL 3385 ( 3386 KernelChannel *pKernelChannel, 3387 NVC36F_CTRL_GPFIFO_SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX_PARAMS *pParams 3388 ) 3389 { 3390 NV_STATUS rmStatus = NV_OK; 3391 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannel); 3392 3393 // 3394 // vGPU: 3395 // 3396 // Since vgpu plugin is required to update notifier for guest, send an RPC 3397 // to host RM for the plugin to hook. 3398 // RPC not needed for SR-IOV vGpu. 3399 // 3400 // GSP-RM: 3401 // 3402 // Notification is done in CPU-RM, so RPC is not made to FW-RM. 3403 // 3404 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 3405 NvBool bIsMIGEnabled = IS_MIG_ENABLED(pGpu); 3406 NvBool bIsVgpuRpcNeeded = IS_VIRTUAL(pGpu) && 3407 !(IS_VIRTUAL_WITH_SRIOV(pGpu) && !bIsMIGEnabled && 3408 kfifoIsPerRunlistChramEnabled(pKernelFifo)); 3409 if (bIsVgpuRpcNeeded) 3410 { 3411 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 3412 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams; 3413 3414 NV_RM_RPC_CONTROL(pGpu, 3415 pRmCtrlParams->hClient, 3416 RES_GET_HANDLE(pKernelChannel), 3417 pRmCtrlParams->cmd, 3418 pRmCtrlParams->pParams, 3419 pRmCtrlParams->paramsSize, 3420 rmStatus); 3421 return rmStatus; 3422 } 3423 3424 rmStatus = kchannelUpdateWorkSubmitTokenNotifIndex(pGpu, pKernelChannel, pParams->index); 3425 return rmStatus; 3426 } 3427 3428 NV_STATUS 3429 kchannelRegisterChild_IMPL 3430 ( 3431 KernelChannel *pKernelChannel, 3432 ChannelDescendant *pObject 3433 ) 3434 { 3435 NvU16 firstObjectClassID; 3436 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannel); 3437 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 3438 3439 // 3440 // On recent GPU architectures such as FERMI, SetObject operations 3441 // require an EngineID:ClassID tuple as an argument, rather than 3442 // an object handle. In order to be able to differentiate between 3443 // different instances of any given software class, the ClassID 3444 // field needs to be unique within the FIFO context. The code below 3445 // attempts to find a qualifying 16-bit ClassID. 3446 // 3447 if (pObject->resourceDesc.engDesc == ENG_SW) 3448 { 3449 RS_ORDERED_ITERATOR it; 3450 RsClient *pClient = RES_GET_CLIENT(pKernelChannel); 3451 ChannelDescendant *pMatchingObject = NULL; 3452 3453 firstObjectClassID = pKernelChannel->nextObjectClassID; 3454 3455 do 3456 { 3457 if (++pKernelChannel->nextObjectClassID == firstObjectClassID) 3458 { 3459 NV_PRINTF(LEVEL_ERROR, "channel %08x:%08x: out of handles!\n", 3460 RES_GET_CLIENT_HANDLE(pKernelChannel), RES_GET_HANDLE(pKernelChannel)); 3461 return NV_ERR_INSUFFICIENT_RESOURCES; 3462 } 3463 if (pKernelChannel->nextObjectClassID == 0) 3464 continue; 3465 3466 it = clientRefOrderedIter(pClient, RES_GET_REF(pKernelChannel), classId(ChannelDescendant), NV_FALSE); 3467 3468 while (clientRefOrderedIterNext(pClient, &it)) 3469 { 3470 pMatchingObject = dynamicCast(it.pResourceRef->pResource, ChannelDescendant); 3471 NV_ASSERT_OR_ELSE(pMatchingObject != NULL, continue); 3472 3473 if ((pMatchingObject->resourceDesc.engDesc == ENG_SW) && 3474 (pMatchingObject->classID == pKernelChannel->nextObjectClassID)) 3475 { 3476 break; 3477 } 3478 3479 pMatchingObject = NULL; 3480 } 3481 } 3482 while (pMatchingObject != NULL); 3483 3484 pObject->classID = pKernelChannel->nextObjectClassID; 3485 } 3486 3487 return kfifoAddObject_HAL(pGpu, pKernelFifo, pObject); 3488 } 3489 3490 NV_STATUS 3491 kchannelDeregisterChild_IMPL 3492 ( 3493 KernelChannel *pKernelChannel, 3494 ChannelDescendant *pObject 3495 ) 3496 { 3497 NV_STATUS status = NV_OK; 3498 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannel); 3499 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 3500 3501 status = kfifoDeleteObject_HAL(pGpu, pKernelFifo, pObject); 3502 if (status != NV_OK) 3503 { 3504 NV_PRINTF(LEVEL_ERROR, "Could not delete hal resources with object\n"); 3505 DBG_BREAKPOINT(); 3506 } 3507 3508 return status; 3509 } 3510 3511 void 3512 kchannelGetChildIterator 3513 ( 3514 KernelChannel *pKernelChannel, 3515 NvU32 classID, 3516 RM_ENGINE_TYPE engineID, 3517 KernelChannelChildIterator *pIter 3518 ) 3519 { 3520 RsClient *pClient = RES_GET_CLIENT(pKernelChannel); 3521 NV_ASSERT_OR_RETURN_VOID(pIter != NULL); 3522 3523 portMemSet(pIter, 0, sizeof(*pIter)); 3524 pIter->classID = classID; 3525 pIter->engineID = engineID; 3526 pIter->rsIter = clientRefOrderedIter(pClient, RES_GET_REF(pKernelChannel), classId(ChannelDescendant), NV_FALSE); 3527 } 3528 3529 ChannelDescendant * 3530 kchannelGetNextChild 3531 ( 3532 KernelChannelChildIterator *pIter 3533 ) 3534 { 3535 ChannelDescendant *pChild; 3536 3537 NV_ASSERT_OR_RETURN(pIter != NULL, NULL); 3538 3539 while (clientRefOrderedIterNext(pIter->rsIter.pClient, &pIter->rsIter)) 3540 { 3541 pChild = dynamicCast(pIter->rsIter.pResourceRef->pResource, ChannelDescendant); 3542 NV_ASSERT_OR_RETURN(pChild != NULL, NULL); 3543 3544 // Continue to the next child if it doesn't match these filters: 3545 if (pIter->engineID != pChild->resourceDesc.engDesc) 3546 continue; 3547 if (pIter->classID != 0) 3548 { 3549 if ((RES_GET_EXT_CLASS_ID(pChild) != pIter->classID) && 3550 (pChild->classID != pIter->classID)) 3551 continue; 3552 } 3553 3554 // Yield this matching child 3555 return pChild; 3556 } 3557 3558 return NULL; 3559 } 3560 3561 ChannelDescendant * 3562 kchannelGetOneChild 3563 ( 3564 KernelChannel *pKernelChannel, 3565 NvU32 classID, 3566 NvU32 engineID 3567 ) 3568 { 3569 KernelChannelChildIterator iter; 3570 3571 kchannelGetChildIterator(pKernelChannel, classID, engineID, &iter); 3572 return kchannelGetNextChild(&iter); 3573 } 3574 3575 /** 3576 * @brief Gets object iterator for a channel or channel group 3577 * 3578 * @param[in] pKernelChannel 3579 * @param[in] classNum 3580 * @param[in] engDesc 3581 * @param[out] pIt 3582 * 3583 */ 3584 void 3585 kchannelGetChildIterOverGroup 3586 ( 3587 KernelChannel *pKernelChannel, 3588 NvU32 classNum, 3589 NvU32 engDesc, 3590 KernelChannelChildIterOverGroup *pIt 3591 ) 3592 { 3593 NV_ASSERT_OR_RETURN_VOID(pIt != NULL); 3594 portMemSet(pIt, 0, sizeof(*pIt)); 3595 3596 NV_ASSERT_OR_RETURN_VOID(pKernelChannel != NULL); 3597 3598 pIt->classNum = classNum; 3599 pIt->engDesc = engDesc; 3600 3601 pIt->channelNode.pKernelChannel = 3602 pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->pChanList->pHead->pKernelChannel; 3603 pIt->channelNode.pNext = 3604 pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->pChanList->pHead->pNext; 3605 3606 kchannelGetChildIterator(pIt->channelNode.pKernelChannel, pIt->classNum, pIt->engDesc, &pIt->kchannelIter); 3607 } 3608 3609 /** 3610 * @brief Get the next object based on given class/engine tag. 3611 * When the class number is 0, it is ignored. 3612 * 3613 * @param[in] pIt 3614 * 3615 * Returns: found child or NULL 3616 * 3617 */ 3618 ChannelDescendant * 3619 kchannelGetNextChildOverGroup 3620 ( 3621 KernelChannelChildIterOverGroup *pIt 3622 ) 3623 { 3624 PCHANNEL_NODE pHead = NULL; 3625 ChannelDescendant *pObject = NULL; 3626 3627 NV_ASSERT_OR_RETURN(pIt != NULL, NULL); 3628 3629 // Start iterating from the given object (if any) of the given channel. 3630 pHead = &pIt->channelNode; 3631 3632 while ((pHead != NULL) && (pHead->pKernelChannel != NULL)) 3633 { 3634 pObject = kchannelGetNextChild(&pIt->kchannelIter); 3635 3636 if (pObject != NULL) 3637 break; 3638 3639 // 3640 // If there are no more objects to inspect in the given channel, 3641 // move to the next channel (if any, for TSGs). 3642 // 3643 pHead = pHead->pNext; 3644 if (pHead != NULL) 3645 { 3646 NV_ASSERT_OR_ELSE(pHead->pKernelChannel != NULL, break); 3647 // Re-initialize the channeldescendant iterator based on this channel 3648 kchannelGetChildIterator(pHead->pKernelChannel, pIt->classNum, pIt->engDesc, &pIt->kchannelIter); 3649 } 3650 } 3651 3652 // Cache off the next channel to start searching from in future iterations. 3653 pIt->channelNode.pKernelChannel = pHead ? pHead->pKernelChannel : NULL; 3654 pIt->channelNode.pNext = pHead ? pHead->pNext : NULL; 3655 3656 return pObject; 3657 } 3658 3659 NV_STATUS 3660 kchannelFindChildByHandle 3661 ( 3662 KernelChannel *pKernelChannel, 3663 NvHandle hResource, 3664 ChannelDescendant **ppObject 3665 ) 3666 { 3667 RsClient *pClient = RES_GET_CLIENT(pKernelChannel); 3668 RsResourceRef *pResourceRef = NULL; 3669 3670 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, clientGetResourceRef(pClient, hResource, &pResourceRef)); 3671 3672 NV_CHECK_OR_RETURN(LEVEL_ERROR, pResourceRef->pParentRef->hResource == RES_GET_HANDLE(pKernelChannel), NV_ERR_OBJECT_NOT_FOUND); 3673 3674 *ppObject = dynamicCast(pResourceRef->pResource, ChannelDescendant); 3675 NV_CHECK_OR_RETURN(LEVEL_ERROR, *ppObject != NULL, NV_ERR_OBJECT_NOT_FOUND); 3676 3677 return NV_OK; 3678 } 3679 3680 static NV_STATUS 3681 _kchannelClearVAList 3682 ( 3683 OBJGPU *pGpu, 3684 VA_LIST *pVaList, 3685 NvBool bUnmap 3686 ) 3687 { 3688 // 3689 // Subcontext handling 3690 // We need to unmap the mappings on all the subcontext, since the this call will be made only on one of the TSG channels. 3691 // 3692 if (bUnmap) 3693 { 3694 OBJVASPACE *pVas; 3695 NvU64 vAddr; 3696 3697 FOR_EACH_IN_VADDR_LIST(pVaList, pVas, vAddr) 3698 { 3699 dmaUnmapBuffer_HAL(pGpu, GPU_GET_DMA(pGpu), pVas, vAddr); 3700 } 3701 FOR_EACH_IN_VADDR_LIST_END(pVaList, pVas, vAddr); 3702 } 3703 3704 vaListClear(pVaList); 3705 3706 return NV_OK; 3707 } 3708 3709 /** 3710 * @brief Set or clear the Engine Context Memdesc. 3711 * 3712 * Should be committed to hardware after this using channelCommitEngineContext(). 3713 * Should be unmapped before cleared/changed using kchannelUnmapEngineCtxBuf() 3714 * 3715 * @param[in] pGpu 3716 * @param[in] pKernelChannel 3717 * @param[in] engDesc 3718 * @param[in] pMemDesc the new memdesc to assign, or NULL to clear 3719 * 3720 * Returns: status 3721 */ 3722 NV_STATUS 3723 kchannelSetEngineContextMemDesc_IMPL 3724 ( 3725 OBJGPU *pGpu, 3726 KernelChannel *pKernelChannel, 3727 NvU32 engDesc, 3728 MEMORY_DESCRIPTOR *pMemDesc 3729 ) 3730 { 3731 NV_STATUS status = NV_OK; 3732 ENGINE_CTX_DESCRIPTOR *pEngCtxDesc; 3733 KernelChannelGroup *pKernelChannelGroup = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup; 3734 3735 NV_PRINTF(LEVEL_INFO, 3736 "ChID %x engDesc 0x%x pMemDesc %p\n", 3737 kchannelGetDebugTag(pKernelChannel), engDesc, pMemDesc); 3738 3739 NV_ASSERT_OR_RETURN(engDesc != ENG_FIFO, NV_ERR_INVALID_PARAMETER); 3740 3741 if (IS_GR(engDesc)) 3742 { 3743 NV_ASSERT_OK_OR_RETURN(kchannelCheckBcStateCurrent(pGpu, pKernelChannel)); 3744 } 3745 3746 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY) 3747 3748 // Get or allocate the EngCtxDesc 3749 pEngCtxDesc = pKernelChannelGroup->ppEngCtxDesc[gpumgrGetSubDeviceInstanceFromGpu(pGpu)]; 3750 3751 if (pEngCtxDesc == NULL && pMemDesc == NULL) 3752 { 3753 // There is no need to clean up or alloc anything. 3754 SLI_LOOP_CONTINUE; 3755 } 3756 3757 if (pEngCtxDesc != NULL) 3758 { 3759 // Cleanup for the engDesc context that existed before 3760 if (pEngCtxDesc->pMemDesc != NULL) 3761 { 3762 memdescFree(pEngCtxDesc->pMemDesc); 3763 memdescDestroy(pEngCtxDesc->pMemDesc); 3764 } 3765 3766 // 3767 } 3768 else 3769 { 3770 NV_ASSERT_OK_OR_ELSE(status, 3771 kchangrpAllocEngineContextDescriptor(pGpu, pKernelChannelGroup), 3772 SLI_LOOP_GOTO(fail)); 3773 pEngCtxDesc = pKernelChannelGroup->ppEngCtxDesc[gpumgrGetSubDeviceInstanceFromGpu(pGpu)]; 3774 NV_ASSERT_OR_ELSE(pEngCtxDesc != NULL, status = NV_ERR_NO_MEMORY; SLI_LOOP_GOTO(fail)); 3775 } 3776 3777 if (pMemDesc != NULL) 3778 { 3779 // We are setting a memdesc 3780 if (pMemDesc->Allocated > 0) 3781 pMemDesc->Allocated++; 3782 memdescAddRef(pMemDesc); 3783 3784 if (memdescGetAddressSpace(pMemDesc) == ADDR_VIRTUAL) 3785 { 3786 NvU64 virtAddr; 3787 3788 // Since the memdesc is already virtual, we do not manage it 3789 status = vaListSetManaged(&pEngCtxDesc->vaList, NV_FALSE); 3790 NV_ASSERT_OR_ELSE(status == NV_OK, SLI_LOOP_GOTO(fail)); 3791 3792 // memdescGetPhysAddr of a virtual memdesc is a virtual addr 3793 virtAddr = memdescGetPhysAddr(pMemDesc, AT_GPU, 0); 3794 status = vaListAddVa(&pEngCtxDesc->vaList, pKernelChannel->pVAS, virtAddr); 3795 NV_ASSERT_OR_ELSE(status == NV_OK, SLI_LOOP_GOTO(fail)); 3796 } 3797 } 3798 3799 // Assign the memdesc (or NULL) 3800 pEngCtxDesc->pMemDesc = pMemDesc; 3801 pEngCtxDesc->engDesc = engDesc; 3802 3803 SLI_LOOP_END 3804 3805 fail: 3806 return status; 3807 } 3808 3809 /** 3810 * @brief Unmaps everything from the Engine Context Memdesc. 3811 * 3812 * @param[in] pGpu 3813 * @param[in] pKernelChannel 3814 * @param[in] engDesc 3815 * 3816 * Returns: status 3817 */ 3818 NV_STATUS 3819 kchannelUnmapEngineCtxBuf_IMPL 3820 ( 3821 OBJGPU *pGpu, 3822 KernelChannel *pKernelChannel, 3823 NvU32 engDesc 3824 ) 3825 { 3826 NV_STATUS status = NV_OK; 3827 ENGINE_CTX_DESCRIPTOR *pEngCtxDesc; 3828 3829 NV_PRINTF(LEVEL_INFO, 3830 "ChID %x engDesc 0x%x\n", 3831 kchannelGetDebugTag(pKernelChannel), engDesc); 3832 3833 NV_ASSERT_OR_RETURN(engDesc != ENG_FIFO, NV_ERR_INVALID_PARAMETER); 3834 3835 if (IS_GR(engDesc)) 3836 { 3837 NV_ASSERT_OK_OR_RETURN(kchannelCheckBcStateCurrent(pGpu, pKernelChannel)); 3838 } 3839 3840 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY) 3841 pEngCtxDesc = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->ppEngCtxDesc[gpumgrGetSubDeviceInstanceFromGpu(pGpu)]; 3842 3843 // EngCtxDesc and MemDesc will be here, or else nothing can be mapped 3844 if ((pEngCtxDesc == NULL) || (pEngCtxDesc->pMemDesc == NULL)) 3845 { 3846 SLI_LOOP_CONTINUE; 3847 } 3848 3849 // Clear VA list, including unmap if managed 3850 status = _kchannelClearVAList(pGpu, &pEngCtxDesc->vaList, vaListGetManaged(&pEngCtxDesc->vaList)); 3851 NV_ASSERT_OR_ELSE(status == NV_OK, SLI_LOOP_GOTO(fail)); 3852 3853 SLI_LOOP_END 3854 3855 fail: 3856 return status; 3857 } 3858 3859 // Check that BcState stays consistent for GR channel engine context 3860 NV_STATUS 3861 kchannelCheckBcStateCurrent_IMPL 3862 ( 3863 OBJGPU *pGpu, 3864 KernelChannel *pKernelChannel 3865 ) 3866 { 3867 #define KERNEL_CHANNEL_BCSTATE_UNINITIALIZED (0) 3868 #define KERNEL_CHANNEL_BCSTATE_DISABLED (1) 3869 #define KERNEL_CHANNEL_BCSTATE_ENABLED (2) 3870 3871 NvBool bBcState = gpumgrGetBcEnabledStatus(pGpu); 3872 NvU8 channelBcStateEnum = bBcState ? KERNEL_CHANNEL_BCSTATE_ENABLED : KERNEL_CHANNEL_BCSTATE_DISABLED; 3873 3874 NV_PRINTF( 3875 LEVEL_INFO, 3876 "GPU = %d, ChID = %d, bcStateCurrent = %d, channelBcStateEnum = %d\n", 3877 pGpu->gpuInstance, 3878 kchannelGetDebugTag(pKernelChannel), 3879 pKernelChannel->bcStateCurrent, 3880 channelBcStateEnum); 3881 3882 // Check that the BC status did not change - 0 = first call, 1 = disable, 2 = enable. 3883 if (pKernelChannel->bcStateCurrent == KERNEL_CHANNEL_BCSTATE_UNINITIALIZED) 3884 { 3885 pKernelChannel->bcStateCurrent = channelBcStateEnum; 3886 } 3887 NV_ASSERT_OR_RETURN(pKernelChannel->bcStateCurrent == channelBcStateEnum, NV_ERR_INVALID_STATE); 3888 3889 return NV_OK; 3890 } 3891 3892 // Map the Engine Context Memdesc and add it's VAddr 3893 NV_STATUS 3894 kchannelMapEngineCtxBuf_IMPL 3895 ( 3896 OBJGPU *pGpu, 3897 KernelChannel *pKernelChannel, 3898 NvU32 engDesc 3899 ) 3900 { 3901 OBJVASPACE *pVAS = NULL; 3902 NV_STATUS status = NV_OK; 3903 ENGINE_CTX_DESCRIPTOR *pEngCtx; 3904 NvU64 addr; 3905 MEMORY_DESCRIPTOR *pTempMemDesc; 3906 OBJGVASPACE *pGVAS; 3907 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 3908 3909 NV_ASSERT_OR_RETURN(engDesc != ENG_FIFO, NV_ERR_INVALID_ARGUMENT); 3910 3911 if (IS_GR(engDesc)) 3912 { 3913 NV_ASSERT_OK_OR_RETURN(kchannelCheckBcStateCurrent(pGpu, pKernelChannel)); 3914 } 3915 3916 NV_PRINTF(LEVEL_INFO, "ChID %d engDesc %s (0x%x) \n", 3917 kchannelGetDebugTag(pKernelChannel), 3918 kfifoGetEngineName_HAL(GPU_GET_KERNEL_FIFO(pGpu), ENGINE_INFO_TYPE_ENG_DESC, engDesc), 3919 engDesc); 3920 3921 pVAS = pKernelChannel->pVAS; 3922 pGVAS = dynamicCast(pVAS, OBJGVASPACE); 3923 NV_ASSERT_OR_RETURN(pGVAS != NULL, NV_ERR_INVALID_STATE); 3924 3925 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 3926 3927 pEngCtx = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->ppEngCtxDesc[gpumgrGetSubDeviceInstanceFromGpu(pGpu)]; 3928 NV_ASSERT_OR_ELSE(pEngCtx != NULL, status = NV_ERR_INVALID_STATE; goto fail); 3929 3930 pTempMemDesc = pEngCtx->pMemDesc; 3931 NV_ASSERT_OR_ELSE(pTempMemDesc != NULL, status = NV_ERR_INVALID_STATE; goto fail); 3932 3933 // 3934 // For virtual context, UMD has already alloced/mapped the engine context. 3935 // So simply get the vaddr 3936 // 3937 3938 status = vaListFindVa(&pEngCtx->vaList, pVAS, &addr); 3939 if (status == NV_OK) 3940 { 3941 // VAddr already exists and needs no action 3942 SLI_LOOP_CONTINUE; 3943 } 3944 else if (status == NV_ERR_OBJECT_NOT_FOUND) 3945 { 3946 NvU32 flags = DMA_ALLOC_VASPACE_NONE; 3947 if (gvaspaceIsExternallyOwned(pGVAS)) 3948 { 3949 // We should never land up here if VA space is externally owned! 3950 NV_ASSERT_FAILED("Externally owned object not found"); 3951 status = NV_ERR_INVALID_OPERATION; 3952 goto fail; 3953 } 3954 3955 kfifoGetCtxBufferMapFlags_HAL(pGpu, pKernelFifo, engDesc, &flags); 3956 3957 status = dmaMapBuffer_HAL(pGpu, GPU_GET_DMA(pGpu), pVAS, pTempMemDesc, &addr, 3958 flags, DMA_UPDATE_VASPACE_FLAGS_NONE); 3959 if (status != NV_OK) 3960 { 3961 NV_PRINTF(LEVEL_ERROR, 3962 "Could not map context buffer for engDesc 0x%x\n", 3963 engDesc); 3964 goto fail; 3965 } 3966 else 3967 { 3968 status = vaListAddVa(&pEngCtx->vaList, pVAS, addr); 3969 NV_ASSERT(status == NV_OK); 3970 } 3971 } 3972 else 3973 { 3974 NV_ASSERT_OK_FAILED("vaListFindVa", status); 3975 goto fail; 3976 } 3977 3978 fail: 3979 if (status != NV_OK) 3980 { 3981 SLI_LOOP_BREAK; 3982 } 3983 SLI_LOOP_END 3984 3985 return status; 3986 } 3987 3988 /** 3989 * @brief Updates the notifier index with which to update the work submit 3990 * notifier on request. 3991 * 3992 * @param[IN] pGpu OBJGPU 3993 * @param[in] pKernelChannel KernelChannel 3994 * @param[in] index Updated notifier index 3995 * 3996 * @return NV_OK 3997 * NV_ERR_OUT_OF_RANGE if index is beyond the bounds of the notifier 3998 */ 3999 NV_STATUS 4000 kchannelUpdateWorkSubmitTokenNotifIndex_IMPL 4001 ( 4002 OBJGPU *pGpu, 4003 KernelChannel *pKernelChannel, 4004 NvU32 index 4005 ) 4006 { 4007 NvHandle hNotifier; 4008 RsClient *pClient = RES_GET_CLIENT(pKernelChannel); 4009 Memory *pMemory; 4010 ContextDma *pContextDma; 4011 NvU32 addressSpace; 4012 NvU64 notificationBufferSize; 4013 Device *pDevice; 4014 4015 hNotifier = pKernelChannel->hErrorContext; 4016 4017 // Clobbering error notifier index is illegal 4018 NV_CHECK_OR_RETURN(LEVEL_INFO, index != NV_CHANNELGPFIFO_NOTIFICATION_TYPE_ERROR, 4019 NV_ERR_INVALID_ARGUMENT); 4020 4021 // If key rotation is enabled then clobbering key rotation notifier is disallowed 4022 ConfidentialCompute *pConfCompute = GPU_GET_CONF_COMPUTE(pGpu); 4023 if ((pConfCompute != NULL) && 4024 (pConfCompute->getProperty(pConfCompute, PDB_PROP_CONFCOMPUTE_KEY_ROTATION_SUPPORTED))) 4025 { 4026 NV_CHECK_OR_RETURN(LEVEL_ERROR, index != NV_CHANNELGPFIFO_NOTIFICATION_TYPE_KEY_ROTATION_STATUS, 4027 NV_ERR_INVALID_ARGUMENT); 4028 } 4029 4030 // Check for integer overflows 4031 if (((index + 1) < index) || 4032 !portSafeMulU64(index + 1, sizeof(NvNotification), ¬ificationBufferSize)) 4033 { 4034 return NV_ERR_OUT_OF_RANGE; 4035 } 4036 4037 pDevice = GPU_RES_GET_DEVICE(pKernelChannel); 4038 4039 if (NV_OK == memGetByHandleAndDevice(pClient, hNotifier, RES_GET_HANDLE(pDevice), &pMemory)) 4040 { 4041 addressSpace = memdescGetAddressSpace(pMemory->pMemDesc); 4042 4043 NV_CHECK_OR_RETURN(LEVEL_INFO, pMemory->Length >= notificationBufferSize, 4044 NV_ERR_OUT_OF_RANGE); 4045 switch (addressSpace) 4046 { 4047 case ADDR_VIRTUAL: 4048 { 4049 NvU64 physAddr = memdescGetPhysAddr(pMemory->pMemDesc, AT_GPU_VA, 0); 4050 PCLI_DMA_MAPPING_INFO pDmaMappingInfo; 4051 4052 NV_CHECK_OR_RETURN(LEVEL_INFO, 4053 CliGetDmaMappingInfo(pClient, 4054 RES_GET_HANDLE(pDevice), 4055 RES_GET_HANDLE(pMemory), 4056 physAddr, 4057 gpumgrGetDeviceGpuMask(pGpu->deviceInstance), 4058 &pDmaMappingInfo), 4059 NV_ERR_GENERIC); 4060 4061 NV_CHECK_OR_RETURN(LEVEL_INFO, pDmaMappingInfo->pMemDesc->Size >= notificationBufferSize, 4062 NV_ERR_OUT_OF_RANGE); 4063 break; 4064 } 4065 case ADDR_FBMEM: 4066 // fall through 4067 case ADDR_SYSMEM: 4068 // Covered by check prior to switch/case 4069 break; 4070 default: 4071 return NV_ERR_NOT_SUPPORTED; 4072 } 4073 } 4074 else if (NV_OK == ctxdmaGetByHandle(pClient, hNotifier, &pContextDma)) 4075 { 4076 NV_CHECK_OR_RETURN(LEVEL_INFO, pContextDma->Limit >= (notificationBufferSize - 1), 4077 NV_ERR_OUT_OF_RANGE); 4078 } 4079 else 4080 { 4081 return NV_ERR_OBJECT_NOT_FOUND; 4082 } 4083 4084 pKernelChannel->notifyIndex[NV_CHANNELGPFIFO_NOTIFICATION_TYPE_WORK_SUBMIT_TOKEN] 4085 = index; 4086 4087 return NV_OK; 4088 } 4089 4090 /** 4091 * @brief Updates the work submit notifier passed to the channel during channel 4092 * creation with the new work submit token. 4093 * 4094 * @param[IN] pGpu OBJGPU 4095 * @param[in] pKernelChannel KernelChannel 4096 * @param[in] token Work submit token to notify clients of 4097 * 4098 * @return NV_OK on successful notify 4099 * NV_OK if client has not set up the doorbell notifier. This should 4100 * be an error once all clients have been updated. 4101 */ 4102 NV_STATUS 4103 kchannelNotifyWorkSubmitToken_IMPL 4104 ( 4105 OBJGPU *pGpu, 4106 KernelChannel *pKernelChannel, 4107 NvU32 token 4108 ) 4109 { 4110 NvU16 notifyStatus = 0x0; 4111 NvU32 index = pKernelChannel->notifyIndex[NV_CHANNELGPFIFO_NOTIFICATION_TYPE_WORK_SUBMIT_TOKEN]; 4112 4113 notifyStatus = 4114 FLD_SET_DRF(_CHANNELGPFIFO, _NOTIFICATION_STATUS, _IN_PROGRESS, _TRUE, notifyStatus); 4115 notifyStatus = 4116 FLD_SET_DRF_NUM(_CHANNELGPFIFO, _NOTIFICATION_STATUS, _VALUE, 0xFFFF, notifyStatus); 4117 4118 return kchannelUpdateNotifierMem(pKernelChannel, index, token, 0, notifyStatus); 4119 } 4120 4121 /** 4122 * @brief Alloc and set up pNotifyActions 4123 * 4124 * @param[in] pKernelChannel 4125 * @param[in] classNuml Channel class 4126 * 4127 * @return NV_OK or error code 4128 */ 4129 static NV_STATUS 4130 _kchannelSetupNotifyActions 4131 ( 4132 KernelChannel *pKernelChannel, 4133 NvU32 classNum 4134 ) 4135 { 4136 CLI_CHANNEL_CLASS_INFO classInfo; 4137 4138 // Allocate notifier action table for the maximum supported by this class 4139 CliGetChannelClassInfo(classNum, &classInfo); 4140 if (classInfo.notifiersMaxCount > 0) 4141 { 4142 pKernelChannel->pNotifyActions = portMemAllocNonPaged( 4143 classInfo.notifiersMaxCount * sizeof(*pKernelChannel->pNotifyActions)); 4144 if (pKernelChannel->pNotifyActions == NULL) 4145 return NV_ERR_NO_MEMORY; 4146 4147 portMemSet(pKernelChannel->pNotifyActions, 0, 4148 classInfo.notifiersMaxCount * sizeof(*pKernelChannel->pNotifyActions)); 4149 } 4150 4151 return NV_OK; 4152 } // end of _kchannelSetupNotifyActions() 4153 4154 /** 4155 * @brief Cleans up pNotifyActions 4156 * 4157 * @param[in] pKernelChannel 4158 */ 4159 static void 4160 _kchannelCleanupNotifyActions 4161 ( 4162 KernelChannel *pKernelChannel 4163 ) 4164 { 4165 // free memory associated with notify actions table 4166 portMemFree(pKernelChannel->pNotifyActions); 4167 pKernelChannel->pNotifyActions = NULL; 4168 } // end of _kchannelCleanupNotifyActions() 4169 4170 static NV_STATUS 4171 _kchannelNotifyOfChid 4172 ( 4173 OBJGPU *pGpu, 4174 KernelChannel *pKernelChannel, 4175 RsClient *pRsClient 4176 ) 4177 { 4178 ContextDma *pContextDma; 4179 4180 // 4181 // Return the chid to the drivers in the error context DMA 4182 // 4183 // We need to update this when virtual channel gets mapped in. 4184 // 4185 4186 if ((ctxdmaGetByHandle(pRsClient, pKernelChannel->hErrorContext, &pContextDma)) == NV_OK) 4187 { 4188 NV_CHECK_OR_RETURN(LEVEL_INFO, pContextDma->Limit >= sizeof(NvNotification) - 1, NV_ERR_INVALID_ARGUMENT); 4189 notifyFillNotifier(pGpu, pContextDma, pKernelChannel->ChID, 0, NV_OK); 4190 } 4191 4192 return NV_OK; 4193 } 4194 4195 NvU32 4196 kchannelGetGfid_IMPL 4197 ( 4198 KernelChannel *pKernelChannel 4199 ) 4200 { 4201 return pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->gfid; 4202 } 4203 4204 NvBool 4205 kchannelIsCpuMapped 4206 ( 4207 OBJGPU *pGpu, 4208 KernelChannel *pKernelChannel 4209 ) 4210 { 4211 return !!(pKernelChannel->swState[gpumgrGetSubDeviceInstanceFromGpu(pGpu)] & 4212 KERNEL_CHANNEL_SW_STATE_CPU_MAP); 4213 } 4214 4215 void 4216 kchannelSetCpuMapped 4217 ( 4218 OBJGPU *pGpu, 4219 KernelChannel *pKernelChannel, 4220 NvBool bCpuMapped 4221 ) 4222 { 4223 if (bCpuMapped) 4224 { 4225 pKernelChannel->swState[gpumgrGetSubDeviceInstanceFromGpu(pGpu)] |= 4226 KERNEL_CHANNEL_SW_STATE_CPU_MAP; 4227 } 4228 else 4229 { 4230 pKernelChannel->swState[gpumgrGetSubDeviceInstanceFromGpu(pGpu)] &= 4231 ~(KERNEL_CHANNEL_SW_STATE_CPU_MAP); 4232 } 4233 } 4234 4235 NvBool 4236 kchannelIsRunlistSet 4237 ( 4238 OBJGPU *pGpu, 4239 KernelChannel *pKernelChannel 4240 ) 4241 { 4242 return !!(pKernelChannel->swState[gpumgrGetSubDeviceInstanceFromGpu(pGpu)] & 4243 KERNEL_CHANNEL_SW_STATE_RUNLIST_SET); 4244 } 4245 4246 void 4247 kchannelSetRunlistSet 4248 ( 4249 OBJGPU *pGpu, 4250 KernelChannel *pKernelChannel, 4251 NvBool bRunlistSet 4252 ) 4253 { 4254 if (bRunlistSet) 4255 { 4256 pKernelChannel->swState[gpumgrGetSubDeviceInstanceFromGpu(pGpu)] |= 4257 KERNEL_CHANNEL_SW_STATE_RUNLIST_SET; 4258 } 4259 else 4260 { 4261 pKernelChannel->swState[gpumgrGetSubDeviceInstanceFromGpu(pGpu)] &= 4262 ~(KERNEL_CHANNEL_SW_STATE_RUNLIST_SET); 4263 } 4264 } 4265 4266 NV_STATUS 4267 kchannelGetChannelPhysicalState_KERNEL 4268 ( 4269 OBJGPU *pGpu, 4270 KernelChannel *pKernelChannel, 4271 NV208F_CTRL_FIFO_GET_CHANNEL_STATE_PARAMS *pChannelStateParams 4272 ) 4273 { 4274 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 4275 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams->pLegacyParams; 4276 NV_STATUS status = NV_OK; 4277 4278 // Get the physical state from GSP 4279 NV_RM_RPC_CONTROL(pGpu, 4280 pRmCtrlParams->hClient, 4281 pRmCtrlParams->hObject, 4282 pRmCtrlParams->cmd, 4283 pRmCtrlParams->pParams, 4284 pRmCtrlParams->paramsSize, 4285 status); 4286 NV_ASSERT_OK_OR_RETURN(status); 4287 4288 return NV_OK; 4289 } 4290 4291 NV_STATUS 4292 kchannelMapUserD_IMPL 4293 ( 4294 OBJGPU *pGpu, 4295 KernelChannel *pKernelChannel, 4296 RS_PRIV_LEVEL privLevel, 4297 NvU64 offset, 4298 NvU32 protect, 4299 NvP64 *ppCpuVirtAddr, 4300 NvP64 *ppPriv 4301 ) 4302 { 4303 NV_STATUS status = NV_OK; 4304 NvU64 userBase; 4305 NvU64 userOffset; 4306 NvU64 userSize; 4307 NvU32 cachingMode = NV_MEMORY_UNCACHED; 4308 4309 // if USERD is allocated by client 4310 if (pKernelChannel->bClientAllocatedUserD) 4311 { 4312 return NV_OK; 4313 } 4314 4315 status = kchannelGetUserdInfo_HAL(pGpu, pKernelChannel, 4316 &userBase, &userOffset, &userSize); 4317 4318 if (status != NV_OK) 4319 return status; 4320 4321 4322 if (userBase == pGpu->busInfo.gpuPhysAddr) 4323 { 4324 // Create a mapping of BAR0 4325 status = osMapGPU(pGpu, privLevel, NvU64_LO32(userOffset+offset), 4326 NvU64_LO32(userSize), protect, ppCpuVirtAddr, ppPriv); 4327 goto done; 4328 } 4329 4330 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING)) 4331 { 4332 cachingMode = NV_MEMORY_CACHED; 4333 } 4334 4335 // 4336 // If userBase is not bar0, then it is bar1 and we create a regular memory 4337 // mapping. 4338 // 4339 if (privLevel >= RS_PRIV_LEVEL_KERNEL) 4340 { 4341 status = osMapPciMemoryKernel64(pGpu, userBase + userOffset + offset, 4342 userSize, protect, ppCpuVirtAddr, cachingMode); 4343 } 4344 else 4345 { 4346 status = osMapPciMemoryUser(pGpu->pOsGpuInfo, 4347 userBase + userOffset + offset, 4348 userSize, protect, ppCpuVirtAddr, 4349 ppPriv, cachingMode); 4350 } 4351 if (!((status == NV_OK) && *ppCpuVirtAddr)) 4352 { 4353 NV_PRINTF(LEVEL_ERROR, 4354 "BAR1 offset 0x%llx for USERD of channel %x could not be cpu mapped\n", 4355 userOffset, kchannelGetDebugTag(pKernelChannel)); 4356 } 4357 4358 done: 4359 4360 // Indicate channel is mapped 4361 if (status == NV_OK) 4362 { 4363 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY) 4364 kchannelSetCpuMapped(pGpu, pKernelChannel, NV_TRUE); 4365 SLI_LOOP_END 4366 } 4367 4368 return status; 4369 } 4370 4371 void 4372 kchannelUnmapUserD_IMPL 4373 ( 4374 OBJGPU *pGpu, 4375 KernelChannel *pKernelChannel, 4376 RS_PRIV_LEVEL privLevel, 4377 NvP64 *ppCpuVirtAddr, 4378 NvP64 *ppPriv 4379 ) 4380 { 4381 NV_STATUS status; 4382 NvU64 userBase; 4383 NvU64 userOffset; 4384 NvU64 userSize; 4385 4386 if (pKernelChannel->bClientAllocatedUserD) 4387 { 4388 return; 4389 } 4390 4391 status = kchannelGetUserdInfo_HAL(pGpu, pKernelChannel, 4392 &userBase, &userOffset, &userSize); 4393 4394 NV_ASSERT_OR_RETURN_VOID(status == NV_OK); 4395 4396 if (userBase == pGpu->busInfo.gpuPhysAddr) 4397 { 4398 osUnmapGPU(pGpu->pOsGpuInfo, privLevel, *ppCpuVirtAddr, 4399 NvU64_LO32(userSize), *ppPriv); 4400 } 4401 else 4402 { 4403 // GF100+ 4404 // Unmap Cpu virt mapping 4405 if (privLevel >= RS_PRIV_LEVEL_KERNEL) 4406 { 4407 osUnmapPciMemoryKernel64(pGpu, *ppCpuVirtAddr); 4408 } 4409 else 4410 { 4411 osUnmapPciMemoryUser(pGpu->pOsGpuInfo, *ppCpuVirtAddr, 4412 userSize, *ppPriv); 4413 } 4414 } 4415 4416 // Indicate channel is !mapped 4417 kchannelSetCpuMapped(pGpu, pKernelChannel, NV_FALSE); 4418 return; 4419 } 4420 4421 static NV_STATUS 4422 _kchannelGetUserMemDesc 4423 ( 4424 OBJGPU *pGpu, 4425 KernelChannel *pKernelChannel, 4426 PMEMORY_DESCRIPTOR *ppMemDesc 4427 ) 4428 { 4429 NV_ASSERT_OR_RETURN(ppMemDesc != NULL, NV_ERR_INVALID_STATE); 4430 *ppMemDesc = NULL; 4431 4432 NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_STATE); 4433 4434 *ppMemDesc = pKernelChannel->pInstSubDeviceMemDesc[gpumgrGetSubDeviceInstanceFromGpu(pGpu)]; 4435 4436 return *ppMemDesc ? NV_OK : NV_ERR_INVALID_STATE; 4437 } 4438 4439 /*! 4440 * @brief Retrieve a KernelChannel from either a KernelChannel or TSG handle. KernelChannel is 4441 * checked first. If TSG is provided, the head of the TSG is returned. 4442 * 4443 * @param[in] pClient Client object 4444 * @param[in] hDual NvHandle either to TSG or to KernelChannel 4445 * @param[out] ppKernelChannel Referenced KernelChannel 4446 */ 4447 NV_STATUS 4448 kchannelGetFromDualHandle_IMPL 4449 ( 4450 RsClient *pClient, 4451 NvHandle hDual, 4452 KernelChannel **ppKernelChannel 4453 ) 4454 { 4455 KernelChannel *pKernelChannel; 4456 RsResourceRef *pChanGrpRef; 4457 4458 NV_ASSERT_OR_RETURN(ppKernelChannel != NULL, NV_ERR_INVALID_ARGUMENT); 4459 4460 *ppKernelChannel = NULL; 4461 4462 if (CliGetKernelChannel(pClient, hDual, &pKernelChannel) == NV_OK) 4463 { 4464 *ppKernelChannel = pKernelChannel; 4465 return NV_OK; 4466 } 4467 4468 if (CliGetChannelGroup(pClient->hClient, hDual, &pChanGrpRef, NULL) == NV_OK) 4469 { 4470 KernelChannelGroupApi *pKernelChannelGroupApi = dynamicCast( 4471 pChanGrpRef->pResource, 4472 KernelChannelGroupApi); 4473 4474 NV_ASSERT_OR_RETURN( 4475 (pKernelChannelGroupApi != NULL) && 4476 (pKernelChannelGroupApi->pKernelChannelGroup != NULL), 4477 NV_ERR_INVALID_ARGUMENT); 4478 4479 if (pKernelChannelGroupApi->pKernelChannelGroup->chanCount == 0) 4480 return NV_ERR_INVALID_ARGUMENT; 4481 4482 *ppKernelChannel = 4483 pKernelChannelGroupApi->pKernelChannelGroup->pChanList->pHead->pKernelChannel; 4484 NV_ASSERT_OR_RETURN(*ppKernelChannel != NULL, NV_ERR_INVALID_STATE); 4485 4486 return NV_OK; 4487 } 4488 4489 return NV_ERR_OBJECT_NOT_FOUND; 4490 } 4491 4492 /*! 4493 * @brief Retrieve a KernelChannel from either a KernelChannel or TSG handle. KernelChannel is 4494 * checked first. If TSG is provided, the head of the TSG is returned. If 4495 * KernelChannel handle is provided, it must not be part of a client-allocated TSG. 4496 * 4497 * @param[in] pClient Client object 4498 * @param[in] hDual NvHandle either to TSG or to bare Channel 4499 * @param[out] ppKernelChannel Referenced KernelChannel 4500 */ 4501 NV_STATUS 4502 kchannelGetFromDualHandleRestricted_IMPL 4503 ( 4504 RsClient *pClient, 4505 NvHandle hDual, 4506 KernelChannel **ppKernelChannel 4507 ) 4508 { 4509 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 4510 kchannelGetFromDualHandle(pClient, hDual, ppKernelChannel)); 4511 if ((RES_GET_HANDLE(*ppKernelChannel) == hDual) && 4512 (((*ppKernelChannel)->pKernelChannelGroupApi->pKernelChannelGroup != NULL) && 4513 !(*ppKernelChannel)->pKernelChannelGroupApi->pKernelChannelGroup->bAllocatedByRm)) 4514 { 4515 NV_PRINTF(LEVEL_ERROR, "channel handle 0x%08x is part of a channel group, not allowed!\n", 4516 RES_GET_HANDLE(*ppKernelChannel)); 4517 return NV_ERR_INVALID_ARGUMENT; 4518 } 4519 return NV_OK; 4520 } 4521 4522 static void 4523 _kchannelUpdateFifoMapping 4524 ( 4525 KernelChannel *pKernelChannel, 4526 OBJGPU *pGpu, 4527 NvBool bKernel, 4528 NvP64 cpuAddress, 4529 NvP64 priv, 4530 NvU64 cpuMapLength, 4531 NvU32 flags, 4532 NvHandle hSubdevice, 4533 RsCpuMapping *pMapping 4534 ) 4535 { 4536 pMapping->pPrivate->pGpu = pGpu; 4537 pMapping->pPrivate->bKernel = bKernel; 4538 pMapping->processId = osGetCurrentProcess(); 4539 pMapping->pLinearAddress = cpuAddress; 4540 pMapping->pPrivate->pPriv = priv; 4541 pMapping->length = cpuMapLength; 4542 pMapping->flags = flags; 4543 pMapping->pContext = (void*)(NvUPtr)pKernelChannel->ChID; 4544 } 4545 4546 NV_STATUS kchannelRetrieveKmb_KERNEL 4547 ( 4548 OBJGPU *pGpu, 4549 KernelChannel *pKernelChannel, 4550 ROTATE_IV_TYPE rotateOperation, 4551 NvBool bIncludeIvOrNonce, 4552 CC_KMB *keyMaterialBundle 4553 ) 4554 { 4555 ConfidentialCompute *pCC = GPU_GET_CONF_COMPUTE(pGpu); 4556 4557 NV_ASSERT(pCC != NULL); 4558 4559 return (confComputeKeyStoreRetrieveViaChannel_HAL(pCC, pKernelChannel, rotateOperation, 4560 bIncludeIvOrNonce, keyMaterialBundle)); 4561 } 4562 4563 /*! 4564 * @brief Get KMB for secure channel 4565 * 4566 * @param[in] pKernelChannnel 4567 * @param[out] pGetKmbParams 4568 */ 4569 NV_STATUS 4570 kchannelCtrlCmdGetKmb_KERNEL 4571 ( 4572 KernelChannel *pKernelChannel, 4573 NVC56F_CTRL_CMD_GET_KMB_PARAMS *pGetKmbParams 4574 ) 4575 { 4576 if (!pKernelChannel->bCCSecureChannel) 4577 { 4578 return NV_ERR_NOT_SUPPORTED; 4579 } 4580 4581 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannel); 4582 ConfidentialCompute *pConfCompute = GPU_GET_CONF_COMPUTE(pGpu); 4583 if (pConfCompute->getProperty(pConfCompute, PDB_PROP_CONFCOMPUTE_KEY_ROTATION_SUPPORTED)) 4584 { 4585 KEY_ROTATION_STATUS state; 4586 NvU32 h2dKey; 4587 NV_ASSERT_OK_OR_RETURN(confComputeGetKeyPairByChannel(pGpu, pConfCompute, pKernelChannel, &h2dKey, NULL)); 4588 NV_ASSERT_OK_OR_RETURN(confComputeGetKeyRotationStatus(pConfCompute, h2dKey, &state)); 4589 if ((state != KEY_ROTATION_STATUS_IDLE) || 4590 (kchannelIsDisabledForKeyRotation(pGpu, pKernelChannel))) 4591 { 4592 return NV_ERR_KEY_ROTATION_IN_PROGRESS; 4593 } 4594 } 4595 4596 portMemCopy((void*)(&pGetKmbParams->kmb), sizeof(CC_KMB), 4597 (const void*)(&pKernelChannel->clientKmb), sizeof(CC_KMB)); 4598 4599 if (pConfCompute->getProperty(pConfCompute, PDB_PROP_CONFCOMPUTE_KEY_ROTATION_SUPPORTED)) 4600 { 4601 // 4602 // If this is the first time GET_KMB is called on a context 4603 // then setup the encrypt stats buffer. 4604 // 4605 if (pKernelChannel->pEncStatsBufMemDesc == NULL) 4606 { 4607 RsClient *pRsClient = NULL; 4608 RsResourceRef *pResourceRef = NULL; 4609 Memory *pMemory = NULL; 4610 MEMORY_DESCRIPTOR *pMemDesc = NULL; 4611 NvHandle hClient = RES_GET_CLIENT_HANDLE(pKernelChannel); 4612 4613 NV_ASSERT_OK_OR_RETURN(serverGetClientUnderLock(&g_resServ, hClient, &pRsClient)); 4614 if (clientGetResourceRef(pRsClient, pGetKmbParams->hMemory, &pResourceRef) != NV_OK) 4615 { 4616 // TODO: Make this fatal once all cients move to using hMemory 4617 return NV_OK; 4618 } 4619 pMemory = dynamicCast(pResourceRef->pResource, Memory); 4620 pMemDesc = pMemory->pMemDesc; 4621 NV_ASSERT_OR_RETURN(pMemDesc != NULL, NV_ERR_INVALID_ARGUMENT); 4622 pKernelChannel->pEncStatsBufMemDesc = pMemDesc; 4623 NV_ASSERT_OK_OR_RETURN(kchannelSetEncryptionStatsBuffer_HAL(pGpu, pKernelChannel, NV_TRUE)); 4624 } 4625 4626 // 4627 // Reset statistics every time GET_KMB is called 4628 // TODO CONFCOMP-984: Make this fatal if this ptr is NULL 4629 // 4630 if (pKernelChannel->pEncStatsBuf != NULL) 4631 portMemSet(pKernelChannel->pEncStatsBuf, 0, sizeof(CC_CRYPTOBUNDLE_STATS)); 4632 } 4633 4634 return NV_OK; 4635 return NV_ERR_NOT_SUPPORTED; 4636 } 4637 4638 /*! 4639 * @brief Rotate the IVs for the given secure channel 4640 * 4641 * @param[in] pKernelChannel 4642 * @param[out] pRotateIvParams 4643 * 4644 * @return NV_OK on success 4645 * @return NV_ERR_NOT_SUPPORTED if channel is not a secure channel. 4646 */ 4647 NV_STATUS 4648 kchannelCtrlRotateSecureChannelIv_KERNEL 4649 ( 4650 KernelChannel *pKernelChannel, 4651 NVC56F_CTRL_ROTATE_SECURE_CHANNEL_IV_PARAMS *pRotateIvParams 4652 ) 4653 { 4654 NV_STATUS status = NV_OK; 4655 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannel); 4656 ConfidentialCompute *pCC = GPU_GET_CONF_COMPUTE(pGpu); 4657 ROTATE_IV_TYPE rotateIvOperation = pRotateIvParams->rotateIvType; 4658 4659 if (!pKernelChannel->bCCSecureChannel) 4660 { 4661 return NV_ERR_NOT_SUPPORTED; 4662 } 4663 4664 if (pCC->getProperty(pCC, PDB_PROP_CONFCOMPUTE_KEY_ROTATION_SUPPORTED)) 4665 { 4666 KEY_ROTATION_STATUS state; 4667 NvU32 h2dKey; 4668 NV_ASSERT_OK_OR_RETURN(confComputeGetKeyPairByChannel(pGpu, pCC, pKernelChannel, &h2dKey, NULL)); 4669 NV_ASSERT_OK_OR_RETURN(confComputeGetKeyRotationStatus(pCC, h2dKey, &state)); 4670 if ((state != KEY_ROTATION_STATUS_IDLE) || 4671 (kchannelIsDisabledForKeyRotation(pGpu, pKernelChannel))) 4672 { 4673 return NV_ERR_KEY_ROTATION_IN_PROGRESS; 4674 } 4675 } 4676 4677 NV_PRINTF(LEVEL_INFO, "Rotating IV in CPU-RM.\n"); 4678 4679 status = confComputeKeyStoreRetrieveViaChannel_HAL( 4680 pCC, pKernelChannel, rotateIvOperation, NV_TRUE, &pKernelChannel->clientKmb); 4681 4682 if (status != NV_OK) 4683 { 4684 return status; 4685 } 4686 4687 portMemSet(pRotateIvParams, 0, sizeof(*pRotateIvParams)); 4688 4689 portMemCopy(pRotateIvParams->updatedKmb.encryptBundle.iv, 4690 sizeof(pRotateIvParams->updatedKmb.encryptBundle.iv), 4691 pKernelChannel->clientKmb.encryptBundle.iv, 4692 sizeof(pKernelChannel->clientKmb.encryptBundle.iv)); 4693 4694 portMemCopy(pRotateIvParams->updatedKmb.decryptBundle.iv, 4695 sizeof(pRotateIvParams->updatedKmb.decryptBundle.iv), 4696 pKernelChannel->clientKmb.decryptBundle.iv, 4697 sizeof(pKernelChannel->clientKmb.decryptBundle.iv)); 4698 4699 pRotateIvParams->rotateIvType = rotateIvOperation; 4700 4701 NV_RM_RPC_CONTROL(pGpu, 4702 RES_GET_CLIENT_HANDLE(pKernelChannel), 4703 RES_GET_HANDLE(pKernelChannel), 4704 NVC56F_CTRL_ROTATE_SECURE_CHANNEL_IV, 4705 pRotateIvParams, 4706 sizeof(*pRotateIvParams), 4707 status); 4708 4709 if (status != NV_OK) 4710 { 4711 return status; 4712 } 4713 4714 if ((rotateIvOperation == ROTATE_IV_ALL_VALID) || (rotateIvOperation == ROTATE_IV_ENCRYPT)) 4715 { 4716 portMemCopy(&pRotateIvParams->updatedKmb.encryptBundle, 4717 sizeof(pRotateIvParams->updatedKmb.encryptBundle), 4718 &pKernelChannel->clientKmb.encryptBundle, 4719 sizeof(pKernelChannel->clientKmb.encryptBundle)); 4720 } 4721 4722 if ((rotateIvOperation == ROTATE_IV_ALL_VALID) || (rotateIvOperation == ROTATE_IV_DECRYPT)) 4723 { 4724 portMemCopy(&pRotateIvParams->updatedKmb.decryptBundle, 4725 sizeof(pRotateIvParams->updatedKmb.decryptBundle), 4726 &pKernelChannel->clientKmb.decryptBundle, 4727 sizeof(pKernelChannel->clientKmb.decryptBundle)); 4728 } 4729 4730 return NV_OK; 4731 return NV_ERR_NOT_SUPPORTED; 4732 } 4733 4734 NV_STATUS 4735 kchannelCtrlRotateSecureChannelIv_PHYSICAL 4736 ( 4737 KernelChannel *pKernelChannel, 4738 NVC56F_CTRL_ROTATE_SECURE_CHANNEL_IV_PARAMS *pRotateIvParams 4739 ) 4740 { 4741 NV_STATUS status; 4742 4743 NV_PRINTF(LEVEL_INFO, "Rotating IV in GSP-RM.\n"); 4744 4745 // CPU-side encrypt IV corresponds to GPU-side decrypt IV. 4746 // CPU-side decrypt IV corresponds to GPU-side encrypt IV. 4747 status = 4748 kchannelRotateSecureChannelIv_HAL(pKernelChannel, 4749 pRotateIvParams->rotateIvType, 4750 pRotateIvParams->updatedKmb.decryptBundle.iv, 4751 pRotateIvParams->updatedKmb.encryptBundle.iv); 4752 if (status != NV_OK) 4753 { 4754 return status; 4755 } 4756 4757 return NV_OK; 4758 } 4759 4760 /*! 4761 * Fill in per-channel MMU exception data and allocate memory for this data if 4762 * necessary 4763 * 4764 * @param[inout] pKernelChannel 4765 * @param[in] pMmuExceptionData MMU exception data to be copied 4766 */ 4767 void 4768 kchannelFillMmuExceptionInfo_IMPL 4769 ( 4770 KernelChannel *pKernelChannel, 4771 FIFO_MMU_EXCEPTION_DATA *pMmuExceptionData 4772 ) 4773 { 4774 NV_STATUS status = NV_OK; 4775 4776 NV_ASSERT_OR_RETURN_VOID(pKernelChannel); 4777 4778 if (pKernelChannel->pMmuExceptionData == NULL) 4779 { 4780 pKernelChannel->pMmuExceptionData = portMemAllocNonPaged(sizeof(FIFO_MMU_EXCEPTION_DATA)); 4781 if (pKernelChannel->pMmuExceptionData == NULL) 4782 status = NV_ERR_NO_MEMORY; 4783 } 4784 4785 if (status == NV_OK) 4786 { 4787 portMemCopy(pKernelChannel->pMmuExceptionData, 4788 sizeof(FIFO_MMU_EXCEPTION_DATA), 4789 pMmuExceptionData, 4790 sizeof(FIFO_MMU_EXCEPTION_DATA)); 4791 } 4792 } 4793 4794 /*! 4795 * Free per-channel MMU exception data if it exists 4796 * 4797 * @param[inout] pKernelChannel 4798 */ 4799 void 4800 kchannelFreeMmuExceptionInfo_IMPL 4801 ( 4802 KernelChannel *pKernelChannel 4803 ) 4804 { 4805 portMemFree(pKernelChannel->pMmuExceptionData); 4806 pKernelChannel->pMmuExceptionData = NULL; 4807 } 4808 4809 /*! 4810 * Check if channel is disabled for key rotation 4811 */ 4812 NvBool kchannelIsDisabledForKeyRotation 4813 ( 4814 OBJGPU *pGpu, 4815 KernelChannel *pKernelChannel 4816 ) 4817 { 4818 return !!(pKernelChannel->swState[gpumgrGetSubDeviceInstanceFromGpu(pGpu)] & 4819 KERNEL_CHANNEL_SW_STATE_DISABLED_FOR_KEY_ROTATION); 4820 } 4821 4822 /*! 4823 * Mark channel disabled for key rotation 4824 */ 4825 void kchannelDisableForKeyRotation 4826 ( 4827 OBJGPU *pGpu, 4828 KernelChannel *pKernelChannel, 4829 NvBool bDisable 4830 ) 4831 { 4832 if (bDisable) 4833 { 4834 pKernelChannel->swState[gpumgrGetSubDeviceInstanceFromGpu(pGpu)] |= 4835 KERNEL_CHANNEL_SW_STATE_DISABLED_FOR_KEY_ROTATION; 4836 } 4837 else 4838 { 4839 pKernelChannel->swState[gpumgrGetSubDeviceInstanceFromGpu(pGpu)] &= 4840 ~KERNEL_CHANNEL_SW_STATE_DISABLED_FOR_KEY_ROTATION; 4841 } 4842 } 4843 4844 /*! 4845 * Check if channel needs to be enabled after key rotation 4846 */ 4847 NvBool kchannelIsEnableAfterKeyRotation 4848 ( 4849 OBJGPU *pGpu, 4850 KernelChannel *pKernelChannel 4851 ) 4852 { 4853 return !!(pKernelChannel->swState[gpumgrGetSubDeviceInstanceFromGpu(pGpu)] & 4854 KERNEL_CHANNEL_SW_STATE_ENABLE_AFTER_KEY_ROTATION); 4855 } 4856 4857 /*! 4858 * Mark channel to be re-enabled after key rotation completes 4859 */ 4860 void kchannelEnableAfterKeyRotation 4861 ( 4862 OBJGPU *pGpu, 4863 KernelChannel *pKernelChannel, 4864 NvBool bEnable 4865 ) 4866 { 4867 if (bEnable) 4868 { 4869 pKernelChannel->swState[gpumgrGetSubDeviceInstanceFromGpu(pGpu)] |= 4870 KERNEL_CHANNEL_SW_STATE_ENABLE_AFTER_KEY_ROTATION; 4871 } 4872 else 4873 { 4874 pKernelChannel->swState[gpumgrGetSubDeviceInstanceFromGpu(pGpu)] &= 4875 ~KERNEL_CHANNEL_SW_STATE_ENABLE_AFTER_KEY_ROTATION; 4876 } 4877 } 4878 4879 /*! 4880 * Creates/destroys persistent mappings for key rotation notifier 4881 */ 4882 NV_STATUS 4883 kchannelSetKeyRotationNotifier_KERNEL 4884 ( 4885 OBJGPU *pGpu, 4886 KernelChannel *pKernelChannel, 4887 NvBool bSet 4888 ) 4889 { 4890 NV_STATUS status = NV_OK; 4891 MEMORY_DESCRIPTOR *pNotifierMemDesc = pKernelChannel->pErrContextMemDesc; 4892 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 4893 TRANSFER_SURFACE surf = {0}; 4894 NV_ASSERT_OR_RETURN(pNotifierMemDesc != NULL, NV_ERR_INVALID_STATE); 4895 NV_ADDRESS_SPACE addressSpace = memdescGetAddressSpace(pNotifierMemDesc); 4896 NvU32 notifyIndex = NV_CHANNELGPFIFO_NOTIFICATION_TYPE_KEY_ROTATION_STATUS; 4897 4898 if (bSet) 4899 { 4900 NV_ASSERT_OR_RETURN(memdescGetSize(pNotifierMemDesc) >= ((notifyIndex + 1) * sizeof(NvNotification)), 4901 NV_ERR_INVALID_ARGUMENT); 4902 4903 NV_ASSERT_OR_RETURN(addressSpace == ADDR_SYSMEM, NV_ERR_NOT_SUPPORTED); 4904 if (pKernelChannel->pKeyRotationNotifierMemDesc == NULL) 4905 { 4906 NV_ASSERT_OK_OR_RETURN(memdescCreateSubMem(&pKernelChannel->pKeyRotationNotifierMemDesc, 4907 pNotifierMemDesc, pGpu, notifyIndex * sizeof(NvNotification), 4908 sizeof(NvNotification))); 4909 surf.pMemDesc = pKernelChannel->pKeyRotationNotifierMemDesc; 4910 surf.offset = 0; 4911 4912 pKernelChannel->pKeyRotationNotifier = 4913 (NvNotification *) memmgrMemBeginTransfer(pMemoryManager, &surf, 4914 sizeof(NvNotification), 4915 TRANSFER_FLAGS_SHADOW_ALLOC); 4916 NV_ASSERT_OR_ELSE(pKernelChannel->pKeyRotationNotifier != NULL, status = NV_ERR_INVALID_STATE; goto done;); 4917 4918 portMemSet((void*)pKernelChannel->pKeyRotationNotifier, 0, sizeof(NvNotification)); 4919 } 4920 } 4921 else 4922 { 4923 if (pKernelChannel->pKeyRotationNotifierMemDesc != NULL) 4924 { 4925 if (pKernelChannel->pKeyRotationNotifier != NULL) 4926 { 4927 surf.pMemDesc = pKernelChannel->pKeyRotationNotifierMemDesc; 4928 surf.offset = 0; 4929 memmgrMemEndTransfer(pMemoryManager, &surf, sizeof(NvNotification), 0); 4930 pKernelChannel->pKeyRotationNotifier = NULL; 4931 } 4932 memdescDestroy(pKernelChannel->pKeyRotationNotifierMemDesc); 4933 pKernelChannel->pKeyRotationNotifierMemDesc = NULL; 4934 } 4935 } 4936 4937 done: 4938 if (status != NV_OK) 4939 { 4940 if (pKernelChannel->pKeyRotationNotifierMemDesc != NULL) 4941 { 4942 memdescDestroy(pKernelChannel->pKeyRotationNotifierMemDesc); 4943 pKernelChannel->pKeyRotationNotifierMemDesc = NULL; 4944 } 4945 } 4946 return status; 4947 } 4948 4949 /*! 4950 * Creates/destroys persistent mappings for encryption stats buffer 4951 */ 4952 NV_STATUS 4953 kchannelSetEncryptionStatsBuffer_KERNEL 4954 ( 4955 OBJGPU *pGpu, 4956 KernelChannel *pKernelChannel, 4957 NvBool bSet 4958 ) 4959 { 4960 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 4961 TRANSFER_SURFACE surf = {0}; 4962 if (bSet) 4963 { 4964 NV_ASSERT_OR_RETURN(pKernelChannel->pEncStatsBuf == NULL, NV_ERR_INVALID_STATE); 4965 // 4966 // we rely on persistent mapping for encryption statistics buffer 4967 // since these will be used in top half and mappings are not allowed 4968 // in contexts that can't sleep on KVM or similar HCC systems. 4969 // 4970 surf.pMemDesc = pKernelChannel->pEncStatsBufMemDesc; 4971 surf.offset = 0; 4972 pKernelChannel->pEncStatsBuf = (CC_CRYPTOBUNDLE_STATS*)memmgrMemBeginTransfer(pMemoryManager, &surf, 4973 sizeof(CC_CRYPTOBUNDLE_STATS), 4974 TRANSFER_FLAGS_SHADOW_ALLOC); 4975 4976 NV_ASSERT_OR_RETURN(pKernelChannel->pEncStatsBuf != NULL, NV_ERR_INVALID_STATE); 4977 portMemSet(pKernelChannel->pEncStatsBuf, 0, sizeof(CC_CRYPTOBUNDLE_STATS)); 4978 } 4979 else 4980 { 4981 // 4982 // Free persistent mappings for encryption stats buffer 4983 // TODO CONFCOMP-984: Make this fatal if this ptr is NULL 4984 // 4985 if (pKernelChannel->pEncStatsBufMemDesc != NULL) 4986 { 4987 surf.pMemDesc = pKernelChannel->pEncStatsBufMemDesc; 4988 surf.offset = 0; 4989 memmgrMemEndTransfer(pMemoryManager, &surf, sizeof(CC_CRYPTOBUNDLE_STATS), 0); 4990 } 4991 } 4992 return NV_OK; 4993 } 4994 4995 static NvNotification* 4996 _kchannelGetKeyRotationNotifier(KernelChannel *pKernelChannel) 4997 { 4998 return pKernelChannel->pKeyRotationNotifier; 4999 } 5000