1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "kernel/gpu/fifo/kernel_channel_group_api.h"
25
26 #include "kernel/core/locks.h"
27 #include "kernel/gpu/fifo/kernel_channel_group.h"
28 #include "kernel/gpu/mem_mgr/mem_mgr.h"
29 #include "kernel/gpu/gr/kernel_graphics.h"
30 #include "kernel/gpu/falcon/kernel_falcon.h"
31 #include "kernel/gpu/rc/kernel_rc.h"
32 #include "platform/sli/sli.h"
33
34 #include "kernel/gpu/conf_compute/conf_compute.h"
35
36 #include "class/cl0090.h" // KERNEL_GRAPHICS_CONTEXT
37 #include "class/cl9067.h" // FERMI_CONTEXT_SHARE_A
38
39 #include "libraries/utils/nvprintf.h"
40 #include "gpu/gpu.h"
41 #include "gpu/device/device.h"
42 #include "kernel/gpu/mig_mgr/kernel_mig_manager.h"
43 #include "gpu/mem_mgr/vaspace_api.h"
44 #include "vgpu/rpc.h"
45 #include "rmapi/rs_utils.h"
46
47 NV_STATUS
kchangrpapiConstruct_IMPL(KernelChannelGroupApi * pKernelChannelGroupApi,CALL_CONTEXT * pCallContext,RS_RES_ALLOC_PARAMS_INTERNAL * pParams)48 kchangrpapiConstruct_IMPL
49 (
50 KernelChannelGroupApi *pKernelChannelGroupApi,
51 CALL_CONTEXT *pCallContext,
52 RS_RES_ALLOC_PARAMS_INTERNAL *pParams
53 )
54 {
55 NvBool bTsgAllocated = NV_FALSE;
56 RsResourceRef *pResourceRef = pCallContext->pResourceRef;
57 NV_STATUS rmStatus;
58 OBJVASPACE *pVAS = NULL;
59 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannelGroupApi);
60 KernelMIGManager *pKernelMIGManager = NULL;
61 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu);
62 NvHandle hVASpace = NV01_NULL_OBJECT;
63 Device *pDevice = NULL;
64 NvU32 gfid = GPU_GFID_PF;
65 RsShared *pShared = NULL;
66 RsClient *pClient;
67 NvBool bLockAcquired = NV_FALSE;
68 Heap *pHeap = GPU_GET_HEAP(pGpu);
69 NvBool bMIGInUse = NV_FALSE;
70 CTX_BUF_INFO *bufInfoList = NULL;
71 NvU32 bufCount = 0;
72 NvBool bReserveMem = NV_FALSE;
73 MIG_INSTANCE_REF ref;
74 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
75 KernelChannelGroup *pKernelChannelGroup = NULL;
76 NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS *pAllocParams = NULL;
77 RM_ENGINE_TYPE rmEngineType;
78
79
80 NV_PRINTF(LEVEL_INFO,
81 "hClient: 0x%x, hParent: 0x%x, hObject:0x%x, hClass: 0x%x\n",
82 pParams->hClient, pParams->hParent, pParams->hResource,
83 pParams->externalClassId);
84
85 if (RS_IS_COPY_CTOR(pParams))
86 {
87 NV_ASSERT_OK_OR_GOTO(rmStatus,
88 rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_FIFO),
89 done);
90 bLockAcquired = NV_TRUE;
91 rmStatus = kchangrpapiCopyConstruct_IMPL(pKernelChannelGroupApi,
92 pCallContext, pParams);
93 goto done;
94 }
95
96 //
97 // Make sure this GPU is not already locked by this thread
98 // Ideally this thread shouldn't have locked any GPU in the system but
99 // checking this is sufficient as memory allocation from PMA requires
100 // current GPU's lock not to be held
101 //
102 if (rmDeviceGpuLockIsOwner(pGpu->gpuInstance))
103 {
104 NV_PRINTF(LEVEL_ERROR, "TSG alloc should be called without acquiring GPU lock\n");
105 LOCK_ASSERT_AND_RETURN(0);
106 }
107
108 bufInfoList = portMemAllocNonPaged(NV_ENUM_SIZE(GR_CTX_BUFFER) * sizeof(*bufInfoList));
109 if (bufInfoList == NULL)
110 {
111 return NV_ERR_NO_MEMORY;
112 }
113
114 // Acquire the lock *only after* PMA is done allocating.
115 NV_ASSERT_OK_OR_GOTO(rmStatus,
116 rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_FIFO),
117 done);
118 bLockAcquired = NV_TRUE;
119
120 pAllocParams = pParams->pAllocParams;
121 hVASpace = pAllocParams->hVASpace;
122
123 NV_ASSERT_OK_OR_GOTO(rmStatus,
124 serverAllocShareWithHalspecParent(&g_resServ, classInfo(KernelChannelGroup),
125 &pShared, staticCast(pGpu, Object)),
126 failed);
127
128 pKernelChannelGroup = dynamicCast(pShared, KernelChannelGroup);
129 pKernelChannelGroupApi->pKernelChannelGroup = pKernelChannelGroup;
130
131 if (!gpuIsClassSupported(pGpu, pResourceRef->externalClassId))
132 {
133 NV_PRINTF(LEVEL_ERROR, "class %x not supported\n",
134 pResourceRef->externalClassId);
135 rmStatus = NV_ERR_NOT_SUPPORTED;
136 goto failed;
137 }
138
139 pKernelChannelGroupApi->hVASpace = hVASpace;
140
141 rmStatus = serverGetClientUnderLock(&g_resServ, pParams->hClient, &pClient);
142 if (rmStatus != NV_OK)
143 {
144 NV_PRINTF(LEVEL_ERROR, "Invalid client handle!\n");
145 rmStatus = NV_ERR_INVALID_ARGUMENT;
146 goto failed;
147 }
148
149 rmStatus = deviceGetByHandle(pClient, pParams->hParent, &pDevice);
150 if (rmStatus != NV_OK)
151 {
152 NV_PRINTF(LEVEL_ERROR, "Invalid parent/device handle!\n");
153 rmStatus = NV_ERR_INVALID_ARGUMENT;
154 goto failed;
155 }
156
157 pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu);
158 bMIGInUse = IS_MIG_IN_USE(pGpu);
159
160 rmEngineType = gpuGetRmEngineType(pAllocParams->engineType);
161
162 if (kfifoIsPerRunlistChramSupportedInHw(pKernelFifo))
163 {
164 if (!RM_ENGINE_TYPE_IS_VALID(rmEngineType))
165 {
166 NV_PRINTF(LEVEL_NOTICE, "Valid engine Id must be specified while allocating TSGs or bare channels!\n");
167 rmStatus = NV_ERR_INVALID_ARGUMENT;
168 goto failed;
169 }
170
171 //
172 // If we have a separate channel RAM for each runlist then we need
173 // to determine runlistId from engineId passed by client. This
174 // runlistId is used to associate all future channels in this TSG to
175 // that runlist. Setting the engineType will cause the runlist
176 // corresponding to that engine to be chosen.
177 //
178 pKernelChannelGroup->engineType = rmEngineType;
179 }
180
181 //
182 // If MIG is enabled, client passes a logical engineId w.r.t its own GPU instance
183 // we need to convert this logical Id to a physical engine Id as we use it
184 // to set runlistId
185 //
186 if (bMIGInUse)
187 {
188 // Engine type must be valid for MIG
189 NV_CHECK_OR_ELSE(LEVEL_NOTICE, RM_ENGINE_TYPE_IS_VALID(pKernelChannelGroup->engineType),
190 rmStatus = NV_ERR_INVALID_STATE; goto failed);
191
192 NV_CHECK_OK_OR_GOTO(
193 rmStatus,
194 LEVEL_ERROR,
195 kmigmgrGetInstanceRefFromDevice(pGpu, pKernelMIGManager,
196 pDevice, &ref),
197 failed);
198
199 NV_CHECK_OK_OR_GOTO(
200 rmStatus,
201 LEVEL_ERROR,
202 kmigmgrGetLocalToGlobalEngineType(pGpu, pKernelMIGManager, ref,
203 rmEngineType,
204 &rmEngineType),
205 failed);
206
207 // Rewrite the engineType with the global engine type
208 pKernelChannelGroup->engineType = rmEngineType;
209 pHeap = ref.pKernelMIGGpuInstance->pMemoryPartitionHeap;
210 }
211 else
212 {
213 // Only GR0 is allowed without MIG
214 if ((RM_ENGINE_TYPE_IS_GR(rmEngineType)) && (rmEngineType != RM_ENGINE_TYPE_GR0))
215 {
216 rmStatus = NV_ERR_INVALID_ARGUMENT;
217 goto failed;
218 }
219 }
220
221 if((pDevice->vaMode != NV_DEVICE_ALLOCATION_VAMODE_MULTIPLE_VASPACES) || (hVASpace != 0))
222 {
223 NV_ASSERT_OK_OR_GOTO(rmStatus,
224 vaspaceGetByHandleOrDeviceDefault(pClient, pParams->hParent, hVASpace, &pVAS),
225 failed);
226
227 if (pVAS == NULL)
228 {
229 rmStatus = NV_ERR_INVALID_STATE;
230 goto failed;
231 }
232 }
233
234
235 // vGpu plugin context flag should only be set on host if context is plugin
236 if (gpuIsSriovEnabled(pGpu))
237 pKernelChannelGroup->bIsCallingContextVgpuPlugin = pAllocParams->bIsCallingContextVgpuPlugin;
238
239 if (pKernelChannelGroup->bIsCallingContextVgpuPlugin)
240 gfid = GPU_GFID_PF;
241 else
242 {
243 NV_ASSERT_OK_OR_GOTO(rmStatus, vgpuGetCallingContextGfid(pGpu, &gfid), failed);
244 }
245
246 if (!RMCFG_FEATURE_PLATFORM_GSP)
247 {
248 NvHandle hRcWatchdog;
249
250 //
251 // WAR for 4217716 - Force allocations made on behalf of watchdog client to
252 // RM reserved heap. This avoids a constant memory allocation from appearing
253 // due to the ctxBufPool reservation out of PMA.
254 //
255 rmStatus = krcWatchdogGetClientHandle(GPU_GET_KERNEL_RC(pGpu), &hRcWatchdog);
256 if ((rmStatus != NV_OK) || (pParams->hClient != hRcWatchdog))
257 {
258 NV_ASSERT_OK_OR_GOTO(rmStatus,
259 ctxBufPoolInit(pGpu, pHeap, &pKernelChannelGroup->pCtxBufPool),
260 failed);
261
262 NV_ASSERT_OK_OR_GOTO(rmStatus,
263 ctxBufPoolInit(pGpu, pHeap, &pKernelChannelGroup->pChannelBufPool),
264 failed);
265 }
266 else
267 {
268 NV_PRINTF(LEVEL_INFO, "Skipping ctxBufPoolInit for RC watchdog\n");
269 }
270 }
271
272 NV_ASSERT_OK_OR_GOTO(rmStatus,
273 kchangrpInit(pGpu, pKernelChannelGroup, pVAS, gfid),
274 failed);
275 bTsgAllocated = NV_TRUE;
276
277 pKernelChannelGroupApi->hLegacykCtxShareSync = 0;
278 pKernelChannelGroupApi->hLegacykCtxShareAsync = 0;
279
280 if (hVASpace != 0)
281 {
282 RsResourceRef *pVASpaceRef;
283 rmStatus = clientGetResourceRef(pCallContext->pClient, hVASpace, &pVASpaceRef);
284 NV_ASSERT(rmStatus == NV_OK);
285 if (rmStatus == NV_OK)
286 refAddDependant(pVASpaceRef, pResourceRef);
287 }
288
289 pKernelChannelGroupApi->hErrorContext = pAllocParams->hObjectError;
290 pKernelChannelGroupApi->hEccErrorContext = pAllocParams->hObjectEccError;
291
292 // Default interleave level
293 NV_ASSERT_OK_OR_GOTO(
294 rmStatus,
295 kchangrpSetInterleaveLevel(pGpu, pKernelChannelGroup,
296 NVA06C_CTRL_INTERLEAVE_LEVEL_MEDIUM),
297 failed);
298
299 ConfidentialCompute *pConfCompute = GPU_GET_CONF_COMPUTE(pGpu);
300 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
301 if ((pConfCompute != NULL) &&
302 (pConfCompute->getProperty(pCC, PDB_PROP_CONFCOMPUTE_CC_FEATURE_ENABLED)))
303 {
304 // TODO: jira CONFCOMP-1621: replace this with actual flag for TSG alloc that skips scrub
305 if ((pMemoryManager->bScrubChannelSetupInProgress) &&
306 (pKernelChannelGroup->pChannelBufPool != NULL) &&
307 (pKernelChannelGroup->pCtxBufPool != NULL))
308 {
309 if (pCallContext->secInfo.privLevel < RS_PRIV_LEVEL_KERNEL)
310 {
311 rmStatus = NV_ERR_INVALID_ARGUMENT;
312 NV_PRINTF(LEVEL_ERROR, "Only kernel priv clients can skip scrubber\n");
313 goto failed;
314 }
315 ctxBufPoolSetScrubSkip(pKernelChannelGroup->pChannelBufPool, NV_TRUE);
316 ctxBufPoolSetScrubSkip(pKernelChannelGroup->pCtxBufPool, NV_TRUE);
317 NV_PRINTF(LEVEL_INFO, "Skipping scrubber for all allocations on this context\n");
318 }
319 }
320
321 //
322 // If ctx buf pools are enabled, filter out partitionable engines
323 // that aren't part of our instance.
324 //
325 // Memory needs to be reserved in the pool only for buffers for
326 // engines in instance.
327 //
328
329 //
330 // Size of memory that will be calculated for ctxBufPool reservation if ctxBufPool is enabled and MIG is disabled
331 // or current engine belongs to this MIG instance and MIG is enabled
332 //
333 if (pKernelChannelGroup->pCtxBufPool != NULL &&
334 (!bMIGInUse || kmigmgrIsEngineInInstance(pGpu, pKernelMIGManager, pKernelChannelGroup->engineType, ref)))
335 {
336 // GR Buffers
337 if (RM_ENGINE_TYPE_IS_GR(pKernelChannelGroup->engineType))
338 {
339 KernelGraphics *pKernelGraphics = GPU_GET_KERNEL_GRAPHICS(pGpu, RM_ENGINE_TYPE_GR_IDX(pKernelChannelGroup->engineType));
340 NvU32 bufId = 0;
341 portMemSet(&bufInfoList[0], 0, sizeof(CTX_BUF_INFO) * NV_ENUM_SIZE(GR_CTX_BUFFER));
342 bufCount = 0;
343
344 kgraphicsDiscoverMaxLocalCtxBufferSize(pGpu, pKernelGraphics);
345
346 FOR_EACH_IN_ENUM(GR_CTX_BUFFER, bufId)
347 {
348 // TODO expose engine class capabilities to kernel RM
349 if (kgrmgrIsCtxBufSupported(bufId, NV_FALSE))
350 {
351 const CTX_BUF_INFO *pBufInfo = kgraphicsGetCtxBufferInfo(pGpu, pKernelGraphics, bufId);
352 bufInfoList[bufCount] = *pBufInfo;
353 NV_PRINTF(LEVEL_INFO, "Reserving 0x%llx bytes for GR ctx bufId = %d\n",
354 bufInfoList[bufCount].size, bufId);
355 bufCount++;
356 }
357 }
358 FOR_EACH_IN_ENUM_END;
359 bReserveMem = NV_TRUE;
360 }
361 else
362 {
363 // Allocate falcon context buffers if engine has (Kernel) Falcon object
364 NvU32 ctxBufferSize;
365 if (IS_GSP_CLIENT(pGpu))
366 {
367 ENGDESCRIPTOR engDesc;
368 KernelFalcon *pKernelFalcon = NULL;
369
370 NV_ASSERT_OK_OR_GOTO(rmStatus,
371 gpuXlateClientEngineIdToEngDesc(pGpu,
372 pKernelChannelGroup->engineType,
373 &engDesc),
374 failed);
375
376 pKernelFalcon = kflcnGetKernelFalconForEngine(pGpu, engDesc);
377 if (pKernelFalcon != NULL)
378 {
379 ctxBufferSize = pKernelFalcon->ctxBufferSize;
380 bReserveMem = NV_TRUE;
381 }
382 }
383
384 if (bReserveMem)
385 {
386 bufInfoList[0].size = ctxBufferSize;
387 bufInfoList[0].align = RM_PAGE_SIZE;
388 bufInfoList[0].attr = RM_ATTR_PAGE_SIZE_4KB;
389 bufInfoList[0].bContig = NV_TRUE;
390 NV_PRINTF(LEVEL_INFO, "Reserving 0x%llx bytes for engineType %d (%d) flcn ctx buffer\n",
391 bufInfoList[0].size, gpuGetNv2080EngineType(pKernelChannelGroup->engineType),
392 pKernelChannelGroup->engineType);
393 bufCount++;
394 }
395 else
396 {
397 NV_PRINTF(LEVEL_INFO, "No buffer reserved for engineType %d (%d) in ctx_buf_pool\n",
398 gpuGetNv2080EngineType(pKernelChannelGroup->engineType),
399 pKernelChannelGroup->engineType);
400 }
401 }
402 }
403
404 if ((!bMIGInUse || RM_ENGINE_TYPE_IS_GR(pKernelChannelGroup->engineType))
405 && !IsT234D(pGpu))
406 {
407 NV_ASSERT_OK_OR_GOTO(rmStatus,
408 pRmApi->AllocWithSecInfo(pRmApi,
409 pParams->hClient,
410 RES_GET_HANDLE(pKernelChannelGroupApi),
411 &pKernelChannelGroupApi->hKernelGraphicsContext,
412 KERNEL_GRAPHICS_CONTEXT,
413 NvP64_NULL,
414 0,
415 RMAPI_ALLOC_FLAGS_SKIP_RPC,
416 NvP64_NULL,
417 &pRmApi->defaultSecInfo),
418 failed);
419 }
420
421 NV_PRINTF(LEVEL_INFO, "Adding group Id: %d hClient:0x%x\n",
422 pKernelChannelGroup->grpID, pParams->hClient);
423
424 if ((IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu)) &&
425 !(pParams->allocFlags & RMAPI_ALLOC_FLAGS_SKIP_RPC))
426 {
427 NV_RM_RPC_ALLOC_OBJECT(pGpu,
428 pParams->hClient,
429 pParams->hParent,
430 pParams->hResource,
431 pParams->externalClassId,
432 pAllocParams,
433 sizeof(*pAllocParams),
434 rmStatus);
435 //
436 // Make sure that corresponding RPC occurs when freeing
437 // KernelChannelGroupApi. Resource server checks this variable during
438 // free and ignores any RPC flags set in resource_list.h
439 //
440 staticCast(pKernelChannelGroupApi, RmResource)->bRpcFree = NV_TRUE;
441
442 if (rmStatus != NV_OK)
443 {
444 NV_PRINTF(LEVEL_ERROR,
445 "KernelChannelGroupApi alloc RPC to vGpu Host failed\n");
446 goto failed;
447 }
448
449 if (IS_VIRTUAL_WITH_FULL_SRIOV(pGpu) || IS_GSP_CLIENT(pGpu))
450 {
451 NVA06C_CTRL_INTERNAL_PROMOTE_FAULT_METHOD_BUFFERS_PARAMS params = {
452 0};
453 NvU32 runqueueIdx;
454 NvU32 maxRunqueues = kfifoGetNumRunqueues_HAL(pGpu, pKernelFifo);
455
456 for (runqueueIdx = 0; runqueueIdx < maxRunqueues; ++runqueueIdx)
457 {
458 MEMORY_DESCRIPTOR *pSrcMemDesc;
459 HW_ENG_FAULT_METHOD_BUFFER *pMthdBuffer;
460 pMthdBuffer = &pKernelChannelGroup->pMthdBuffers[runqueueIdx];
461 pSrcMemDesc = pMthdBuffer->pMemDesc;
462
463 params.methodBufferMemdesc[runqueueIdx].size = (
464 pSrcMemDesc->Size);
465 params.methodBufferMemdesc[runqueueIdx].addressSpace = (
466 memdescGetAddressSpace(pSrcMemDesc));
467 params.methodBufferMemdesc[runqueueIdx].cpuCacheAttrib = (
468 memdescGetCpuCacheAttrib(pSrcMemDesc));
469 params.methodBufferMemdesc[runqueueIdx].alignment = 1;
470
471 if (IS_VIRTUAL_WITH_FULL_SRIOV(pGpu))
472 {
473 params.bar2Addr[runqueueIdx] = pMthdBuffer->bar2Addr;
474 params.methodBufferMemdesc[runqueueIdx].base = (
475 memdescGetPhysAddr(pSrcMemDesc, AT_CPU, 0));
476 }
477 else
478 {
479 //
480 // The case of both vGpu full SRIOV + GSP_CLIENT host is not
481 // supported. This else branch considers the case of
482 // GSP_CLIENT only without vGpu.
483 //
484 params.methodBufferMemdesc[runqueueIdx].base = (
485 memdescGetPhysAddr(pSrcMemDesc, AT_GPU, 0));
486 }
487 }
488 params.numValidEntries = runqueueIdx;
489
490 rmStatus = pRmApi->Control(pRmApi,
491 pParams->hClient,
492 RES_GET_HANDLE(pKernelChannelGroupApi),
493 NVA06C_CTRL_CMD_INTERNAL_PROMOTE_FAULT_METHOD_BUFFERS,
494 ¶ms,
495 sizeof params);
496
497 if (rmStatus != NV_OK)
498 {
499 NV_PRINTF(LEVEL_ERROR,
500 "Control call to update method buffer memdesc failed\n");
501 goto failed;
502 }
503 }
504 }
505
506 if (kfifoIsZombieSubctxWarEnabled(pKernelFifo))
507 {
508 kchangrpSetSubcontextZombieState_HAL(pGpu, pKernelChannelGroup, 0, NV_TRUE);
509 kchangrpUpdateSubcontextMask_HAL(pGpu, pKernelChannelGroup, 0, NV_TRUE);
510 }
511
512 // initialize apiObjList with original client's KernelChannelGroupApi object
513 listInit(&pKernelChannelGroup->apiObjList, portMemAllocatorGetGlobalNonPaged());
514
515 if (listAppendValue(&pKernelChannelGroup->apiObjList, &pKernelChannelGroupApi) == NULL)
516 {
517 rmStatus = NV_ERR_INSUFFICIENT_RESOURCES;
518 listClear(&pKernelChannelGroup->apiObjList);
519 goto failed;
520 }
521
522 failed:
523 if (rmStatus != NV_OK)
524 {
525 if (pKernelChannelGroupApi->hKernelGraphicsContext != NV01_NULL_OBJECT)
526 {
527 pRmApi->Free(pRmApi, pParams->hClient,
528 pKernelChannelGroupApi->hKernelGraphicsContext);
529 }
530
531 if (pKernelChannelGroup != NULL)
532 {
533 if (bTsgAllocated)
534 kchangrpDestroy(pGpu, pKernelChannelGroup);
535
536 if (pKernelChannelGroup->pCtxBufPool != NULL)
537 ctxBufPoolDestroy(&pKernelChannelGroup->pCtxBufPool);
538
539 if (pKernelChannelGroup->pChannelBufPool != NULL)
540 ctxBufPoolDestroy(&pKernelChannelGroup->pChannelBufPool);
541
542 }
543
544 if (pShared)
545 serverFreeShare(&g_resServ, pShared);
546 }
547
548 done:
549
550 if (bLockAcquired)
551 rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
552
553 if ((rmStatus == NV_OK) && bReserveMem)
554 {
555 // GPU lock should not be held when reserving memory for ctxBufPool
556 NV_CHECK_OK(rmStatus, LEVEL_ERROR,
557 ctxBufPoolReserve(pGpu, pKernelChannelGroup->pCtxBufPool, bufInfoList, bufCount));
558 if (rmStatus != NV_OK)
559 {
560 // Acquire the lock again for the cleanup path
561 NV_ASSERT_OK_OR_RETURN(rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_FIFO));
562 bLockAcquired = NV_TRUE;
563 goto failed;
564 }
565 }
566
567 portMemFree(bufInfoList);
568
569 return rmStatus;
570 }
571
572 NV_STATUS
kchangrpapiControl_IMPL(KernelChannelGroupApi * pKernelChannelGroupApi,CALL_CONTEXT * pCallContext,RS_RES_CONTROL_PARAMS_INTERNAL * pParams)573 kchangrpapiControl_IMPL
574 (
575 KernelChannelGroupApi *pKernelChannelGroupApi,
576 CALL_CONTEXT *pCallContext,
577 RS_RES_CONTROL_PARAMS_INTERNAL *pParams
578 )
579 {
580 RsResourceRef *pResourceRef = RES_GET_REF(pKernelChannelGroupApi);
581
582 (void)pResourceRef;
583 NV_PRINTF(LEVEL_INFO, "grpID 0x%x handle 0x%x cmd 0x%x\n",
584 pKernelChannelGroupApi->pKernelChannelGroup->grpID,
585 pResourceRef->hResource, pParams->pLegacyParams->cmd);
586
587 return gpuresControl_IMPL(staticCast(pKernelChannelGroupApi, GpuResource),
588 pCallContext, pParams);
589 }
590
591 void
kchangrpapiDestruct_IMPL(KernelChannelGroupApi * pKernelChannelGroupApi)592 kchangrpapiDestruct_IMPL
593 (
594 KernelChannelGroupApi *pKernelChannelGroupApi
595 )
596 {
597 CALL_CONTEXT *pCallContext;
598 RS_RES_FREE_PARAMS_INTERNAL *pParams;
599 RsResourceRef *pResourceRef;
600 RsClient *pClient;
601 KernelChannelGroup *pKernelChannelGroup =
602 pKernelChannelGroupApi->pKernelChannelGroup;
603 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannelGroupApi);
604 NV_STATUS rmStatus = NV_OK;
605 RS_ORDERED_ITERATOR it;
606 RsShared *pShared = staticCast(pKernelChannelGroup, RsShared);
607 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
608
609 resGetFreeParams(staticCast(pKernelChannelGroupApi, RsResource),
610 &pCallContext, &pParams);
611 pResourceRef = pCallContext->pResourceRef;
612 pClient = pCallContext->pClient;
613
614 NV_PRINTF(LEVEL_INFO, "\n");
615
616 // RS-TODO should still free channels?
617 if (serverGetShareRefCount(&g_resServ, pShared) > 1)
618 {
619 // Remove this kchangrpapi object from the list of owners in the shared object
620 listRemoveFirstByValue(&pKernelChannelGroupApi->pKernelChannelGroup->apiObjList, &pKernelChannelGroupApi);
621 goto done;
622 }
623
624 if (pKernelChannelGroup != NULL)
625 kchangrpSetRealtime_HAL(pGpu, pKernelChannelGroup, NV_FALSE);
626
627 // If channels still exist in this group, free them
628 // RS-TODO this can be removed after re-parenting support is added
629 it = kchannelGetIter(pClient, pResourceRef);
630 while (clientRefOrderedIterNext(pClient, &it))
631 {
632 NV_STATUS tmpStatus;
633
634 tmpStatus = pRmApi->Free(pRmApi, pClient->hClient, it.pResourceRef->hResource);
635 if ((tmpStatus != NV_OK) && (rmStatus == NV_OK))
636 rmStatus = tmpStatus;
637 }
638
639 NV_ASSERT(rmStatus == NV_OK);
640
641 if (pKernelChannelGroup != NULL)
642 {
643 kchangrpDestroy(pGpu, pKernelChannelGroup);
644
645 if (pKernelChannelGroup->pCtxBufPool != NULL)
646 {
647 ctxBufPoolRelease(pKernelChannelGroup->pCtxBufPool);
648 ctxBufPoolDestroy(&pKernelChannelGroup->pCtxBufPool);
649 }
650
651 if (pKernelChannelGroup->pChannelBufPool != NULL)
652 {
653 ctxBufPoolRelease(pKernelChannelGroup->pChannelBufPool);
654 ctxBufPoolDestroy(&pKernelChannelGroup->pChannelBufPool);
655 }
656
657 listClear(&pKernelChannelGroup->apiObjList);
658 }
659
660 done:
661 serverFreeShare(&g_resServ, pShared);
662
663 pParams->status = rmStatus;
664 }
665
666 NV_STATUS
kchangrpapiCopyConstruct_IMPL(KernelChannelGroupApi * pKernelChannelGroupApi,CALL_CONTEXT * pCallContext,RS_RES_ALLOC_PARAMS_INTERNAL * pParams)667 kchangrpapiCopyConstruct_IMPL
668 (
669 KernelChannelGroupApi *pKernelChannelGroupApi,
670 CALL_CONTEXT *pCallContext,
671 RS_RES_ALLOC_PARAMS_INTERNAL *pParams
672 )
673 {
674 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
675 RsClient *pDstClient = pCallContext->pClient;
676 RsResourceRef *pDstRef = pCallContext->pResourceRef;
677 RsResourceRef *pSrcRef = pParams->pSrcRef;
678 KernelChannelGroupApi *pChanGrpSrc = dynamicCast(pSrcRef->pResource,
679 KernelChannelGroupApi);
680 RS_ITERATOR iter;
681 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannelGroupApi);
682 NV_STATUS status = NV_OK;
683 RsResourceRef *pVaspaceRef = NULL;
684 VaSpaceApi *pVaspaceApi = NULL;
685
686 pKernelChannelGroupApi->hKernelGraphicsContext = NV01_NULL_OBJECT;
687 pKernelChannelGroupApi->hLegacykCtxShareSync = NV01_NULL_OBJECT;
688 pKernelChannelGroupApi->hLegacykCtxShareAsync = NV01_NULL_OBJECT;
689
690 pKernelChannelGroupApi->pKernelChannelGroup =
691 pChanGrpSrc->pKernelChannelGroup;
692 serverRefShare(&g_resServ,
693 staticCast(pKernelChannelGroupApi->pKernelChannelGroup, RsShared));
694
695 iter = serverutilRefIter(pDstClient->hClient, pDstRef->pParentRef->hResource, classId(VaSpaceApi), RS_ITERATE_DESCENDANTS, NV_TRUE);
696 while (clientRefIterNext(iter.pClient, &iter))
697 {
698 pVaspaceRef = iter.pResourceRef;
699 pVaspaceApi = dynamicCast(pVaspaceRef->pResource, VaSpaceApi);
700 NV_ASSERT_OR_RETURN(pVaspaceApi != NULL, NV_ERR_INVALID_STATE);
701
702 if (pVaspaceApi->pVASpace ==
703 pKernelChannelGroupApi->pKernelChannelGroup->pVAS)
704 {
705 refAddDependant(pVaspaceRef, pDstRef);
706 break;
707 }
708 }
709
710 if (pChanGrpSrc->hKernelGraphicsContext != NV01_NULL_OBJECT)
711 {
712 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR,
713 pRmApi->DupObject(pRmApi,
714 pDstClient->hClient,
715 pDstRef->hResource,
716 &pKernelChannelGroupApi->hKernelGraphicsContext,
717 pParams->pSrcClient->hClient,
718 pChanGrpSrc->hKernelGraphicsContext,
719 0),
720 fail);
721 }
722
723 //
724 // If this channel group is in legacy mode, new client needs its own handles to the
725 // sync and async internally allocated kctxshares
726 //
727 if (pChanGrpSrc->pKernelChannelGroup->bLegacyMode)
728 {
729 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR,
730 pRmApi->DupObject(pRmApi,
731 pDstClient->hClient,
732 pDstRef->hResource,
733 &pKernelChannelGroupApi->hLegacykCtxShareSync,
734 pParams->pSrcClient->hClient,
735 pChanGrpSrc->hLegacykCtxShareSync,
736 0),
737 fail);
738
739 // All chips have SYNC, Some chips won't have an ASYNC kctxshare
740 if (pChanGrpSrc->hLegacykCtxShareAsync != 0)
741 {
742 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR,
743 pRmApi->DupObject(pRmApi,
744 pDstClient->hClient,
745 pDstRef->hResource,
746 &pKernelChannelGroupApi->hLegacykCtxShareAsync,
747 pParams->pSrcClient->hClient,
748 pChanGrpSrc->hLegacykCtxShareAsync,
749 0),
750 fail);
751 }
752 }
753
754 if (IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu))
755 {
756 NV_RM_RPC_DUP_OBJECT(pGpu, pDstClient->hClient, pDstRef->pParentRef->hResource, pDstRef->hResource,
757 pParams->pSrcClient->hClient, pSrcRef->hResource, 0,
758 NV_TRUE, // automatically issue RPC_FREE on object free
759 pDstRef, status);
760
761 if (status != NV_OK)
762 goto fail;
763 }
764
765 if (listAppendValue(&pKernelChannelGroupApi->pKernelChannelGroup->apiObjList, &pKernelChannelGroupApi) == NULL)
766 {
767 status = NV_ERR_INSUFFICIENT_RESOURCES;
768 goto fail;
769 }
770
771 return status;
772
773 fail:
774 if (pKernelChannelGroupApi->hLegacykCtxShareAsync != NV01_NULL_OBJECT)
775 {
776 pRmApi->Free(pRmApi, pDstClient->hClient,
777 pKernelChannelGroupApi->hLegacykCtxShareAsync);
778 }
779 if (pKernelChannelGroupApi->hLegacykCtxShareSync != NV01_NULL_OBJECT)
780 {
781 pRmApi->Free(pRmApi, pDstClient->hClient,
782 pKernelChannelGroupApi->hLegacykCtxShareSync);
783 }
784 if (pKernelChannelGroupApi->hKernelGraphicsContext != NV01_NULL_OBJECT)
785 {
786 pRmApi->Free(pRmApi, pDstClient->hClient,
787 pKernelChannelGroupApi->hKernelGraphicsContext);
788 }
789
790 serverFreeShare(&g_resServ,
791 staticCast(pKernelChannelGroupApi->pKernelChannelGroup, RsShared));
792
793 return status;
794 }
795
796 NvBool
kchangrpapiCanCopy_IMPL(KernelChannelGroupApi * pKernelChannelGroupApi)797 kchangrpapiCanCopy_IMPL
798 (
799 KernelChannelGroupApi *pKernelChannelGroupApi
800 )
801 {
802 return NV_TRUE;
803 }
804
805 NV_STATUS
CliGetChannelGroup(NvHandle hClient,NvHandle hChanGrp,RsResourceRef ** ppChanGrpRef,NvHandle * phDevice)806 CliGetChannelGroup
807 (
808 NvHandle hClient,
809 NvHandle hChanGrp,
810 RsResourceRef **ppChanGrpRef,
811 NvHandle *phDevice
812 )
813 {
814 NV_STATUS status;
815 RsClient *pRsClient;
816 RsResourceRef *pResourceRef;
817 RsResourceRef *pParentRef;
818
819 if (!ppChanGrpRef)
820 {
821 return NV_ERR_INVALID_ARGUMENT;
822 }
823
824 status = serverGetClientUnderLock(&g_resServ, hClient, &pRsClient);
825 NV_ASSERT(status == NV_OK);
826 if (status != NV_OK)
827 return status;
828
829 status = clientGetResourceRefByType(pRsClient, hChanGrp,
830 classId(KernelChannelGroupApi),
831 &pResourceRef);
832 if (status != NV_OK)
833 return status;
834
835 *ppChanGrpRef = pResourceRef;
836
837 if (phDevice)
838 {
839 pParentRef = pResourceRef->pParentRef;
840 *phDevice = pParentRef->hResource;
841 }
842
843 return NV_OK;
844 }
845
846 /*!
847 * @brief Use TSG in legacy mode
848 *
849 * In legacy mode, RM pre-allocates the subcontexts in a TSG.
850 * This is needed for the following reasons:
851 *
852 * 1. We are also using subcontext to represent TSG contexts in pre-VOLTA chips (see below).
853 * But RM clients haven't yet moved to the subcontext model in production code.
854 * So RM implicitly creates it for them, until they make the switch.
855 *
856 * 2. Pre-VOLTA, we only support one address space in a TSG.
857 * Preallocating the subcontext prevents accidental use of multiple address spaces within a TSG.
858 * So we use the vaspace specified/implied at TSG creation to create the subcontexts.
859 *
860 * 3. Tests and clients on VOLTA that don't explicitly specify subcontexts need to behave similar
861 * to previous chips until they allocate the kctxshares themselves.
862 *
863 * Legacy subcontexts are interpreted in the following ways:
864 *
865 * VOLTA+ : subcontext 0 is VEID 0, subcontext 1 is VEID 1
866 * GM20X thru PASCAL : subcontext 0 is SCG type 0, subcontext 1 is SCG type 1
867 * pre-GM20X : just a single subcontext 0; no SCG or VEIDs attached to it.
868 *
869 * @param[in] pKernelChannelGroupApi Channel group pointer
870 * @param[in] pGpu GPU object pointer
871 * @param[in] pKernelFifo FIFO object pointer
872 * @param[in] hClient Client handle
873 *
874 */
875 NV_STATUS
kchangrpapiSetLegacyMode_IMPL(KernelChannelGroupApi * pKernelChannelGroupApi,OBJGPU * pGpu,KernelFifo * pKernelFifo,NvHandle hClient)876 kchangrpapiSetLegacyMode_IMPL
877 (
878 KernelChannelGroupApi *pKernelChannelGroupApi,
879 OBJGPU *pGpu,
880 KernelFifo *pKernelFifo,
881 NvHandle hClient
882 )
883 {
884 KernelChannelGroup *pKernelChannelGroup = pKernelChannelGroupApi->pKernelChannelGroup;
885 NvHandle hTsg = RES_GET_HANDLE(pKernelChannelGroupApi);
886 NvHandle hkCtxShare = 0;
887 NV_STATUS status = NV_OK;
888 NvU32 maxSubctx = 0;
889 NvU64 numMax = 0;
890 NvU64 numFree = 0;
891 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
892 KernelChannelGroupApiListIter it;
893
894 NV_CTXSHARE_ALLOCATION_PARAMETERS kctxshareParams = { 0 };
895
896 ct_assert(NV_CTXSHARE_ALLOCATION_FLAGS_SUBCONTEXT_SYNC == 0);
897 ct_assert(NV_CTXSHARE_ALLOCATION_FLAGS_SUBCONTEXT_ASYNC == 1);
898
899 NV_ASSERT_OK(pKernelChannelGroup->pSubctxIdHeap->eheapGetSize(
900 pKernelChannelGroup->pSubctxIdHeap,
901 &numMax));
902
903 NV_ASSERT_OK(pKernelChannelGroup->pSubctxIdHeap->eheapGetFree(
904 pKernelChannelGroup->pSubctxIdHeap,
905 &numFree));
906
907 NV_ASSERT(numMax ==
908 kfifoChannelGroupGetLocalMaxSubcontext_HAL(pGpu, pKernelFifo,
909 pKernelChannelGroup,
910 NV_FALSE));
911
912 NV_ASSERT_OR_RETURN(numMax == numFree && numMax != 0, NV_ERR_INVALID_STATE);
913
914 pKernelChannelGroup->pSubctxIdHeap->eheapDestruct(
915 pKernelChannelGroup->pSubctxIdHeap);
916 //
917 // There should only be 1 (SYNC) or 2 legacy kctxshares (SYNC + ASYNC),
918 // depending on chip
919 //
920 maxSubctx = kfifoChannelGroupGetLocalMaxSubcontext_HAL(pGpu, pKernelFifo,
921 pKernelChannelGroup,
922 NV_TRUE);
923 NV_ASSERT_OR_RETURN(numMax == numFree, NV_ERR_INVALID_STATE);
924 NV_ASSERT(maxSubctx == 1 || maxSubctx == 2);
925
926 constructObjEHeap(pKernelChannelGroup->pSubctxIdHeap,
927 0, maxSubctx, sizeof(KernelCtxShare *), 0);
928
929 pKernelChannelGroup->bLegacyMode = NV_TRUE;
930
931 // Allocate SYNC
932 hkCtxShare = 0;
933 kctxshareParams.hVASpace = 0;
934 kctxshareParams.flags = NV_CTXSHARE_ALLOCATION_FLAGS_SUBCONTEXT_SYNC;
935 kctxshareParams.subctxId = 0xFFFFFFFF;
936
937 NV_ASSERT_OK_OR_GOTO(status,
938 pRmApi->AllocWithSecInfo(pRmApi,
939 hClient,
940 hTsg,
941 &hkCtxShare,
942 FERMI_CONTEXT_SHARE_A,
943 NV_PTR_TO_NvP64(&kctxshareParams),
944 sizeof(kctxshareParams),
945 RMAPI_ALLOC_FLAGS_SKIP_RPC,
946 NvP64_NULL,
947 &pRmApi->defaultSecInfo),
948 fail);
949
950 NV_ASSERT(kctxshareParams.subctxId == NV_CTXSHARE_ALLOCATION_FLAGS_SUBCONTEXT_SYNC);
951
952 pKernelChannelGroupApi->hLegacykCtxShareSync = hkCtxShare;
953
954 if(maxSubctx == 2)
955 {
956 // Allocate ASYNC
957 hkCtxShare = 0;
958 kctxshareParams.hVASpace = 0;
959 kctxshareParams.flags = NV_CTXSHARE_ALLOCATION_FLAGS_SUBCONTEXT_ASYNC;
960 kctxshareParams.subctxId = 0xFFFFFFFF;
961
962 NV_ASSERT_OK_OR_GOTO(status,
963 pRmApi->AllocWithSecInfo(pRmApi,
964 hClient,
965 hTsg,
966 &hkCtxShare,
967 FERMI_CONTEXT_SHARE_A,
968 NV_PTR_TO_NvP64(&kctxshareParams),
969 sizeof(kctxshareParams),
970 RMAPI_ALLOC_FLAGS_SKIP_RPC,
971 NvP64_NULL,
972 &pRmApi->defaultSecInfo),
973 fail);
974
975 NV_ASSERT(kctxshareParams.subctxId == NV_CTXSHARE_ALLOCATION_FLAGS_SUBCONTEXT_ASYNC);
976
977 pKernelChannelGroupApi->hLegacykCtxShareAsync = hkCtxShare;
978 }
979
980 NV_ASSERT_OK_OR_GOTO(status,
981 pKernelChannelGroup->pSubctxIdHeap->eheapGetFree(
982 pKernelChannelGroup->pSubctxIdHeap,
983 &numFree),
984 fail);
985
986 NV_ASSERT_OR_GOTO(numFree == 0, fail);
987
988 //
989 // If this channel group has been duped, we need to provide kctxshareApi handles to the
990 // other channelGroupApi objects that share this channel group since the handles will
991 // only work for a single client.
992 //
993 it = listIterAll(&pKernelChannelGroup->apiObjList);
994 while (listIterNext(&it))
995 {
996 KernelChannelGroupApi *pChanGrpDest = *it.pValue;
997
998 if(pChanGrpDest == pKernelChannelGroupApi)
999 continue;
1000
1001 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR,
1002 pRmApi->DupObject(pRmApi,
1003 RES_GET_CLIENT_HANDLE(pChanGrpDest),
1004 RES_GET_HANDLE(pChanGrpDest),
1005 &pChanGrpDest->hLegacykCtxShareSync,
1006 RES_GET_CLIENT_HANDLE(pKernelChannelGroupApi),
1007 pKernelChannelGroupApi->hLegacykCtxShareSync,
1008 0),
1009 fail);
1010
1011 if (maxSubctx == 2)
1012 {
1013 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR,
1014 pRmApi->DupObject(pRmApi,
1015 RES_GET_CLIENT_HANDLE(pChanGrpDest),
1016 RES_GET_HANDLE(pChanGrpDest),
1017 &pChanGrpDest->hLegacykCtxShareAsync,
1018 RES_GET_CLIENT_HANDLE(pKernelChannelGroupApi),
1019 pKernelChannelGroupApi->hLegacykCtxShareAsync,
1020 0),
1021 fail);
1022 }
1023 }
1024
1025 return status;
1026
1027 fail:
1028 NV_PRINTF(LEVEL_ERROR, "Failed to set channel group in legacy mode.\n");
1029
1030 pKernelChannelGroup->bLegacyMode = NV_FALSE;
1031
1032 it = listIterAll(&pKernelChannelGroup->apiObjList);
1033
1034 while (listIterNext(&it))
1035 {
1036 KernelChannelGroupApi *pChanGrpIt = *it.pValue;
1037
1038 if (pChanGrpIt->hLegacykCtxShareSync != 0)
1039 {
1040 pRmApi->Free(pRmApi, RES_GET_CLIENT_HANDLE(pChanGrpIt), pChanGrpIt->hLegacykCtxShareSync);
1041 pChanGrpIt->hLegacykCtxShareSync = 0;
1042 }
1043
1044 if (pChanGrpIt->hLegacykCtxShareAsync != 0)
1045 {
1046 pRmApi->Free(pRmApi, RES_GET_CLIENT_HANDLE(pChanGrpIt), pChanGrpIt->hLegacykCtxShareAsync);
1047 pChanGrpIt->hLegacykCtxShareAsync = 0;
1048 }
1049 }
1050
1051 if(status == NV_OK)
1052 {
1053 status = NV_ERR_INVALID_STATE;
1054 }
1055
1056 return status;
1057 }
1058
1059 NV_STATUS
kchangrpapiCtrlCmdGpFifoSchedule_IMPL(KernelChannelGroupApi * pKernelChannelGroupApi,NVA06C_CTRL_GPFIFO_SCHEDULE_PARAMS * pSchedParams)1060 kchangrpapiCtrlCmdGpFifoSchedule_IMPL
1061 (
1062 KernelChannelGroupApi *pKernelChannelGroupApi,
1063 NVA06C_CTRL_GPFIFO_SCHEDULE_PARAMS *pSchedParams
1064 )
1065 {
1066 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannelGroupApi);
1067 RsResourceRef *pResourceRef = RES_GET_REF(pKernelChannelGroupApi);
1068 KernelChannelGroup *pKernelChannelGroup = NULL;
1069 NV_STATUS status = NV_OK;
1070 KernelFifo *pKernelFifo;
1071 CLASSDESCRIPTOR *pClass = NULL;
1072 CHANNEL_NODE *pChanNode = NULL;
1073 CHANNEL_LIST *pChanList = NULL;
1074 NvU32 runlistId = INVALID_RUNLIST_ID;
1075 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
1076
1077 if (pKernelChannelGroupApi->pKernelChannelGroup == NULL)
1078 return NV_ERR_INVALID_OBJECT;
1079 pKernelChannelGroup = pKernelChannelGroupApi->pKernelChannelGroup;
1080
1081 if (gpuGetClassByClassId(pGpu, pResourceRef->externalClassId, &pClass) != NV_OK)
1082 {
1083 NV_PRINTF(LEVEL_ERROR, "class %x not supported\n",
1084 pResourceRef->externalClassId);
1085 }
1086 NV_ASSERT_OR_RETURN((pClass != NULL), NV_ERR_NOT_SUPPORTED);
1087
1088 //
1089 // Bug 1737765: Prevent Externally Owned Channels from running unless bound
1090 // It is possible for clients to allocate and schedule channels while
1091 // skipping the UVM registration step which binds the appropriate
1092 // allocations in RM. We need to fail channel scheduling if the channels
1093 // have not been registered with UVM.
1094 // We include this check for every channel in the group because it is
1095 // expected that Volta+ may use a separate VAS for each channel.
1096 //
1097
1098 pChanList = pKernelChannelGroup->pChanList;
1099
1100 for (pChanNode = pChanList->pHead; pChanNode; pChanNode = pChanNode->pNext)
1101 {
1102 NV_CHECK_OR_RETURN(LEVEL_NOTICE, kchannelIsSchedulable_HAL(pGpu, pChanNode->pKernelChannel),
1103 NV_ERR_INVALID_STATE);
1104 }
1105
1106 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY);
1107 pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu);
1108 pChanList = pKernelChannelGroup->pChanList;
1109
1110 //
1111 // Some channels may not have objects allocated on them, so they won't have
1112 // a runlist committed yet. Force them all onto the same runlist so the
1113 // low level code knows what do to with them.
1114 //
1115 // First we walk through the channels to see if there is a runlist assigned
1116 // already and if so are the channels consistent.
1117 //
1118 runlistId = pKernelChannelGroup->runlistId; // Start with TSG runlistId
1119 for (pChanNode = pChanList->pHead; pChanNode; pChanNode = pChanNode->pNext)
1120 {
1121 KernelChannel *pKernelChannel = pChanNode->pKernelChannel;
1122
1123 NV_ASSERT_OR_ELSE(pKernelChannel != NULL, continue);
1124
1125 if (kchannelIsRunlistSet(pGpu, pKernelChannel))
1126 {
1127 if (runlistId == INVALID_RUNLIST_ID)
1128 {
1129 runlistId = kchannelGetRunlistId(pKernelChannel);
1130 }
1131 else // Catch if 2 channels in the same TSG have different runlistId
1132 {
1133 if (runlistId != kchannelGetRunlistId(pKernelChannel))
1134 {
1135 NV_PRINTF(LEVEL_ERROR,
1136 "Channels in TSG %d have different runlist IDs this should never happen!\n",
1137 pKernelChannelGroup->grpID);
1138 DBG_BREAKPOINT();
1139 }
1140 }
1141 }
1142 }
1143
1144 // If no channels have a runlist set, get the default and use it.
1145 if (runlistId == INVALID_RUNLIST_ID)
1146 {
1147 runlistId = kfifoGetDefaultRunlist_HAL(pGpu, pKernelFifo,
1148 pKernelChannelGroup->engineType);
1149 }
1150
1151 // We can rewrite TSG runlist id just as we will do that for all TSG channels below
1152 pKernelChannelGroup->runlistId = runlistId;
1153
1154 //
1155 // Now go through and force any channels w/o the runlist set to use either
1156 // the default or whatever we found other channels to be allocated on.
1157 //
1158 for (pChanNode = pChanList->pHead; pChanNode; pChanNode = pChanNode->pNext)
1159 {
1160 KernelChannel *pKernelChannel = pChanNode->pKernelChannel;
1161
1162 NV_ASSERT_OR_ELSE(pKernelChannel != NULL, continue);
1163
1164 if (!kchannelIsRunlistSet(pGpu, pKernelChannel))
1165 {
1166 kfifoRunlistSetId_HAL(pGpu, pKernelFifo, pKernelChannel, runlistId);
1167 }
1168 }
1169 SLI_LOOP_END
1170
1171 if (IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu))
1172 {
1173 CALL_CONTEXT *pCallContext = resservGetTlsCallContext();
1174 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams;
1175 NvHandle hClient = RES_GET_CLIENT_HANDLE(pKernelChannelGroupApi);
1176 NvHandle hObject = RES_GET_HANDLE(pKernelChannelGroupApi);
1177
1178 NV_RM_RPC_CONTROL(pGpu,
1179 hClient,
1180 hObject,
1181 pRmCtrlParams->cmd,
1182 pRmCtrlParams->pParams,
1183 pRmCtrlParams->paramsSize,
1184 status);
1185 return status;
1186 }
1187
1188
1189 //
1190 // Do an internal control call to do channel reset
1191 // on Host (Physical) RM
1192 //
1193 status = pRmApi->Control(pRmApi,
1194 RES_GET_CLIENT_HANDLE(pKernelChannelGroupApi),
1195 RES_GET_HANDLE(pKernelChannelGroupApi),
1196 NVA06C_CTRL_CMD_INTERNAL_GPFIFO_SCHEDULE,
1197 pSchedParams,
1198 sizeof(NVA06C_CTRL_GPFIFO_SCHEDULE_PARAMS));
1199
1200 return status;
1201 }
1202
1203 NV_STATUS
kchangrpapiCtrlCmdBind_IMPL(KernelChannelGroupApi * pKernelChannelGroupApi,NVA06C_CTRL_BIND_PARAMS * pParams)1204 kchangrpapiCtrlCmdBind_IMPL
1205 (
1206 KernelChannelGroupApi *pKernelChannelGroupApi,
1207 NVA06C_CTRL_BIND_PARAMS *pParams
1208 )
1209 {
1210 NV_STATUS rmStatus = NV_OK;
1211 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannelGroupApi);
1212 Device *pDevice = GPU_RES_GET_DEVICE(pKernelChannelGroupApi);
1213 CHANNEL_NODE *pChanNode;
1214 RM_ENGINE_TYPE localEngineType;
1215 RM_ENGINE_TYPE globalEngineType;
1216 ENGDESCRIPTOR engineDesc;
1217 NvBool bMIGInUse = IS_MIG_IN_USE(pGpu);
1218
1219 NV_ASSERT_OR_RETURN(pParams != NULL, NV_ERR_INVALID_ARGUMENT);
1220
1221 localEngineType = globalEngineType = gpuGetRmEngineType(pParams->engineType);
1222
1223 if (bMIGInUse)
1224 {
1225 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu);
1226 MIG_INSTANCE_REF ref;
1227
1228 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
1229 kmigmgrGetInstanceRefFromDevice(pGpu, pKernelMIGManager, pDevice, &ref));
1230
1231 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
1232 kmigmgrGetLocalToGlobalEngineType(pGpu, pKernelMIGManager, ref,
1233 localEngineType,
1234 &globalEngineType));
1235 }
1236
1237 NV_PRINTF(LEVEL_INFO,
1238 "Binding TSG %d to Engine %d (%d)\n",
1239 pKernelChannelGroupApi->pKernelChannelGroup->grpID,
1240 gpuGetNv2080EngineType(globalEngineType), globalEngineType);
1241
1242 // Translate globalEnginetype -> enginedesc
1243 NV_ASSERT_OK_OR_CAPTURE_FIRST_ERROR(rmStatus,
1244 gpuXlateClientEngineIdToEngDesc(pGpu, globalEngineType, &engineDesc));
1245
1246 // Translate engineDesc -> runlistId for TSG
1247 NV_ASSERT_OK_OR_CAPTURE_FIRST_ERROR(rmStatus,
1248 kfifoEngineInfoXlate_HAL(pGpu, GPU_GET_KERNEL_FIFO(pGpu),
1249 ENGINE_INFO_TYPE_ENG_DESC,
1250 engineDesc,
1251 ENGINE_INFO_TYPE_RUNLIST,
1252 &pKernelChannelGroupApi->pKernelChannelGroup->runlistId));
1253
1254 for (pChanNode =
1255 pKernelChannelGroupApi->pKernelChannelGroup->pChanList->pHead;
1256 pChanNode != NULL;
1257 pChanNode = pChanNode->pNext)
1258 {
1259 NV_ASSERT_OK_OR_CAPTURE_FIRST_ERROR(rmStatus,
1260 kchannelBindToRunlist(pChanNode->pKernelChannel,
1261 localEngineType,
1262 engineDesc));
1263 if (rmStatus != NV_OK)
1264 {
1265 break;
1266 }
1267 }
1268
1269 return rmStatus;
1270 }
1271
1272 NV_STATUS
kchangrpapiCtrlCmdGetTimeslice_IMPL(KernelChannelGroupApi * pKernelChannelGroupApi,NVA06C_CTRL_TIMESLICE_PARAMS * pTsParams)1273 kchangrpapiCtrlCmdGetTimeslice_IMPL
1274 (
1275 KernelChannelGroupApi *pKernelChannelGroupApi,
1276 NVA06C_CTRL_TIMESLICE_PARAMS *pTsParams
1277 )
1278 {
1279 KernelChannelGroup *pKernelChannelGroup = NULL;
1280
1281 if (pKernelChannelGroupApi->pKernelChannelGroup == NULL)
1282 return NV_ERR_INVALID_OBJECT;
1283 pKernelChannelGroup = pKernelChannelGroupApi->pKernelChannelGroup;
1284
1285 pTsParams->timesliceUs = pKernelChannelGroup->timesliceUs;
1286
1287 return NV_OK;
1288 }
1289
1290 NV_STATUS
kchangrpapiCtrlCmdSetTimeslice_IMPL(KernelChannelGroupApi * pKernelChannelGroupApi,NVA06C_CTRL_TIMESLICE_PARAMS * pTsParams)1291 kchangrpapiCtrlCmdSetTimeslice_IMPL
1292 (
1293 KernelChannelGroupApi *pKernelChannelGroupApi,
1294 NVA06C_CTRL_TIMESLICE_PARAMS *pTsParams
1295 )
1296 {
1297 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannelGroupApi);
1298 RsResourceRef *pResourceRef = RES_GET_REF(pKernelChannelGroupApi);
1299 KernelChannelGroup *pKernelChannelGroup = NULL;
1300 NV_STATUS status = NV_OK;
1301 CLASSDESCRIPTOR *pClass = NULL;
1302 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
1303
1304 if (pKernelChannelGroupApi->pKernelChannelGroup == NULL)
1305 return NV_ERR_INVALID_OBJECT;
1306 pKernelChannelGroup = pKernelChannelGroupApi->pKernelChannelGroup;
1307
1308 if (gpuGetClassByClassId(pGpu, pResourceRef->externalClassId, &pClass) != NV_OK)
1309 {
1310 NV_PRINTF(LEVEL_ERROR, "class %x not supported\n",
1311 pResourceRef->externalClassId);
1312 }
1313 NV_ASSERT_OR_RETURN((pClass != NULL), NV_ERR_NOT_SUPPORTED);
1314
1315 if (IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu))
1316 {
1317 CALL_CONTEXT *pCallContext = resservGetTlsCallContext();
1318 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams;
1319 NvHandle hClient = RES_GET_CLIENT_HANDLE(pKernelChannelGroupApi);
1320 NvHandle hObject = RES_GET_HANDLE(pKernelChannelGroupApi);
1321 NVA06C_CTRL_TIMESLICE_PARAMS *pParams = (NVA06C_CTRL_TIMESLICE_PARAMS *)(pRmCtrlParams->pParams);
1322
1323 NV_RM_RPC_CONTROL(pGpu,
1324 hClient,
1325 hObject,
1326 pRmCtrlParams->cmd,
1327 pRmCtrlParams->pParams,
1328 pRmCtrlParams->paramsSize,
1329 status);
1330
1331 // Update guest RM's internal bookkeeping with the timeslice.
1332 if (status == NV_OK)
1333 {
1334 pKernelChannelGroup->timesliceUs = pParams->timesliceUs;
1335 }
1336
1337 return status;
1338 }
1339
1340 //
1341 // Do an internal control call to do channel reset
1342 // on Host (Physical) RM
1343 //
1344 status = pRmApi->Control(pRmApi,
1345 RES_GET_CLIENT_HANDLE(pKernelChannelGroupApi),
1346 RES_GET_HANDLE(pKernelChannelGroupApi),
1347 NVA06C_CTRL_CMD_INTERNAL_SET_TIMESLICE,
1348 pTsParams,
1349 sizeof(NVA06C_CTRL_TIMESLICE_PARAMS));
1350
1351 return status;
1352 }
1353
1354 NV_STATUS
kchangrpapiCtrlCmdGetInfo_IMPL(KernelChannelGroupApi * pKernelChannelGroupApi,NVA06C_CTRL_GET_INFO_PARAMS * pParams)1355 kchangrpapiCtrlCmdGetInfo_IMPL
1356 (
1357 KernelChannelGroupApi *pKernelChannelGroupApi,
1358 NVA06C_CTRL_GET_INFO_PARAMS *pParams
1359 )
1360 {
1361 KernelChannelGroup *pKernelChannelGroup = NULL;
1362
1363 if (pKernelChannelGroupApi->pKernelChannelGroup == NULL)
1364 return NV_ERR_INVALID_OBJECT;
1365 pKernelChannelGroup = pKernelChannelGroupApi->pKernelChannelGroup;
1366
1367 pParams->tsgID = pKernelChannelGroup->grpID;
1368
1369 return NV_OK;
1370 }
1371
1372 NV_STATUS
kchangrpapiCtrlCmdSetInterleaveLevel_IMPL(KernelChannelGroupApi * pKernelChannelGroupApi,NVA06C_CTRL_INTERLEAVE_LEVEL_PARAMS * pParams)1373 kchangrpapiCtrlCmdSetInterleaveLevel_IMPL
1374 (
1375 KernelChannelGroupApi *pKernelChannelGroupApi,
1376 NVA06C_CTRL_INTERLEAVE_LEVEL_PARAMS *pParams
1377 )
1378 {
1379 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannelGroupApi);
1380 RsResourceRef *pResourceRef = RES_GET_REF(pKernelChannelGroupApi);
1381 KernelChannelGroup *pKernelChannelGroup =
1382 pKernelChannelGroupApi->pKernelChannelGroup;
1383 PCLASSDESCRIPTOR pClass = NULL;
1384 NV_STATUS status = NV_OK;
1385
1386 if (gpuGetClassByClassId(pGpu, pResourceRef->externalClassId, &pClass) != NV_OK)
1387 {
1388 NV_PRINTF(LEVEL_ERROR, "class %x not supported\n",
1389 pResourceRef->externalClassId);
1390 }
1391 NV_ASSERT_OR_RETURN((pClass != NULL), NV_ERR_NOT_SUPPORTED);
1392
1393 if (IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu))
1394 {
1395 CALL_CONTEXT *pCallContext = resservGetTlsCallContext();
1396 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams;
1397 NvHandle hClient = RES_GET_CLIENT_HANDLE(pKernelChannelGroupApi);
1398 NvHandle hObject = RES_GET_HANDLE(pKernelChannelGroupApi);
1399
1400 NV_RM_RPC_CONTROL(pGpu,
1401 hClient,
1402 hObject,
1403 pRmCtrlParams->cmd,
1404 pRmCtrlParams->pParams,
1405 pRmCtrlParams->paramsSize,
1406 status);
1407 NV_CHECK_OR_RETURN(LEVEL_INFO, status == NV_OK, NV_ERR_NOT_SUPPORTED);
1408 }
1409
1410 status = kchangrpSetInterleaveLevel(pGpu, pKernelChannelGroup, pParams->tsgInterleaveLevel);
1411
1412 return status;
1413 }
1414
1415 NV_STATUS
kchangrpapiCtrlCmdGetInterleaveLevel_IMPL(KernelChannelGroupApi * pKernelChannelGroupApi,NVA06C_CTRL_INTERLEAVE_LEVEL_PARAMS * pParams)1416 kchangrpapiCtrlCmdGetInterleaveLevel_IMPL
1417 (
1418 KernelChannelGroupApi *pKernelChannelGroupApi,
1419 NVA06C_CTRL_INTERLEAVE_LEVEL_PARAMS *pParams
1420 )
1421 {
1422 KernelChannelGroup *pKernelChannelGroup = NULL;
1423 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannelGroupApi);
1424 NvU32 subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu);
1425
1426 if (pKernelChannelGroupApi->pKernelChannelGroup == NULL)
1427 return NV_ERR_INVALID_OBJECT;
1428 pKernelChannelGroup = pKernelChannelGroupApi->pKernelChannelGroup;
1429
1430 pParams->tsgInterleaveLevel = pKernelChannelGroup->pInterleaveLevel[subdevInst];
1431
1432 return NV_OK;
1433 }
1434
1435 /*!
1436 * @brief Handler for NVA06C_CTRL_CMD_INTERNAL_PROMOTE_FAULT_METHOD_BUFFERS
1437 *
1438 * This is currently un-implemented as split change for bug 200691429
1439 */
1440 NV_STATUS
kchangrpapiCtrlCmdInternalPromoteFaultMethodBuffers_IMPL(KernelChannelGroupApi * pKernelChannelGroupApi,NVA06C_CTRL_INTERNAL_PROMOTE_FAULT_METHOD_BUFFERS_PARAMS * pParams)1441 kchangrpapiCtrlCmdInternalPromoteFaultMethodBuffers_IMPL
1442 (
1443 KernelChannelGroupApi *pKernelChannelGroupApi,
1444 NVA06C_CTRL_INTERNAL_PROMOTE_FAULT_METHOD_BUFFERS_PARAMS *pParams
1445 )
1446 {
1447 NV_PRINTF(LEVEL_INFO,
1448 "bug 200691429: kchangrpapiCtrlCmdInternalPromoteFaultMethodBuffers_IMPL received\n");
1449 return NV_OK;
1450 }
1451