1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 #define NVOC_KERNEL_GRAPHICS_MANAGER_H_PRIVATE_ACCESS_ALLOWED
25
26 #include "kernel/gpu/gr/kernel_graphics_manager.h"
27 #include "kernel/gpu/gr/kernel_graphics.h"
28 #include "kernel/gpu/gr/fecs_event_list.h"
29
30 #include "kernel/gpu/device/device.h"
31 #include "kernel/gpu/fifo/kernel_channel_group_api.h"
32 #include "kernel/gpu/fifo/kernel_channel_group.h"
33
34 #include "kernel/rmapi/client.h"
35 #include "kernel/rmapi/client_resource.h"
36 #include "nvRmReg.h"
37
38 // COMPUTE
39 #include "class/clb0c0.h"
40 #include "class/clb1c0.h"
41 #include "class/clc0c0.h"
42 #include "class/clc1c0.h"
43 #include "class/clc3c0.h"
44 #include "class/clc5c0.h"
45 #include "class/clc6c0.h"
46 #include "class/clc7c0.h"
47 #include "class/clc9c0.h"
48 #include "class/clcbc0.h"
49 #include "class/clcdc0.h"
50 // GFX
51 #include "class/clb097.h"
52 #include "class/clb197.h"
53 #include "class/clc097.h"
54 #include "class/clc197.h"
55 #include "class/clc397.h"
56 #include "class/clc597.h"
57 #include "class/clc697.h"
58 #include "class/clc797.h"
59 #include "class/clc997.h"
60 #include "class/clcb97.h"
61 #include "class/clcd97.h"
62 // TWOD
63 #include "class/cl902d.h"
64
65 // MEM2MEM
66 #include "class/cla140.h"
67 #include "class/clcd40.h"
68
69 static NvBool
_kgrmgrGPUInstanceHasComputeInstances(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,KERNEL_MIG_GPU_INSTANCE * pKernelMIGGpuInstance)70 _kgrmgrGPUInstanceHasComputeInstances
71 (
72 OBJGPU *pGpu,
73 KernelGraphicsManager *pKernelGraphicsManager,
74 KERNEL_MIG_GPU_INSTANCE *pKernelMIGGpuInstance
75 )
76 {
77 NV_ASSERT_OR_RETURN(pKernelMIGGpuInstance != NULL, NV_FALSE);
78 NvU32 computeInstanceIdx;
79
80 for (computeInstanceIdx = 0;
81 computeInstanceIdx < NV_ARRAY_ELEMENTS(pKernelMIGGpuInstance->MIGComputeInstance);
82 ++computeInstanceIdx)
83 {
84 if (pKernelMIGGpuInstance->MIGComputeInstance[computeInstanceIdx].bValid)
85 return NV_TRUE;
86 }
87
88 return NV_FALSE;
89 }
90
91 /*!
92 * @brief Get GR object type from the class number
93 *
94 * @param[IN] classNum external class number
95 * @param[OUT] pObjectType GR class subtype
96 */
97 void
kgrmgrGetGrObjectType_IMPL(NvU32 classNum,NvU32 * pObjectType)98 kgrmgrGetGrObjectType_IMPL
99 (
100 NvU32 classNum,
101 NvU32 *pObjectType
102 )
103 {
104 switch (classNum)
105 {
106 case MAXWELL_COMPUTE_A:
107 case MAXWELL_COMPUTE_B:
108 case PASCAL_COMPUTE_A:
109 case PASCAL_COMPUTE_B:
110 case VOLTA_COMPUTE_A:
111 case TURING_COMPUTE_A:
112 case AMPERE_COMPUTE_A:
113 case AMPERE_COMPUTE_B:
114 case ADA_COMPUTE_A:
115 case HOPPER_COMPUTE_A:
116 case BLACKWELL_COMPUTE_A:
117 *pObjectType = GR_OBJECT_TYPE_COMPUTE;
118 break;
119 case MAXWELL_A:
120 case MAXWELL_B:
121 case PASCAL_A:
122 case PASCAL_B:
123 case VOLTA_A:
124 case TURING_A:
125 case AMPERE_A:
126 case AMPERE_B:
127 case ADA_A:
128 case HOPPER_A:
129 case BLACKWELL_A:
130 *pObjectType = GR_OBJECT_TYPE_3D;
131 break;
132 case FERMI_TWOD_A:
133 *pObjectType = GR_OBJECT_TYPE_2D;
134 break;
135 case KEPLER_INLINE_TO_MEMORY_B:
136 case BLACKWELL_INLINE_TO_MEMORY_A:
137 *pObjectType = GR_OBJECT_TYPE_MEM;
138 break;
139 default:
140 *pObjectType = GR_OBJECT_TYPE_INVALID;
141 break;
142 }
143 }
144
145 /*!
146 * @brief Is local ctx buffer supported
147 *
148 * @param[IN] bufId buffer Id
149 * @param[IN] bClassSupported2D Is 2D class supported
150 */
151 NvBool
kgrmgrIsCtxBufSupported_IMPL(GR_CTX_BUFFER bufId,NvBool bClassSupported2D)152 kgrmgrIsCtxBufSupported_IMPL
153 (
154 GR_CTX_BUFFER bufId,
155 NvBool bClassSupported2D
156 )
157 {
158 NvBool bSupported = NV_FALSE;
159
160 NV_ASSERT_OR_RETURN(NV_ENUM_IS(GR_CTX_BUFFER, bufId), NV_FALSE);
161
162 // All buffers are supported when 2D class is supported
163 if (bClassSupported2D)
164 {
165 return NV_TRUE;
166 }
167
168 switch (bufId)
169 {
170 case GR_CTX_BUFFER_ZCULL:
171 // fall-through
172 case GR_CTX_BUFFER_PREEMPT:
173 // fall-through
174 case GR_CTX_BUFFER_SPILL:
175 // fall-through
176 case GR_CTX_BUFFER_BETA_CB:
177 // fall-through
178 case GR_CTX_BUFFER_PAGEPOOL:
179 // fall-through
180 case GR_CTX_BUFFER_RTV_CB:
181 // fall-through
182 case GR_CTX_BUFFER_SETUP:
183 bSupported = NV_FALSE;
184 break;
185
186 case GR_CTX_BUFFER_PM:
187 // fall-through
188 case GR_CTX_BUFFER_MAIN:
189 // fall-through
190 case GR_CTX_BUFFER_PATCH:
191 bSupported = NV_TRUE;
192 break;
193
194 // No default case - compiler enforces update if enum changes
195 }
196 return bSupported;
197 }
198
199 /*!
200 * @brief Is globalctx buffer supported
201 *
202 * @param[IN] bufId buffer Id
203 * @param[IN] bClassSupported2D Is 2D class supported
204 */
205 NvBool
kgrmgrIsGlobalCtxBufSupported_IMPL(GR_GLOBALCTX_BUFFER bufId,NvBool bClassSupported2D)206 kgrmgrIsGlobalCtxBufSupported_IMPL
207 (
208 GR_GLOBALCTX_BUFFER bufId,
209 NvBool bClassSupported2D
210 )
211 {
212 NvBool bSupported = NV_FALSE;
213
214 NV_ASSERT_OR_RETURN(NV_ENUM_IS(GR_GLOBALCTX_BUFFER, bufId), NV_FALSE);
215
216 // All buffers are supported when 2D class is supported
217 if (bClassSupported2D)
218 {
219 return NV_TRUE;
220 }
221
222 switch (bufId)
223 {
224 case GR_GLOBALCTX_BUFFER_BUNDLE_CB:
225 // fall-through
226 case GR_GLOBALCTX_BUFFER_PAGEPOOL:
227 // fall-through
228 case GR_GLOBALCTX_BUFFER_ATTRIBUTE_CB:
229 // fall-through
230 case GR_GLOBALCTX_BUFFER_RTV_CB:
231 // fall-through
232 case GR_GLOBALCTX_BUFFER_GFXP_POOL:
233 // fall-through
234 case GR_GLOBALCTX_BUFFER_GFXP_CTRL_BLK:
235 bSupported = NV_FALSE;
236 break;
237
238 case GR_GLOBALCTX_BUFFER_PRIV_ACCESS_MAP:
239 // fall-through
240 case GR_GLOBALCTX_BUFFER_UNRESTRICTED_PRIV_ACCESS_MAP:
241 // fall-through
242 case GR_GLOBALCTX_BUFFER_FECS_EVENT:
243 // fall-through
244 case GR_GLOBAL_BUFFER_GLOBAL_PRIV_ACCESS_MAP:
245 bSupported = NV_TRUE;
246 break;
247
248 // No default case - compiler enforces update if enum changes
249 }
250
251 return bSupported;
252 }
253
254 /*!
255 * @brief Sets swizzID and engineID on routing info if they don't already exist.
256 *
257 * @param[in] engID GR engine ID
258 * @param[in, out] pRouteInfo Client provided routing info
259 */
260 void
kgrmgrCtrlSetEngineID_IMPL(NvU32 engID,NV2080_CTRL_GR_ROUTE_INFO * pRouteInfo)261 kgrmgrCtrlSetEngineID_IMPL
262 (
263 NvU32 engID,
264 NV2080_CTRL_GR_ROUTE_INFO *pRouteInfo
265 )
266 {
267 if (NULL == pRouteInfo)
268 {
269 return;
270 }
271 else if (NV2080_CTRL_GR_ROUTE_INFO_FLAGS_TYPE_NONE ==
272 DRF_VAL(2080_CTRL_GR, _ROUTE_INFO_FLAGS, _TYPE, pRouteInfo->flags))
273 {
274 pRouteInfo->flags = DRF_DEF(2080_CTRL_GR, _ROUTE_INFO_FLAGS, _TYPE, _ENGID);
275 pRouteInfo->route = DRF_NUM64(2080_CTRL_GR, _ROUTE_INFO_DATA, _ENGID, engID);
276 }
277 }
278
279 /*!
280 * @brief Sets channel handle on routing info if it doesn't already exist.
281 *
282 * @param[in] hChannel Channel handle
283 * @param[in, out] pRouteInfo Client provided routing info
284 */
285 void
kgrmgrCtrlSetChannelHandle_IMPL(NvHandle hChannel,NV2080_CTRL_GR_ROUTE_INFO * pRouteInfo)286 kgrmgrCtrlSetChannelHandle_IMPL
287 (
288 NvHandle hChannel,
289 NV2080_CTRL_GR_ROUTE_INFO *pRouteInfo
290 )
291 {
292 if (NULL == pRouteInfo)
293 {
294 return;
295 }
296 else if (NV2080_CTRL_GR_ROUTE_INFO_FLAGS_TYPE_NONE ==
297 DRF_VAL(2080_CTRL_GR, _ROUTE_INFO_FLAGS, _TYPE, pRouteInfo->flags))
298 {
299 pRouteInfo->flags = DRF_DEF(2080_CTRL_GR, _ROUTE_INFO_FLAGS, _TYPE, _CHANNEL);
300 pRouteInfo->route = DRF_NUM64(2080_CTRL_GR, _ROUTE_INFO_DATA, _CHANNEL_HANDLE, hChannel);
301 }
302 }
303
304 static void
_kgrmgrInitRegistryOverrides(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager)305 _kgrmgrInitRegistryOverrides
306 (
307 OBJGPU *pGpu,
308 KernelGraphicsManager *pKernelGraphicsManager
309 )
310 {
311 }
312
313 NV_STATUS
kgrmgrConstructEngine_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,ENGDESCRIPTOR engDesc)314 kgrmgrConstructEngine_IMPL
315 (
316 OBJGPU *pGpu,
317 KernelGraphicsManager *pKernelGraphicsManager,
318 ENGDESCRIPTOR engDesc
319 )
320 {
321 _kgrmgrInitRegistryOverrides(pGpu, pKernelGraphicsManager);
322
323 fecsGlobalLoggingInit(pGpu, pKernelGraphicsManager, &pKernelGraphicsManager->pFecsGlobalTraceInfo);
324
325 return NV_OK;
326 }
327
328 void
kgrmgrDestruct_IMPL(KernelGraphicsManager * pKernelGraphicsManager)329 kgrmgrDestruct_IMPL
330 (
331 KernelGraphicsManager *pKernelGraphicsManager
332 )
333 {
334 OBJGPU *pGpu = ENG_GET_GPU(pKernelGraphicsManager);
335
336 fecsGlobalLoggingTeardown(pGpu, pKernelGraphicsManager);
337
338 portMemSet(&pKernelGraphicsManager->legacyKgraphicsStaticInfo.floorsweepingMasks, 0,
339 sizeof(pKernelGraphicsManager->legacyKgraphicsStaticInfo.floorsweepingMasks));
340 portMemFree(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pPpcMasks);
341 pKernelGraphicsManager->legacyKgraphicsStaticInfo.pPpcMasks = NULL;
342 portMemFree(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo);
343 pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo = NULL;
344 pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized = NV_FALSE;
345 }
346
347 void
kgrmgrStateDestroy_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager)348 kgrmgrStateDestroy_IMPL
349 (
350 OBJGPU *pGpu,
351 KernelGraphicsManager *pKernelGraphicsManager
352 )
353 {
354 fecsRemoveAllBindpointsForGpu(pGpu);
355 }
356
357 /*!
358 * @brief Set legacy Kgraphics Static Info (i.e. state of GR0)
359 */
360 void
kgrmgrSetLegacyKgraphicsStaticInfo_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,KernelGraphics * pKernelGraphics)361 kgrmgrSetLegacyKgraphicsStaticInfo_IMPL
362 (
363 OBJGPU *pGpu,
364 KernelGraphicsManager *pKernelGraphicsManager,
365 KernelGraphics *pKernelGraphics
366 )
367 {
368 NV_CHECK_OR_RETURN_VOID(LEVEL_INFO, !pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized);
369 NV_ASSERT_OR_RETURN_VOID((pKernelGraphics != NULL) && (kgraphicsGetInstance(pGpu, pKernelGraphics) == 0));
370 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics);
371 NV_ASSERT_OR_RETURN_VOID(pKernelGraphicsStaticInfo != NULL);
372
373 portMemCopy(&pKernelGraphicsManager->legacyKgraphicsStaticInfo.floorsweepingMasks, sizeof(pKernelGraphicsStaticInfo->floorsweepingMasks),
374 &pKernelGraphicsStaticInfo->floorsweepingMasks, sizeof(pKernelGraphicsStaticInfo->floorsweepingMasks));
375
376 if (pKernelGraphicsStaticInfo->pPpcMasks != NULL)
377 {
378 pKernelGraphicsManager->legacyKgraphicsStaticInfo.pPpcMasks = portMemAllocNonPaged(sizeof(*pKernelGraphicsStaticInfo->pPpcMasks));
379 NV_ASSERT_OR_GOTO(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pPpcMasks != NULL, cleanup);
380
381 portMemCopy(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pPpcMasks, sizeof(*pKernelGraphicsStaticInfo->pPpcMasks),
382 pKernelGraphicsStaticInfo->pPpcMasks, sizeof(*pKernelGraphicsStaticInfo->pPpcMasks));
383 }
384
385 if (pKernelGraphicsStaticInfo->pGrInfo != NULL)
386 {
387 pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo = portMemAllocNonPaged(sizeof(*pKernelGraphicsStaticInfo->pGrInfo));
388 NV_ASSERT_OR_GOTO(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo != NULL, cleanup);
389
390 portMemCopy(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo, sizeof(*pKernelGraphicsStaticInfo->pGrInfo),
391 pKernelGraphicsStaticInfo->pGrInfo, sizeof(*pKernelGraphicsStaticInfo->pGrInfo));
392 }
393
394 pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized = NV_TRUE;
395 return;
396
397 cleanup:
398 portMemFree(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pPpcMasks);
399 pKernelGraphicsManager->legacyKgraphicsStaticInfo.pPpcMasks = NULL;
400 portMemFree(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo);
401 pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo = NULL;
402 }
403
404 /*!
405 * @brief Retrieves associated KernelGraphics engine for given device / route info
406 *
407 * @param[in] pGpu
408 * @param[in] pKernelGraphicsManager
409 * @param[in] pDevice
410 * @param[in] grRouteInfo Client-provided info to direct GR accesses
411 * @param[out] ppKernelGraphics (Optional) Ptr to store appropriate KernelGraphics *, if desired.
412 */
413 NV_STATUS
kgrmgrCtrlRouteKGRWithDevice_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,Device * pDevice,const NV2080_CTRL_GR_ROUTE_INFO * pGrRouteInfo,KernelGraphics ** ppKernelGraphics)414 kgrmgrCtrlRouteKGRWithDevice_IMPL
415 (
416 OBJGPU *pGpu,
417 KernelGraphicsManager *pKernelGraphicsManager,
418 Device *pDevice,
419 const NV2080_CTRL_GR_ROUTE_INFO *pGrRouteInfo,
420 KernelGraphics **ppKernelGraphics
421 )
422 {
423 MIG_INSTANCE_REF ref;
424 KernelGraphics *pKernelGraphics;
425 NvU32 type;
426 NV_STATUS status = NV_OK;
427 NvU32 grIdx;
428 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo = *pGrRouteInfo;
429 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu);
430 RsClient *pClient;
431
432 if (!IS_MIG_IN_USE(pGpu))
433 {
434 grIdx = 0;
435 goto done;
436 }
437
438 pClient = RES_GET_CLIENT(pDevice);
439
440 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
441 kmigmgrGetInstanceRefFromDevice(pGpu, pKernelMIGManager, pDevice, &ref));
442
443 //
444 // Compute instances always have 1 GR engine, so automatically fill in
445 // the route info when subscribed to a compute instance
446 //
447 if (ref.pMIGComputeInstance != NULL)
448 {
449 portMemSet(&grRouteInfo, 0, sizeof(NV2080_CTRL_GR_ROUTE_INFO));
450 kgrmgrCtrlSetEngineID(0, &grRouteInfo);
451 }
452 else
453 {
454 RmClient *pRmClient = dynamicCast(pClient, RmClient);
455
456 if (pRmClient == NULL)
457 return NV_ERR_INVALID_OBJECT_HANDLE;
458
459 RS_PRIV_LEVEL privLevel = rmclientGetCachedPrivilege(pRmClient);
460 if (!rmclientIsAdmin(pRmClient, privLevel) &&
461 _kgrmgrGPUInstanceHasComputeInstances(pGpu, pKernelGraphicsManager, ref.pKernelMIGGpuInstance))
462 {
463 return NV_ERR_INSUFFICIENT_PERMISSIONS;
464 }
465 }
466
467 type = DRF_VAL(2080_CTRL_GR, _ROUTE_INFO_FLAGS, _TYPE, grRouteInfo.flags);
468 switch (type)
469 {
470 case NV2080_CTRL_GR_ROUTE_INFO_FLAGS_TYPE_NONE:
471 NV_PRINTF(LEVEL_ERROR,
472 "Cannot give GR Route flag of TYPE_NONE with MIG enabled!\n");
473 return NV_ERR_INVALID_ARGUMENT;
474
475 case NV2080_CTRL_GR_ROUTE_INFO_FLAGS_TYPE_ENGID:
476 {
477 NvU32 localGrIdx = DRF_VAL64(2080_CTRL_GR, _ROUTE_INFO_DATA,
478 _ENGID, grRouteInfo.route);
479 RM_ENGINE_TYPE globalRmEngType;
480
481 NV_CHECK_OK_OR_RETURN(
482 LEVEL_ERROR,
483 kmigmgrGetLocalToGlobalEngineType(pGpu, pKernelMIGManager, ref,
484 RM_ENGINE_TYPE_GR(localGrIdx),
485 &globalRmEngType));
486 NV_ASSERT_OR_RETURN(RM_ENGINE_TYPE_IS_GR(globalRmEngType), NV_ERR_INVALID_STATE);
487 grIdx = RM_ENGINE_TYPE_GR_IDX(globalRmEngType);
488
489 break;
490 }
491
492 case NV2080_CTRL_GR_ROUTE_INFO_FLAGS_TYPE_CHANNEL:
493 {
494 KernelChannel *pKernelChannel;
495 RM_ENGINE_TYPE rmEngineType;
496 NvHandle hChannel = DRF_VAL64(2080_CTRL_GR, _ROUTE_INFO_DATA,
497 _CHANNEL_HANDLE, grRouteInfo.route);
498
499 status = CliGetKernelChannel(pClient, hChannel, &pKernelChannel);
500 if (status != NV_OK)
501 {
502 RsResourceRef *pChanGrpRef;
503 KernelChannelGroupApi *pKernelChannelGroupApi = NULL;
504 KernelChannelGroup *pKernelChannelGroup = NULL;
505
506 //
507 // If retrieving a channel with the given hChannel doesn't work,
508 // try interpreting it as a handle to a channel group instead.
509 //
510 status = CliGetChannelGroup(pClient->hClient, hChannel,
511 &pChanGrpRef, NULL);
512 if (NV_OK != status)
513 {
514 NV_PRINTF(LEVEL_ERROR,
515 "Failed to find a channel or TSG with given handle 0x%08x associated with hClient=0x%08x\n",
516 hChannel, pClient->hClient);
517 return NV_ERR_INVALID_ARGUMENT;
518 }
519
520 pKernelChannelGroupApi = dynamicCast(pChanGrpRef->pResource,
521 KernelChannelGroupApi);
522 NV_ASSERT_OR_RETURN(
523 (pKernelChannelGroupApi != NULL &&
524 pKernelChannelGroupApi->pKernelChannelGroup != NULL),
525 NV_ERR_INVALID_STATE);
526 pKernelChannelGroup =
527 pKernelChannelGroupApi->pKernelChannelGroup;
528
529 NV_PRINTF(LEVEL_INFO,
530 "Found TSG with given handle 0x%08x, using this to determine GR engine ID\n",
531 hChannel);
532 rmEngineType = pKernelChannelGroup->engineType;
533 }
534 else
535 {
536 NV_PRINTF(LEVEL_INFO,
537 "Found channel with given handle 0x%08x, using this to determine GR engine ID\n",
538 hChannel);
539 rmEngineType = kchannelGetEngineType(pKernelChannel);
540 }
541
542 if (!RM_ENGINE_TYPE_IS_GR(rmEngineType))
543 {
544 NV_PRINTF(LEVEL_ERROR,
545 "Failed to route GR using non-GR engine type 0x%x (0x%x)\n",
546 gpuGetNv2080EngineType(rmEngineType), rmEngineType);
547 return NV_ERR_INVALID_ARGUMENT;
548 }
549
550 grIdx = RM_ENGINE_TYPE_GR_IDX(rmEngineType);
551 break;
552 }
553 default:
554 NV_PRINTF(LEVEL_ERROR,
555 "Unrecognized GR Route flag type 0x%x!\n", type);
556 return NV_ERR_INVALID_ARGUMENT;
557 }
558
559 done:
560 pKernelGraphics = GPU_GET_KERNEL_GRAPHICS(pGpu, grIdx);
561 NV_ASSERT_OR_RETURN(pKernelGraphics != NULL, NV_ERR_INVALID_STATE);
562
563 if (ppKernelGraphics != NULL)
564 *ppKernelGraphics = pKernelGraphics;
565
566 return status;
567 }
568
569 /*!
570 * @return legacy GPC mask enumerated by this chip
571 */
572 NvU32
kgrmgrGetLegacyGpcMask_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager)573 kgrmgrGetLegacyGpcMask_IMPL
574 (
575 OBJGPU *pGpu,
576 KernelGraphicsManager *pKernelGraphicsManager
577 )
578 {
579 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized, 0);
580
581 return pKernelGraphicsManager->legacyKgraphicsStaticInfo.floorsweepingMasks.gpcMask;
582 }
583
584 /*!
585 * @return legacy physical GFX GPC mask enumerated by this chip
586 */
587 NvU32
kgrmgrGetLegacyPhysGfxGpcMask_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager)588 kgrmgrGetLegacyPhysGfxGpcMask_IMPL
589 (
590 OBJGPU *pGpu,
591 KernelGraphicsManager *pKernelGraphicsManager
592 )
593 {
594 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized, 0);
595
596 return pKernelGraphicsManager->legacyKgraphicsStaticInfo.floorsweepingMasks.physGfxGpcMask;
597 }
598
599 /*!
600 * @return legacy TPC mask for certain GPC
601 *
602 * @param[in] pGpu
603 * @param[in] KernelGraphicsManager
604 * @param[in] gpcId Indicates logical GPC ID when MIG enabled or physical
605 * GPC ID when MIG disabled
606 */
607 NvU32
kgrmgrGetLegacyTpcMask_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,NvU32 gpcId)608 kgrmgrGetLegacyTpcMask_IMPL
609 (
610 OBJGPU *pGpu,
611 KernelGraphicsManager *pKernelGraphicsManager,
612 NvU32 gpcId
613 )
614 {
615 NvU32 maxNumGpcs;
616
617 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized, 0);
618 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo != NULL, 0);
619
620 maxNumGpcs = pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo->infoList[NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_GPCS].data;
621 NV_CHECK_OR_RETURN(LEVEL_ERROR, (gpcId < maxNumGpcs), 0);
622
623 return pKernelGraphicsManager->legacyKgraphicsStaticInfo.floorsweepingMasks.tpcMask[gpcId];
624 }
625
626 /*!
627 * @brief Get legacy PPC mask for certain GPC
628 */
629 NV_STATUS
kgrmgrGetLegacyPpcMask_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,NvU32 physGpcId,NvU32 * pPpcMask)630 kgrmgrGetLegacyPpcMask_IMPL
631 (
632 OBJGPU *pGpu,
633 KernelGraphicsManager *pKernelGraphicsManager,
634 NvU32 physGpcId,
635 NvU32 *pPpcMask
636 )
637 {
638 NvU32 maxNumGpcs;
639
640 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized, NV_ERR_INVALID_STATE);
641 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo != NULL, NV_ERR_INVALID_STATE);
642 maxNumGpcs = pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo->infoList[NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_GPCS].data;
643 NV_CHECK_OR_RETURN(LEVEL_ERROR, (physGpcId < maxNumGpcs), NV_ERR_INVALID_ARGUMENT);
644 NV_ASSERT_OR_RETURN((pPpcMask != NULL), NV_ERR_INVALID_ARGUMENT);
645 NV_ASSERT_OR_RETURN((pKernelGraphicsManager->legacyKgraphicsStaticInfo.pPpcMasks != NULL), NV_ERR_NOT_SUPPORTED);
646
647 *pPpcMask = pKernelGraphicsManager->legacyKgraphicsStaticInfo.pPpcMasks->mask[physGpcId];
648
649 return NV_OK;
650 }
651
652 /*!
653 * @brief Returns legacy zcull mask for specific gpc
654 */
655 NvU32
kgrmgrGetLegacyZcullMask_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,NvU32 physGpcId)656 kgrmgrGetLegacyZcullMask_IMPL
657 (
658 OBJGPU *pGpu,
659 KernelGraphicsManager *pKernelGraphicsManager,
660 NvU32 physGpcId
661 )
662 {
663 NvU32 maxNumGpcs;
664
665 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized, 0);
666 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo != NULL, NV_ERR_INVALID_STATE);
667 maxNumGpcs = pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo->infoList[NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_GPCS].data;
668 NV_CHECK_OR_RETURN(LEVEL_ERROR, (physGpcId < maxNumGpcs), NV_ERR_INVALID_ARGUMENT);
669
670 return pKernelGraphicsManager->legacyKgraphicsStaticInfo.floorsweepingMasks.zcullMask[physGpcId];
671 }
672
673 /*!
674 * @brief Function to Alloc VEIDs for a GR engine
675 *
676 * @param[IN] pGpu
677 * @param[IN] pKernelGraphicsManager
678 * @param[IN] grIdx phys gr idx
679 * @param[IN] veidSpanOffset Offset of spans localized to a GPU instance to make the VEID request from
680 * @param[IN] veidCount Total VEIDs connected to this GR engine
681 * @param[IN] pKernelMIGGPUInstance
682 */
683 NV_STATUS
kgrmgrAllocVeidsForGrIdx_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,NvU32 grIdx,NvU32 veidSpanOffset,NvU32 veidCount,KERNEL_MIG_GPU_INSTANCE * pKernelMIGGPUInstance)684 kgrmgrAllocVeidsForGrIdx_IMPL
685 (
686 OBJGPU *pGpu,
687 KernelGraphicsManager *pKernelGraphicsManager,
688 NvU32 grIdx,
689 NvU32 veidSpanOffset,
690 NvU32 veidCount,
691 KERNEL_MIG_GPU_INSTANCE *pKernelMIGGPUInstance
692 )
693 {
694 NvU32 veidSizePerSpan;
695 NvU32 veidStart = 0;
696 NvU32 veidEnd = 0;
697 NvU32 GPUInstanceVeidEnd;
698 NvU64 GPUInstanceVeidMask;
699 NvU64 GPUInstanceFreeVeidMask;
700 NvU64 reqVeidMask;
701
702 // This GR should not be already configured to use any VEIDs
703 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->grIdxVeidMask[grIdx] == 0, NV_ERR_INVALID_STATE);
704
705 NV_ASSERT_OK_OR_RETURN(
706 kgrmgrGetVeidSizePerSpan(pGpu, pKernelGraphicsManager, &veidSizePerSpan));
707
708 // We statically assign VEIDs to a GR based on the number of GPCs connected to it
709 if (veidCount % veidSizePerSpan != 0)
710 {
711 NV_PRINTF(LEVEL_ERROR, "veidCount %d is not aligned to veidSizePerSpan=%d\n", veidCount, veidSizePerSpan);
712 return NV_ERR_INVALID_ARGUMENT;
713 }
714
715 NV_ASSERT_OR_RETURN(veidSpanOffset != KMIGMGR_SPAN_OFFSET_INVALID, NV_ERR_INVALID_ARGUMENT);
716
717 veidStart = (veidSpanOffset * veidSizePerSpan) + pKernelMIGGPUInstance->resourceAllocation.veidOffset;
718 veidEnd = veidStart + veidCount - 1;
719
720 NV_ASSERT_OR_RETURN(veidStart < veidEnd, NV_ERR_INVALID_STATE);
721 NV_ASSERT_OR_RETURN(veidStart < 64, NV_ERR_INVALID_ARGUMENT);
722 NV_ASSERT_OR_RETURN(veidEnd < 64, NV_ERR_INVALID_ARGUMENT);
723
724 reqVeidMask = DRF_SHIFTMASK64(veidEnd:veidStart);
725
726 NV_ASSERT_OR_RETURN(reqVeidMask != 0x0, NV_ERR_INVALID_STATE);
727
728 // Create a mask for VEIDs associated with this GPU instance
729 GPUInstanceVeidEnd = pKernelMIGGPUInstance->resourceAllocation.veidOffset + pKernelMIGGPUInstance->resourceAllocation.veidCount - 1;
730 GPUInstanceVeidMask = DRF_SHIFTMASK64(GPUInstanceVeidEnd:pKernelMIGGPUInstance->resourceAllocation.veidOffset);
731
732 NV_ASSERT_OR_RETURN(GPUInstanceVeidMask != 0x0, NV_ERR_INVALID_STATE);
733
734 GPUInstanceFreeVeidMask = ~(pKernelGraphicsManager->veidInUseMask) & GPUInstanceVeidMask;
735
736 // VEID range should not overlap with existing VEIDs in use
737 NV_ASSERT_OR_RETURN((pKernelGraphicsManager->veidInUseMask & reqVeidMask) == 0, NV_ERR_STATE_IN_USE);
738
739 //
740 // For swizzId 0, client can use any range of VEIDs if not used, so we don't
741 // need to do the range check
742 //
743 NV_ASSERT_OR_RETURN(((pKernelMIGGPUInstance->swizzId == 0) ||
744 ((GPUInstanceFreeVeidMask & reqVeidMask) == reqVeidMask)),
745 NV_ERR_INSUFFICIENT_RESOURCES);
746
747 // mark each VEID in the range as "in use"
748 pKernelGraphicsManager->veidInUseMask |= reqVeidMask;
749 pKernelGraphicsManager->grIdxVeidMask[grIdx] |= reqVeidMask;
750
751 return NV_OK;
752 }
753
754 /*!
755 * @brief Function to Clear Gr Engine to VEIDs mapping
756 */
757 void
kgrmgrClearVeidsForGrIdx_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,NvU32 grIdx)758 kgrmgrClearVeidsForGrIdx_IMPL
759 (
760 OBJGPU *pGpu,
761 KernelGraphicsManager *pKernelGraphicsManager,
762 NvU32 grIdx
763 )
764 {
765 NvU64 veidMask = pKernelGraphicsManager->grIdxVeidMask[grIdx];
766
767 // mark all VEIDs of this GR engine as "not in use"
768 NV_ASSERT((pKernelGraphicsManager->veidInUseMask & veidMask) == veidMask);
769 pKernelGraphicsManager->veidInUseMask &= ~veidMask;
770 pKernelGraphicsManager->grIdxVeidMask[grIdx] = 0;
771 }
772
773 /*!
774 * @brief Get VEID step size
775 */
776 NV_STATUS
kgrmgrGetVeidStepSize_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,NvU32 * pVeidStepSize)777 kgrmgrGetVeidStepSize_IMPL
778 (
779 OBJGPU *pGpu,
780 KernelGraphicsManager *pKernelGraphicsManager,
781 NvU32 *pVeidStepSize
782 )
783 {
784 NV_ASSERT_OR_RETURN(pVeidStepSize != NULL, NV_ERR_INVALID_ARGUMENT);
785 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized, NV_ERR_INVALID_STATE);
786 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo != NULL, NV_ERR_INVALID_STATE);
787
788 *pVeidStepSize = pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo->infoList[NV0080_CTRL_GR_INFO_INDEX_LITTER_MIN_SUBCTX_PER_SMC_ENG].data;
789
790 return NV_OK;
791 }
792
793 /*!
794 * @brief Get VEID size for each span
795 */
796 NV_STATUS
kgrmgrGetVeidSizePerSpan_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,NvU32 * pVeidSizePerSpan)797 kgrmgrGetVeidSizePerSpan_IMPL
798 (
799 OBJGPU *pGpu,
800 KernelGraphicsManager *pKernelGraphicsManager,
801 NvU32 *pVeidSizePerSpan
802 )
803 {
804 NV_ASSERT_OR_RETURN(pVeidSizePerSpan != NULL, NV_ERR_INVALID_ARGUMENT);
805
806 // VEIDs for each span should be the VEID size we assign to the smallest GPC count
807 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
808 kgrmgrGetVeidsFromGpcCount_HAL(pGpu, pKernelGraphicsManager, 1, pVeidSizePerSpan));
809
810 return NV_OK;
811 }
812
813 /*!
814 * @brief Function to get starting VEID for a Gr Engine
815 */
816 NV_STATUS
kgrmgrGetVeidBaseForGrIdx_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,NvU32 grIdx,NvU32 * pVeidStart)817 kgrmgrGetVeidBaseForGrIdx_IMPL
818 (
819 OBJGPU *pGpu,
820 KernelGraphicsManager *pKernelGraphicsManager,
821 NvU32 grIdx,
822 NvU32 *pVeidStart
823 )
824 {
825 NvU64 veidMask;
826 NV_ASSERT_OR_RETURN(pVeidStart != NULL, NV_ERR_INVALID_ARGUMENT);
827 NV_ASSERT_OR_RETURN(grIdx != KGRMGR_MAX_GR, NV_ERR_INVALID_ARGUMENT);
828
829 *pVeidStart = 0;
830
831 veidMask = pKernelGraphicsManager->grIdxVeidMask[grIdx];
832
833 //
834 // If a GR is not configured, VEID mask for it will be "0" and counting
835 // "0" in a zero-based mask will result in max bit-width size.
836 //
837 if (veidMask != 0x0)
838 *pVeidStart = portUtilCountTrailingZeros64(veidMask);
839
840 return NV_OK;
841 }
842
843 /*!
844 * @brief Function to get GR index for a VEID
845 */
846 NV_STATUS
kgrmgrGetGrIdxForVeid_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,NvU32 veid,NvU32 * pGrIdx)847 kgrmgrGetGrIdxForVeid_IMPL
848 (
849 OBJGPU *pGpu,
850 KernelGraphicsManager *pKernelGraphicsManager,
851 NvU32 veid,
852 NvU32 *pGrIdx
853 )
854 {
855 NvU32 i;
856 NvU64 veidMask = NVBIT64(veid);
857
858 NV_ASSERT_OR_RETURN(pGrIdx != NULL, NV_ERR_INVALID_ARGUMENT);
859 for (i = 0; i < KGRMGR_MAX_GR; ++i)
860 {
861 if ((pKernelGraphicsManager->grIdxVeidMask[i] & veidMask) != 0)
862 break;
863 }
864 NV_ASSERT_OR_RETURN(i != KGRMGR_MAX_GR, NV_ERR_OBJECT_NOT_FOUND);
865 *pGrIdx = i;
866 return NV_OK;
867 }
868
869 /*!
870 * @brief discovers maximum size of local ctx buffers.
871 */
872 NV_STATUS
kgrmgrDiscoverMaxLocalCtxBufInfo_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,KernelGraphics * pKernelGraphics,NvU32 swizzId)873 kgrmgrDiscoverMaxLocalCtxBufInfo_IMPL
874 (
875 OBJGPU *pGpu,
876 KernelGraphicsManager *pKernelGraphicsManager,
877 KernelGraphics *pKernelGraphics,
878 NvU32 swizzId
879 )
880 {
881 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu);
882
883 //
884 // Most of GR is stub'd so context related things are not needed in AMODEL.
885 // But this function can be called in some MODS test, so return OK directly
886 // to avoid failing the test.
887 //
888 if (IS_MODS_AMODEL(pGpu))
889 return NV_OK;
890
891 NV_CHECK_OR_RETURN(LEVEL_SILENT,
892 kmigmgrIsMemoryPartitioningNeeded_HAL(pGpu, pKernelMIGManager, swizzId), NV_OK);
893
894 return kgraphicsDiscoverMaxLocalCtxBufferSize(pGpu, pKernelGraphics);
895 }
896
897 /*!
898 * @brief Get ctxbufpool info for the global ctx buffer
899 */
900 const CTX_BUF_INFO *
kgrmgrGetGlobalCtxBufInfo_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,GR_GLOBALCTX_BUFFER buf)901 kgrmgrGetGlobalCtxBufInfo_IMPL
902 (
903 OBJGPU *pGpu,
904 KernelGraphicsManager *pKernelGraphicsManager,
905 GR_GLOBALCTX_BUFFER buf
906 )
907 {
908 NV_ASSERT_OR_RETURN(NV_ENUM_IS(GR_GLOBALCTX_BUFFER, buf), NULL);
909 return &pKernelGraphicsManager->globalCtxBufInfo[buf];
910 }
911
912 /*!
913 * @brief Set ctxbufpool parameters for the global ctx buffer
914 */
915 void
kgrmgrSetGlobalCtxBufInfo_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,GR_GLOBALCTX_BUFFER buf,NvU64 size,NvU64 align,RM_ATTR_PAGE_SIZE attr,NvBool bContiguous)916 kgrmgrSetGlobalCtxBufInfo_IMPL
917 (
918 OBJGPU *pGpu,
919 KernelGraphicsManager *pKernelGraphicsManager,
920 GR_GLOBALCTX_BUFFER buf,
921 NvU64 size,
922 NvU64 align,
923 RM_ATTR_PAGE_SIZE attr,
924 NvBool bContiguous
925 )
926 {
927 CTX_BUF_INFO *pInfo;
928 NV_ASSERT_OR_RETURN_VOID(NV_ENUM_IS(GR_GLOBALCTX_BUFFER, buf));
929
930 pInfo = &pKernelGraphicsManager->globalCtxBufInfo[buf];
931 pInfo->size = size;
932 pInfo->align = align;
933 pInfo->attr = attr;
934 pInfo->bContig = bContiguous;
935 }
936
937 /*!
938 * @brief Gets maximum size of GR global ctx buffers.
939 * These are sizes of buffer for GR0 in legacy mode with all GPCs
940 * connected to GR0.
941 */
942 NV_STATUS
kgrmgrDiscoverMaxGlobalCtxBufSizes_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,KernelGraphics * pKernelGraphics,NvBool bMemoryPartitioningNeeded)943 kgrmgrDiscoverMaxGlobalCtxBufSizes_IMPL
944 (
945 OBJGPU *pGpu,
946 KernelGraphicsManager *pKernelGraphicsManager,
947 KernelGraphics *pKernelGraphics,
948 NvBool bMemoryPartitioningNeeded
949 )
950 {
951 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo;
952 GR_GLOBALCTX_BUFFER bufId;
953
954 //
955 // Most of GR is stub'd so context related things are not needed in AMODEL.
956 // But this function can be called in some MODS test, so return OK directly
957 // to avoid failing the test.
958 //
959 if (IS_MODS_AMODEL(pGpu))
960 return NV_OK;
961
962 NV_ASSERT_OR_RETURN(!IS_MIG_IN_USE(pGpu), NV_ERR_INVALID_STATE);
963
964 //
965 // Bug 2915422: Eventually we expect this check to be replaced by ctxBufPoolIsSupported
966 // we can't use that check today because PDB_PROP_GPU_MOVE_CTX_BUFFERS_TO_PMA is not enabled
967 // when this function is called because
968 // kgraphicsInitializeDeferredStaticData below will eventually lead to
969 // global ctx buffer allocation from ctxBufPools even before these pools are
970 // populated which happens later during GPU instance creation. Once we are
971 // able to rip out global buffer alloc from
972 // kgraphicsInitializeDeferredStaticData, we can enable the above property
973 // early.
974 //
975 NV_CHECK_OR_RETURN(LEVEL_SILENT, bMemoryPartitioningNeeded, NV_OK);
976
977 // Make sure sizes of all buffers are setup
978 NV_ASSERT_OK_OR_RETURN(
979 kgraphicsInitializeDeferredStaticData(pGpu, pKernelGraphics, NV01_NULL_OBJECT, NV01_NULL_OBJECT));
980
981 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics);
982 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE);
983 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pContextBuffersInfo != NULL, NV_ERR_INVALID_STATE);
984
985 // Get sizes of global ctx buffers
986 FOR_EACH_IN_ENUM(GR_GLOBALCTX_BUFFER, bufId)
987 {
988 NvU32 fifoEngineId;
989 NV_ASSERT_OK_OR_RETURN(
990 kgrctxGlobalCtxBufferToFifoEngineId(bufId, &fifoEngineId));
991
992 //
993 // contiguity is determined later before reservation as it depends on settings
994 // that take effect after this point.
995 //
996 kgrmgrSetGlobalCtxBufInfo(pGpu, pKernelGraphicsManager, bufId,
997 pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[fifoEngineId].size,
998 pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[fifoEngineId].alignment,
999 RM_ATTR_PAGE_SIZE_4KB, NV_FALSE);
1000 }
1001 FOR_EACH_IN_ENUM_END;
1002
1003 return NV_OK;
1004 }
1005
1006 /*!
1007 * @return legacy TPC count for certain GPC
1008 *
1009 * @param[in] pGpu
1010 * @param[in] KernelGraphicsManager
1011 * @param[in] gpcId Indicates logical GPC ID
1012 */
1013 NvU32
kgrmgrGetLegacyGpcTpcCount_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,NvU32 gpcId)1014 kgrmgrGetLegacyGpcTpcCount_IMPL
1015 (
1016 OBJGPU *pGpu,
1017 KernelGraphicsManager *pKernelGraphicsManager,
1018 NvU32 gpcId
1019 )
1020 {
1021 NvU32 maxNumGpcs;
1022
1023 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized, 0);
1024 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo != NULL, 0);
1025
1026 maxNumGpcs = nvPopCount32(pKernelGraphicsManager->legacyKgraphicsStaticInfo.floorsweepingMasks.gpcMask);
1027 NV_CHECK_OR_RETURN(LEVEL_ERROR, (gpcId < maxNumGpcs), 0);
1028
1029 return pKernelGraphicsManager->legacyKgraphicsStaticInfo.floorsweepingMasks.tpcCount[gpcId];
1030 }
1031
1032 /*!
1033 * @brief Function to Alloc VEIDs with either a specific placement or first-fit placement
1034 * depending upon the value passed into pSpanStart
1035 *
1036 * @param[IN] pGpu
1037 * @param[IN] pKernelGraphicsManager
1038 * @param[IN/OUT] pInUseMask Mask of VEIDs in-use, updated with newly set VEIDs
1039 * @param[IN] veidCount Total VEIDs connected to this GR engine
1040 * @param[IN/OUT] pSpanStart GPU Instance local span offset to begin assigning VEIDs from
1041 * @param[IN] pKernelMIGGPUInstance
1042 */
1043 NV_STATUS
kgrmgrCheckVeidsRequest_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,NvU64 * pInUseMask,NvU32 veidCount,NvU32 * pSpanStart,KERNEL_MIG_GPU_INSTANCE * pKernelMIGGPUInstance)1044 kgrmgrCheckVeidsRequest_IMPL
1045 (
1046 OBJGPU *pGpu,
1047 KernelGraphicsManager *pKernelGraphicsManager,
1048 NvU64 *pInUseMask,
1049 NvU32 veidCount,
1050 NvU32 *pSpanStart,
1051 KERNEL_MIG_GPU_INSTANCE *pKernelMIGGPUInstance
1052 )
1053 {
1054 NvU32 veidSizePerSpan;
1055 NvU32 veidStart = 0;
1056 NvU32 veidEnd = 0;
1057 NvU32 GPUInstanceVeidEnd;
1058 NvU64 GPUInstanceVeidMask;
1059 NvU64 GPUInstanceFreeVeidMask;
1060 NvU64 veidMask;
1061
1062 NV_ASSERT_OK_OR_RETURN(
1063 kgrmgrGetVeidSizePerSpan(pGpu, pKernelGraphicsManager, &veidSizePerSpan));
1064
1065 NV_ASSERT_OR_RETURN(pSpanStart != NULL, NV_ERR_INVALID_ARGUMENT);
1066 NV_ASSERT_OR_RETURN(pInUseMask != NULL, NV_ERR_INVALID_ARGUMENT);
1067
1068 // We statically assign VEIDs to a GR based on the number of GPCs connected to it
1069 if (veidCount % veidSizePerSpan != 0)
1070 {
1071 NV_PRINTF(LEVEL_ERROR, "veidCount %d is not aligned to veidSizePerSpan=%d\n", veidCount, veidSizePerSpan);
1072 return NV_ERR_INVALID_ARGUMENT;
1073 }
1074
1075 // Create a mask for VEIDs associated with this GPU instance
1076 GPUInstanceVeidEnd = pKernelMIGGPUInstance->resourceAllocation.veidOffset + pKernelMIGGPUInstance->resourceAllocation.veidCount - 1;
1077 GPUInstanceVeidMask = DRF_SHIFTMASK64(GPUInstanceVeidEnd:pKernelMIGGPUInstance->resourceAllocation.veidOffset);
1078
1079 NV_ASSERT_OR_RETURN(GPUInstanceVeidMask != 0x0, NV_ERR_INVALID_STATE);
1080
1081 GPUInstanceFreeVeidMask = ~(*pInUseMask) & GPUInstanceVeidMask;
1082
1083 if (*pSpanStart != KMIGMGR_SPAN_OFFSET_INVALID)
1084 {
1085 veidStart = (*pSpanStart * veidSizePerSpan) + pKernelMIGGPUInstance->resourceAllocation.veidOffset;
1086 veidEnd = veidStart + veidCount - 1;
1087
1088 NV_ASSERT_OR_RETURN(veidStart < veidEnd, NV_ERR_INVALID_STATE);
1089 NV_ASSERT_OR_RETURN(veidStart < 64, NV_ERR_INVALID_ARGUMENT);
1090 NV_ASSERT_OR_RETURN(veidEnd < 64, NV_ERR_INVALID_ARGUMENT);
1091 }
1092 else
1093 {
1094 NvU64 reqVeidMask = DRF_SHIFTMASK64(veidCount - 1:0);
1095 NvU32 i;
1096
1097 for (i = pKernelMIGGPUInstance->resourceAllocation.veidOffset; i <= GPUInstanceVeidEnd; i += veidSizePerSpan)
1098 {
1099 // See if requested slots are available within this range
1100 if (((GPUInstanceFreeVeidMask >> i) & reqVeidMask) == reqVeidMask)
1101 {
1102 veidStart = i;
1103 veidEnd = veidStart + veidCount - 1;
1104 break;
1105 }
1106 }
1107
1108 NV_CHECK_OR_RETURN(LEVEL_SILENT, i <= GPUInstanceVeidEnd,
1109 NV_ERR_INSUFFICIENT_RESOURCES);
1110 }
1111
1112 veidMask = DRF_SHIFTMASK64(veidEnd:veidStart);
1113 NV_ASSERT_OR_RETURN(veidMask != 0x0, NV_ERR_INVALID_STATE);
1114
1115 //
1116 // For swizzId 0, client can use any range of VEIDs if not used, so we don't
1117 // need to do the range check
1118 //
1119 NV_CHECK_OR_RETURN(LEVEL_ERROR,
1120 ((pKernelMIGGPUInstance->swizzId == 0) ||
1121 (GPUInstanceVeidMask & veidMask) == veidMask),
1122 NV_ERR_INSUFFICIENT_RESOURCES);
1123
1124 // VEID range should not overlap with existing VEIDs in use
1125 NV_ASSERT_OR_RETURN((*pInUseMask & veidMask) == 0, NV_ERR_STATE_IN_USE);
1126
1127 NV_ASSERT(veidStart >= pKernelMIGGPUInstance->resourceAllocation.veidOffset);
1128
1129 *pSpanStart = (veidStart - pKernelMIGGPUInstance->resourceAllocation.veidOffset) / veidSizePerSpan;
1130 *pInUseMask |= veidMask;
1131 return NV_OK;
1132 }
1133
1134