1 /*
2  * SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3  * SPDX-License-Identifier: MIT
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21  * DEALINGS IN THE SOFTWARE.
22  */
23 
24 #define NVOC_KERNEL_GRAPHICS_MANAGER_H_PRIVATE_ACCESS_ALLOWED
25 
26 #include "kernel/gpu/gr/kernel_graphics_manager.h"
27 #include "kernel/gpu/gr/kernel_graphics.h"
28 #include "kernel/gpu/gr/fecs_event_list.h"
29 
30 #include "kernel/gpu/device/device.h"
31 #include "kernel/gpu/fifo/kernel_channel_group_api.h"
32 #include "kernel/gpu/fifo/kernel_channel_group.h"
33 
34 #include "kernel/rmapi/client.h"
35 #include "kernel/rmapi/client_resource.h"
36 #include "nvRmReg.h"
37 
38 // COMPUTE
39 #include "class/clb0c0.h"
40 #include "class/clb1c0.h"
41 #include "class/clc0c0.h"
42 #include "class/clc1c0.h"
43 #include "class/clc3c0.h"
44 #include "class/clc5c0.h"
45 #include "class/clc6c0.h"
46 #include "class/clc7c0.h"
47 #include "class/clc9c0.h"
48 #include "class/clcbc0.h"
49 #include "class/clcdc0.h"
50 // GFX
51 #include "class/clb097.h"
52 #include "class/clb197.h"
53 #include "class/clc097.h"
54 #include "class/clc197.h"
55 #include "class/clc397.h"
56 #include "class/clc597.h"
57 #include "class/clc697.h"
58 #include "class/clc797.h"
59 #include "class/clc997.h"
60 #include "class/clcb97.h"
61 #include "class/clcd97.h"
62 // TWOD
63 #include "class/cl902d.h"
64 
65 // MEM2MEM
66 #include "class/cla140.h"
67 #include "class/clcd40.h"
68 
69 static NvBool
_kgrmgrGPUInstanceHasComputeInstances(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,KERNEL_MIG_GPU_INSTANCE * pKernelMIGGpuInstance)70 _kgrmgrGPUInstanceHasComputeInstances
71 (
72     OBJGPU *pGpu,
73     KernelGraphicsManager *pKernelGraphicsManager,
74     KERNEL_MIG_GPU_INSTANCE *pKernelMIGGpuInstance
75 )
76 {
77     NV_ASSERT_OR_RETURN(pKernelMIGGpuInstance != NULL, NV_FALSE);
78     NvU32 computeInstanceIdx;
79 
80     for (computeInstanceIdx = 0;
81          computeInstanceIdx < NV_ARRAY_ELEMENTS(pKernelMIGGpuInstance->MIGComputeInstance);
82          ++computeInstanceIdx)
83     {
84         if (pKernelMIGGpuInstance->MIGComputeInstance[computeInstanceIdx].bValid)
85             return NV_TRUE;
86     }
87 
88     return NV_FALSE;
89 }
90 
91 /*!
92  * @brief Get GR object type from the class number
93  *
94  * @param[IN]   classNum        external class number
95  * @param[OUT]  pObjectType     GR class subtype
96  */
97 void
kgrmgrGetGrObjectType_IMPL(NvU32 classNum,NvU32 * pObjectType)98 kgrmgrGetGrObjectType_IMPL
99 (
100     NvU32 classNum,
101     NvU32 *pObjectType
102 )
103 {
104     switch (classNum)
105     {
106         case MAXWELL_COMPUTE_A:
107         case MAXWELL_COMPUTE_B:
108         case PASCAL_COMPUTE_A:
109         case PASCAL_COMPUTE_B:
110         case VOLTA_COMPUTE_A:
111         case TURING_COMPUTE_A:
112         case AMPERE_COMPUTE_A:
113         case AMPERE_COMPUTE_B:
114         case ADA_COMPUTE_A:
115         case HOPPER_COMPUTE_A:
116         case BLACKWELL_COMPUTE_A:
117             *pObjectType = GR_OBJECT_TYPE_COMPUTE;
118             break;
119         case MAXWELL_A:
120         case MAXWELL_B:
121         case PASCAL_A:
122         case PASCAL_B:
123         case VOLTA_A:
124         case TURING_A:
125         case AMPERE_A:
126         case AMPERE_B:
127         case ADA_A:
128         case HOPPER_A:
129         case BLACKWELL_A:
130             *pObjectType = GR_OBJECT_TYPE_3D;
131             break;
132         case FERMI_TWOD_A:
133             *pObjectType = GR_OBJECT_TYPE_2D;
134             break;
135         case KEPLER_INLINE_TO_MEMORY_B:
136         case BLACKWELL_INLINE_TO_MEMORY_A:
137             *pObjectType = GR_OBJECT_TYPE_MEM;
138             break;
139         default:
140             *pObjectType = GR_OBJECT_TYPE_INVALID;
141             break;
142     }
143 }
144 
145 /*!
146  * @brief Is local ctx buffer supported
147  *
148  * @param[IN]   bufId             buffer Id
149  * @param[IN]   bClassSupported2D Is 2D class supported
150  */
151 NvBool
kgrmgrIsCtxBufSupported_IMPL(GR_CTX_BUFFER bufId,NvBool bClassSupported2D)152 kgrmgrIsCtxBufSupported_IMPL
153 (
154     GR_CTX_BUFFER bufId,
155     NvBool bClassSupported2D
156 )
157 {
158     NvBool bSupported = NV_FALSE;
159 
160     NV_ASSERT_OR_RETURN(NV_ENUM_IS(GR_CTX_BUFFER, bufId), NV_FALSE);
161 
162     // All buffers are supported when 2D class is supported
163     if (bClassSupported2D)
164     {
165         return NV_TRUE;
166     }
167 
168     switch (bufId)
169     {
170         case GR_CTX_BUFFER_ZCULL:
171             // fall-through
172         case GR_CTX_BUFFER_PREEMPT:
173             // fall-through
174         case GR_CTX_BUFFER_SPILL:
175             // fall-through
176         case GR_CTX_BUFFER_BETA_CB:
177             // fall-through
178         case GR_CTX_BUFFER_PAGEPOOL:
179             // fall-through
180         case GR_CTX_BUFFER_RTV_CB:
181             // fall-through
182         case GR_CTX_BUFFER_SETUP:
183             bSupported = NV_FALSE;
184             break;
185 
186         case GR_CTX_BUFFER_PM:
187             // fall-through
188         case GR_CTX_BUFFER_MAIN:
189             // fall-through
190         case GR_CTX_BUFFER_PATCH:
191             bSupported = NV_TRUE;
192             break;
193 
194         // No default case - compiler enforces update if enum changes
195     }
196     return bSupported;
197 }
198 
199 /*!
200  * @brief Is globalctx buffer supported
201  *
202  * @param[IN]   bufId             buffer Id
203  * @param[IN]   bClassSupported2D Is 2D class supported
204  */
205 NvBool
kgrmgrIsGlobalCtxBufSupported_IMPL(GR_GLOBALCTX_BUFFER bufId,NvBool bClassSupported2D)206 kgrmgrIsGlobalCtxBufSupported_IMPL
207 (
208     GR_GLOBALCTX_BUFFER bufId,
209     NvBool bClassSupported2D
210 )
211 {
212     NvBool bSupported = NV_FALSE;
213 
214     NV_ASSERT_OR_RETURN(NV_ENUM_IS(GR_GLOBALCTX_BUFFER, bufId), NV_FALSE);
215 
216     // All buffers are supported when 2D class is supported
217     if (bClassSupported2D)
218     {
219         return NV_TRUE;
220     }
221 
222     switch (bufId)
223     {
224         case GR_GLOBALCTX_BUFFER_BUNDLE_CB:
225             // fall-through
226         case GR_GLOBALCTX_BUFFER_PAGEPOOL:
227             // fall-through
228         case GR_GLOBALCTX_BUFFER_ATTRIBUTE_CB:
229             // fall-through
230         case GR_GLOBALCTX_BUFFER_RTV_CB:
231             // fall-through
232         case GR_GLOBALCTX_BUFFER_GFXP_POOL:
233             // fall-through
234         case GR_GLOBALCTX_BUFFER_GFXP_CTRL_BLK:
235             bSupported = NV_FALSE;
236             break;
237 
238         case GR_GLOBALCTX_BUFFER_PRIV_ACCESS_MAP:
239             // fall-through
240         case GR_GLOBALCTX_BUFFER_UNRESTRICTED_PRIV_ACCESS_MAP:
241             // fall-through
242         case GR_GLOBALCTX_BUFFER_FECS_EVENT:
243             // fall-through
244         case GR_GLOBAL_BUFFER_GLOBAL_PRIV_ACCESS_MAP:
245             bSupported = NV_TRUE;
246             break;
247 
248         // No default case - compiler enforces update if enum changes
249     }
250 
251     return bSupported;
252 }
253 
254 /*!
255  * @brief Sets swizzID and engineID on routing info if they don't already exist.
256  *
257  * @param[in]      engID            GR engine ID
258  * @param[in, out] pRouteInfo       Client provided routing info
259  */
260 void
kgrmgrCtrlSetEngineID_IMPL(NvU32 engID,NV2080_CTRL_GR_ROUTE_INFO * pRouteInfo)261 kgrmgrCtrlSetEngineID_IMPL
262 (
263     NvU32 engID,
264     NV2080_CTRL_GR_ROUTE_INFO *pRouteInfo
265 )
266 {
267     if (NULL == pRouteInfo)
268     {
269         return;
270     }
271     else if (NV2080_CTRL_GR_ROUTE_INFO_FLAGS_TYPE_NONE ==
272              DRF_VAL(2080_CTRL_GR, _ROUTE_INFO_FLAGS, _TYPE, pRouteInfo->flags))
273     {
274         pRouteInfo->flags = DRF_DEF(2080_CTRL_GR, _ROUTE_INFO_FLAGS, _TYPE, _ENGID);
275         pRouteInfo->route = DRF_NUM64(2080_CTRL_GR, _ROUTE_INFO_DATA, _ENGID, engID);
276     }
277 }
278 
279 /*!
280  * @brief Sets channel handle on routing info if it doesn't already exist.
281  *
282  * @param[in]      hChannel         Channel handle
283  * @param[in, out] pRouteInfo       Client provided routing info
284  */
285 void
kgrmgrCtrlSetChannelHandle_IMPL(NvHandle hChannel,NV2080_CTRL_GR_ROUTE_INFO * pRouteInfo)286 kgrmgrCtrlSetChannelHandle_IMPL
287 (
288     NvHandle hChannel,
289     NV2080_CTRL_GR_ROUTE_INFO *pRouteInfo
290 )
291 {
292     if (NULL == pRouteInfo)
293     {
294         return;
295     }
296     else if (NV2080_CTRL_GR_ROUTE_INFO_FLAGS_TYPE_NONE ==
297              DRF_VAL(2080_CTRL_GR, _ROUTE_INFO_FLAGS, _TYPE, pRouteInfo->flags))
298     {
299         pRouteInfo->flags = DRF_DEF(2080_CTRL_GR, _ROUTE_INFO_FLAGS, _TYPE, _CHANNEL);
300         pRouteInfo->route = DRF_NUM64(2080_CTRL_GR, _ROUTE_INFO_DATA, _CHANNEL_HANDLE, hChannel);
301     }
302 }
303 
304 static void
_kgrmgrInitRegistryOverrides(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager)305 _kgrmgrInitRegistryOverrides
306 (
307     OBJGPU *pGpu,
308     KernelGraphicsManager *pKernelGraphicsManager
309 )
310 {
311 }
312 
313 NV_STATUS
kgrmgrConstructEngine_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,ENGDESCRIPTOR engDesc)314 kgrmgrConstructEngine_IMPL
315 (
316     OBJGPU                *pGpu,
317     KernelGraphicsManager *pKernelGraphicsManager,
318     ENGDESCRIPTOR          engDesc
319 )
320 {
321     _kgrmgrInitRegistryOverrides(pGpu, pKernelGraphicsManager);
322 
323     fecsGlobalLoggingInit(pGpu, pKernelGraphicsManager, &pKernelGraphicsManager->pFecsGlobalTraceInfo);
324 
325     return NV_OK;
326 }
327 
328 void
kgrmgrDestruct_IMPL(KernelGraphicsManager * pKernelGraphicsManager)329 kgrmgrDestruct_IMPL
330 (
331     KernelGraphicsManager *pKernelGraphicsManager
332 )
333 {
334     OBJGPU *pGpu = ENG_GET_GPU(pKernelGraphicsManager);
335 
336     fecsGlobalLoggingTeardown(pGpu, pKernelGraphicsManager);
337 
338     portMemSet(&pKernelGraphicsManager->legacyKgraphicsStaticInfo.floorsweepingMasks, 0,
339            sizeof(pKernelGraphicsManager->legacyKgraphicsStaticInfo.floorsweepingMasks));
340     portMemFree(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pPpcMasks);
341     pKernelGraphicsManager->legacyKgraphicsStaticInfo.pPpcMasks = NULL;
342     portMemFree(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo);
343     pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo = NULL;
344     pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized = NV_FALSE;
345 }
346 
347 void
kgrmgrStateDestroy_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager)348 kgrmgrStateDestroy_IMPL
349 (
350     OBJGPU *pGpu,
351     KernelGraphicsManager *pKernelGraphicsManager
352 )
353 {
354     fecsRemoveAllBindpointsForGpu(pGpu);
355 }
356 
357 /*!
358  * @brief Set legacy Kgraphics Static Info (i.e. state of GR0)
359  */
360 void
kgrmgrSetLegacyKgraphicsStaticInfo_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,KernelGraphics * pKernelGraphics)361 kgrmgrSetLegacyKgraphicsStaticInfo_IMPL
362 (
363     OBJGPU *pGpu,
364     KernelGraphicsManager *pKernelGraphicsManager,
365     KernelGraphics *pKernelGraphics
366 )
367 {
368     NV_CHECK_OR_RETURN_VOID(LEVEL_INFO, !pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized);
369     NV_ASSERT_OR_RETURN_VOID((pKernelGraphics != NULL) && (kgraphicsGetInstance(pGpu, pKernelGraphics) == 0));
370     const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics);
371     NV_ASSERT_OR_RETURN_VOID(pKernelGraphicsStaticInfo != NULL);
372 
373     portMemCopy(&pKernelGraphicsManager->legacyKgraphicsStaticInfo.floorsweepingMasks, sizeof(pKernelGraphicsStaticInfo->floorsweepingMasks),
374                 &pKernelGraphicsStaticInfo->floorsweepingMasks, sizeof(pKernelGraphicsStaticInfo->floorsweepingMasks));
375 
376     if (pKernelGraphicsStaticInfo->pPpcMasks != NULL)
377     {
378         pKernelGraphicsManager->legacyKgraphicsStaticInfo.pPpcMasks = portMemAllocNonPaged(sizeof(*pKernelGraphicsStaticInfo->pPpcMasks));
379         NV_ASSERT_OR_GOTO(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pPpcMasks != NULL, cleanup);
380 
381         portMemCopy(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pPpcMasks, sizeof(*pKernelGraphicsStaticInfo->pPpcMasks),
382                     pKernelGraphicsStaticInfo->pPpcMasks, sizeof(*pKernelGraphicsStaticInfo->pPpcMasks));
383     }
384 
385     if (pKernelGraphicsStaticInfo->pGrInfo != NULL)
386     {
387         pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo = portMemAllocNonPaged(sizeof(*pKernelGraphicsStaticInfo->pGrInfo));
388         NV_ASSERT_OR_GOTO(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo != NULL, cleanup);
389 
390         portMemCopy(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo, sizeof(*pKernelGraphicsStaticInfo->pGrInfo),
391                     pKernelGraphicsStaticInfo->pGrInfo, sizeof(*pKernelGraphicsStaticInfo->pGrInfo));
392     }
393 
394     pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized = NV_TRUE;
395     return;
396 
397 cleanup:
398     portMemFree(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pPpcMasks);
399     pKernelGraphicsManager->legacyKgraphicsStaticInfo.pPpcMasks = NULL;
400     portMemFree(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo);
401     pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo = NULL;
402 }
403 
404 /*!
405  * @brief Retrieves associated KernelGraphics engine for given device / route info
406  *
407  * @param[in]  pGpu
408  * @param[in]  pKernelGraphicsManager
409  * @param[in]  pDevice
410  * @param[in]  grRouteInfo                   Client-provided info to direct GR accesses
411  * @param[out] ppKernelGraphics (Optional)   Ptr to store appropriate KernelGraphics *, if desired.
412  */
413 NV_STATUS
kgrmgrCtrlRouteKGRWithDevice_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,Device * pDevice,const NV2080_CTRL_GR_ROUTE_INFO * pGrRouteInfo,KernelGraphics ** ppKernelGraphics)414 kgrmgrCtrlRouteKGRWithDevice_IMPL
415 (
416     OBJGPU *pGpu,
417     KernelGraphicsManager *pKernelGraphicsManager,
418     Device *pDevice,
419     const NV2080_CTRL_GR_ROUTE_INFO *pGrRouteInfo,
420     KernelGraphics **ppKernelGraphics
421 )
422 {
423     MIG_INSTANCE_REF ref;
424     KernelGraphics *pKernelGraphics;
425     NvU32 type;
426     NV_STATUS status = NV_OK;
427     NvU32 grIdx;
428     NV2080_CTRL_GR_ROUTE_INFO grRouteInfo = *pGrRouteInfo;
429     KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu);
430     RsClient *pClient;
431 
432     if (!IS_MIG_IN_USE(pGpu))
433     {
434         grIdx = 0;
435         goto done;
436     }
437 
438     pClient = RES_GET_CLIENT(pDevice);
439 
440     NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
441         kmigmgrGetInstanceRefFromDevice(pGpu, pKernelMIGManager, pDevice, &ref));
442 
443     //
444     // Compute instances always have 1 GR engine, so automatically fill in
445     // the route info when subscribed to a compute instance
446     //
447     if (ref.pMIGComputeInstance != NULL)
448     {
449         portMemSet(&grRouteInfo, 0, sizeof(NV2080_CTRL_GR_ROUTE_INFO));
450         kgrmgrCtrlSetEngineID(0, &grRouteInfo);
451     }
452     else
453     {
454         RmClient *pRmClient = dynamicCast(pClient, RmClient);
455 
456         if (pRmClient == NULL)
457             return NV_ERR_INVALID_OBJECT_HANDLE;
458 
459         RS_PRIV_LEVEL privLevel = rmclientGetCachedPrivilege(pRmClient);
460         if (!rmclientIsAdmin(pRmClient, privLevel) &&
461              _kgrmgrGPUInstanceHasComputeInstances(pGpu, pKernelGraphicsManager, ref.pKernelMIGGpuInstance))
462         {
463             return NV_ERR_INSUFFICIENT_PERMISSIONS;
464         }
465     }
466 
467     type = DRF_VAL(2080_CTRL_GR, _ROUTE_INFO_FLAGS, _TYPE, grRouteInfo.flags);
468     switch (type)
469     {
470         case NV2080_CTRL_GR_ROUTE_INFO_FLAGS_TYPE_NONE:
471             NV_PRINTF(LEVEL_ERROR,
472                       "Cannot give GR Route flag of TYPE_NONE with MIG enabled!\n");
473             return NV_ERR_INVALID_ARGUMENT;
474 
475         case NV2080_CTRL_GR_ROUTE_INFO_FLAGS_TYPE_ENGID:
476         {
477             NvU32 localGrIdx = DRF_VAL64(2080_CTRL_GR, _ROUTE_INFO_DATA,
478                                          _ENGID, grRouteInfo.route);
479             RM_ENGINE_TYPE globalRmEngType;
480 
481             NV_CHECK_OK_OR_RETURN(
482                 LEVEL_ERROR,
483                 kmigmgrGetLocalToGlobalEngineType(pGpu, pKernelMIGManager, ref,
484                                                   RM_ENGINE_TYPE_GR(localGrIdx),
485                                                   &globalRmEngType));
486             NV_ASSERT_OR_RETURN(RM_ENGINE_TYPE_IS_GR(globalRmEngType), NV_ERR_INVALID_STATE);
487             grIdx = RM_ENGINE_TYPE_GR_IDX(globalRmEngType);
488 
489             break;
490         }
491 
492         case NV2080_CTRL_GR_ROUTE_INFO_FLAGS_TYPE_CHANNEL:
493         {
494             KernelChannel *pKernelChannel;
495             RM_ENGINE_TYPE rmEngineType;
496             NvHandle hChannel = DRF_VAL64(2080_CTRL_GR, _ROUTE_INFO_DATA,
497                                           _CHANNEL_HANDLE, grRouteInfo.route);
498 
499             status = CliGetKernelChannel(pClient, hChannel, &pKernelChannel);
500             if (status != NV_OK)
501             {
502                 RsResourceRef         *pChanGrpRef;
503                 KernelChannelGroupApi *pKernelChannelGroupApi = NULL;
504                 KernelChannelGroup    *pKernelChannelGroup    = NULL;
505 
506                 //
507                 // If retrieving a channel with the given hChannel doesn't work,
508                 // try interpreting it as a handle to a channel group instead.
509                 //
510                 status = CliGetChannelGroup(pClient->hClient, hChannel,
511                     &pChanGrpRef, NULL);
512                 if (NV_OK != status)
513                 {
514                     NV_PRINTF(LEVEL_ERROR,
515                               "Failed to find a channel or TSG with given handle 0x%08x associated with hClient=0x%08x\n",
516                               hChannel, pClient->hClient);
517                     return NV_ERR_INVALID_ARGUMENT;
518                 }
519 
520                 pKernelChannelGroupApi = dynamicCast(pChanGrpRef->pResource,
521                                                      KernelChannelGroupApi);
522                 NV_ASSERT_OR_RETURN(
523                     (pKernelChannelGroupApi != NULL &&
524                      pKernelChannelGroupApi->pKernelChannelGroup != NULL),
525                     NV_ERR_INVALID_STATE);
526                 pKernelChannelGroup =
527                     pKernelChannelGroupApi->pKernelChannelGroup;
528 
529                 NV_PRINTF(LEVEL_INFO,
530                           "Found TSG with given handle 0x%08x, using this to determine GR engine ID\n",
531                           hChannel);
532                 rmEngineType = pKernelChannelGroup->engineType;
533             }
534             else
535             {
536                 NV_PRINTF(LEVEL_INFO,
537                           "Found channel with given handle 0x%08x, using this to determine GR engine ID\n",
538                           hChannel);
539                 rmEngineType = kchannelGetEngineType(pKernelChannel);
540             }
541 
542             if (!RM_ENGINE_TYPE_IS_GR(rmEngineType))
543             {
544                 NV_PRINTF(LEVEL_ERROR,
545                           "Failed to route GR using non-GR engine type 0x%x (0x%x)\n",
546                           gpuGetNv2080EngineType(rmEngineType), rmEngineType);
547                 return NV_ERR_INVALID_ARGUMENT;
548             }
549 
550             grIdx = RM_ENGINE_TYPE_GR_IDX(rmEngineType);
551             break;
552         }
553         default:
554             NV_PRINTF(LEVEL_ERROR,
555                       "Unrecognized GR Route flag type 0x%x!\n", type);
556             return NV_ERR_INVALID_ARGUMENT;
557     }
558 
559 done:
560     pKernelGraphics = GPU_GET_KERNEL_GRAPHICS(pGpu, grIdx);
561     NV_ASSERT_OR_RETURN(pKernelGraphics != NULL, NV_ERR_INVALID_STATE);
562 
563     if (ppKernelGraphics != NULL)
564         *ppKernelGraphics = pKernelGraphics;
565 
566     return status;
567 }
568 
569 /*!
570  * @return legacy GPC mask enumerated by this chip
571  */
572 NvU32
kgrmgrGetLegacyGpcMask_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager)573 kgrmgrGetLegacyGpcMask_IMPL
574 (
575     OBJGPU *pGpu,
576     KernelGraphicsManager *pKernelGraphicsManager
577 )
578 {
579     NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized, 0);
580 
581     return pKernelGraphicsManager->legacyKgraphicsStaticInfo.floorsweepingMasks.gpcMask;
582 }
583 
584 /*!
585  * @return legacy physical GFX GPC mask enumerated by this chip
586  */
587 NvU32
kgrmgrGetLegacyPhysGfxGpcMask_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager)588 kgrmgrGetLegacyPhysGfxGpcMask_IMPL
589 (
590     OBJGPU *pGpu,
591     KernelGraphicsManager *pKernelGraphicsManager
592 )
593 {
594     NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized, 0);
595 
596     return pKernelGraphicsManager->legacyKgraphicsStaticInfo.floorsweepingMasks.physGfxGpcMask;
597 }
598 
599 /*!
600  * @return legacy TPC mask for certain GPC
601  *
602  * @param[in]  pGpu
603  * @param[in]  KernelGraphicsManager
604  * @param[in]  gpcId                 Indicates logical GPC ID when MIG enabled or physical
605  *                                   GPC ID when MIG disabled
606  */
607 NvU32
kgrmgrGetLegacyTpcMask_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,NvU32 gpcId)608 kgrmgrGetLegacyTpcMask_IMPL
609 (
610     OBJGPU *pGpu,
611     KernelGraphicsManager *pKernelGraphicsManager,
612     NvU32 gpcId
613 )
614 {
615     NvU32 maxNumGpcs;
616 
617     NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized, 0);
618     NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo != NULL, 0);
619 
620     maxNumGpcs = pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo->infoList[NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_GPCS].data;
621     NV_CHECK_OR_RETURN(LEVEL_ERROR, (gpcId < maxNumGpcs), 0);
622 
623     return pKernelGraphicsManager->legacyKgraphicsStaticInfo.floorsweepingMasks.tpcMask[gpcId];
624 }
625 
626 /*!
627  * @brief Get legacy PPC mask for certain GPC
628  */
629 NV_STATUS
kgrmgrGetLegacyPpcMask_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,NvU32 physGpcId,NvU32 * pPpcMask)630 kgrmgrGetLegacyPpcMask_IMPL
631 (
632     OBJGPU *pGpu,
633     KernelGraphicsManager *pKernelGraphicsManager,
634     NvU32 physGpcId,
635     NvU32 *pPpcMask
636 )
637 {
638     NvU32 maxNumGpcs;
639 
640     NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized, NV_ERR_INVALID_STATE);
641     NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo != NULL, NV_ERR_INVALID_STATE);
642     maxNumGpcs = pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo->infoList[NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_GPCS].data;
643     NV_CHECK_OR_RETURN(LEVEL_ERROR, (physGpcId < maxNumGpcs), NV_ERR_INVALID_ARGUMENT);
644     NV_ASSERT_OR_RETURN((pPpcMask != NULL), NV_ERR_INVALID_ARGUMENT);
645     NV_ASSERT_OR_RETURN((pKernelGraphicsManager->legacyKgraphicsStaticInfo.pPpcMasks != NULL), NV_ERR_NOT_SUPPORTED);
646 
647     *pPpcMask = pKernelGraphicsManager->legacyKgraphicsStaticInfo.pPpcMasks->mask[physGpcId];
648 
649     return NV_OK;
650 }
651 
652 /*!
653  * @brief Returns legacy zcull mask for specific gpc
654  */
655 NvU32
kgrmgrGetLegacyZcullMask_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,NvU32 physGpcId)656 kgrmgrGetLegacyZcullMask_IMPL
657 (
658     OBJGPU *pGpu,
659     KernelGraphicsManager *pKernelGraphicsManager,
660     NvU32 physGpcId
661 )
662 {
663     NvU32 maxNumGpcs;
664 
665     NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized, 0);
666     NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo != NULL, NV_ERR_INVALID_STATE);
667     maxNumGpcs = pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo->infoList[NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_GPCS].data;
668     NV_CHECK_OR_RETURN(LEVEL_ERROR, (physGpcId < maxNumGpcs), NV_ERR_INVALID_ARGUMENT);
669 
670     return pKernelGraphicsManager->legacyKgraphicsStaticInfo.floorsweepingMasks.zcullMask[physGpcId];
671 }
672 
673 /*!
674  * @brief   Function to Alloc VEIDs for a GR engine
675  *
676  * @param[IN]   pGpu
677  * @param[IN]   pKernelGraphicsManager
678  * @param[IN]   grIdx                  phys gr idx
679  * @param[IN]   veidSpanOffset         Offset of spans localized to a GPU instance to make the VEID request from
680  * @param[IN]   veidCount              Total VEIDs connected to this GR engine
681  * @param[IN]   pKernelMIGGPUInstance
682  */
683 NV_STATUS
kgrmgrAllocVeidsForGrIdx_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,NvU32 grIdx,NvU32 veidSpanOffset,NvU32 veidCount,KERNEL_MIG_GPU_INSTANCE * pKernelMIGGPUInstance)684 kgrmgrAllocVeidsForGrIdx_IMPL
685 (
686     OBJGPU *pGpu,
687     KernelGraphicsManager *pKernelGraphicsManager,
688     NvU32 grIdx,
689     NvU32 veidSpanOffset,
690     NvU32 veidCount,
691     KERNEL_MIG_GPU_INSTANCE *pKernelMIGGPUInstance
692 )
693 {
694     NvU32 veidSizePerSpan;
695     NvU32 veidStart = 0;
696     NvU32 veidEnd = 0;
697     NvU32 GPUInstanceVeidEnd;
698     NvU64 GPUInstanceVeidMask;
699     NvU64 GPUInstanceFreeVeidMask;
700     NvU64 reqVeidMask;
701 
702     // This GR should not be already configured to use any VEIDs
703     NV_ASSERT_OR_RETURN(pKernelGraphicsManager->grIdxVeidMask[grIdx] == 0, NV_ERR_INVALID_STATE);
704 
705     NV_ASSERT_OK_OR_RETURN(
706         kgrmgrGetVeidSizePerSpan(pGpu, pKernelGraphicsManager, &veidSizePerSpan));
707 
708     // We statically assign VEIDs to a GR based on the number of GPCs connected to it
709     if (veidCount % veidSizePerSpan != 0)
710     {
711         NV_PRINTF(LEVEL_ERROR, "veidCount %d is not aligned to veidSizePerSpan=%d\n", veidCount, veidSizePerSpan);
712         return NV_ERR_INVALID_ARGUMENT;
713     }
714 
715     NV_ASSERT_OR_RETURN(veidSpanOffset != KMIGMGR_SPAN_OFFSET_INVALID, NV_ERR_INVALID_ARGUMENT);
716 
717     veidStart = (veidSpanOffset * veidSizePerSpan) + pKernelMIGGPUInstance->resourceAllocation.veidOffset;
718     veidEnd = veidStart + veidCount - 1;
719 
720     NV_ASSERT_OR_RETURN(veidStart < veidEnd, NV_ERR_INVALID_STATE);
721     NV_ASSERT_OR_RETURN(veidStart < 64, NV_ERR_INVALID_ARGUMENT);
722     NV_ASSERT_OR_RETURN(veidEnd < 64, NV_ERR_INVALID_ARGUMENT);
723 
724     reqVeidMask = DRF_SHIFTMASK64(veidEnd:veidStart);
725 
726     NV_ASSERT_OR_RETURN(reqVeidMask != 0x0, NV_ERR_INVALID_STATE);
727 
728     // Create a mask for VEIDs associated with this GPU instance
729     GPUInstanceVeidEnd = pKernelMIGGPUInstance->resourceAllocation.veidOffset + pKernelMIGGPUInstance->resourceAllocation.veidCount - 1;
730     GPUInstanceVeidMask = DRF_SHIFTMASK64(GPUInstanceVeidEnd:pKernelMIGGPUInstance->resourceAllocation.veidOffset);
731 
732     NV_ASSERT_OR_RETURN(GPUInstanceVeidMask != 0x0, NV_ERR_INVALID_STATE);
733 
734     GPUInstanceFreeVeidMask = ~(pKernelGraphicsManager->veidInUseMask) & GPUInstanceVeidMask;
735 
736     // VEID range should not overlap with existing VEIDs in use
737     NV_ASSERT_OR_RETURN((pKernelGraphicsManager->veidInUseMask & reqVeidMask) == 0, NV_ERR_STATE_IN_USE);
738 
739     //
740     // For swizzId 0, client can use any range of VEIDs if not used, so we don't
741     // need to do the range check
742     //
743     NV_ASSERT_OR_RETURN(((pKernelMIGGPUInstance->swizzId == 0) ||
744                          ((GPUInstanceFreeVeidMask & reqVeidMask) == reqVeidMask)),
745                         NV_ERR_INSUFFICIENT_RESOURCES);
746 
747     // mark each VEID in the range as "in use"
748     pKernelGraphicsManager->veidInUseMask |= reqVeidMask;
749     pKernelGraphicsManager->grIdxVeidMask[grIdx] |= reqVeidMask;
750 
751     return NV_OK;
752 }
753 
754 /*!
755  * @brief   Function to Clear Gr Engine to VEIDs mapping
756  */
757 void
kgrmgrClearVeidsForGrIdx_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,NvU32 grIdx)758 kgrmgrClearVeidsForGrIdx_IMPL
759 (
760     OBJGPU *pGpu,
761     KernelGraphicsManager *pKernelGraphicsManager,
762     NvU32 grIdx
763 )
764 {
765     NvU64 veidMask = pKernelGraphicsManager->grIdxVeidMask[grIdx];
766 
767     // mark all VEIDs of this GR engine as "not in use"
768     NV_ASSERT((pKernelGraphicsManager->veidInUseMask & veidMask) == veidMask);
769     pKernelGraphicsManager->veidInUseMask &= ~veidMask;
770     pKernelGraphicsManager->grIdxVeidMask[grIdx] = 0;
771 }
772 
773 /*!
774  * @brief Get VEID step size
775  */
776 NV_STATUS
kgrmgrGetVeidStepSize_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,NvU32 * pVeidStepSize)777 kgrmgrGetVeidStepSize_IMPL
778 (
779     OBJGPU *pGpu,
780     KernelGraphicsManager *pKernelGraphicsManager,
781     NvU32 *pVeidStepSize
782 )
783 {
784     NV_ASSERT_OR_RETURN(pVeidStepSize != NULL, NV_ERR_INVALID_ARGUMENT);
785     NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized, NV_ERR_INVALID_STATE);
786     NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo != NULL, NV_ERR_INVALID_STATE);
787 
788     *pVeidStepSize = pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo->infoList[NV0080_CTRL_GR_INFO_INDEX_LITTER_MIN_SUBCTX_PER_SMC_ENG].data;
789 
790     return NV_OK;
791 }
792 
793 /*!
794  * @brief Get VEID size for each span
795  */
796 NV_STATUS
kgrmgrGetVeidSizePerSpan_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,NvU32 * pVeidSizePerSpan)797 kgrmgrGetVeidSizePerSpan_IMPL
798 (
799     OBJGPU *pGpu,
800     KernelGraphicsManager *pKernelGraphicsManager,
801     NvU32 *pVeidSizePerSpan
802 )
803 {
804     NV_ASSERT_OR_RETURN(pVeidSizePerSpan != NULL, NV_ERR_INVALID_ARGUMENT);
805 
806     // VEIDs for each span should be the VEID size we assign to the smallest GPC count
807     NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
808         kgrmgrGetVeidsFromGpcCount_HAL(pGpu, pKernelGraphicsManager, 1, pVeidSizePerSpan));
809 
810     return NV_OK;
811 }
812 
813 /*!
814  * @brief   Function to get starting VEID for a Gr Engine
815  */
816 NV_STATUS
kgrmgrGetVeidBaseForGrIdx_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,NvU32 grIdx,NvU32 * pVeidStart)817 kgrmgrGetVeidBaseForGrIdx_IMPL
818 (
819     OBJGPU *pGpu,
820     KernelGraphicsManager *pKernelGraphicsManager,
821     NvU32 grIdx,
822     NvU32 *pVeidStart
823 )
824 {
825     NvU64 veidMask;
826     NV_ASSERT_OR_RETURN(pVeidStart != NULL, NV_ERR_INVALID_ARGUMENT);
827     NV_ASSERT_OR_RETURN(grIdx != KGRMGR_MAX_GR, NV_ERR_INVALID_ARGUMENT);
828 
829     *pVeidStart = 0;
830 
831     veidMask = pKernelGraphicsManager->grIdxVeidMask[grIdx];
832 
833     //
834     // If a GR is not configured, VEID mask for it will be "0" and counting
835     // "0" in a zero-based mask will result in max bit-width size.
836     //
837     if (veidMask != 0x0)
838         *pVeidStart = portUtilCountTrailingZeros64(veidMask);
839 
840     return NV_OK;
841 }
842 
843 /*!
844  * @brief   Function to get GR index for a VEID
845  */
846 NV_STATUS
kgrmgrGetGrIdxForVeid_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,NvU32 veid,NvU32 * pGrIdx)847 kgrmgrGetGrIdxForVeid_IMPL
848 (
849     OBJGPU *pGpu,
850     KernelGraphicsManager *pKernelGraphicsManager,
851     NvU32 veid,
852     NvU32 *pGrIdx
853 )
854 {
855     NvU32 i;
856     NvU64 veidMask = NVBIT64(veid);
857 
858     NV_ASSERT_OR_RETURN(pGrIdx != NULL, NV_ERR_INVALID_ARGUMENT);
859     for (i = 0; i < KGRMGR_MAX_GR; ++i)
860     {
861         if ((pKernelGraphicsManager->grIdxVeidMask[i] & veidMask) != 0)
862             break;
863     }
864     NV_ASSERT_OR_RETURN(i != KGRMGR_MAX_GR, NV_ERR_OBJECT_NOT_FOUND);
865     *pGrIdx = i;
866     return NV_OK;
867 }
868 
869 /*!
870  * @brief discovers maximum size of local ctx buffers.
871  */
872 NV_STATUS
kgrmgrDiscoverMaxLocalCtxBufInfo_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,KernelGraphics * pKernelGraphics,NvU32 swizzId)873 kgrmgrDiscoverMaxLocalCtxBufInfo_IMPL
874 (
875     OBJGPU *pGpu,
876     KernelGraphicsManager *pKernelGraphicsManager,
877     KernelGraphics *pKernelGraphics,
878     NvU32 swizzId
879 )
880 {
881     KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu);
882 
883     //
884     // Most of GR is stub'd so context related things are not needed in AMODEL.
885     // But this function can be called in some MODS test, so return OK directly
886     // to avoid failing the test.
887     //
888     if (IS_MODS_AMODEL(pGpu))
889         return NV_OK;
890 
891     NV_CHECK_OR_RETURN(LEVEL_SILENT,
892         kmigmgrIsMemoryPartitioningNeeded_HAL(pGpu, pKernelMIGManager, swizzId), NV_OK);
893 
894     return kgraphicsDiscoverMaxLocalCtxBufferSize(pGpu, pKernelGraphics);
895 }
896 
897 /*!
898  * @brief Get ctxbufpool info for the global ctx buffer
899  */
900 const CTX_BUF_INFO *
kgrmgrGetGlobalCtxBufInfo_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,GR_GLOBALCTX_BUFFER buf)901 kgrmgrGetGlobalCtxBufInfo_IMPL
902 (
903     OBJGPU *pGpu,
904     KernelGraphicsManager *pKernelGraphicsManager,
905     GR_GLOBALCTX_BUFFER buf
906 )
907 {
908     NV_ASSERT_OR_RETURN(NV_ENUM_IS(GR_GLOBALCTX_BUFFER, buf), NULL);
909     return &pKernelGraphicsManager->globalCtxBufInfo[buf];
910 }
911 
912 /*!
913  * @brief Set ctxbufpool parameters for the global ctx buffer
914  */
915 void
kgrmgrSetGlobalCtxBufInfo_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,GR_GLOBALCTX_BUFFER buf,NvU64 size,NvU64 align,RM_ATTR_PAGE_SIZE attr,NvBool bContiguous)916 kgrmgrSetGlobalCtxBufInfo_IMPL
917 (
918     OBJGPU *pGpu,
919     KernelGraphicsManager *pKernelGraphicsManager,
920     GR_GLOBALCTX_BUFFER buf,
921     NvU64 size,
922     NvU64 align,
923     RM_ATTR_PAGE_SIZE attr,
924     NvBool bContiguous
925 )
926 {
927     CTX_BUF_INFO *pInfo;
928     NV_ASSERT_OR_RETURN_VOID(NV_ENUM_IS(GR_GLOBALCTX_BUFFER, buf));
929 
930     pInfo = &pKernelGraphicsManager->globalCtxBufInfo[buf];
931     pInfo->size    = size;
932     pInfo->align   = align;
933     pInfo->attr    = attr;
934     pInfo->bContig = bContiguous;
935 }
936 
937 /*!
938  * @brief Gets maximum size of GR global ctx buffers.
939  *        These are sizes of buffer for GR0 in legacy mode with all GPCs
940  *        connected to GR0.
941  */
942 NV_STATUS
kgrmgrDiscoverMaxGlobalCtxBufSizes_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,KernelGraphics * pKernelGraphics,NvBool bMemoryPartitioningNeeded)943 kgrmgrDiscoverMaxGlobalCtxBufSizes_IMPL
944 (
945     OBJGPU *pGpu,
946     KernelGraphicsManager *pKernelGraphicsManager,
947     KernelGraphics *pKernelGraphics,
948     NvBool bMemoryPartitioningNeeded
949 )
950 {
951     const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo;
952     GR_GLOBALCTX_BUFFER bufId;
953 
954     //
955     // Most of GR is stub'd so context related things are not needed in AMODEL.
956     // But this function can be called in some MODS test, so return OK directly
957     // to avoid failing the test.
958     //
959     if (IS_MODS_AMODEL(pGpu))
960         return NV_OK;
961 
962     NV_ASSERT_OR_RETURN(!IS_MIG_IN_USE(pGpu), NV_ERR_INVALID_STATE);
963 
964     //
965     // Bug 2915422: Eventually we expect this check to be replaced by ctxBufPoolIsSupported
966     // we can't use that check today because PDB_PROP_GPU_MOVE_CTX_BUFFERS_TO_PMA is not enabled
967     // when this function is called because
968     // kgraphicsInitializeDeferredStaticData below will eventually lead to
969     // global ctx buffer allocation from ctxBufPools even before these pools are
970     // populated which happens later during GPU instance creation. Once we are
971     // able to rip out global buffer alloc from
972     // kgraphicsInitializeDeferredStaticData, we can enable the above property
973     // early.
974     //
975     NV_CHECK_OR_RETURN(LEVEL_SILENT, bMemoryPartitioningNeeded, NV_OK);
976 
977     // Make sure sizes of all buffers are setup
978     NV_ASSERT_OK_OR_RETURN(
979         kgraphicsInitializeDeferredStaticData(pGpu, pKernelGraphics, NV01_NULL_OBJECT, NV01_NULL_OBJECT));
980 
981     pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics);
982     NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE);
983     NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pContextBuffersInfo != NULL, NV_ERR_INVALID_STATE);
984 
985     // Get sizes of global ctx buffers
986     FOR_EACH_IN_ENUM(GR_GLOBALCTX_BUFFER, bufId)
987     {
988         NvU32 fifoEngineId;
989         NV_ASSERT_OK_OR_RETURN(
990             kgrctxGlobalCtxBufferToFifoEngineId(bufId, &fifoEngineId));
991 
992         //
993         // contiguity is determined later before reservation as it depends on settings
994         // that take effect after this point.
995         //
996         kgrmgrSetGlobalCtxBufInfo(pGpu, pKernelGraphicsManager, bufId,
997                                   pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[fifoEngineId].size,
998                                   pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[fifoEngineId].alignment,
999                                   RM_ATTR_PAGE_SIZE_4KB, NV_FALSE);
1000     }
1001     FOR_EACH_IN_ENUM_END;
1002 
1003     return NV_OK;
1004 }
1005 
1006 /*!
1007  * @return legacy TPC count for certain GPC
1008  *
1009  * @param[in]  pGpu
1010  * @param[in]  KernelGraphicsManager
1011  * @param[in]  gpcId                 Indicates logical GPC ID
1012  */
1013 NvU32
kgrmgrGetLegacyGpcTpcCount_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,NvU32 gpcId)1014 kgrmgrGetLegacyGpcTpcCount_IMPL
1015 (
1016     OBJGPU *pGpu,
1017     KernelGraphicsManager *pKernelGraphicsManager,
1018     NvU32 gpcId
1019 )
1020 {
1021     NvU32 maxNumGpcs;
1022 
1023     NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized, 0);
1024     NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo != NULL, 0);
1025 
1026     maxNumGpcs = nvPopCount32(pKernelGraphicsManager->legacyKgraphicsStaticInfo.floorsweepingMasks.gpcMask);
1027     NV_CHECK_OR_RETURN(LEVEL_ERROR, (gpcId < maxNumGpcs), 0);
1028 
1029     return pKernelGraphicsManager->legacyKgraphicsStaticInfo.floorsweepingMasks.tpcCount[gpcId];
1030 }
1031 
1032 /*!
1033  * @brief   Function to Alloc VEIDs with either a specific placement or first-fit placement
1034  *          depending upon the value passed into pSpanStart
1035  *
1036  * @param[IN]     pGpu
1037  * @param[IN]     pKernelGraphicsManager
1038  * @param[IN/OUT] pInUseMask             Mask of VEIDs in-use, updated with newly set VEIDs
1039  * @param[IN]     veidCount              Total VEIDs connected to this GR engine
1040  * @param[IN/OUT] pSpanStart             GPU Instance local span offset to begin assigning VEIDs from
1041  * @param[IN]     pKernelMIGGPUInstance
1042  */
1043 NV_STATUS
kgrmgrCheckVeidsRequest_IMPL(OBJGPU * pGpu,KernelGraphicsManager * pKernelGraphicsManager,NvU64 * pInUseMask,NvU32 veidCount,NvU32 * pSpanStart,KERNEL_MIG_GPU_INSTANCE * pKernelMIGGPUInstance)1044 kgrmgrCheckVeidsRequest_IMPL
1045 (
1046     OBJGPU *pGpu,
1047     KernelGraphicsManager *pKernelGraphicsManager,
1048     NvU64 *pInUseMask,
1049     NvU32 veidCount,
1050     NvU32 *pSpanStart,
1051     KERNEL_MIG_GPU_INSTANCE *pKernelMIGGPUInstance
1052 )
1053 {
1054     NvU32 veidSizePerSpan;
1055     NvU32 veidStart = 0;
1056     NvU32 veidEnd = 0;
1057     NvU32 GPUInstanceVeidEnd;
1058     NvU64 GPUInstanceVeidMask;
1059     NvU64 GPUInstanceFreeVeidMask;
1060     NvU64 veidMask;
1061 
1062     NV_ASSERT_OK_OR_RETURN(
1063         kgrmgrGetVeidSizePerSpan(pGpu, pKernelGraphicsManager, &veidSizePerSpan));
1064 
1065     NV_ASSERT_OR_RETURN(pSpanStart != NULL, NV_ERR_INVALID_ARGUMENT);
1066     NV_ASSERT_OR_RETURN(pInUseMask != NULL, NV_ERR_INVALID_ARGUMENT);
1067 
1068     // We statically assign VEIDs to a GR based on the number of GPCs connected to it
1069     if (veidCount % veidSizePerSpan != 0)
1070     {
1071         NV_PRINTF(LEVEL_ERROR, "veidCount %d is not aligned to veidSizePerSpan=%d\n", veidCount, veidSizePerSpan);
1072         return NV_ERR_INVALID_ARGUMENT;
1073     }
1074 
1075     // Create a mask for VEIDs associated with this GPU instance
1076     GPUInstanceVeidEnd = pKernelMIGGPUInstance->resourceAllocation.veidOffset + pKernelMIGGPUInstance->resourceAllocation.veidCount - 1;
1077     GPUInstanceVeidMask = DRF_SHIFTMASK64(GPUInstanceVeidEnd:pKernelMIGGPUInstance->resourceAllocation.veidOffset);
1078 
1079     NV_ASSERT_OR_RETURN(GPUInstanceVeidMask != 0x0, NV_ERR_INVALID_STATE);
1080 
1081     GPUInstanceFreeVeidMask = ~(*pInUseMask) & GPUInstanceVeidMask;
1082 
1083     if (*pSpanStart != KMIGMGR_SPAN_OFFSET_INVALID)
1084     {
1085         veidStart = (*pSpanStart * veidSizePerSpan) + pKernelMIGGPUInstance->resourceAllocation.veidOffset;
1086         veidEnd = veidStart + veidCount - 1;
1087 
1088         NV_ASSERT_OR_RETURN(veidStart < veidEnd, NV_ERR_INVALID_STATE);
1089         NV_ASSERT_OR_RETURN(veidStart < 64, NV_ERR_INVALID_ARGUMENT);
1090         NV_ASSERT_OR_RETURN(veidEnd < 64, NV_ERR_INVALID_ARGUMENT);
1091     }
1092     else
1093     {
1094         NvU64 reqVeidMask = DRF_SHIFTMASK64(veidCount - 1:0);
1095         NvU32 i;
1096 
1097         for (i = pKernelMIGGPUInstance->resourceAllocation.veidOffset; i <= GPUInstanceVeidEnd; i += veidSizePerSpan)
1098         {
1099             // See if requested slots are available within this range
1100             if (((GPUInstanceFreeVeidMask >> i) & reqVeidMask) == reqVeidMask)
1101             {
1102                 veidStart = i;
1103                 veidEnd = veidStart + veidCount - 1;
1104                 break;
1105             }
1106         }
1107 
1108         NV_CHECK_OR_RETURN(LEVEL_SILENT, i <= GPUInstanceVeidEnd,
1109                            NV_ERR_INSUFFICIENT_RESOURCES);
1110     }
1111 
1112     veidMask = DRF_SHIFTMASK64(veidEnd:veidStart);
1113     NV_ASSERT_OR_RETURN(veidMask != 0x0, NV_ERR_INVALID_STATE);
1114 
1115     //
1116     // For swizzId 0, client can use any range of VEIDs if not used, so we don't
1117     // need to do the range check
1118     //
1119     NV_CHECK_OR_RETURN(LEVEL_ERROR,
1120                        ((pKernelMIGGPUInstance->swizzId == 0) ||
1121                         (GPUInstanceVeidMask & veidMask) == veidMask),
1122                        NV_ERR_INSUFFICIENT_RESOURCES);
1123 
1124     // VEID range should not overlap with existing VEIDs in use
1125     NV_ASSERT_OR_RETURN((*pInUseMask & veidMask) == 0, NV_ERR_STATE_IN_USE);
1126 
1127     NV_ASSERT(veidStart >= pKernelMIGGPUInstance->resourceAllocation.veidOffset);
1128 
1129     *pSpanStart = (veidStart - pKernelMIGGPUInstance->resourceAllocation.veidOffset) / veidSizePerSpan;
1130     *pInUseMask |= veidMask;
1131     return NV_OK;
1132 }
1133 
1134