1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 /***************************** HW State Routines ***************************\
25 * GPU Manager *
26 \***************************************************************************/
27
28
29
30 #include "core/system.h"
31 #include "core/locks.h"
32 #include "platform/sli/sli.h"
33 #include "gpu_mgr/gpu_mgr.h"
34 #include "gpu/device/device.h"
35 #include "gpu/gpu.h"
36 #include "tls/tls.h"
37 #include "nvrm_registry.h"
38 #include "nv_ref.h"
39 #include "nvlimits.h"
40 #include "nv-firmware-registry.h"
41
42 #include "kernel/gpu/intr/intr.h"
43 #include "vgpu/sdk-structures.h"
44 #include "virtualization/hypervisor/hypervisor.h"
45 #include "platform/chipset/chipset.h"
46 #include "published/pcie_switch/pcie_switch_ref.h"
47 #include "kernel/gpu/nvlink/kernel_nvlink.h"
48 #include "gpu/mem_sys/kern_mem_sys.h"
49 #include "gpu/conf_compute/conf_compute.h"
50 #include "gpu/gpu_fabric_probe.h"
51 #include "gpu/mig_mgr/gpu_instance_subscription.h"
52 #include "ctrl/ctrlc56f.h"
53
54 // local static funcs
55 static void gpumgrSetAttachInfo(OBJGPU *, GPUATTACHARG *);
56 static void gpumgrGetGpuHalFactor(NvU32 *pChipId0, NvU32 *pChipId1, NvU32 *pSocChipId0, RM_RUNTIME_VARIANT *pRmVariant, TEGRA_CHIP_TYPE *pTegraType, GPUATTACHARG *pAttachArg);
57 static NvBool _gpumgrGetPcieP2PCapsFromCache(NvU32 gpuMask, NvU8* pP2PWriteCapsStatus, NvU8* pP2PReadCapsStatus);
58
59 static void
_gpumgrUnregisterRmCapsForGpuUnderLock(NvU64 gpuDomainBusDevice)60 _gpumgrUnregisterRmCapsForGpuUnderLock(NvU64 gpuDomainBusDevice)
61 {
62 OBJSYS *pSys = SYS_GET_INSTANCE();
63 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
64 NvU32 i;
65
66 // SMC partition caps must be destroyed before GPU caps.
67 gpumgrUnregisterRmCapsForMIGGI(gpuDomainBusDevice);
68
69 for (i = 0; i < NV_ARRAY_ELEMENTS(pGpuMgr->probedGpus); i++)
70 {
71 PROBEDGPU *pProbedGpu = &pGpuMgr->probedGpus[i];
72
73 if (pProbedGpu->gpuDomainBusDevice == gpuDomainBusDevice &&
74 pProbedGpu->gpuId != NV0000_CTRL_GPU_INVALID_ID)
75 {
76 osRmCapUnregister(&pProbedGpu->pOsRmCaps);
77 break;
78 }
79 }
80 }
81
82 static void
_gpumgrUnregisterRmCapsForGpu(NvU64 gpuDomainBusDevice)83 _gpumgrUnregisterRmCapsForGpu(NvU64 gpuDomainBusDevice)
84 {
85 OBJSYS *pSys = SYS_GET_INSTANCE();
86 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
87
88 portSyncMutexAcquire(pGpuMgr->probedGpusLock);
89 _gpumgrUnregisterRmCapsForGpuUnderLock(gpuDomainBusDevice);
90 portSyncMutexRelease(pGpuMgr->probedGpusLock);
91 }
92
93 static NV_STATUS
_gpumgrRegisterRmCapsForGpu(OBJGPU * pGpu)94 _gpumgrRegisterRmCapsForGpu(OBJGPU *pGpu)
95 {
96 NV_STATUS status = NV_ERR_INVALID_STATE;
97 OBJSYS *pSys = SYS_GET_INSTANCE();
98 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
99 NvU32 i;
100
101 portSyncMutexAcquire(pGpuMgr->probedGpusLock);
102
103 for (i = 0; i < NV_ARRAY_ELEMENTS(pGpuMgr->probedGpus); i++)
104 {
105 PROBEDGPU *pProbedGpu = &pGpuMgr->probedGpus[i];
106
107 if (((pProbedGpu->gpuDomainBusDevice == 0) && pGpu->bIsSOC) ||
108 (pProbedGpu->gpuDomainBusDevice == gpuGetDBDF(pGpu) &&
109 pProbedGpu->gpuId != NV0000_CTRL_GPU_INVALID_ID))
110 {
111 if (pProbedGpu->pOsRmCaps == NULL)
112 {
113 status = osRmCapRegisterGpu(pGpu->pOsGpuInfo,
114 &pProbedGpu->pOsRmCaps);
115 }
116 else
117 {
118 status = NV_OK;
119 }
120
121 pGpu->pOsRmCaps = pProbedGpu->pOsRmCaps;
122 break;
123 }
124 }
125
126 NV_ASSERT(status == NV_OK);
127
128 portSyncMutexRelease(pGpuMgr->probedGpusLock);
129
130 return status;
131 }
132
133 static NV_STATUS
_gpumgrDetermineConfComputeCapabilities(OBJGPUMGR * pGpuMgr,OBJGPU * pGpu)134 _gpumgrDetermineConfComputeCapabilities
135 (
136 OBJGPUMGR *pGpuMgr,
137 OBJGPU *pGpu
138 )
139 {
140 NvBool bCCFeatureEnabled = NV_FALSE;
141 ConfidentialCompute *pCC = GPU_GET_CONF_COMPUTE(pGpu);
142 bCCFeatureEnabled = (pCC != NULL) &&
143 pCC->getProperty(pCC, PDB_PROP_CONFCOMPUTE_ENABLED);
144
145 // First GPU
146 if (ONEBITSET(pGpuMgr->gpuAttachMask))
147 {
148 pGpuMgr->ccCaps.bApmFeatureCapable = NV_FALSE;
149 pGpuMgr->ccCaps.bHccFeatureCapable = pGpu->getProperty(pGpu, PDB_PROP_GPU_CC_FEATURE_CAPABLE);
150 pGpuMgr->ccCaps.bCCFeatureEnabled = bCCFeatureEnabled;
151 pGpuMgr->ccCaps.bDevToolsModeEnabled = gpuIsCCDevToolsModeEnabled(pGpu);
152 pGpuMgr->ccCaps.bMultiGpuProtectedPcieModeEnabled = gpuIsCCMultiGpuProtectedPcieModeEnabled(pGpu);
153
154 if (pGpuMgr->ccCaps.bDevToolsModeEnabled)
155 {
156 pGpuMgr->ccCaps.bAcceptClientRequest = NV_TRUE;
157 }
158 else
159 {
160 pGpuMgr->ccCaps.bAcceptClientRequest = NV_FALSE;
161 }
162 }
163 else
164 {
165 NV_ASSERT_OR_RETURN(pGpuMgr->ccCaps.bCCFeatureEnabled ==
166 bCCFeatureEnabled, NV_ERR_INVALID_STATE);
167 if (pGpuMgr->ccCaps.bCCFeatureEnabled)
168 {
169 NV_ASSERT_OR_RETURN(pGpuMgr->ccCaps.bHccFeatureCapable ==
170 pGpu->getProperty(pGpu, PDB_PROP_GPU_CC_FEATURE_CAPABLE),
171 NV_ERR_INVALID_STATE);
172 }
173 //
174 // If one of the GPUs is not Hopper CC capable, the system as a whole
175 // is not Hopper CC capable
176 //
177 pGpuMgr->ccCaps.bHccFeatureCapable &=
178 pGpu->getProperty(pGpu, PDB_PROP_GPU_CC_FEATURE_CAPABLE);
179
180 NV_ASSERT_OR_RETURN(pGpuMgr->ccCaps.bDevToolsModeEnabled ==
181 gpuIsCCDevToolsModeEnabled(pGpu),
182 NV_ERR_INVALID_STATE);
183
184 NV_ASSERT_OR_RETURN(pGpuMgr->ccCaps.bMultiGpuProtectedPcieModeEnabled ==
185 gpuIsCCMultiGpuProtectedPcieModeEnabled(pGpu),
186 NV_ERR_INVALID_STATE);
187 }
188
189 return NV_OK;
190 }
191
192 static void
_gpumgrCacheClearMIGGpuIdInfo(NvU32 gpuId)193 _gpumgrCacheClearMIGGpuIdInfo(NvU32 gpuId)
194 {
195 OBJSYS *pSys = SYS_GET_INSTANCE();
196 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
197 NvU32 i;
198
199 portSyncRwLockAcquireWrite(pGpuMgr->cachedMIGInfoLock);
200
201 for (i = 0; i < NV_MAX_DEVICES; i++)
202 {
203 if (pGpuMgr->cachedMIGInfo[i].gpuId == gpuId)
204 {
205 portMemSet(&pGpuMgr->cachedMIGInfo[i], 0x0, sizeof(pGpuMgr->cachedMIGInfo[i]));
206 break;
207 }
208 }
209
210 portSyncRwLockReleaseWrite(pGpuMgr->cachedMIGInfoLock);
211 }
212
213 //
214 // ODB functions
215 //
216 NV_STATUS
gpumgrConstruct_IMPL(OBJGPUMGR * pGpuMgr)217 gpumgrConstruct_IMPL(OBJGPUMGR *pGpuMgr)
218 {
219 NvU32 i;
220
221 NV_PRINTF(LEVEL_INFO, "gpumgrConstruct\n");
222
223 pGpuMgr->numGpuHandles = 0;
224
225 for (i = 0; i < NV_MAX_DEVICES; i++)
226 pGpuMgr->gpuHandleIDList[i].gpuInstance = NV_MAX_DEVICES;
227
228 pGpuMgr->probedGpusLock = portSyncMutexCreate(portMemAllocatorGetGlobalNonPaged());
229
230 if (pGpuMgr->probedGpusLock == NULL)
231 return NV_ERR_INSUFFICIENT_RESOURCES;
232
233 for (i = 0; i < NV_ARRAY_ELEMENTS(pGpuMgr->probedGpus); i++)
234 {
235 portMemSet(&pGpuMgr->probedGpus[i], 0, sizeof(PROBEDGPU));
236 pGpuMgr->probedGpus[i].gpuId = NV0000_CTRL_GPU_INVALID_ID;
237 }
238
239 pGpuMgr->gpuAttachCount = 0;
240 pGpuMgr->gpuAttachMask = 0;
241
242 pGpuMgr->deviceCount = 0;
243 for (i = 0; i < NV_ARRAY_ELEMENTS(pGpuMgr->pGpuGrpTable); i++)
244 {
245 pGpuMgr->pGpuGrpTable[i] = NULL;
246 }
247
248 pGpuMgr->powerDisconnectedGpuCount = 0;
249
250 NV_ASSERT_OK_OR_RETURN(gpumgrInitPcieP2PCapsCache(pGpuMgr));
251
252 pGpuMgr->cachedMIGInfoLock =
253 portSyncRwLockCreate(portMemAllocatorGetGlobalNonPaged());
254 NV_ASSERT_OR_RETURN(pGpuMgr->cachedMIGInfoLock != NULL,
255 NV_ERR_INSUFFICIENT_RESOURCES);
256
257 portMemSet(pGpuMgr->cachedMIGInfo, 0, sizeof(pGpuMgr->cachedMIGInfo));
258
259 pGpuMgr->ccAttackerAdvantage = SECURITY_POLICY_ATTACKER_ADVANTAGE_DEFAULT;
260
261 return NV_OK;
262 }
263
264
265 void
gpumgrDestruct_IMPL(OBJGPUMGR * pGpuMgr)266 gpumgrDestruct_IMPL(OBJGPUMGR *pGpuMgr)
267 {
268 NV_PRINTF(LEVEL_INFO, "gpumgrDestruct\n");
269
270 portSyncMutexDestroy(pGpuMgr->probedGpusLock);
271
272 portSyncRwLockDestroy(pGpuMgr->cachedMIGInfoLock);
273
274 gpumgrDestroyPcieP2PCapsCache(pGpuMgr);
275 }
276
277 static NvBool
_gpumgrThreadHasExpandedGpuVisibilityInTls(void)278 _gpumgrThreadHasExpandedGpuVisibilityInTls(void)
279 {
280 NvP64 entry = tlsEntryGet(TLS_ENTRY_ID_GPUMGR_EXPANDED_GPU_VISIBILITY);
281 return (entry != NvP64_NULL) && (entry == ((NvP64) 1));
282 }
283
284 NvBool
gpumgrThreadHasExpandedGpuVisibility(void)285 gpumgrThreadHasExpandedGpuVisibility(void)
286 {
287 if (RMCFG_FEATURE_PLATFORM_UNIX)
288 {
289 return _gpumgrThreadHasExpandedGpuVisibilityInTls();
290 }
291 else
292 {
293 //
294 // Bug 4376209
295 // Non-UNIX platforms have expanded GPU visibility by default for now
296 // (this is OK as they do not have parallel init yet).
297 //
298 return NV_TRUE;
299 }
300 }
301
302 NV_STATUS
gpumgrThreadEnableExpandedGpuVisibility(void)303 gpumgrThreadEnableExpandedGpuVisibility(void)
304 {
305 NvP64 *pEntry = tlsEntryAcquire(TLS_ENTRY_ID_GPUMGR_EXPANDED_GPU_VISIBILITY);
306 if (pEntry == NULL)
307 {
308 return NV_ERR_INSUFFICIENT_RESOURCES;
309 }
310
311 *pEntry = (NvP64) 1;
312 return NV_OK;
313 }
314
315 void
gpumgrThreadDisableExpandedGpuVisibility(void)316 gpumgrThreadDisableExpandedGpuVisibility(void)
317 {
318 if (_gpumgrThreadHasExpandedGpuVisibilityInTls())
319 {
320 NvU32 refCount = tlsEntryRelease(TLS_ENTRY_ID_GPUMGR_EXPANDED_GPU_VISIBILITY);
321 NV_ASSERT(refCount == 0);
322 }
323 }
324
325 //
326 // gpumgrAllocGpuInstance
327 //
328 // This interface returns the next available gpu number.
329 //
330 NV_STATUS
gpumgrAllocGpuInstance(NvU32 * pGpuInstance)331 gpumgrAllocGpuInstance(NvU32 *pGpuInstance)
332 {
333 OBJSYS *pSys = SYS_GET_INSTANCE();
334 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
335
336 NvU32 i;
337 NvU64 availableIDs = ((1ULL << NV_MAX_DEVICES) - 1);
338
339 if (pGpuMgr->numGpuHandles == 0)
340 {
341 *pGpuInstance = 0;
342 return NV_OK;
343 }
344 else if (pGpuMgr->numGpuHandles == NV_MAX_DEVICES)
345 {
346 *pGpuInstance = NV_MAX_DEVICES;
347 return NV_ERR_INSUFFICIENT_RESOURCES;
348 }
349
350 for (i = 0; i < pGpuMgr->numGpuHandles; i++)
351 availableIDs &= ~NVBIT(pGpuMgr->gpuHandleIDList[i].gpuInstance);
352
353 for (i = 0; ((availableIDs & (1ULL << i)) == 0); i++)
354 ;
355
356 *pGpuInstance = i;
357
358 return NV_OK;
359 }
360
361 //
362 // During destruction of a GPU the handle list needs to be modified.
363 // Since we cannot guarantee the _last_ GPU will always be the one
364 // destroyed we have to compact the handle list so we have no gaps
365 // and can simply decrement numGpuHandles.
366 //
367 static void
_gpumgrShiftDownGpuHandles(NvU32 startIndex)368 _gpumgrShiftDownGpuHandles(NvU32 startIndex)
369 {
370 OBJSYS *pSys = SYS_GET_INSTANCE();
371 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
372 NvU32 i, lastMovedIndex = startIndex;
373
374 for (i = startIndex; i < (NV_MAX_DEVICES - 1); i++)
375 {
376 if (pGpuMgr->gpuHandleIDList[i + 1].pGpu != 0)
377 {
378 lastMovedIndex = i + 1;
379 pGpuMgr->gpuHandleIDList[i].gpuInstance =
380 pGpuMgr->gpuHandleIDList[i + 1].gpuInstance;
381
382 pGpuMgr->gpuHandleIDList[i].pGpu =
383 pGpuMgr->gpuHandleIDList[i + 1].pGpu;
384 }
385 }
386
387 pGpuMgr->gpuHandleIDList[lastMovedIndex].gpuInstance = NV_MAX_DEVICES;
388 pGpuMgr->gpuHandleIDList[lastMovedIndex].pGpu = reinterpretCast(NULL, OBJGPU *);
389 pGpuMgr->numGpuHandles--;
390 }
391
392 static void
_gpumgrDestroyGpu(NvU32 gpuInstance)393 _gpumgrDestroyGpu(NvU32 gpuInstance)
394 {
395 OBJSYS *pSys = SYS_GET_INSTANCE();
396 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
397 OBJGPU *pGpu;
398 NvU32 i;
399
400 osSyncWithGpuDestroy(NV_TRUE);
401
402 pGpu = gpumgrGetGpu(gpuInstance);
403
404 objDelete(pGpu);
405
406 for (i = 0; i < pGpuMgr->numGpuHandles; i++)
407 {
408 if (pGpuMgr->gpuHandleIDList[i].gpuInstance == gpuInstance)
409 {
410 pGpuMgr->gpuHandleIDList[i].gpuInstance = NV_MAX_DEVICES;
411 pGpuMgr->gpuHandleIDList[i].pGpu = reinterpretCast(NULL, OBJGPU *);
412 _gpumgrShiftDownGpuHandles(i);
413 break;
414 }
415 }
416
417 osSyncWithGpuDestroy(NV_FALSE);
418 }
419
420 POBJGPU
gpumgrGetGpu(NvU32 gpuInstance)421 gpumgrGetGpu(NvU32 gpuInstance)
422 {
423 OBJSYS *pSys = SYS_GET_INSTANCE();
424 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
425 NvU32 i;
426
427 for (i = 0; i < pGpuMgr->numGpuHandles; i++)
428 {
429 if (pGpuMgr->gpuHandleIDList[i].gpuInstance == gpuInstance)
430 {
431 OBJGPU *pGpu = pGpuMgr->gpuHandleIDList[i].pGpu;
432 if (pGpu != NULL)
433 {
434 if (gpumgrThreadHasExpandedGpuVisibility() ||
435 pGpu->getProperty(pGpu, PDB_PROP_GPU_STATE_INITIALIZED))
436 {
437 return pGpu;
438 }
439 }
440 }
441 }
442
443 return NULL;
444 }
445
446 POBJGPU
gpumgrGetSomeGpu(void)447 gpumgrGetSomeGpu(void)
448 {
449 OBJGPU *pGpu = NULL;
450 NvU32 gpuMask = 0;
451 NvU32 gpuIndex = 0;
452 NvU32 gpuCount = 0;
453
454 // Get some gpu to get the SLI Display Parent
455 gpumgrGetGpuAttachInfo(&gpuCount, &gpuMask);
456 pGpu = gpumgrGetNextGpu(gpuMask, &gpuIndex);
457
458 if (pGpu == NULL)
459 {
460 // None of the GPUs are initialized - Too early
461 NV_PRINTF(LEVEL_ERROR,
462 "Failed to retrieve pGpu - Too early call!.\n");
463 NV_ASSERT(NV_FALSE);
464 return pGpu;
465 }
466 return pGpu;
467 }
468
469
470 //
471 // gpumgrAllocDeviceInstance
472 //
473 // This interface returns the next available broadcast device number.
474 // This broadcast device number is used to uniquely identify this set
475 // of gpu(s) both internally in the RM (e.g. OBJGPUGRP handle) as well
476 // as via the architecture (e.g., for the 'deviceId' parameter of
477 // NV0080_ALLOC_PARAMETERS).
478 //
479 NV_STATUS
gpumgrAllocDeviceInstance(NvU32 * pDeviceInstance)480 gpumgrAllocDeviceInstance(NvU32 *pDeviceInstance)
481 {
482 OBJSYS *pSys = SYS_GET_INSTANCE();
483 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
484 NvU32 i;
485
486 for (i = 0; i < NV_ARRAY_ELEMENTS(pGpuMgr->pGpuGrpTable); i++)
487 if (pGpuMgr->pGpuGrpTable[i] == NULL)
488 break;
489
490 if (i == NV_MAX_DEVICES)
491 {
492 *pDeviceInstance = NV_MAX_DEVICES;
493 return NV_ERR_GENERIC;
494 }
495
496 *pDeviceInstance = i;
497
498 return NV_OK;
499 }
500
501 //
502 // gpumgrGetGpuAttachInfo
503 //
504 // Returns current gpu attach info.
505 //
506 NV_STATUS
gpumgrGetGpuAttachInfo(NvU32 * pGpuCnt,NvU32 * pGpuMask)507 gpumgrGetGpuAttachInfo(NvU32 *pGpuCnt, NvU32 *pGpuMask)
508 {
509 OBJSYS *pSys = SYS_GET_INSTANCE();
510 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
511
512 NvU32 outCount = 0;
513 NvU32 outMask = 0;
514
515 NvU32 gpuMask = pGpuMgr->gpuAttachMask;
516 NvU32 gpuInstance = 0;
517
518 while (gpuInstance != NV_MAX_DEVICES)
519 {
520 OBJGPU *pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance);
521 if (pGpu == NULL)
522 continue;
523
524 if (gpumgrThreadHasExpandedGpuVisibility() ||
525 pGpu->getProperty(pGpu, PDB_PROP_GPU_STATE_INITIALIZED))
526 {
527 // adjust gpuInstance down one as gpumgrGetNextGpu increments one
528 outMask |= NVBIT(gpuInstance - 1);
529 outCount++;
530 }
531 }
532
533 // caller can pass in NULL for outparams that it doesn't need.
534 if (pGpuCnt != NULL) *pGpuCnt = outCount;
535 if (pGpuMask != NULL) *pGpuMask = outMask;
536
537 return NV_OK;
538 }
539
540 NvU32
gpumgrGetDeviceGpuMask(NvU32 deviceInstance)541 gpumgrGetDeviceGpuMask(NvU32 deviceInstance)
542 {
543 OBJGPUGRP *pGpuGrp = gpumgrGetGpuGrpFromInstance(deviceInstance);
544
545 if (pGpuGrp == NULL)
546 {
547 NV_PRINTF(LEVEL_WARNING,
548 "Could not find GPU Group for deviceInstance 0x%x!\n",
549 deviceInstance);
550 return 0;
551 }
552
553 return gpugrpGetGpuMask(pGpuGrp);
554 }
555
556 NV_STATUS
gpumgrIsDeviceInstanceValid(NvU32 deviceInstance)557 gpumgrIsDeviceInstanceValid(NvU32 deviceInstance)
558 {
559 OBJSYS *pSys = SYS_GET_INSTANCE();
560 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
561 OBJGPUGRP *pGpuGrp = NULL;
562
563 if (deviceInstance >= NV_MAX_DEVICES)
564 return NV_ERR_INVALID_ARGUMENT;
565
566 pGpuGrp = pGpuMgr->pGpuGrpTable[deviceInstance];
567 if (NULL == pGpuGrp)
568 return NV_ERR_INVALID_DATA;
569
570 if (0 == gpugrpGetGpuMask(pGpuGrp))
571 return NV_ERR_INVALID_ARGUMENT;
572
573 return NV_OK;
574 }
575
576 NvBool
gpumgrIsSubDeviceInstanceValid(NvU32 subDeviceInstance)577 gpumgrIsSubDeviceInstanceValid(NvU32 subDeviceInstance)
578 {
579 if (subDeviceInstance >= NV2080_MAX_SUBDEVICES)
580 return NV_FALSE;
581
582 return NV_TRUE;
583 }
584
gpumgrGetPrimaryForDevice(NvU32 deviceInstance)585 NvU32 gpumgrGetPrimaryForDevice(NvU32 deviceInstance)
586 {
587 NvU32 gpuMask, gpuInstance = 0;
588 OBJGPU *pGpu = NULL;
589
590 gpuMask = gpumgrGetDeviceGpuMask(deviceInstance);
591
592 if (gpumgrIsSubDeviceCountOne(gpuMask))
593 {
594 pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance);
595 if (pGpu != NULL)
596 {
597 return pGpu->gpuInstance;
598 }
599 }
600 else
601 {
602 while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
603 {
604 if (gpumgrIsParentGPU(pGpu))
605 {
606 return pGpu->gpuInstance;
607 }
608 }
609 }
610
611 NV_PRINTF(LEVEL_ERROR,
612 "deviceInstance 0x%x does not exist!\n", deviceInstance);
613
614 return 0; // this should not happen, never
615 }
616
617 NvBool
gpumgrIsDeviceEnabled(NvU32 deviceInstance)618 gpumgrIsDeviceEnabled(NvU32 deviceInstance)
619 {
620 NvU32 gpuMask, gpuInstance = 0;
621 NvBool bEnabled;
622
623 gpuMask = gpumgrGetDeviceGpuMask(deviceInstance);
624
625 if (gpuMask == 0)
626 {
627 return NV_FALSE;
628 }
629 /*
630 * Check if this device
631 * - has been disabled via Power-SLI
632 * - is in the "drain" state
633 */
634 if (gpumgrIsSubDeviceCountOne(gpuMask))
635 {
636 OBJGPU *pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance);
637
638 if (pGpu == NULL)
639 return NV_FALSE;
640
641 if ((gpumgrQueryGpuDrainState(pGpu->gpuId, &bEnabled, NULL) == NV_OK)
642 && bEnabled)
643 {
644 return NV_FALSE;
645 }
646 }
647
648 return NV_TRUE;
649 }
650
651 //
652 // gpumgrRegisterGpuId
653 //
654 // This interface is used by os-dependent code to insert a probed
655 // gpu into the table of probed gpus known to the RM.
656 //
657 NV_STATUS
gpumgrRegisterGpuId(NvU32 gpuId,NvU64 gpuDomainBusDevice)658 gpumgrRegisterGpuId(NvU32 gpuId, NvU64 gpuDomainBusDevice)
659 {
660 NV_STATUS status = NV_ERR_INSUFFICIENT_RESOURCES;
661 OBJSYS *pSys = SYS_GET_INSTANCE();
662 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
663 NvU32 i;
664
665 portSyncMutexAcquire(pGpuMgr->probedGpusLock);
666
667 for (i = 0; i < NV_ARRAY_ELEMENTS(pGpuMgr->probedGpus); i++)
668 {
669 if (pGpuMgr->probedGpus[i].gpuId == gpuId)
670 {
671 NV_PRINTF(LEVEL_ERROR,
672 "GPU id 0x%x already registered at index %u\n",
673 gpuId, i);
674
675 // Duplicate gpu
676 status = NV_ERR_IN_USE;
677 goto done;
678 }
679
680 if (pGpuMgr->probedGpus[i].gpuId == NV0000_CTRL_GPU_INVALID_ID)
681 {
682 pGpuMgr->probedGpus[i].gpuId = gpuId;
683 pGpuMgr->probedGpus[i].gpuDomainBusDevice = gpuDomainBusDevice;
684 pGpuMgr->probedGpus[i].bInitAttempted = NV_FALSE;
685 pGpuMgr->probedGpus[i].bExcluded = NV_FALSE;
686 pGpuMgr->probedGpus[i].bUuidValid = NV_FALSE;
687 pGpuMgr->probedGpus[i].pOsRmCaps = NULL;
688 status = NV_OK;
689 goto done;
690 }
691 }
692
693 done:
694 portSyncMutexRelease(pGpuMgr->probedGpusLock);
695 return status;
696 }
697
698 //
699 // gpumgrUnregisterGpuId
700 //
701 // This interface is used by os-dependent code to remove a gpu
702 // from the table of probed gpus known to the RM.
703 //
704 NV_STATUS
gpumgrUnregisterGpuId(NvU32 gpuId)705 gpumgrUnregisterGpuId(NvU32 gpuId)
706 {
707 OBJSYS *pSys = SYS_GET_INSTANCE();
708 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
709 NvU32 i;
710
711 portSyncMutexAcquire(pGpuMgr->probedGpusLock);
712
713 for (i = 0; i < NV_ARRAY_ELEMENTS(pGpuMgr->probedGpus); i++)
714 {
715 PROBEDGPU *pProbedGpu = &pGpuMgr->probedGpus[i];
716
717 if (pProbedGpu->gpuId == gpuId)
718 {
719 _gpumgrCacheClearMIGGpuIdInfo(gpuId);
720 gpumgrRemovePcieP2PCapsFromCache(pProbedGpu->gpuId);
721 _gpumgrUnregisterRmCapsForGpuUnderLock(pProbedGpu->gpuDomainBusDevice);
722 pProbedGpu->gpuId = NV0000_CTRL_GPU_INVALID_ID;
723 pProbedGpu->bDrainState = NV_FALSE;
724 pProbedGpu->bRemoveIdle = NV_FALSE;
725 pProbedGpu->bExcluded = NV_FALSE;
726 pProbedGpu->bUuidValid = NV_FALSE;
727 goto done;
728 }
729 }
730
731 done:
732 portSyncMutexRelease(pGpuMgr->probedGpusLock);
733 return NV_OK;
734 }
735
736 //
737 // gpumgrExcludeGpuId
738 //
739 // This interface is used by os-dependent code to 'exclude' a gpu.
740 //
741 // gpuId: the device to exclude
742 //
743 NV_STATUS
gpumgrExcludeGpuId(NvU32 gpuId)744 gpumgrExcludeGpuId(NvU32 gpuId)
745 {
746 OBJSYS *pSys = SYS_GET_INSTANCE();
747 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
748 NvU32 i;
749
750 portSyncMutexAcquire(pGpuMgr->probedGpusLock);
751
752 for (i = 0; i < NV_ARRAY_ELEMENTS(pGpuMgr->probedGpus); i++)
753 {
754 PROBEDGPU *pProbedGpu = &pGpuMgr->probedGpus[i];
755
756 if (pProbedGpu->gpuId == gpuId)
757 {
758 pProbedGpu->bExcluded = NV_TRUE;
759 goto done;
760 }
761 }
762
763 done:
764 portSyncMutexRelease(pGpuMgr->probedGpusLock);
765 return NV_OK;
766 }
767
768 //
769 // gpumgrSetUuid
770 //
771 // This interface is used by os-dependent code to pass the UUID for a gpu.
772 // The UUID is a 16-byte raw UUID/GID.
773 //
774 NV_STATUS
gpumgrSetUuid(NvU32 gpuId,NvU8 * uuid)775 gpumgrSetUuid(NvU32 gpuId, NvU8 *uuid)
776 {
777 OBJSYS *pSys = SYS_GET_INSTANCE();
778 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
779 NvU32 i;
780
781 if (uuid == NULL)
782 return NV_ERR_INVALID_DATA;
783
784 portSyncMutexAcquire(pGpuMgr->probedGpusLock);
785
786 for (i = 0; i < NV_ARRAY_ELEMENTS(pGpuMgr->probedGpus); i++)
787 {
788 PROBEDGPU *pProbedGpu = &pGpuMgr->probedGpus[i];
789
790 if (pProbedGpu->gpuId == gpuId)
791 {
792 portMemCopy(pProbedGpu->uuid, RM_SHA1_GID_SIZE, uuid, RM_SHA1_GID_SIZE);
793 pProbedGpu->bUuidValid = NV_TRUE;
794 goto done;
795 }
796 }
797
798 done:
799 portSyncMutexRelease(pGpuMgr->probedGpusLock);
800 return NV_OK;
801 }
802
803 //
804 // gpumgrGetCachedUuid
805 //
806 // Lookup the cached UUID for a GPU
807 //
808 static NV_STATUS
gpumgrGetCachedUuid(NvU32 gpuId,NvU8 * uuid,unsigned int len)809 gpumgrGetCachedUuid(NvU32 gpuId, NvU8 *uuid, unsigned int len)
810 {
811 OBJSYS *pSys = SYS_GET_INSTANCE();
812 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
813 NvU32 i;
814 NV_STATUS status = NV_ERR_OBJECT_NOT_FOUND;
815
816 if (uuid == NULL || len < RM_SHA1_GID_SIZE)
817 return NV_ERR_INVALID_ARGUMENT;
818
819 portSyncMutexAcquire(pGpuMgr->probedGpusLock);
820
821 for (i = 0; i < NV_ARRAY_ELEMENTS(pGpuMgr->probedGpus); i++)
822 {
823 PROBEDGPU *pProbedGpu = &pGpuMgr->probedGpus[i];
824
825 if (pProbedGpu->gpuId == gpuId)
826 {
827 if (pProbedGpu->bUuidValid)
828 {
829 portMemCopy(uuid, RM_SHA1_GID_SIZE, pProbedGpu->uuid, RM_SHA1_GID_SIZE);
830 status = NV_OK;
831 }
832 else
833 status = NV_ERR_NOT_SUPPORTED;
834 goto done;
835 }
836 }
837
838 done:
839 portSyncMutexRelease(pGpuMgr->probedGpusLock);
840 return status;
841 }
842
843 NV_STATUS
gpumgrGetGpuUuidInfo(NvU32 gpuId,NvU8 ** ppUuidStr,NvU32 * pUuidStrLen,NvU32 uuidFlags)844 gpumgrGetGpuUuidInfo(NvU32 gpuId, NvU8 **ppUuidStr, NvU32 *pUuidStrLen, NvU32 uuidFlags)
845 {
846 NvU8 *pUuid;
847 NV_STATUS status;
848
849 if (ppUuidStr == NULL || pUuidStrLen == NULL)
850 return NV_ERR_INVALID_DATA;
851
852 // gpumgr only supports SHA1 format; error out if requesting otherwise
853 if (!FLD_TEST_DRF(0000_CTRL_CMD, _GPU_GET_UUID_FROM_GPU_ID_FLAGS, _TYPE, _SHA1, uuidFlags))
854 return NV_ERR_INVALID_ARGUMENT;
855
856 pUuid = portMemAllocNonPaged(RM_SHA1_GID_SIZE);
857 if (pUuid == NULL)
858 return NV_ERR_NO_MEMORY;
859
860 status = gpumgrGetCachedUuid(gpuId, pUuid, RM_SHA1_GID_SIZE);
861 if (status != NV_OK)
862 {
863 portMemFree(pUuid);
864 return status;
865 }
866
867 if (FLD_TEST_DRF(0000_CTRL_CMD, _GPU_GET_UUID_FROM_GPU_ID_FLAGS, _FORMAT, _BINARY, uuidFlags))
868 {
869 // Binary case - pUuid is freed by the caller
870 *ppUuidStr = pUuid;
871 *pUuidStrLen = RM_SHA1_GID_SIZE;
872 }
873 else
874 {
875 // Conversion to ASCII or UNICODE
876 status = transformGidToUserFriendlyString(pUuid, RM_SHA1_GID_SIZE,
877 ppUuidStr, pUuidStrLen, uuidFlags);
878 portMemFree(pUuid);
879 }
880
881 return status;
882 }
883
884 static NV_STATUS
gpumgrGetRegisteredIds(NvU64 gpuDomainBusDevice,NvU32 * pGpuId,NvUuid * pUuid,NvBool * pbUuidValid)885 gpumgrGetRegisteredIds
886 (
887 NvU64 gpuDomainBusDevice,
888 NvU32 *pGpuId,
889 NvUuid *pUuid,
890 NvBool *pbUuidValid
891 )
892 {
893 OBJSYS *pSys = SYS_GET_INSTANCE();
894 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
895 NV_STATUS status = NV_ERR_OBJECT_NOT_FOUND;
896
897 portSyncMutexAcquire(pGpuMgr->probedGpusLock);
898
899 for (NvU32 i = 0; i < NV_ARRAY_ELEMENTS(pGpuMgr->probedGpus); i++)
900 {
901 PROBEDGPU *pProbedGpu = &pGpuMgr->probedGpus[i];
902 if (pProbedGpu->gpuId == NV0000_CTRL_GPU_INVALID_ID)
903 continue;
904
905 if (pProbedGpu->gpuDomainBusDevice == gpuDomainBusDevice)
906 {
907 *pGpuId = pProbedGpu->gpuId;
908 *pbUuidValid = pProbedGpu->bUuidValid;
909 if (pProbedGpu->bUuidValid)
910 portMemCopy(pUuid->uuid, sizeof(pUuid->uuid),
911 pProbedGpu->uuid, sizeof(pProbedGpu->uuid));
912
913 status = NV_OK;
914 break;
915 }
916 }
917
918 portSyncMutexRelease(pGpuMgr->probedGpusLock);
919 return status;
920 }
921
922 static void
gpumgrGetGpuHalFactorOfVirtual(NvBool * pIsVirtual,GPUATTACHARG * pAttachArg)923 gpumgrGetGpuHalFactorOfVirtual(NvBool *pIsVirtual, GPUATTACHARG *pAttachArg)
924 {
925 DEVICE_MAPPING gpuDevMapping = {0};
926 NvU32 pmcBoot1;
927
928 gpuDevMapping.gpuNvAddr = pAttachArg->regBaseAddr;
929 gpuDevMapping.gpuNvLength = pAttachArg->regLength;
930
931 *pIsVirtual = NV_FALSE;
932
933 // Check register NV_PMC_BOOT_1
934 pmcBoot1 = osDevReadReg032(/*pGpu=*/ NULL, &gpuDevMapping, NV_PMC_BOOT_1);
935
936 // VGPU with SRIOV
937 if (FLD_TEST_DRF(_PMC, _BOOT_1, _VGPU, _VF, pmcBoot1))
938 {
939 *pIsVirtual = NV_TRUE;
940 }
941 }
942
gpumgrGetRmFirmwareLogsEnabled(NvU32 enableFirmwareLogsRegVal)943 NvBool gpumgrGetRmFirmwareLogsEnabled
944 (
945 NvU32 enableFirmwareLogsRegVal
946 )
947 {
948 // Check for logs
949 if (enableFirmwareLogsRegVal == NV_REG_ENABLE_GPU_FIRMWARE_LOGS_ENABLE)
950 {
951 return NV_TRUE;
952 }
953 #if defined(DEBUG) || defined(DEVELOP)
954 if (enableFirmwareLogsRegVal == NV_REG_ENABLE_GPU_FIRMWARE_LOGS_ENABLE_ON_DEBUG)
955 {
956 return NV_TRUE;
957 }
958 #endif // defined(DEBUG) || defined(DEVELOP)
959
960 return NV_FALSE;
961 }
962
gpumgrGetRmFirmwarePolicy(NvU32 chipId,NvU32 pmcBoot42,NvBool bIsSoc,NvU32 enableFirmwareRegVal,NvBool * pbRequestFirmware,NvBool * pbAllowFallbackToMonolithicRm)963 void gpumgrGetRmFirmwarePolicy
964 (
965 NvU32 chipId,
966 NvU32 pmcBoot42,
967 NvBool bIsSoc,
968 NvU32 enableFirmwareRegVal,
969 NvBool *pbRequestFirmware,
970 NvBool *pbAllowFallbackToMonolithicRm
971 )
972 {
973 NvBool bFirmwareCapable = NV_FALSE;
974 NvBool bEnableByDefault = NV_FALSE;
975 NvU32 regkeyFirmwareMode;
976
977 regkeyFirmwareMode =
978 enableFirmwareRegVal & NV_REG_ENABLE_GPU_FIRMWARE_MODE_MASK;
979 *pbAllowFallbackToMonolithicRm =
980 !!(enableFirmwareRegVal & NV_REG_ENABLE_GPU_FIRMWARE_POLICY_ALLOW_FALLBACK);
981
982 bFirmwareCapable = gpumgrIsDeviceRmFirmwareCapable(chipId, pmcBoot42,
983 bIsSoc,
984 &bEnableByDefault);
985
986 *pbRequestFirmware =
987 (bFirmwareCapable &&
988 ((regkeyFirmwareMode == NV_REG_ENABLE_GPU_FIRMWARE_MODE_ENABLED) ||
989 (bEnableByDefault && (regkeyFirmwareMode != NV_REG_ENABLE_GPU_FIRMWARE_MODE_DISABLED))));
990 }
991
_gpumgrIsRmFirmwareCapableChip(NvU32 pmcBoot42)992 static NvBool _gpumgrIsRmFirmwareCapableChip(NvU32 pmcBoot42)
993 {
994 return (DRF_VAL(_PMC, _BOOT_42, _ARCHITECTURE, pmcBoot42) >= NV_PMC_BOOT_42_ARCHITECTURE_TU100);
995 }
996
gpumgrIsVgxRmFirmwareCapableChip(NvU32 pmcBoot42)997 NvBool gpumgrIsVgxRmFirmwareCapableChip(NvU32 pmcBoot42)
998 {
999 if (FLD_TEST_DRF(_PMC, _BOOT_42, _ARCHITECTURE, _GH100, pmcBoot42))
1000 return NV_TRUE;
1001
1002 if (FLD_TEST_DRF(_PMC, _BOOT_42, _ARCHITECTURE, _AD100, pmcBoot42))
1003 return NV_TRUE;
1004
1005 return NV_FALSE;
1006 }
1007
_gpumgrIsRmFirmwareDefaultChip(NvU32 pmcBoot42)1008 static NvBool _gpumgrIsRmFirmwareDefaultChip(NvU32 pmcBoot42)
1009 {
1010 return FLD_TEST_DRF(_PMC, _BOOT_42, _ARCHITECTURE, _GH100, pmcBoot42);
1011 }
1012
_gpumgrIsVgxRmFirmwareDefaultChip(NvU32 pmcBoot42)1013 static NvBool _gpumgrIsVgxRmFirmwareDefaultChip(NvU32 pmcBoot42)
1014 {
1015 return gpumgrIsVgxRmFirmwareCapableChip(pmcBoot42);
1016 }
1017
gpumgrIsDeviceRmFirmwareCapable(NvU16 devId,NvU32 pmcBoot42,NvBool bIsSoc,NvBool * pbEnabledByDefault)1018 NvBool gpumgrIsDeviceRmFirmwareCapable
1019 (
1020 NvU16 devId,
1021 NvU32 pmcBoot42,
1022 NvBool bIsSoc,
1023 NvBool *pbEnabledByDefault
1024 )
1025 {
1026 NvBool bEnabledByDefault = NV_FALSE;
1027 NvBool bFirmwareCapable = NV_TRUE;
1028
1029 // SoC is treated as always firmware capable and not enabled by default
1030 if (bIsSoc)
1031 goto finish;
1032
1033 if (!hypervisorIsVgxHyper() && !_gpumgrIsRmFirmwareCapableChip(pmcBoot42))
1034 {
1035 bFirmwareCapable = NV_FALSE;
1036 goto finish;
1037 }
1038 else if (hypervisorIsVgxHyper() && !gpumgrIsVgxRmFirmwareCapableChip(pmcBoot42))
1039 {
1040 bFirmwareCapable = NV_FALSE;
1041 goto finish;
1042 }
1043
1044 #if NVCPU_IS_FAMILY_PPC
1045 // Disable default enablement for GSP on PowerPC until it is fully tested
1046 bEnabledByDefault = NV_FALSE;
1047 goto finish;
1048 #else
1049 {
1050 static const NvU16 defaultGspRmGpus[] = {
1051 0x1E37, // GFN
1052 0x1E38, // TU102
1053 0x1EB4, // T4G
1054 0x1EB8, // T4
1055 0x1EB9, // T4
1056
1057 0x20B0, // A100
1058 0x20B1, // A100
1059 0x20B2, // A100
1060 0x20B3, // A200
1061 0x20B5, // A100 80GB
1062 0x20B6, // A100
1063 0x20B7, // A30
1064 0x20B8, // A100X SKU230/231 Roy-100
1065 0x20B9, // A30X SKU205/206 Roy-30
1066 0x20F0, // A100
1067 0x20F1, // A100
1068 0x20F2, // A100
1069 0x2235, // A40
1070 0x2236, // A10 SKU215 Pris-24
1071 0x2237, // A10G SKU215 Pris-24
1072 0x25B6, // A16
1073 0x20F5, // A800-80
1074 0x20F6, // A800-40
1075 0x20FD, // A100T RoyB
1076
1077 0x26B5, // L40
1078 0x26B8, // L40G
1079 0x26F5, // L40-CNX
1080 0x27B7, // L16
1081 0x27B8, // L4 (both SKUs)
1082 };
1083
1084 if (hypervisorIsVgxHyper() && _gpumgrIsVgxRmFirmwareDefaultChip(pmcBoot42))
1085 {
1086 bEnabledByDefault = NV_TRUE;
1087 }
1088 else if (!hypervisorIsVgxHyper() && _gpumgrIsRmFirmwareDefaultChip(pmcBoot42))
1089 {
1090 bEnabledByDefault = NV_TRUE;
1091 }
1092 else if (!hypervisorIsVgxHyper() || RMCFG_FEATURE_PLATFORM_GSP)
1093 {
1094 for (NvU32 i = 0; i < NV_ARRAY_ELEMENTS(defaultGspRmGpus); i++)
1095 {
1096 if (defaultGspRmGpus[i] == devId)
1097 {
1098 bEnabledByDefault = NV_TRUE;
1099 break;
1100 }
1101 }
1102 }
1103 }
1104 #endif
1105
1106 finish:
1107 if (pbEnabledByDefault != NULL)
1108 {
1109 *pbEnabledByDefault = bEnabledByDefault;
1110
1111 if (bEnabledByDefault)
1112 {
1113 NV_PRINTF(LEVEL_INFO,
1114 "DevId 0x%x is GSP-RM enabled by default\n",
1115 devId);
1116 }
1117 }
1118
1119 return bFirmwareCapable;
1120 }
1121
gpumgrCheckRmFirmwarePolicy(NvU64 nvDomainBusDeviceFunc,NvBool bRequestFwClientRm,NvU32 pmcBoot42)1122 static NvBool gpumgrCheckRmFirmwarePolicy
1123 (
1124 NvU64 nvDomainBusDeviceFunc,
1125 NvBool bRequestFwClientRm,
1126 NvU32 pmcBoot42
1127 )
1128 {
1129
1130 if (!bRequestFwClientRm)
1131 return NV_FALSE;
1132
1133 if (!_gpumgrIsRmFirmwareCapableChip(pmcBoot42))
1134 {
1135 NV_PRINTF(LEVEL_ERROR, "Disabling GSP offload -- GPU not supported\n");
1136 return NV_FALSE;
1137 }
1138
1139 return NV_TRUE;
1140 }
1141
1142 //
1143 // gpumgrGetGpuHalFactor
1144 //
1145 // Get Gpu Hal factors those are used to init Hal binding
1146 //
1147 static void
gpumgrGetGpuHalFactor(NvU32 * pChipId0,NvU32 * pChipId1,NvU32 * pSocChipId0,RM_RUNTIME_VARIANT * pRmVariant,TEGRA_CHIP_TYPE * pTegraType,GPUATTACHARG * pAttachArg)1148 gpumgrGetGpuHalFactor
1149 (
1150 NvU32 *pChipId0,
1151 NvU32 *pChipId1,
1152 NvU32 *pSocChipId0,
1153 RM_RUNTIME_VARIANT *pRmVariant,
1154 TEGRA_CHIP_TYPE *pTegraType,
1155 GPUATTACHARG *pAttachArg
1156 )
1157 {
1158 NvBool isVirtual;
1159 NvBool isFwClient;
1160
1161 *pTegraType = TEGRA_CHIP_TYPE_DEFAULT;
1162
1163 // get ChipId0 and ChipId1
1164 if (pAttachArg->socDeviceArgs.specified)
1165 {
1166 *pChipId0 = pAttachArg->socDeviceArgs.socChipId0;
1167
1168 // iGPU has a PMC_BOOT_0, Display does not
1169 if (pAttachArg->socDeviceArgs.bIsIGPU)
1170 {
1171 DEVICE_MAPPING *pGpuDevMapping = NULL;
1172 pGpuDevMapping = &pAttachArg->socDeviceArgs.deviceMapping[DEVICE_INDEX_GPU];
1173 *pChipId0 = osDevReadReg032(/*pGpu=*/ NULL, pGpuDevMapping, NV_PMC_BOOT_0);
1174 }
1175
1176 *pChipId1 = 0;
1177 *pSocChipId0 = pAttachArg->socDeviceArgs.socChipId0;
1178 isVirtual = NV_FALSE;
1179 }
1180 else if (pAttachArg->bIsSOC)
1181 {
1182 // This path is only taken for ARCH MODS iGPU verification.
1183
1184 *pChipId0 = pAttachArg->socChipId0;
1185 *pChipId1 = 0;
1186 *pSocChipId0 = pAttachArg->socChipId0;
1187 isVirtual = NV_FALSE;
1188 }
1189 else
1190 {
1191 DEVICE_MAPPING gpuDevMapping = {0};
1192 gpuDevMapping.gpuNvAddr = pAttachArg->regBaseAddr;
1193 gpuDevMapping.gpuNvLength = pAttachArg->regLength;
1194
1195 //
1196 // PMC_BOOT_42 register is added above G94+ chips which is internal to NVIDIA
1197 // Earlier we used PMC_BOOT_0 as Internal ID which is now exposed to customers
1198 //
1199 *pChipId0 = osDevReadReg032(/*pGpu=*/ NULL, &gpuDevMapping, NV_PMC_BOOT_0);
1200 *pChipId1 = osDevReadReg032(/*pGpu=*/ NULL, &gpuDevMapping, NV_PMC_BOOT_42);
1201 *pSocChipId0 = 0;
1202
1203 gpumgrGetGpuHalFactorOfVirtual(&isVirtual, pAttachArg);
1204
1205 //
1206 // If socChipId0 has valid value, then running environment is SOCV.
1207 // The Tegra chip after Ampere arch is using PCIE interface which connects
1208 // iGPU to SoC for BAR and control accesses (interrupt).
1209 // The code between TEGRA_CHIP_TYPE_PCIE and TEGRA_CHIP_TYPE_SOC
1210 // shares same dGPU ARCH specific HAL mostly except manual differences due to
1211 // latency of manual updates between nvgpu (Standlone iGPU/Full Chip Verification)
1212 // and nvmobile (SOC) trees.
1213 //
1214 if (pAttachArg->socChipId0 != 0)
1215 {
1216 *pTegraType = TEGRA_CHIP_TYPE_SOC;
1217 }
1218 }
1219
1220 isFwClient = gpumgrCheckRmFirmwarePolicy(pAttachArg->nvDomainBusDeviceFunc,
1221 pAttachArg->bRequestFwClientRm,
1222 *pChipId1);
1223
1224 if (RMCFG_FEATURE_PLATFORM_GSP || RMCFG_FEATURE_PLATFORM_DCE)
1225 *pRmVariant = RM_RUNTIME_VARIANT_UCODE;
1226 else if (isVirtual)
1227 *pRmVariant = RM_RUNTIME_VARIANT_VF;
1228 else if (isFwClient)
1229 *pRmVariant = RM_RUNTIME_VARIANT_PF_KERNEL_ONLY;
1230 else
1231 *pRmVariant = RM_RUNTIME_VARIANT_PF_MONOLITHIC; // default, monolithic mode
1232
1233 NV_PRINTF(LEVEL_INFO,
1234 "ChipId0[0x%x] ChipId1[0x%x] SocChipId0[0x%x] isFwClient[%d] RmVariant[%d] tegraType[%d]\n",
1235 *pChipId0, *pChipId1, *pSocChipId0, isFwClient, *pRmVariant, *pTegraType);
1236 }
1237
1238
1239 //
1240 // _gpumgrCreateGpu
1241 //
1242 // Former _sysCreateGpu(). The function is moved to Gpumgr for hinding struct
1243 // GPUATTACHARG from SYS. SYS is still the parent object of both GPUMGR and
1244 // GPU.
1245 //
1246 static NV_STATUS
_gpumgrCreateGpu(NvU32 gpuInstance,GPUATTACHARG * pAttachArg)1247 _gpumgrCreateGpu(NvU32 gpuInstance, GPUATTACHARG *pAttachArg)
1248 {
1249 OBJSYS *pSys = SYS_GET_INSTANCE();
1250 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
1251 OBJGPU *pGpu;
1252 NV_STATUS status;
1253 RM_RUNTIME_VARIANT rmVariant;
1254 TEGRA_CHIP_TYPE tegraType;
1255 NvU32 chipId0; // 32-bit chipId (pmcBoot0 on GPU)
1256 NvU32 chipId1; // 32-bit chipId (pmcBoot42 on GPU)
1257 NvU32 socChipId0; // 32-bit SOC chipId
1258 NvU32 hidrev, majorRev;
1259 NvU32 gpuId;
1260 NvUuid gpuUuid;
1261 NvBool bGpuUuidValid = NV_FALSE;
1262
1263 gpumgrGetGpuHalFactor(&chipId0, &chipId1, &socChipId0, &rmVariant, &tegraType, pAttachArg);
1264
1265 hidrev = DRF_VAL(_PAPB_MISC, _GP_HIDREV, _CHIPID, socChipId0);
1266 majorRev = DRF_VAL(_PAPB_MISC, _GP_HIDREV, _MAJORREV, socChipId0);
1267
1268 // WAR: The majorrev of t234 shows 0xa on fmodel instead of 0x4
1269 if ((hidrev == 0x23) && (majorRev == 0xa))
1270 {
1271 majorRev = 0x4;
1272 }
1273
1274 hidrev = (hidrev << 4) | majorRev;
1275
1276 //
1277 // The OS layer must have previously registered the GPU ID, and may have already registered
1278 // the UUID. Pull out the registered IDs for this device from the probed GPU info to pass to
1279 // the OBJGPU constructor.
1280 //
1281 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, gpumgrGetRegisteredIds(pAttachArg->nvDomainBusDeviceFunc,
1282 &gpuId, &gpuUuid, &bGpuUuidValid));
1283
1284 // create OBJGPU with halspec factor initialization value
1285 status = objCreate(&pGpu, pSys, OBJGPU,
1286 /* ChipHal_arch = */ DRF_VAL(_PMC, _BOOT_42, _ARCHITECTURE, chipId1),
1287 /* ChipHal_impl = */ DRF_VAL(_PMC, _BOOT_42, _IMPLEMENTATION, chipId1),
1288 /* ChipHal_hidrev = */ hidrev,
1289 /* RmVariantHal_rmVariant = */ rmVariant,
1290 /* TegraChipHal_tegraType = */ tegraType,
1291 /* DispIpHal_ipver = */ 0, // initialized later
1292 /* ctor.gpuInstance = */ gpuInstance,
1293 /* ctor.gpuId = */ gpuId,
1294 /* ctor.pUuid = */ bGpuUuidValid ? &gpuUuid : NULL);
1295 if (status != NV_OK)
1296 {
1297 return status;
1298 }
1299
1300 // legacy chip-config Hal registration path
1301 status = gpuBindHalLegacy(pGpu, chipId0, chipId1, socChipId0);
1302 if (status != NV_OK)
1303 {
1304 objDelete(pGpu);
1305 return status;
1306 }
1307
1308 //
1309 // Save away the public ID associated with the handle just returned
1310 // from create object.
1311 //
1312 pGpuMgr->gpuHandleIDList[pGpuMgr->numGpuHandles].gpuInstance = gpuInstance;
1313 pGpuMgr->gpuHandleIDList[pGpuMgr->numGpuHandles].pGpu = pGpu;
1314
1315 pGpuMgr->numGpuHandles++;
1316
1317 return status;
1318 }
1319
1320
1321 static void
_gpumgrGetEncSessionStatsReportingState(OBJGPU * pGpu)1322 _gpumgrGetEncSessionStatsReportingState(OBJGPU *pGpu)
1323 {
1324 NvU32 numClasses = 0;
1325
1326 // Set NvEnc Session Stats reporting state flag to 'not supported' on GPUs with no encoder engine.
1327 NV_STATUS nvStatus = gpuGetClassList(pGpu, &numClasses, NULL, MKENGDESC(classId(OBJMSENC), 0));
1328 if ((nvStatus != NV_OK) || (numClasses == 0))
1329 {
1330 pGpu->encSessionStatsReportingState = NV2080_CTRL_GPU_INFO_NVENC_STATS_REPORTING_STATE_NOT_SUPPORTED;
1331 return;
1332 }
1333
1334 pGpu->encSessionStatsReportingState = NV2080_CTRL_GPU_INFO_NVENC_STATS_REPORTING_STATE_DISABLED;
1335
1336 #if defined(NVCPU_AARCH64)
1337 pGpu->encSessionStatsReportingState = NV_REG_STR_RM_NVENC_SESSION_STATS_REPORTING_STATE_DISABLED;
1338 #else
1339 VGPU_STATIC_INFO *pVSI = GPU_GET_STATIC_INFO(pGpu);
1340 if (IS_VIRTUAL(pGpu))
1341 {
1342 if (pVSI)
1343 {
1344 pGpu->encSessionStatsReportingState = pVSI->vgpuStaticProperties.encSessionStatsReportingState;
1345 }
1346 else
1347 {
1348 pGpu->encSessionStatsReportingState = NV2080_CTRL_GPU_INFO_NVENC_STATS_REPORTING_STATE_ENABLED;
1349 }
1350 }
1351 else
1352 {
1353 NvU32 encSessionStatsReportingState = NV_REG_STR_RM_NVENC_SESSION_STATS_REPORTING_STATE_DISABLED;;
1354
1355 // Encoder Sessions Stats report is enabled by default
1356 pGpu->encSessionStatsReportingState = NV2080_CTRL_GPU_INFO_NVENC_STATS_REPORTING_STATE_ENABLED;
1357
1358 // Override the value with the value specified in registry.
1359 if (osReadRegistryDword(pGpu,
1360 NV_REG_STR_RM_NVENC_SESSION_STATS_REPORTING_STATE,
1361 &encSessionStatsReportingState) == NV_OK)
1362 {
1363 switch(encSessionStatsReportingState)
1364 {
1365 case NV_REG_STR_RM_NVENC_SESSION_STATS_REPORTING_STATE_DISABLED:
1366 pGpu->encSessionStatsReportingState = NV2080_CTRL_GPU_INFO_NVENC_STATS_REPORTING_STATE_DISABLED;
1367 break;
1368 case NV_REG_STR_RM_NVENC_SESSION_STATS_REPORTING_STATE_ENABLED:
1369 pGpu->encSessionStatsReportingState = NV2080_CTRL_GPU_INFO_NVENC_STATS_REPORTING_STATE_ENABLED;
1370 break;
1371 default:
1372 break;
1373 }
1374 }
1375 }
1376 #endif // defined(NVCPU_AARCH64)
1377 }
1378
1379 //
1380 // gpumgrAttachGpu
1381 //
1382 // This interface is used by os-dependent code to attach a new gpu
1383 // to the pool managed by the RM. Construction of OBJGPU and it's
1384 // descendants is handled here, along with any other necessary prep
1385 // for the subsequent gpu preinit/init stages.
1386 //
1387 NV_STATUS
gpumgrAttachGpu(NvU32 gpuInstance,GPUATTACHARG * pAttachArg)1388 gpumgrAttachGpu(NvU32 gpuInstance, GPUATTACHARG *pAttachArg)
1389 {
1390 OBJSYS *pSys = SYS_GET_INSTANCE();
1391 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
1392 OBJGPU *pGpu = NULL;
1393 NV_STATUS status;
1394
1395 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner());
1396
1397 // create the new OBJGPU
1398 if ((status = _gpumgrCreateGpu(gpuInstance, pAttachArg)) != NV_OK)
1399 {
1400 goto gpumgrAttach_error_and_exit;
1401 }
1402
1403 // get a pointer to the new OBJGPU
1404 pGpu = gpumgrGetGpu(gpuInstance);
1405
1406 // load up attach parameters
1407 gpumgrSetAttachInfo(pGpu, pAttachArg);
1408
1409 // Load OOR check address mode based on arch
1410 #if defined(NVCPU_X86_64)
1411 pGpu->busInfo.oorArch = OOR_ARCH_X86_64;
1412 #elif defined(NVCPU_PPC64LE)
1413 pGpu->busInfo.oorArch = OOR_ARCH_PPC64LE;
1414 #elif defined(NVCPU_ARM)
1415 pGpu->busInfo.oorArch = OOR_ARCH_ARM;
1416 #elif defined(NVCPU_AARCH64)
1417 pGpu->busInfo.oorArch = OOR_ARCH_AARCH64;
1418 #else
1419 pGpu->busInfo.oorArch = OOR_ARCH_NONE;
1420 #endif
1421
1422 pGpu->pOS = SYS_GET_OS(pSys);
1423
1424 // let os fill in dpc details before we get into engine construction
1425 if ((status = osDpcAttachGpu(pGpu, pAttachArg->pOsAttachArg)) != NV_OK)
1426 {
1427 goto gpumgrAttach_error_and_exit;
1428 }
1429
1430 // let os fill in what it needs before we get into engine construction
1431 if ((status = osAttachGpu(pGpu, pAttachArg->pOsAttachArg)) != NV_OK)
1432 {
1433 goto gpumgrAttach_error_and_exit;
1434 }
1435
1436 NV_ASSERT((pGpuMgr->gpuAttachMask & NVBIT(gpuInstance)) == 0);
1437 pGpuMgr->gpuAttachMask |= NVBIT(gpuInstance);
1438 pGpuMgr->gpuAttachCount++;
1439
1440 status = _gpumgrRegisterRmCapsForGpu(pGpu);
1441 if (status != NV_OK)
1442 {
1443 goto gpumgrAttach_error_and_exit;
1444 }
1445
1446 // finish gpu construction
1447 if ((status = gpuPostConstruct(pGpu, pAttachArg)) != NV_OK)
1448 {
1449 goto gpumgrAttach_error_and_exit;
1450 }
1451
1452 _gpumgrGetEncSessionStatsReportingState(pGpu);
1453
1454 Intr *pIntr = GPU_GET_INTR(pGpu);
1455 // On some boards, vbios enables interrupt early before RM
1456 // initialize pGpu so that hotplug intrs can be serviced on
1457 // the mfg line. Disable interrupt here for this case.
1458 if (pIntr != NULL)
1459 {
1460 if (intrGetIntrEnFromHw_HAL(pGpu, pIntr, NULL) != INTERRUPT_TYPE_DISABLED)
1461 {
1462 intrSetIntrEnInHw_HAL(pGpu, pIntr, INTERRUPT_TYPE_DISABLED, NULL);
1463 intrSetStall_HAL(pGpu, pIntr, INTERRUPT_TYPE_DISABLED, NULL);
1464 }
1465 }
1466
1467 // Add entry into system nvlink topo array
1468 gpumgrAddSystemNvlinkTopo(pAttachArg->nvDomainBusDeviceFunc);
1469 // Add entry into system partition topo array
1470 gpumgrAddSystemMIGInstanceTopo(pAttachArg->nvDomainBusDeviceFunc);
1471
1472 // Determine conf compute params
1473 NV_ASSERT_OK_OR_RETURN(_gpumgrDetermineConfComputeCapabilities(pGpuMgr, pGpu));
1474
1475 if (!IS_GSP_CLIENT(pGpu))
1476 pGpuMgr->gpuMonolithicRmMask |= NVBIT(gpuInstance);
1477
1478 return status;
1479
1480 gpumgrAttach_error_and_exit:
1481 if ((pGpuMgr->gpuAttachMask & NVBIT(gpuInstance)) != 0)
1482 {
1483 pGpuMgr->gpuAttachMask &= ~NVBIT(gpuInstance);
1484 pGpuMgr->gpuAttachCount--;
1485 }
1486
1487 if (pGpu != NULL)
1488 {
1489 _gpumgrUnregisterRmCapsForGpu(gpuGetDBDF(pGpu));
1490 }
1491
1492 osDpcDetachGpu(pGpu);
1493 _gpumgrDestroyGpu(gpuInstance);
1494 return status;
1495 }
1496
1497 //
1498 // gpumgrDetachGpu
1499 //
1500 // This entry point detaches a gpu from the RM. The corresponding
1501 // OBJGPU and any of it's offspring are released, etc.
1502 //
1503 NV_STATUS
gpumgrDetachGpu(NvU32 gpuInstance)1504 gpumgrDetachGpu(NvU32 gpuInstance)
1505 {
1506 OBJSYS *pSys = SYS_GET_INSTANCE();
1507 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
1508 OBJGPU *pGpu = gpumgrGetGpu(gpuInstance);
1509 NvBool bDelClientResourcesFromGpuMask = !pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_TIMEOUT_RECOVERY);
1510
1511 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner());
1512
1513 // Mark for deletion the stale clients related to the GPU mask
1514 if (bDelClientResourcesFromGpuMask)
1515 {
1516 rmapiSetDelPendingClientResourcesFromGpuMask(NVBIT(gpuInstance));
1517 }
1518
1519 osDpcDetachGpu(pGpu);
1520
1521 pGpu->pOsRmCaps = NULL;
1522
1523 // release pDev
1524 _gpumgrDestroyGpu(gpuInstance);
1525
1526 // Delete the marked clients related to the GPU mask
1527 if (bDelClientResourcesFromGpuMask)
1528 {
1529 rmapiDelPendingDevices(NVBIT(gpuInstance));
1530 rmapiDelPendingClients();
1531 }
1532
1533 pGpuMgr->gpuMonolithicRmMask &= ~NVBIT(gpuInstance);
1534 NV_ASSERT(pGpuMgr->gpuAttachMask & NVBIT(gpuInstance));
1535 pGpuMgr->gpuAttachMask &= ~NVBIT(gpuInstance);
1536 pGpuMgr->gpuAttachCount--;
1537
1538 return NV_OK;
1539 }
1540
1541 //
1542 // gpumgrCreateDevice
1543 //
1544 // Create a broadcast device. The set of one or more gpus
1545 // comprising the broadcast device is described by gpuMask.
1546 //
1547 NV_STATUS
gpumgrCreateDevice(NvU32 * pDeviceInstance,NvU32 gpuMask,NvU32 * pGpuIdsOrdinal)1548 gpumgrCreateDevice(NvU32 *pDeviceInstance, NvU32 gpuMask, NvU32 *pGpuIdsOrdinal)
1549 {
1550 OBJSYS *pSys = SYS_GET_INSTANCE();
1551 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
1552 OBJGPU *pParentGpu = NULL;
1553 NvU32 gpuInstance;
1554 NV_STATUS status = NV_ERR_INVALID_REQUEST;
1555 OBJGPUGRP *pGpuGrp = NULL;
1556
1557 pGpuMgr->deviceCount++;
1558
1559 NV_ASSERT(gpuMask != 0);
1560
1561 // if only 1 gpu in the set, we're done
1562 if (gpumgrIsSubDeviceCountOne(gpuMask))
1563 {
1564 // alloc new broadcast device instance
1565 status = gpumgrAllocDeviceInstance(pDeviceInstance);
1566 if (status != NV_OK)
1567 {
1568 goto gpumgrCreateDevice_exit;
1569 }
1570
1571 gpumgrConstructGpuGrpObject(pGpuMgr, gpuMask,
1572 &pGpuMgr->pGpuGrpTable[*pDeviceInstance]);
1573 //
1574 // Set up parent gpu state. pParentGpu == NULL during boot when
1575 // we're first creating this device because the GPU attach process
1576 // has not yet completed. pParentGpu != NULL when we're coming
1577 // out of SLI (unlinking).
1578 //
1579 gpuInstance = 0;
1580 pParentGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance);
1581 if (pParentGpu)
1582 {
1583 gpumgrSetParentGPU(pParentGpu, pParentGpu);
1584 }
1585
1586 gpumgrAddDeviceMaskToGpuInstTable(gpuMask);
1587 status = NV_OK;
1588 goto gpumgrCreateDevice_exit;
1589 }
1590
1591 gpumgrCreateDevice_exit:
1592 if (status != NV_OK)
1593 {
1594 // Device creation failed
1595 pGpuMgr->deviceCount--;
1596 }
1597 else
1598 {
1599 pGpuGrp = pGpuMgr->pGpuGrpTable[*pDeviceInstance];
1600 if (gpugrpGetGpuMask(pGpuGrp) != gpuMask)
1601 {
1602 NV_ASSERT(0);
1603 gpumgrDestroyDevice(*pDeviceInstance);
1604 return NV_ERR_INVALID_DATA;
1605 }
1606 NV_PRINTF(LEVEL_INFO,
1607 "gpumgrCreateDevice: deviceInst 0x%x mask 0x%x\n",
1608 *pDeviceInstance, gpuMask);
1609 }
1610 return status;
1611 }
1612
1613 NV_STATUS
gpumgrDestroyDevice(NvU32 deviceInstance)1614 gpumgrDestroyDevice(NvU32 deviceInstance)
1615 {
1616 OBJSYS *pSys = SYS_GET_INSTANCE();
1617 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
1618 NV_STATUS status = NV_OK;
1619 OBJGPUGRP *pGpuGrp = pGpuMgr->pGpuGrpTable[deviceInstance];
1620 NvU32 gpuMask;
1621
1622 NV_ASSERT_OR_RETURN(pGpuGrp != NULL, NV_ERR_OBJECT_NOT_FOUND);
1623 gpuMask = gpugrpGetGpuMask(pGpuGrp);
1624
1625 NV_ASSERT(gpuMask != 0);
1626
1627 // if we only have one subdevice we're done
1628 if (gpumgrIsSubDeviceCountOne(gpuMask))
1629 {
1630 gpugrpDestroy(pGpuGrp);
1631 pGpuMgr->pGpuGrpTable[deviceInstance] = NULL;
1632 gpumgrClearDeviceMaskFromGpuInstTable(gpuMask);
1633 goto gpumgrDestroyDevice_exit;
1634 }
1635
1636 gpumgrDestroyDevice_exit:
1637 pGpuMgr->deviceCount--;
1638
1639 return status;
1640 }
1641
1642 //
1643 // gpumgrGetDeviceInstanceMask
1644 //
1645 // Returns mask of enabled (or valid) device instances.
1646 // This mask tells clients which NV01_DEVICE class
1647 // instances are valid.
1648 //
1649 NvU32
gpumgrGetDeviceInstanceMask(void)1650 gpumgrGetDeviceInstanceMask(void)
1651 {
1652 NvU32 i, deviceInstanceMask = 0;
1653
1654 // for every broadcast device...
1655 for (i = 0; i < NV_MAX_DEVICES; i++)
1656 {
1657 // ...add it to our mask if it's enabled
1658 if (NV_OK == gpumgrIsDeviceInstanceValid(i))
1659 deviceInstanceMask |= NVBIT(i);
1660 }
1661
1662 return deviceInstanceMask;
1663 }
1664
1665 NvU32
gpumgrGetGpuMask(OBJGPU * pGpu)1666 gpumgrGetGpuMask(OBJGPU *pGpu)
1667 {
1668 NvU32 deviceInstance = gpuGetDeviceInstance(pGpu);
1669
1670 NV_ASSERT(deviceInstance < NV_MAX_DEVICES);
1671
1672 return gpumgrGetDeviceGpuMask(deviceInstance);
1673 }
1674
1675 //
1676 // gpumgrGetSubDeviceCount
1677 //
1678 NvU32
gpumgrGetSubDeviceCount(NvU32 gpuMask)1679 gpumgrGetSubDeviceCount(NvU32 gpuMask)
1680 {
1681 NvU32 subDeviceCount = 0;
1682
1683 // tally # of gpus in the set
1684 while (gpuMask != 0)
1685 {
1686 subDeviceCount ++;
1687 gpuMask &= (gpuMask-1); // remove lowest bit in gpuMask
1688 }
1689 return subDeviceCount;
1690 }
1691
1692 //
1693 // gpumgrGetSubDeviceCountFromGpu
1694 // ATTENTION: When using with SLI Next / RM Unlinked SLI, the
1695 // subdevice count is always 1 for each GPU. This can cause
1696 // bugs, buffer overflows with arrays based on subdevice instances as
1697 // with RM Unlinked SLI:
1698 // - subdevice count is always 1 (the GPUs are not linked)
1699 // - GPU subdevice instance can be non zero
1700 // For subdevice instance arrays, please use
1701 // gpumgrGetSubDeviceMaxValuePlus1()
1702 //
1703 NvU32
gpumgrGetSubDeviceCountFromGpu(OBJGPU * pGpu)1704 gpumgrGetSubDeviceCountFromGpu(OBJGPU *pGpu)
1705 {
1706 NvU32 gpuMask = gpumgrGetGpuMask(pGpu);
1707 NvU32 subDeviceCount = gpumgrGetSubDeviceCount(gpuMask);
1708
1709 NV_ASSERT(subDeviceCount > 0);
1710 return subDeviceCount;
1711 }
1712
1713 //
1714 // gpumgrGetSubDeviceMaxValuePlus1
1715 // SLI disabled: return 1
1716 // SLI enabled with RM linked in SLI: returns 2 or more
1717 // SLI enabled with RM unlinked: return current subdeviceInstance + 1
1718 // Life of the function: until a full transition to SLI Next / RM Unlinked SLI.
1719 //
1720 NvU32
gpumgrGetSubDeviceMaxValuePlus1(OBJGPU * pGpu)1721 gpumgrGetSubDeviceMaxValuePlus1(OBJGPU *pGpu)
1722 {
1723 if (!IsSLIEnabled(pGpu))
1724 {
1725 // SLI disabled: return 1 as all GPU subdevice instances are 0
1726 // Unkinked SLI: returns the current subdevice instance + 1
1727 return gpumgrGetSubDeviceInstanceFromGpu(pGpu) + 1;
1728 }
1729 else
1730 {
1731 // SLI Enabled in RM: The count of subdevice instances for that GPU/device
1732 return gpumgrGetSubDeviceCountFromGpu(pGpu);
1733 }
1734 }
1735
1736 static void
gpumgrSetAttachInfo(OBJGPU * pGpu,GPUATTACHARG * pAttachArg)1737 gpumgrSetAttachInfo(OBJGPU *pGpu, GPUATTACHARG *pAttachArg)
1738 {
1739 if (pAttachArg->socDeviceArgs.specified)
1740 {
1741 NvU32 idx;
1742 NvU32 maxIdx;
1743 // This path is taken for Tegra Display and iGPU
1744
1745 //
1746 // TODO: This existing field is specifically used to safeguard
1747 // iGPU-specific code paths within RM, and should actually be NV_FALSE for
1748 // T234D+.
1749 //
1750 // See JIRA TDS-5101 for more details.
1751 //
1752 pGpu->bIsSOC = NV_TRUE;
1753 maxIdx = SOC_DEV_MAPPING_MAX;
1754
1755 for (idx = 0; idx < maxIdx; idx++)
1756 {
1757 pGpu->deviceMappings[idx] = pAttachArg->socDeviceArgs.deviceMapping[idx];
1758 }
1759
1760 pGpu->busInfo.iovaspaceId = pAttachArg->socDeviceArgs.iovaspaceId;
1761 {
1762 pGpu->busInfo.gpuPhysAddr = pGpu->deviceMappings[SOC_DEV_MAPPING_DISP].gpuNvPAddr;
1763 pGpu->gpuDeviceMapCount = 1;
1764 }
1765
1766 //
1767 // TODO bug 2100708: a fake DBDF is used on SOC to opt out of some
1768 // RM paths that cause issues otherwise, see the bug for details.
1769 //
1770 pGpu->busInfo.nvDomainBusDeviceFunc = pAttachArg->nvDomainBusDeviceFunc;
1771 }
1772 else if (pAttachArg->bIsSOC)
1773 {
1774 // This path is only taken for ARCH MODS iGPU verification.
1775
1776 NV_ASSERT(sizeof(pGpu->deviceMappings) == sizeof(pAttachArg->socDeviceMappings));
1777 pGpu->bIsSOC = NV_TRUE;
1778 pGpu->idInfo.PCIDeviceID = pAttachArg->socId;
1779 pGpu->idInfo.PCISubDeviceID = pAttachArg->socSubId;
1780 pGpu->busInfo.iovaspaceId = pAttachArg->iovaspaceId;
1781 if (RMCFG_FEATURE_PLATFORM_MODS)
1782 {
1783 NV_ASSERT(sizeof(pGpu->deviceMappings) == sizeof(pAttachArg->socDeviceMappings));
1784 portMemCopy(pGpu->deviceMappings, sizeof(pGpu->deviceMappings), pAttachArg->socDeviceMappings, sizeof(pGpu->deviceMappings));
1785 pGpu->gpuDeviceMapCount = pAttachArg->socDeviceCount;
1786
1787 //
1788 // TODO bug 2100708: a fake DBDF is used on SOC to opt out of some
1789 // RM paths that cause issues otherwise, see the bug for details.
1790 //
1791 pGpu->busInfo.nvDomainBusDeviceFunc = pAttachArg->nvDomainBusDeviceFunc;
1792 }
1793 }
1794 else
1795 {
1796 //
1797 // Set this gpu's hardware register access address pointers
1798 // from the contents of mappingInfo.
1799 //
1800 pGpu->bIsSOC = NV_FALSE;
1801
1802 pGpu->deviceMappings[0].gpuNvAddr = pAttachArg->regBaseAddr;
1803 pGpu->registerAccess.gpuFbAddr = pAttachArg->fbBaseAddr;
1804 pGpu->busInfo.gpuPhysAddr = pAttachArg->devPhysAddr;
1805 pGpu->busInfo.gpuPhysFbAddr = pAttachArg->fbPhysAddr;
1806 pGpu->busInfo.gpuPhysInstAddr = pAttachArg->instPhysAddr;
1807 pGpu->busInfo.gpuPhysIoAddr = pAttachArg->ioPhysAddr;
1808 pGpu->busInfo.iovaspaceId = pAttachArg->iovaspaceId;
1809 pGpu->busInfo.nvDomainBusDeviceFunc = pAttachArg->nvDomainBusDeviceFunc;
1810 pGpu->deviceMappings[0].gpuNvLength = pAttachArg->regLength;
1811 pGpu->fbLength = pAttachArg->fbLength;
1812 pGpu->busInfo.IntLine = pAttachArg->intLine;
1813 pGpu->gpuDeviceMapCount = 1;
1814 pGpu->cpuNumaNodeId = pAttachArg->cpuNumaNodeId;
1815
1816 if ( ! pAttachArg->instBaseAddr )
1817 {
1818 //
1819 // The OS init goo didn't map a separate region for instmem.
1820 // So instead use the 1M mapping in bar0.
1821 //
1822 pGpu->instSetViaAttachArg = NV_FALSE;
1823 pGpu->registerAccess.gpuInstAddr = (GPUHWREG*)(((NvU8*)pGpu->deviceMappings[0].gpuNvAddr) + 0x00700000); // aka NV_PRAMIN.
1824 if (!pGpu->busInfo.gpuPhysInstAddr)
1825 {
1826 //
1827 // Only use the bar0 window physical address if the OS didn't
1828 // specify a bar2 physical address.
1829 //
1830 pGpu->busInfo.gpuPhysInstAddr = pGpu->busInfo.gpuPhysAddr + 0x00700000; // aka NV_PRAMIN
1831 }
1832 pGpu->instLength = 0x100000; // 1MB
1833 }
1834 else
1835 {
1836 pGpu->instSetViaAttachArg = NV_TRUE;
1837 pGpu->registerAccess.gpuInstAddr = pAttachArg->instBaseAddr;
1838 pGpu->instLength = pAttachArg->instLength;
1839 }
1840 }
1841 }
1842
1843 //
1844 // gpumgrStatePreInitGpu & gpumgrStateInitGpu
1845 //
1846 // These routines handle unicast gpu initialization.
1847 //
1848 NV_STATUS
gpumgrStatePreInitGpu(OBJGPU * pGpu)1849 gpumgrStatePreInitGpu(OBJGPU *pGpu)
1850 {
1851 NV_STATUS status;
1852
1853 // LOCK: acquire GPUs lock
1854 status = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_INIT);
1855 if (status == NV_OK)
1856 {
1857 if (FULL_GPU_SANITY_CHECK(pGpu))
1858 {
1859 // pre-init phase done in UC mode
1860 status = gpuStatePreInit(pGpu);
1861 }
1862 else
1863 {
1864 status = NV_ERR_GPU_IS_LOST;
1865 DBG_BREAKPOINT();
1866 }
1867 // UNLOCK: release GPUs lock
1868 rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
1869 }
1870
1871 // save the init status for later client queries
1872 gpumgrSetGpuInitStatus(pGpu->gpuId, status);
1873
1874 return status;
1875 }
1876
1877 NV_STATUS
gpumgrStateInitGpu(OBJGPU * pGpu)1878 gpumgrStateInitGpu(OBJGPU *pGpu)
1879 {
1880 NV_STATUS status;
1881
1882 // LOCK: acquire GPUs lock
1883 status = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_INIT);
1884 if (status == NV_OK)
1885 {
1886 // init phase
1887 status = gpuStateInit(pGpu);
1888
1889 // UNLOCK: release GPUs lock
1890 rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
1891 }
1892
1893 // save the init status for later client queries
1894 gpumgrSetGpuInitStatus(pGpu->gpuId, status);
1895
1896 return status;
1897 }
1898
1899 NV_STATUS
gpumgrStateLoadGpu(OBJGPU * pGpu,NvU32 flags)1900 gpumgrStateLoadGpu(OBJGPU *pGpu, NvU32 flags)
1901 {
1902 NV_STATUS status;
1903
1904 // LOCK: acquire GPUs lock
1905 status = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_INIT);
1906 if (status == NV_OK)
1907 {
1908 // Load phase
1909 status = gpuStateLoad(pGpu, flags);
1910
1911 // UNLOCK: release GPUs lock
1912 rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
1913 }
1914
1915 if (status != NV_OK)
1916 goto gpumgrStateLoadGpu_exit;
1917
1918 gpumgrStateLoadGpu_exit:
1919 // save the init status for later client queries
1920 gpumgrSetGpuInitStatus(pGpu->gpuId, status);
1921
1922 return status;
1923 }
1924
1925 //
1926 // gpumgrGetNextGpu
1927 //
1928 // This routine searches subDeviceMask for the next gpu by using
1929 // the start index value as a beginning bit position. If a gpu is
1930 // found, the start index value is bumped to the next bit position
1931 // in the mask.
1932 //
1933 POBJGPU
gpumgrGetNextGpu(NvU32 subDeviceMask,NvU32 * pStartIndex)1934 gpumgrGetNextGpu(NvU32 subDeviceMask, NvU32 *pStartIndex)
1935 {
1936 NvU32 i;
1937
1938 if (*pStartIndex > NV_MAX_DEVICES)
1939 {
1940 *pStartIndex = NV_MAX_DEVICES;
1941 return NULL;
1942 }
1943
1944 for (i = *pStartIndex; i < NV_MAX_DEVICES; i++)
1945 {
1946 if (subDeviceMask & NVBIT(i))
1947 {
1948 *pStartIndex = i+1;
1949 return gpumgrGetGpu(i);
1950 }
1951 }
1952
1953 *pStartIndex = NV_MAX_DEVICES;
1954 return NULL;
1955 }
1956
1957
1958 //
1959 // gpumgrIsGpuPointerValid - Validates pGpu is initialized without dereferencing it.
1960 //
1961 NvBool
gpumgrIsGpuPointerValid(OBJGPU * pGpu)1962 gpumgrIsGpuPointerValid(OBJGPU *pGpu)
1963 {
1964 OBJGPU *pTempGpu = NULL;
1965 NvU32 gpuMask = 0;
1966 NvU32 gpuCount = 0;
1967 NvU32 gpuIndex = 0;
1968
1969 gpumgrGetGpuAttachInfo(&gpuCount, &gpuMask);
1970 pTempGpu = gpumgrGetNextGpu(gpuMask, &gpuIndex);
1971
1972 while(pTempGpu)
1973 {
1974 if (pTempGpu->getProperty(pTempGpu, PDB_PROP_GPU_STATE_INITIALIZED))
1975 {
1976 if (pTempGpu == pGpu)
1977 {
1978 return NV_TRUE;
1979 }
1980 }
1981
1982 pTempGpu = gpumgrGetNextGpu(gpuMask, &gpuIndex);
1983 }
1984
1985 return NV_FALSE;
1986 }
1987
1988 //
1989 // gpumgrIsGpuPointerAttached - Validates pGpu is attached without dereferencing it.
1990 //
1991 NvBool
gpumgrIsGpuPointerAttached(OBJGPU * pGpu)1992 gpumgrIsGpuPointerAttached(OBJGPU *pGpu)
1993 {
1994 OBJGPU *pTempGpu = NULL;
1995 NvU32 gpuMask = 0;
1996 NvU32 gpuCount = 0;
1997 NvU32 gpuIndex = 0;
1998
1999 gpumgrGetGpuAttachInfo(&gpuCount, &gpuMask);
2000 pTempGpu = gpumgrGetNextGpu(gpuMask, &gpuIndex);
2001
2002 while(pTempGpu)
2003 {
2004 if (pTempGpu == pGpu)
2005 {
2006 return NV_TRUE;
2007 }
2008 pTempGpu = gpumgrGetNextGpu(gpuMask, &gpuIndex);
2009 }
2010
2011 return NV_FALSE;
2012 }
2013
2014
gpumgrIsGpuDisplayParent(OBJGPU * pGpu)2015 NvBool gpumgrIsGpuDisplayParent(OBJGPU *pGpu)
2016 {
2017 OBJGPUGRP *pGpuGrp = gpumgrGetGpuGrpFromGpu(pGpu);
2018 NvBool rc = NV_FALSE;
2019 NvU32 gpuMask;
2020
2021 NV_ASSERT_OR_RETURN(pGpuGrp != NULL, NV_FALSE);
2022 gpuMask = gpugrpGetGpuMask(pGpuGrp);
2023
2024 // If there's only one GPU in the device, then of course it's the display parent!
2025 if (gpumgrIsSubDeviceCountOne(gpuMask))
2026 {
2027 rc = NV_TRUE;
2028 }
2029 //
2030 // If the gpuInstance argument is the first gpuInstance in the ordering,
2031 // then it's the display parent!
2032 //
2033 else if (pGpu->gpuInstance == pGpuGrp->SliLinkOrder[0].gpuInstance)
2034 {
2035 rc = NV_TRUE;
2036 }
2037
2038 // Otherwise it isn't.
2039 return rc;
2040 }
2041
gpumgrGetDisplayParent(OBJGPU * pGpu)2042 OBJGPU *gpumgrGetDisplayParent(OBJGPU *pGpu)
2043 {
2044 OBJGPUGRP *pGpuGrp = gpumgrGetGpuGrpFromGpu(pGpu);
2045 NvU32 gpuCount;
2046 NvU32 gpuMask;
2047 NvU32 gpuInstance;
2048
2049 NV_ASSERT_OR_RETURN(pGpuGrp != NULL, NULL);
2050 gpuMask = gpugrpGetGpuMask(pGpuGrp);
2051 gpuCount = gpumgrGetSubDeviceCount(gpuMask);
2052
2053 if (gpuCount > 1)
2054 {
2055 gpuInstance = pGpuGrp->SliLinkOrder[0].gpuInstance;
2056 pGpu = gpumgrGetGpu(gpuInstance);
2057 }
2058
2059 gpumgrSetBcEnabledStatus(pGpu, NV_FALSE);
2060
2061 return pGpu;
2062 }
2063
2064 //
2065 // gpumgrGetProbedGpuIds
2066 //
2067 // This routine services the NV0000_CTRL_GPU_GET_PROBED_IDS command.
2068 // The passed in gpuIds table is filled in with valid gpuId info
2069 // for each probed gpu. Invalid entries in the table are set to the
2070 // invalid id value.
2071 //
2072 NV_STATUS
gpumgrGetProbedGpuIds(NV0000_CTRL_GPU_GET_PROBED_IDS_PARAMS * pGpuIdsParams)2073 gpumgrGetProbedGpuIds(NV0000_CTRL_GPU_GET_PROBED_IDS_PARAMS *pGpuIdsParams)
2074 {
2075 OBJSYS *pSys = SYS_GET_INSTANCE();
2076 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
2077 NvU32 i, j, k;
2078
2079 ct_assert(NV_MAX_DEVICES == NV0000_CTRL_GPU_MAX_PROBED_GPUS);
2080
2081 portSyncMutexAcquire(pGpuMgr->probedGpusLock);
2082
2083 for (i = 0, j = 0, k = 0; i < NV_ARRAY_ELEMENTS(pGpuMgr->probedGpus); i++)
2084 {
2085 if (pGpuMgr->probedGpus[i].gpuId != NV0000_CTRL_GPU_INVALID_ID)
2086 {
2087 if (pGpuMgr->probedGpus[i].bExcluded)
2088 pGpuIdsParams->excludedGpuIds[k++] = pGpuMgr->probedGpus[i].gpuId;
2089 else
2090 pGpuIdsParams->gpuIds[j++] = pGpuMgr->probedGpus[i].gpuId;
2091 }
2092 }
2093
2094 portSyncMutexRelease(pGpuMgr->probedGpusLock);
2095
2096 for (i = j; i < NV_ARRAY_ELEMENTS(pGpuIdsParams->gpuIds); i++)
2097 pGpuIdsParams->gpuIds[i] = NV0000_CTRL_GPU_INVALID_ID;
2098
2099 for (i = k; i < NV_ARRAY_ELEMENTS(pGpuIdsParams->excludedGpuIds); i++)
2100 pGpuIdsParams->excludedGpuIds[i] = NV0000_CTRL_GPU_INVALID_ID;
2101
2102 return NV_OK;
2103 }
2104
2105 //
2106 // gpumgrGetAttachedGpuIds
2107 //
2108 // This routine services the NV0000_CTRL_GPU_GET_ATTACHED_IDS command.
2109 // The passed in gpuIds table is filled in with valid gpuId info
2110 // for each attached gpu. Any remaining entries in the table are set to
2111 // the invalid id value.
2112 //
2113 NV_STATUS
gpumgrGetAttachedGpuIds(NV0000_CTRL_GPU_GET_ATTACHED_IDS_PARAMS * pGpuIdsParams)2114 gpumgrGetAttachedGpuIds(NV0000_CTRL_GPU_GET_ATTACHED_IDS_PARAMS *pGpuIdsParams)
2115 {
2116 OBJGPU *pGpu;
2117 NvU32 gpuAttachCnt, gpuAttachMask, i, cnt;
2118 NvU32 *pGpuIds = &pGpuIdsParams->gpuIds[0];
2119
2120 // fill the table w/valid entries
2121 gpumgrGetGpuAttachInfo(&gpuAttachCnt, &gpuAttachMask);
2122 for (cnt = 0, i = 0; i < NV_MAX_DEVICES; i++)
2123 {
2124 if (gpuAttachMask & NVBIT(i))
2125 {
2126 pGpu = gpumgrGetGpu(i);
2127 pGpuIds[cnt++] = pGpu->gpuId;
2128 }
2129 }
2130
2131 // invalidate rest of the entries
2132 while (cnt < NV0000_CTRL_GPU_MAX_ATTACHED_GPUS)
2133 pGpuIds[cnt++] = NV0000_CTRL_GPU_INVALID_ID;
2134
2135 return NV_OK;
2136 }
2137
2138 //
2139 // gpumgrGetSubDeviceDeviceInstanceFromGpu
2140 //
2141 // Given a pGpu return the corresponding subdevice instance value.
2142 //
2143 NvU32
gpumgrGetSubDeviceInstanceFromGpu(OBJGPU * pGpu)2144 gpumgrGetSubDeviceInstanceFromGpu(OBJGPU *pGpu)
2145 {
2146 return pGpu->subdeviceInstance;
2147 }
2148
2149 //
2150 // gpumgrGetParentGPU
2151 //
2152 POBJGPU
gpumgrGetParentGPU(OBJGPU * pGpu)2153 gpumgrGetParentGPU(OBJGPU *pGpu)
2154 {
2155 OBJGPUGRP *pGpuGrp = gpumgrGetGpuGrpFromGpu(pGpu);
2156 NvU32 gpuMask;
2157
2158 NV_ASSERT_OR_RETURN(pGpuGrp != NULL, NULL);
2159 gpuMask = gpugrpGetGpuMask(pGpuGrp);
2160
2161 if (gpumgrIsSubDeviceCountOne(gpuMask))
2162 {
2163 return pGpu;
2164 }
2165 else
2166 {
2167 return gpugrpGetParentGpu(pGpuGrp);
2168 }
2169 }
2170
2171 //
2172 // gpumgrSetParentGPU
2173 //
2174 void
gpumgrSetParentGPU(OBJGPU * pGpu,OBJGPU * pParentGpu)2175 gpumgrSetParentGPU(OBJGPU *pGpu, OBJGPU *pParentGpu)
2176 {
2177 OBJGPUGRP *pGpuGrp = gpumgrGetGpuGrpFromGpu(pGpu);
2178
2179 NV_ASSERT_OR_RETURN_VOID(pGpuGrp != NULL);
2180 gpugrpSetParentGpu(pGpuGrp, pParentGpu);
2181 }
2182
2183 //
2184 // gpumgrGetGpuFromId
2185 //
2186 // Find the specified gpu from it's gpuId.
2187 //
2188 POBJGPU
gpumgrGetGpuFromId(NvU32 gpuId)2189 gpumgrGetGpuFromId(NvU32 gpuId)
2190 {
2191 OBJGPU *pGpu;
2192 NvU32 gpuAttachCnt, gpuAttachMask;
2193 NvU32 i;
2194
2195 gpumgrGetGpuAttachInfo(&gpuAttachCnt, &gpuAttachMask);
2196 for (i = 0; i < NV_MAX_DEVICES; i++)
2197 {
2198 if (gpuAttachMask & NVBIT(i))
2199 {
2200 pGpu = gpumgrGetGpu(i);
2201
2202 // found it
2203 if (pGpu->gpuId == gpuId)
2204 return pGpu;
2205 }
2206 }
2207
2208 // didn't find it
2209 return NULL;
2210 }
2211
2212 //
2213 // gpumgrGetGpuFromUuid()
2214 //
2215 // Get GPUOBJECT from UUID. Returns NULL if it cannot find a GPU with the
2216 // requested UUID.
2217 //
2218 POBJGPU
gpumgrGetGpuFromUuid(const NvU8 * pGpuUuid,NvU32 flags)2219 gpumgrGetGpuFromUuid(const NvU8 *pGpuUuid, NvU32 flags)
2220 {
2221 OBJGPU *pGpu;
2222 NvU32 attachedGpuCount;
2223 NvU32 attachedGpuMask;
2224 NvU32 gpuIndex;
2225 NvU32 gidStrLen;
2226 NvU8 *pGidString = NULL;
2227 NV_STATUS rmStatus;
2228
2229 // get all attached GPUs
2230 rmStatus = gpumgrGetGpuAttachInfo(&attachedGpuCount, &attachedGpuMask);
2231
2232 gpuIndex = 0;
2233
2234 for(pGpu = gpumgrGetNextGpu(attachedGpuMask, &gpuIndex);
2235 pGpu != NULL;
2236 pGpu = gpumgrGetNextGpu(attachedGpuMask, &gpuIndex))
2237 {
2238 //
2239 // get the GPU's UUID
2240 //
2241 // This implementation relies on the fact that gpuGetGidInfo() only
2242 // allocates memory if it succeeds.
2243 //
2244 rmStatus = gpuGetGidInfo(pGpu, &pGidString, &gidStrLen, flags);
2245 if (NV_OK != rmStatus)
2246 return NULL;
2247
2248 // check if it matches
2249 if (0 == portMemCmp(pGidString, pGpuUuid, gidStrLen))
2250 {
2251 portMemFree(pGidString);
2252 return pGpu;
2253 }
2254 else
2255 {
2256 // if it doesn't match, clean up allocated memory for next iteration
2257 portMemFree(pGidString);
2258 }
2259 }
2260
2261 return NULL; // Failed to find a GPU with the requested UUID
2262 }
2263
2264 //
2265 // gpumgrGetGpuFromBusInfo
2266 //
2267 // Find the specified GPU using its PCI bus info.
2268 //
2269 POBJGPU
gpumgrGetGpuFromBusInfo(NvU32 domain,NvU8 bus,NvU8 device)2270 gpumgrGetGpuFromBusInfo(NvU32 domain, NvU8 bus, NvU8 device)
2271 {
2272 NV_STATUS status;
2273 OBJGPU *pGpu;
2274 NvU32 attachedGpuCount;
2275 NvU32 attachedGpuMask;
2276 NvU32 gpuIndex = 0;
2277
2278 status = gpumgrGetGpuAttachInfo(&attachedGpuCount, &attachedGpuMask);
2279 NV_ASSERT_OR_RETURN(status == NV_OK, NULL);
2280
2281 for (pGpu = gpumgrGetNextGpu(attachedGpuMask, &gpuIndex);
2282 pGpu != NULL;
2283 pGpu = gpumgrGetNextGpu(attachedGpuMask, &gpuIndex))
2284 {
2285 if ((gpuGetDomain(pGpu) == domain) &&
2286 (gpuGetBus(pGpu) == bus) &&
2287 (gpuGetDevice(pGpu) == device))
2288 {
2289 return pGpu;
2290 }
2291 }
2292
2293 return NULL;
2294 }
2295
2296 //
2297 // gpumgrSetGpuId
2298 //
2299 // This routine assigns the specified gpuId to the specified gpu.
2300 //
2301 void
gpumgrSetGpuId(OBJGPU * pGpu,NvU32 gpuId)2302 gpumgrSetGpuId(OBJGPU *pGpu, NvU32 gpuId)
2303 {
2304 pGpu->gpuId = gpuId;
2305
2306 // if boardId is unassigned then give it a default value now
2307 if (pGpu->boardId == 0xffffffff)
2308 {
2309 pGpu->boardId = gpuId;
2310 }
2311 }
2312
2313 //
2314 // gpumgrGetGpuIdInfo
2315 //
2316 // Special purpose routine that handles NV0000_CTRL_CMD_GPU_GET_ID_INFO
2317 // requests from clients.
2318 // NV0000_CTRL_CMD_GPU_GET_ID_INFO is deprecated in favour of
2319 // NV0000_CTRL_CMD_GPU_GET_ID_INFO_V2, per comments in ctrl0000gpu.h
2320 //
2321 NV_STATUS
gpumgrGetGpuIdInfoV2(NV0000_CTRL_GPU_GET_ID_INFO_V2_PARAMS * pGpuInfo)2322 gpumgrGetGpuIdInfoV2(NV0000_CTRL_GPU_GET_ID_INFO_V2_PARAMS *pGpuInfo)
2323 {
2324 OBJGPU *pGpu;
2325 NvU32 deviceInstance, subDeviceInstance;
2326
2327 // start by making sure client request specifies a valid gpu
2328 pGpu = gpumgrGetGpuFromId(pGpuInfo->gpuId);
2329 if (pGpu == NULL)
2330 {
2331 NV_PRINTF(LEVEL_INFO,
2332 "gpumgrGetGpuInfoV2: bad gpuid spec: 0x%x\n",
2333 pGpuInfo->gpuId);
2334 return NV_ERR_INVALID_ARGUMENT;
2335 }
2336
2337 NV_ASSERT_OR_RETURN(gpumgrIsSafeToReadGpuInfo(), NV_ERR_INVALID_LOCK_STATE);
2338
2339 //
2340 // We have a valid gpuInstance, so now let's get the corresponding
2341 // deviceInstance/subDeviceInstance pair.
2342 //
2343 deviceInstance = gpuGetDeviceInstance(pGpu);
2344 if (deviceInstance == NV_MAX_DEVICES)
2345 {
2346 NV_PRINTF(LEVEL_ERROR,
2347 "gpumgrGetGpuInfoV2: deviceInstance not found\n");
2348 return NV_ERR_INVALID_ARGUMENT;
2349 }
2350
2351 subDeviceInstance = gpumgrGetSubDeviceInstanceFromGpu(pGpu);
2352
2353 pGpuInfo->gpuInstance = pGpu->gpuInstance;
2354 pGpuInfo->deviceInstance = deviceInstance;
2355 pGpuInfo->subDeviceInstance = subDeviceInstance;
2356 pGpuInfo->boardId = pGpu->boardId;
2357
2358 //
2359 // Setup gpu info flags; see ctrl0000gpu.h for list of flags.
2360 //
2361 pGpuInfo->gpuFlags = 0;
2362 pGpuInfo->numaId = NV0000_CTRL_NO_NUMA_NODE;
2363 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_ATS_SUPPORTED))
2364 {
2365 pGpuInfo->gpuFlags |= DRF_NUM(0000, _CTRL_GPU_ID_INFO, _ATS_ENABLED,
2366 NV0000_CTRL_GPU_ID_INFO_ATS_ENABLED_TRUE);
2367
2368 if (gpuIsSelfHosted(pGpu) && IS_MIG_IN_USE(pGpu))
2369 {
2370 //
2371 // We do not have enough information here to route to correct MIG instance.
2372 // There may even not be enough state to do this lookup at all as this is a 0x0000 control
2373 // but we need to get to Subdevice child. Try extra hard to preserve originally
2374 // implemented behavior. See bug 4251227.
2375 //
2376 CALL_CONTEXT *pCallContext = resservGetTlsCallContext();
2377 RS_ITERATOR it = clientRefIter(pCallContext->pClient, NULL, classId(GPUInstanceSubscription),
2378 RS_ITERATE_DESCENDANTS, NV_TRUE);
2379
2380 while (clientRefIterNext(it.pClient, &it))
2381 {
2382 NvU32 swizzId;
2383 GPUInstanceSubscription *pGPUInstanceSubscription =
2384 dynamicCast(it.pResourceRef->pResource, GPUInstanceSubscription);
2385
2386 if (GPU_RES_GET_GPU(pGPUInstanceSubscription) != pGpu)
2387 {
2388 continue;
2389 }
2390
2391 swizzId = gisubscriptionGetMIGGPUInstance(pGPUInstanceSubscription)->swizzId;
2392
2393 if (swizzId == NVC637_DEVICE_PROFILING_SWIZZID)
2394 {
2395 continue;
2396 }
2397 if (GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu)->memPartitionNumaInfo[swizzId].bInUse)
2398 {
2399 pGpuInfo->numaId = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu)->memPartitionNumaInfo[swizzId].numaNodeId;
2400 }
2401 break;
2402 }
2403 }
2404 else
2405 {
2406 pGpuInfo->numaId = pGpu->numaNodeId;
2407 }
2408 }
2409
2410 // is this gpu in use?
2411 pGpuInfo->gpuFlags |= DRF_NUM(0000, _CTRL_GPU_ID_INFO, _IN_USE, gpuIsInUse(pGpu));
2412
2413 // is this gpu part of a sli device?
2414 pGpuInfo->gpuFlags |= DRF_NUM(0000, _CTRL_GPU_ID_INFO, _LINKED_INTO_SLI_DEVICE, IsSLIEnabled(pGpu));
2415
2416 // is this gpu a mobile gpu?
2417 if (IsMobile(pGpu))
2418 {
2419 pGpuInfo->gpuFlags |= DRF_DEF(0000, _CTRL_GPU_ID_INFO, _MOBILE, _TRUE);
2420 }
2421
2422 // is this gpu the boot primary?
2423 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_PRIMARY_DEVICE))
2424 {
2425 pGpuInfo->gpuFlags |= DRF_DEF(0000, _CTRL_GPU_ID_INFO, _BOOT_MASTER, _TRUE);
2426 }
2427
2428 // is this GPU part of an SOC
2429 if (pGpu->bIsSOC)
2430 {
2431 pGpuInfo->gpuFlags |= DRF_DEF(0000, _CTRL_GPU_ID_INFO, _SOC, _TRUE);
2432 }
2433
2434 // GPU specific SLI status
2435 pGpuInfo->sliStatus = pGpu->sliStatus;
2436
2437 NV_PRINTF(LEVEL_INFO,
2438 "gpumgrGetGpuInfoV2: gpu[0x%x]: device 0x%x subdevice 0x%x\n",
2439 pGpuInfo->gpuId, pGpuInfo->deviceInstance,
2440 pGpuInfo->subDeviceInstance);
2441
2442 return NV_OK;
2443 }
2444 NV_STATUS
gpumgrGetGpuIdInfo(NV0000_CTRL_GPU_GET_ID_INFO_PARAMS * pGpuInfo)2445 gpumgrGetGpuIdInfo(NV0000_CTRL_GPU_GET_ID_INFO_PARAMS *pGpuInfo)
2446 {
2447 NV_STATUS status;
2448 NV0000_CTRL_GPU_GET_ID_INFO_V2_PARAMS pGpuInfoV2 = {0};
2449 pGpuInfoV2.gpuId = pGpuInfo->gpuId;
2450
2451 status = gpumgrGetGpuIdInfoV2(&pGpuInfoV2);
2452
2453 if (status != NV_OK)
2454 {
2455 return status;
2456 }
2457 pGpuInfo->gpuFlags = pGpuInfoV2.gpuFlags;
2458 pGpuInfo->deviceInstance = pGpuInfoV2.deviceInstance;
2459 pGpuInfo->subDeviceInstance = pGpuInfoV2.subDeviceInstance;
2460 pGpuInfo->sliStatus = pGpuInfoV2.sliStatus;
2461 pGpuInfo->boardId = pGpuInfoV2.boardId;
2462 pGpuInfo->gpuInstance = pGpuInfoV2.gpuInstance;
2463 pGpuInfo->numaId = pGpuInfoV2.numaId;
2464
2465 return status;
2466 }
2467
2468 //
2469 // gpumgrGetGpuInitStatus
2470 //
2471 // Special purpose routine that handles NV0000_CTRL_CMD_GET_GPU_INIT_STATUS
2472 // requests from clients.
2473 //
2474 NV_STATUS
gpumgrGetGpuInitStatus(NV0000_CTRL_GPU_GET_INIT_STATUS_PARAMS * pGpuInitStatus)2475 gpumgrGetGpuInitStatus(NV0000_CTRL_GPU_GET_INIT_STATUS_PARAMS *pGpuInitStatus)
2476 {
2477 OBJSYS *pSys = SYS_GET_INSTANCE();
2478 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
2479 NvU32 i;
2480 NV_STATUS rmStatus;
2481
2482 portSyncMutexAcquire(pGpuMgr->probedGpusLock);
2483
2484 for (i = 0; i < NV_ARRAY_ELEMENTS(pGpuMgr->probedGpus); i++)
2485 {
2486 if (pGpuMgr->probedGpus[i].gpuId == pGpuInitStatus->gpuId)
2487 {
2488 if (pGpuMgr->probedGpus[i].bInitAttempted)
2489 {
2490 pGpuInitStatus->status = pGpuMgr->probedGpus[i].initStatus;
2491 rmStatus = NV_OK;
2492 }
2493 else
2494 {
2495 //
2496 // No init has been attempted on this GPU yet, so this request
2497 // doesn't make any sense.
2498 //
2499 rmStatus = NV_ERR_INVALID_STATE;
2500 }
2501 goto done;
2502 }
2503 }
2504
2505 // We couldn't find a probed gpuId matching the requested one.
2506 rmStatus = NV_ERR_INVALID_ARGUMENT;
2507 done:
2508 portSyncMutexRelease(pGpuMgr->probedGpusLock);
2509 return rmStatus;
2510 }
2511
2512 NV_STATUS
gpumgrGetProbedGpuDomainBusDevice(NvU32 gpuId,NvU64 * gpuDomainBusDevice)2513 gpumgrGetProbedGpuDomainBusDevice(NvU32 gpuId, NvU64 *gpuDomainBusDevice)
2514 {
2515 OBJSYS *pSys = SYS_GET_INSTANCE();
2516 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
2517 NvU32 i;
2518 NV_STATUS rmStatus;
2519
2520 portSyncMutexAcquire(pGpuMgr->probedGpusLock);
2521
2522 for (i = 0; i < NV_ARRAY_ELEMENTS(pGpuMgr->probedGpus); i++)
2523 {
2524 if (pGpuMgr->probedGpus[i].gpuId == gpuId)
2525 {
2526 *gpuDomainBusDevice = pGpuMgr->probedGpus[i].gpuDomainBusDevice;
2527 rmStatus = NV_OK;
2528 goto done;
2529 }
2530 }
2531
2532 //
2533 // We couldn't find a probed gpuId matching the requested one.
2534 //
2535 // This used to return a generic NV_ERR_INVALID_ARGUMENT, but we want to be
2536 // more specific as at least nvml wants to be able to tell this case apart
2537 // from other errors. This case is expected when GPUs are removed from the
2538 // driver (e.g. through unbind on Linux) after a client queries for the
2539 // probed GPUs, but before getting the PCI info for all of them.
2540 //
2541 rmStatus = NV_ERR_OBJECT_NOT_FOUND;
2542
2543 done:
2544 portSyncMutexRelease(pGpuMgr->probedGpusLock);
2545 return rmStatus;
2546 }
2547
2548 //
2549 // gpumgrSetGpuInitStatus
2550 //
2551 // Marks initialization of the gpu in question as attempted and stores the
2552 // status.
2553 //
2554 void
gpumgrSetGpuInitStatus(NvU32 gpuId,NV_STATUS status)2555 gpumgrSetGpuInitStatus(NvU32 gpuId, NV_STATUS status)
2556 {
2557 OBJSYS *pSys = SYS_GET_INSTANCE();
2558 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
2559 NvU32 i;
2560
2561 portSyncMutexAcquire(pGpuMgr->probedGpusLock);
2562
2563 for (i = 0; i < NV_ARRAY_ELEMENTS(pGpuMgr->probedGpus); i++)
2564 {
2565 if (pGpuMgr->probedGpus[i].gpuId == gpuId)
2566 {
2567 // Overwrite any previous init status
2568 pGpuMgr->probedGpus[i].bInitAttempted = NV_TRUE;
2569 pGpuMgr->probedGpus[i].initStatus = status;
2570 break;
2571 }
2572 }
2573
2574 portSyncMutexRelease(pGpuMgr->probedGpusLock);
2575 }
2576
2577 //
2578 // gpumgrUpdateBoardId
2579 //
2580 // Update gpu boardIds. By default the boardId will already be set to
2581 // the gpuId. This routine fetches the upstream bridge port and sets the
2582 // new boardId to the pGpu
2583 //
2584 void
gpumgrUpdateBoardId_IMPL(OBJGPU * pGpu)2585 gpumgrUpdateBoardId_IMPL(OBJGPU *pGpu)
2586 {
2587 OBJSYS *pSys = SYS_GET_INSTANCE();
2588 OBJCL *pCl = SYS_GET_CL(pSys);
2589 void *handleUp;
2590 NvU16 vendorIDUp, deviceIDUp;
2591 NvU8 busUp, deviceUp, funcUp;
2592 NvU32 domain;
2593 NvU32 boardId;
2594
2595 if (!pGpu->getProperty(pGpu, PDB_PROP_GPU_BEHIND_BRIDGE))
2596 {
2597 return;
2598 }
2599
2600 domain = gpuGetDomain(pGpu);
2601 handleUp = clFindBrdgUpstreamPort(pGpu, pCl, NV_TRUE,
2602 &busUp, &deviceUp, &funcUp,
2603 &vendorIDUp, &deviceIDUp,
2604 NULL);
2605 if (!handleUp)
2606 return;
2607
2608 if (!(IS_SUPPORTED_PCIE_SWITCH(vendorIDUp, deviceIDUp)))
2609 {
2610 return;
2611 }
2612
2613 boardId = gpuGenerate32BitId(domain, busUp, deviceUp);
2614 pGpu->boardId = boardId;
2615 }
2616
2617 //
2618 // gpumgrGetDefaultPrimaryGpu
2619 //
2620 // This routine looks at the set of GPUs and picks a the primary (parent)
2621 // with the following rules, in this order:
2622 // 1- If a primary GPU has been passed in an SLI config by a client
2623 // 2- If there is a boot primary in the GPU mask
2624 // 3- The first VGA device attached (not 3d controller)
2625 //
2626 NvU32
gpumgrGetDefaultPrimaryGpu(NvU32 gpuMask)2627 gpumgrGetDefaultPrimaryGpu
2628 (
2629 NvU32 gpuMask
2630 )
2631 {
2632 OBJGPU *pGpu = NULL;
2633 NvU32 gpuInstance;
2634
2635 if (gpuMask == 0)
2636 {
2637 NV_ASSERT(gpuMask);
2638 return 0;
2639 }
2640
2641 // Find masterFromSLIConfig, set when a RM client passes a primary GPU
2642 // index from a SLI config
2643 gpuInstance = 0;
2644 while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
2645 {
2646 if (pGpu->masterFromSLIConfig)
2647 {
2648 break;
2649 }
2650 }
2651
2652 // default to boot primary
2653 if (pGpu == NULL)
2654 {
2655 gpuInstance = 0;
2656 while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
2657 {
2658 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_PRIMARY_DEVICE))
2659 {
2660 break;
2661 }
2662 }
2663 }
2664
2665 if (pGpu)
2666 {
2667 return gpuGetInstance(pGpu);
2668 }
2669
2670 // otherwise the primary is the first non 3d controller in the set attached to the RM
2671 gpuInstance = 0;
2672 while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
2673 {
2674 if (!pGpu->getProperty(pGpu, PDB_PROP_GPU_3D_CONTROLLER))
2675 {
2676 break;
2677 }
2678 }
2679
2680 if (!pGpu)
2681 {
2682 // The GPU mask contains only 3d Controllers.
2683 // Choose first one in the set attached to the RM.
2684 gpuInstance = 0;
2685 pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance);
2686 }
2687
2688 if (pGpu == NULL)
2689 {
2690 return 0; // This should never happen
2691 }
2692
2693 return gpuGetInstance(pGpu);
2694 }
2695
2696 void
gpumgrServiceInterrupts_IMPL(NvU32 gpuMask,MC_ENGINE_BITVECTOR * engineMask,NvBool bExhaustive)2697 gpumgrServiceInterrupts_IMPL(NvU32 gpuMask, MC_ENGINE_BITVECTOR *engineMask, NvBool bExhaustive)
2698 {
2699 OBJGPU *pGpu;
2700 NvU32 gpuInstance = 0;
2701
2702 while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
2703 {
2704 if (gpuIsGpuFullPower(pGpu))
2705 {
2706 Intr *pIntr = GPU_GET_INTR(pGpu);
2707
2708 //
2709 // On SLI, one OBJGPU's StateInit functions could attempt to service
2710 // interrupts on another OBJGPU which has not yet started StateInit.
2711 // However Intr is not prepared to service interrupts until after
2712 // intrStateInit. To avoid calling in too early, check that the
2713 // interrupt table is initialized.
2714 //
2715 if (vectIsEmpty(&pIntr->intrTable))
2716 {
2717 continue;
2718 }
2719
2720 //
2721 // Service interrupts for the specified engines now.
2722 // A non-false value for 'bExhaustive' indicates that intrServiceStallList
2723 // should repreatedly service all specified engines, until it finds
2724 // no further pending interrupt work remains on those engines.
2725 //
2726 intrServiceStallList_HAL(pGpu, pIntr, engineMask, bExhaustive);
2727 }
2728 }
2729 }
2730
2731 NV_STATUS
gpumgrGetGpuLockAndDrPorts(OBJGPU * pGpu,OBJGPU * pPeerGpu,NvU32 * pPinsetOut,NvU32 * pPinsetIn)2732 gpumgrGetGpuLockAndDrPorts
2733 (
2734 OBJGPU *pGpu,
2735 OBJGPU *pPeerGpu,
2736 NvU32 *pPinsetOut,
2737 NvU32 *pPinsetIn
2738 )
2739 {
2740
2741 OBJSYS *pSys = SYS_GET_INSTANCE();
2742 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
2743 OBJGPUGRP *pGpuGrp = NULL;
2744 NvU32 pinsetIndex = 0, childPinset = drPinSet_None;
2745 NvU32 i;
2746 NvU32 gpuInstance = pGpu->gpuInstance;
2747 NV_STATUS rmStatus;
2748
2749 *pPinsetOut = drPinSet_None;
2750 *pPinsetIn = drPinSet_None;
2751
2752 if (pPeerGpu == NULL)
2753 {
2754 pGpuGrp = gpumgrGetGpuGrpFromGpu(pGpu);
2755 NV_ASSERT_OR_RETURN(pGpuGrp != NULL, NV_ERR_INVALID_DATA);
2756
2757 for (i = 0; i < NV2080_MAX_SUBDEVICES; i++)
2758 {
2759 if (pGpuGrp->SliLinkOrder[i].gpuInstance == gpuInstance)
2760 {
2761 break;
2762 }
2763 }
2764
2765 if (i == NV2080_MAX_SUBDEVICES)
2766 {
2767 *pPinsetOut = 0;
2768 *pPinsetIn = 0;
2769 }
2770 else
2771 {
2772 *pPinsetOut = pGpuGrp->SliLinkOrder[i].ChildDrPort;
2773 *pPinsetIn = pGpuGrp->SliLinkOrder[i].ParentDrPort;
2774 }
2775
2776 return NV_OK;
2777 }
2778 else
2779 {
2780 NvU32 childPinsetIndex;
2781
2782 for (pinsetIndex = 0; pinsetIndex < DR_PINSET_COUNT; pinsetIndex++)
2783 {
2784 if (!FLD_TEST_DRF(_SLILINK, _ROUTE, _INPUT, _PASSED,
2785 pGpuMgr->gpuSliLinkRoute[pGpuMgr->gpuBridgeType][gpuInstance][pPeerGpu->gpuInstance][pinsetIndex]))
2786 {
2787 continue;
2788 }
2789
2790 childPinset = DRF_VAL(_SLILINK, _ROUTE, _INPUT_CHILD_PORT,
2791 pGpuMgr->gpuSliLinkRoute[pGpuMgr->gpuBridgeType][gpuInstance][pPeerGpu->gpuInstance][pinsetIndex]);
2792
2793 rmStatus = gpumgrPinsetToPinsetTableIndex(childPinset, &childPinsetIndex);
2794 if (rmStatus != NV_OK)
2795 {
2796 return rmStatus;
2797 }
2798 if (!FLD_TEST_DRF(_SLILINK, _ROUTE, _OUTPUT, _PASSED,
2799 pGpuMgr->gpuSliLinkRoute[pGpuMgr->gpuBridgeType][pPeerGpu->gpuInstance][gpuInstance][childPinsetIndex]))
2800 {
2801 continue;
2802 }
2803 *pPinsetOut |= childPinset;
2804 *pPinsetIn |= NVBIT(pinsetIndex);
2805 }
2806 return ((*pPinsetOut != drPinSet_None) && (*pPinsetIn != drPinSet_None)) ? NV_OK : NV_ERR_INVALID_ARGUMENT;
2807 }
2808
2809 }
2810
2811 //
2812 // Stores the address of the boot primary in pGpu
2813 // Returns NV_OK on success NV_ERR_GENERIC otherwise.
2814 //
2815 NV_STATUS
gpumgrGetBootPrimary(OBJGPU ** ppGpu)2816 gpumgrGetBootPrimary(OBJGPU **ppGpu)
2817 {
2818 NvU32 gpuCount, gpuMask, idx1;
2819 OBJGPU *pGpu = NULL;
2820
2821 // Find boot primary
2822 idx1 = 0;
2823 gpumgrGetGpuAttachInfo(&gpuCount, &gpuMask);
2824 while ((pGpu = gpumgrGetNextGpu(gpuMask, &idx1)) != NULL)
2825 {
2826 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_PRIMARY_DEVICE))
2827 break;
2828 }
2829 *ppGpu = pGpu;
2830
2831 // No boot primary
2832 if (pGpu == NULL)
2833 {
2834 return NV_ERR_GENERIC;
2835 }
2836
2837 return NV_OK;
2838 }
2839
2840 //
2841 // Returns the mGpu
2842 //
gpumgrGetMGpu(void)2843 OBJGPU *gpumgrGetMGpu (void)
2844 {
2845 OBJGPU *pGpu;
2846 NvU32 gpuCount, gpuMask, gpuIndex = 0;
2847 // Parse through all the GPUs
2848
2849 gpumgrGetGpuAttachInfo(&gpuCount, &gpuMask);
2850 while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuIndex)))
2851 {
2852 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_HYBRID_MGPU))
2853 {
2854 break;
2855 }
2856 }
2857
2858 return pGpu;
2859 }
2860
2861 //
2862 // Get PhysFbAddr for the given GPU which may be different if
2863 // the GPU is broadcast or chipset broadcast are enabled or not:
2864 // - BC GPU + no CL BC -> returns gpu address
2865 // - UC GPU -> returns GPU address
2866 // - BC GPU + CL BC -> returns broadcast address
2867 //
gpumgrGetGpuPhysFbAddr(OBJGPU * pGpu)2868 RmPhysAddr gpumgrGetGpuPhysFbAddr(OBJGPU *pGpu)
2869 {
2870 RmPhysAddr physFbAddr;
2871
2872 physFbAddr = pGpu->busInfo.gpuPhysFbAddr;
2873
2874 NV_ASSERT(pGpu->getProperty(pGpu, PDB_PROP_GPU_ZERO_FB) || physFbAddr);
2875 return physFbAddr;
2876 }
2877
2878
2879 //
2880 // Get GPU object from subdevice instance
2881 //
2882 POBJGPU
gpumgrGetGpuFromSubDeviceInst(NvU32 deviceInst,NvU32 subDeviceInst)2883 gpumgrGetGpuFromSubDeviceInst(NvU32 deviceInst, NvU32 subDeviceInst)
2884 {
2885 OBJGPU *pGpu = NULL;
2886 OBJGPUGRP *pGpuGrp = NULL;
2887 NvU32 gpuInst = 0;
2888 NvU32 gpuMask;
2889
2890 pGpuGrp = gpumgrGetGpuGrpFromInstance(deviceInst);
2891 NV_ASSERT_OR_RETURN(pGpuGrp != NULL, NULL);
2892
2893 gpuMask = gpugrpGetGpuMask(pGpuGrp);
2894
2895 // check for single GPU case
2896 if (gpumgrGetSubDeviceCount(gpuMask) == 1)
2897 return gpumgrGetNextGpu(gpuMask, &gpuInst);
2898
2899 while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInst)) != NULL)
2900 {
2901 if (gpumgrGetSubDeviceInstanceFromGpu(pGpu) == subDeviceInst)
2902 {
2903 break;
2904 }
2905 }
2906
2907 NV_ASSERT(pGpu);
2908
2909 return pGpu;
2910 }
2911
2912 /*!
2913 * @brief sets the device instance pGpu->deviceInstance for the GPUs indicated by the gpu mask
2914 *
2915 * Only remove the device instance if it is the last GPU to be removed.
2916 *
2917 * At RM initialization we fill in the software feature values for this GPU.
2918 * The values are determined from the software feature database
2919 *
2920 * @param[in] gpuMask NvU32 value
2921 *
2922 * @return NV_OK or NV_ERR_OBJECT_NOT_FOUND if no GPU has been found
2923 *
2924 */
2925 NV_STATUS
gpumgrAddDeviceInstanceToGpus(NvU32 gpuMask)2926 gpumgrAddDeviceInstanceToGpus(NvU32 gpuMask)
2927 {
2928 OBJSYS *pSys = SYS_GET_INSTANCE();
2929 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
2930 NV_STATUS rmStatus = NV_ERR_OBJECT_NOT_FOUND;
2931 OBJGPU *pGpu = NULL;
2932 NvU32 i, gpuIndex = 0;
2933 OBJGPUGRP *pGpuGrp = NULL;
2934
2935 // Add the device instance to the GPU objects in the mask
2936 while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuIndex)))
2937 {
2938 rmStatus = NV_ERR_OBJECT_NOT_FOUND;
2939 for (i = 0; i < NV_ARRAY_ELEMENTS(pGpuMgr->pGpuGrpTable); i++)
2940 {
2941 pGpuGrp = pGpuMgr->pGpuGrpTable[i];
2942 // if it contains the specified gpu...
2943 if ((pGpuGrp != NULL) &&
2944 (gpugrpGetGpuMask(pGpuGrp) & NVBIT(pGpu->gpuInstance)))
2945 {
2946 pGpu->deviceInstance = i;
2947 rmStatus = NV_OK;
2948 break;
2949 }
2950 }
2951 NV_ASSERT_OK_OR_RETURN(rmStatus);
2952 }
2953
2954 return rmStatus;
2955 }
2956
2957 /*!
2958 * @brief Retrieves the OBJGPUGRP pointer given the instance
2959 *
2960 * @param[in] gpugrpInstance GPUGRP instance
2961 *
2962 * @return GPUGRP pointer on success, NULL on error
2963 *
2964 */
2965 OBJGPUGRP *
gpumgrGetGpuGrpFromInstance(NvU32 gpugrpInstance)2966 gpumgrGetGpuGrpFromInstance
2967 (
2968 NvU32 gpugrpInstance
2969 )
2970 {
2971 OBJSYS *pSys = SYS_GET_INSTANCE();
2972 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
2973 NV_ASSERT_OR_RETURN(gpugrpInstance < NV_MAX_DEVICES, NULL);
2974 return pGpuMgr->pGpuGrpTable[gpugrpInstance];
2975 }
2976
2977 /*!
2978 * @brief Retrieves the OBJGPUGRP pointer given the GPU pointer.
2979 *
2980 * @param[in] pGpu GPU object pointer
2981 *
2982 * @return OBJGPUGRP pointer on success, NULL on error
2983 *
2984 */
2985 OBJGPUGRP *
gpumgrGetGpuGrpFromGpu(OBJGPU * pGpu)2986 gpumgrGetGpuGrpFromGpu
2987 (
2988 OBJGPU *pGpu
2989 )
2990 {
2991 OBJSYS *pSys = SYS_GET_INSTANCE();
2992 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
2993 NvU32 gpugrpInstance = gpuGetDeviceInstance(pGpu);
2994 NV_ASSERT_OR_RETURN(gpugrpInstance < NV_MAX_DEVICES, NULL);
2995
2996 return pGpuMgr->pGpuGrpTable[gpugrpInstance];
2997 }
2998
2999 /*!
3000 * @brief Constructs the GPUGRP object for the given instance
3001 *
3002 * @param[in] pGpu GPU object pointer
3003 * @param[in] gpuMask GpuMask corresponding to this GPUGRP
3004 * @param[out] ppGpuGrp Newly created gpugrp object pointer
3005 *
3006 * @return NV_OK on success, appropriate error on failure.
3007 *
3008 */
3009 NV_STATUS
gpumgrConstructGpuGrpObject(OBJGPUMGR * pGpuMgr,NvU32 gpuMask,OBJGPUGRP ** ppGpuGrp)3010 gpumgrConstructGpuGrpObject
3011 (
3012 OBJGPUMGR *pGpuMgr,
3013 NvU32 gpuMask,
3014 OBJGPUGRP **ppGpuGrp
3015 )
3016 {
3017 NV_STATUS status;
3018
3019 status = objCreate(ppGpuGrp, pGpuMgr, OBJGPUGRP);
3020 if (NV_OK != status)
3021 {
3022 return status;
3023 }
3024
3025 status = gpugrpCreate(*ppGpuGrp, gpuMask);
3026 if (NV_OK != status)
3027 {
3028 return status;
3029 }
3030
3031 return NV_OK;
3032 }
3033
3034 /*!
3035 * @brief Enter/exit "drain" state on a given GPU
3036 *
3037 * @param[in] gpuId Platform specific GPU Id
3038 * @param[in] bEnable NV_TRUE: enter, NV_FALSE: exit
3039 * @param[in] bRemove Ask the OS to forget the GPU, once quiescent
3040 * @param[in] bLinkDisable Shut down the upstream PCIe link after the removal.
3041 * This is done in user-land, we just check that the
3042 * GPU is in the right state.
3043 *
3044 * @return NV_OK on success, appropriate error on failure.
3045 */
3046 NV_STATUS
gpumgrModifyGpuDrainState(NvU32 gpuId,NvBool bEnable,NvBool bRemove,NvBool bLinkDisable)3047 gpumgrModifyGpuDrainState
3048
3049 (
3050 NvU32 gpuId,
3051 NvBool bEnable,
3052 NvBool bRemove,
3053 NvBool bLinkDisable
3054 )
3055 {
3056 OBJSYS *pSys = SYS_GET_INSTANCE();
3057 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
3058 OBJGPU *pGpu;
3059 NvBool bAttached;
3060 NvBool bStateChange = NV_FALSE;
3061 NvU32 i;
3062 NvU32 domain = 0;
3063 NvU8 bus = 0;
3064 NvU8 device = 0;
3065
3066 if (bRemove && !osRemoveGpuSupported())
3067 {
3068 return NV_ERR_NOT_SUPPORTED;
3069 }
3070
3071 bAttached = ((pGpu = gpumgrGetGpuFromId(gpuId)) != NULL);
3072
3073 if (bEnable && bLinkDisable && bAttached)
3074 {
3075 return NV_ERR_IN_USE;
3076 }
3077
3078 portSyncMutexAcquire(pGpuMgr->probedGpusLock);
3079
3080 for (i = 0; i < NV_ARRAY_ELEMENTS(pGpuMgr->probedGpus); ++i)
3081 {
3082 if (pGpuMgr->probedGpus[i].gpuId == gpuId)
3083 {
3084 bStateChange = pGpuMgr->probedGpus[i].bDrainState != bEnable;
3085 pGpuMgr->probedGpus[i].bDrainState = bEnable;
3086 pGpuMgr->probedGpus[i].bRemoveIdle = bEnable && bRemove;
3087 domain = gpuDecodeDomain(pGpuMgr->probedGpus[i].gpuDomainBusDevice);
3088 bus = gpuDecodeBus(pGpuMgr->probedGpus[i].gpuDomainBusDevice);
3089 device = gpuDecodeDevice(pGpuMgr->probedGpus[i].gpuDomainBusDevice);
3090 break;
3091 }
3092 }
3093
3094 portSyncMutexRelease(pGpuMgr->probedGpusLock);
3095
3096 if (i == NV_MAX_DEVICES)
3097 {
3098 return NV_ERR_INVALID_ARGUMENT;
3099 }
3100
3101 //
3102 // If the initial drain state (characterized by enabling draining without
3103 // setting the remove flag) is already enabled, multiple clients may be
3104 // trying to simultaneously manage drain state. Only return success for the
3105 // first to allow them to filter out the others.
3106 //
3107 if (bEnable && !bRemove && !bStateChange)
3108 {
3109 return NV_ERR_IN_USE;
3110 }
3111
3112 if (bEnable && bRemove && !bAttached)
3113 {
3114 osRemoveGpu(domain, bus, device);
3115 }
3116
3117 return NV_OK;
3118 }
3119
3120 /*!
3121 * @brief Query "drain"/remove state on a given GPU
3122 *
3123 * @param[in] gpuId Platform specific GPU Id
3124 * @param[out] pBEnable Drain state ptr
3125 * @param[out] pBRemove Remove flag ptr
3126 *
3127 * @return NV_OK on success, appropriate error on failure.
3128 */
3129 NV_STATUS
gpumgrQueryGpuDrainState(NvU32 gpuId,NvBool * pBEnable,NvBool * pBRemove)3130 gpumgrQueryGpuDrainState
3131
3132 (
3133 NvU32 gpuId,
3134 NvBool *pBEnable,
3135 NvBool *pBRemove
3136 )
3137 {
3138 OBJSYS *pSys = SYS_GET_INSTANCE();
3139 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
3140 NvU32 i;
3141
3142 portSyncMutexAcquire(pGpuMgr->probedGpusLock);
3143
3144 for (i = 0; i < NV_ARRAY_ELEMENTS(pGpuMgr->probedGpus); ++i)
3145 {
3146 if (pGpuMgr->probedGpus[i].gpuId == gpuId)
3147 {
3148 if (pBEnable != NULL)
3149 {
3150 *pBEnable = pGpuMgr->probedGpus[i].bDrainState;
3151 }
3152
3153 if (pBRemove != NULL)
3154 {
3155 *pBRemove = pGpuMgr->probedGpus[i].bRemoveIdle;
3156 }
3157
3158 break;
3159 }
3160 }
3161
3162 portSyncMutexRelease(pGpuMgr->probedGpusLock);
3163
3164 //
3165 // This used to return a generic NV_ERR_INVALID_ARGUMENT on error, but we
3166 // want to be more specific as at least nvml wants to be able to tell this
3167 // case apart from other errors. This case is expected when GPUs are
3168 // removed from the driver (e.g. through unbind on Linux) after a client
3169 // queries for the probed GPUs, but before getting the PCI info for all of
3170 // them.
3171 //
3172 return (i == NV_MAX_DEVICES) ? NV_ERR_OBJECT_NOT_FOUND : NV_OK;
3173 }
3174
3175 /*!
3176 * @brief Retrieves the group gpuMask that contains this gpuInstance.
3177 * Used for locking all gpus under the same device together
3178 *
3179 * @param[in] gpuInstance: unique Index per GPU
3180 *
3181 * @return gpuMask: mask of all GPU that are in the same group
3182 *
3183 */
3184 NvU32
gpumgrGetGrpMaskFromGpuInst(NvU32 gpuInst)3185 gpumgrGetGrpMaskFromGpuInst
3186 (
3187 NvU32 gpuInst
3188 )
3189 {
3190 OBJSYS *pSys = SYS_GET_INSTANCE();
3191 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
3192
3193 NV_ASSERT_OR_RETURN(gpuInst < NV_MAX_DEVICES, 0);
3194
3195 return pGpuMgr->gpuInstMaskTable[gpuInst];
3196 }
3197
3198 /*!
3199 * @brief Updates per GPU isnstance table to contain correct group mask
3200 *
3201 * @param[in] gpuMask: mask of all GPUs that are in the same group
3202 *
3203 */
3204 void
gpumgrAddDeviceMaskToGpuInstTable(NvU32 gpuMask)3205 gpumgrAddDeviceMaskToGpuInstTable
3206 (
3207 NvU32 gpuMask
3208 )
3209 {
3210 OBJSYS *pSys = SYS_GET_INSTANCE();
3211 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
3212 int gpuInst;
3213 NvU32 tempGpuMask = gpuMask;
3214
3215 for (gpuInst = 0; (tempGpuMask != 0) && (gpuInst < NV_MAX_DEVICES); gpuInst++)
3216 {
3217 if (NVBIT(gpuInst) & gpuMask)
3218 pGpuMgr->gpuInstMaskTable[gpuInst] = gpuMask;
3219
3220 tempGpuMask &= ~NVBIT(gpuInst);
3221 }
3222 }
3223
3224 /*!
3225 * @brief Clear group mask from per GPU isnstance table (when group is destroyed)
3226 *
3227 * @param[in] gpuMask: gpu group mask being teared down
3228 *
3229 */
3230 void
gpumgrClearDeviceMaskFromGpuInstTable(NvU32 gpuMask)3231 gpumgrClearDeviceMaskFromGpuInstTable
3232 (
3233 NvU32 gpuMask
3234 )
3235 {
3236 OBJSYS *pSys = SYS_GET_INSTANCE();
3237 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
3238 int gpuInst;
3239 NvU32 tempGpuMask = gpuMask;
3240
3241 for (gpuInst = 0; (tempGpuMask != 0) && (gpuInst < NV_MAX_DEVICES); gpuInst++)
3242 {
3243 if (NVBIT(gpuInst) & gpuMask)
3244 pGpuMgr->gpuInstMaskTable[gpuInst] = 0;
3245
3246 tempGpuMask &= ~NVBIT(gpuInst);
3247 }
3248 }
3249
3250 /*!
3251 * @brief Add an nvlink topo entry by a GPU's PCI DomainBusDevice if not already present
3252 *
3253 * @param[in] DomainBusDevice: the PCI DomainBusDevice for the gpu to be registered
3254 *
3255 */
3256 void
gpumgrAddSystemNvlinkTopo_IMPL(NvU64 DomainBusDevice)3257 gpumgrAddSystemNvlinkTopo_IMPL
3258 (
3259 NvU64 DomainBusDevice
3260 )
3261 {
3262 OBJSYS *pSys = SYS_GET_INSTANCE();
3263 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
3264 NvU32 i;
3265
3266 if (gpumgrGetSystemNvlinkTopo(DomainBusDevice, NULL))
3267 {
3268 // This gpu is already registered
3269 return;
3270 }
3271
3272 for (i = 0; i < NV_MAX_DEVICES; i++)
3273 {
3274 // add new gpu entry into the first slot available
3275 if (!pGpuMgr->nvlinkTopologyInfo[i].valid)
3276 {
3277 pGpuMgr->nvlinkTopologyInfo[i].valid = NV_TRUE;
3278 pGpuMgr->nvlinkTopologyInfo[i].DomainBusDevice = DomainBusDevice;
3279 return;
3280 }
3281 }
3282 }
3283
3284 /*!
3285 * @brief Finds nvlinkTopologyInfo entry info based on DomainBusDevice
3286 *
3287 * @param[in] DomainBusDevice: the PCI DomainBusDevice for the gpu to be registered
3288 * @param[out] pTopoParams: topology params found for the specified GPU
3289 *
3290 * @returns NV_TRUE if entry found
3291 * NV_FALSE otherwise
3292 */
3293 NvBool
gpumgrGetSystemNvlinkTopo_IMPL(NvU64 DomainBusDevice,PNVLINK_TOPOLOGY_PARAMS pTopoParams)3294 gpumgrGetSystemNvlinkTopo_IMPL
3295 (
3296 NvU64 DomainBusDevice,
3297 PNVLINK_TOPOLOGY_PARAMS pTopoParams
3298 )
3299 {
3300 OBJSYS *pSys = SYS_GET_INSTANCE();
3301 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
3302 NvU32 i, idx;
3303
3304 for (i = 0; i < NV_MAX_DEVICES; i++)
3305 {
3306 //
3307 // Choose the correct GPU by comparing PCI BusDomainDevice
3308 // This ensures we are using the same GPU across gpu load/unload
3309 //
3310 if ((pGpuMgr->nvlinkTopologyInfo[i].valid) && (pGpuMgr->nvlinkTopologyInfo[i].DomainBusDevice == DomainBusDevice))
3311 {
3312 if (pTopoParams)
3313 {
3314 pTopoParams->sysmemLinks = pGpuMgr->nvlinkTopologyInfo[i].params.sysmemLinks;
3315 pTopoParams->maxLinksPerPeer = pGpuMgr->nvlinkTopologyInfo[i].params.maxLinksPerPeer;
3316 pTopoParams->bSymmetric = pGpuMgr->nvlinkTopologyInfo[i].params.bSymmetric;
3317 // Pascal Only
3318 pTopoParams->numLinks = pGpuMgr->nvlinkTopologyInfo[i].params.numLinks;
3319 // Volta+
3320 pTopoParams->numPeers = pGpuMgr->nvlinkTopologyInfo[i].params.numPeers;
3321 pTopoParams->bSwitchConfig = pGpuMgr->nvlinkTopologyInfo[i].params.bSwitchConfig;
3322 // Ampere+
3323 for (idx = 0; idx < NV2080_CTRL_MAX_PCES; idx++)
3324 {
3325 pTopoParams->maxPceLceMap[idx] = pGpuMgr->nvlinkTopologyInfo[i].params.maxPceLceMap[idx];
3326 }
3327 for (idx = 0; idx < NV2080_CTRL_MAX_GRCES; idx++)
3328 {
3329 pTopoParams->maxGrceConfig[idx] = pGpuMgr->nvlinkTopologyInfo[i].params.maxGrceConfig[idx];
3330 }
3331 pTopoParams->maxExposeCeMask = pGpuMgr->nvlinkTopologyInfo[i].params.maxExposeCeMask;
3332 pTopoParams->maxTopoIdx = pGpuMgr->nvlinkTopologyInfo[i].params.maxTopoIdx;
3333 for (idx = 0; idx < NV2080_CTRL_CE_MAX_HSHUBS; idx++)
3334 {
3335 pTopoParams->pceAvailableMaskPerHshub[idx] = pGpuMgr->nvlinkTopologyInfo[i].params.pceAvailableMaskPerHshub[idx];
3336 }
3337 }
3338 return NV_TRUE;
3339 }
3340 }
3341
3342 return NV_FALSE;
3343 }
3344
3345 /*!
3346 * @brief Update the nvlinkTopologyInfo entry
3347 *
3348 * @param[in] DomainBusDevice: the PCI DomainBusDevice for the gpu to be registered
3349 * @param[in] pTopoParams: topology params to update the cache with
3350 *
3351 */
3352 void
gpumgrUpdateSystemNvlinkTopo_IMPL(NvU64 DomainBusDevice,PNVLINK_TOPOLOGY_PARAMS pTopoParams)3353 gpumgrUpdateSystemNvlinkTopo_IMPL
3354 (
3355 NvU64 DomainBusDevice,
3356 PNVLINK_TOPOLOGY_PARAMS pTopoParams
3357 )
3358 {
3359 OBJSYS *pSys = SYS_GET_INSTANCE();
3360 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
3361 NvU32 i, idx;
3362
3363 for (i = 0; i < NV_MAX_DEVICES; i++)
3364 {
3365 //
3366 // Choose the correct GPU by comparing PCI BusDomainDevice
3367 // This ensures we are using the same GPU across gpu load/unload
3368 //
3369 if ((pGpuMgr->nvlinkTopologyInfo[i].valid) && (pGpuMgr->nvlinkTopologyInfo[i].DomainBusDevice == DomainBusDevice))
3370 {
3371 pGpuMgr->nvlinkTopologyInfo[i].params.sysmemLinks = pTopoParams->sysmemLinks;
3372 pGpuMgr->nvlinkTopologyInfo[i].params.maxLinksPerPeer = pTopoParams->maxLinksPerPeer;
3373 pGpuMgr->nvlinkTopologyInfo[i].params.numLinks = pTopoParams->numLinks; // Pascal only
3374 pGpuMgr->nvlinkTopologyInfo[i].params.numPeers = pTopoParams->numPeers;
3375 pGpuMgr->nvlinkTopologyInfo[i].params.bSymmetric = pTopoParams->bSymmetric;
3376 pGpuMgr->nvlinkTopologyInfo[i].params.bSwitchConfig = pTopoParams->bSwitchConfig;
3377 // Ampere + only
3378 for (idx = 0; idx < NV2080_CTRL_MAX_PCES; idx++)
3379 {
3380 pGpuMgr->nvlinkTopologyInfo[i].params.maxPceLceMap[idx] = pTopoParams->maxPceLceMap[idx];
3381 }
3382 for (idx = 0; idx < NV2080_CTRL_MAX_GRCES; idx++)
3383 {
3384 pGpuMgr->nvlinkTopologyInfo[i].params.maxGrceConfig[idx] = pTopoParams->maxGrceConfig[idx];
3385 }
3386 pGpuMgr->nvlinkTopologyInfo[i].params.maxExposeCeMask = pTopoParams->maxExposeCeMask;
3387 pGpuMgr->nvlinkTopologyInfo[i].params.maxTopoIdx = pTopoParams->maxTopoIdx;
3388 for (idx = 0; idx < NV2080_CTRL_CE_MAX_HSHUBS; idx++)
3389 {
3390 pGpuMgr->nvlinkTopologyInfo[i].params.pceAvailableMaskPerHshub[idx] = pTopoParams->pceAvailableMaskPerHshub[idx];
3391 }
3392 return;
3393 }
3394 }
3395 }
3396
3397 /*!
3398 * @brief Check if GPUs are indirect peers
3399 *
3400 * @param[in] pGpu
3401 * @param[in] pRemoteGpu
3402 * @returns NV_TRUE if GPUs are indirect peers
3403 *
3404 */
3405 NvBool
gpumgrCheckIndirectPeer_IMPL(OBJGPU * pGpu,OBJGPU * pRemoteGpu)3406 gpumgrCheckIndirectPeer_IMPL
3407 (
3408 OBJGPU *pGpu,
3409 OBJGPU *pRemoteGpu
3410 )
3411 {
3412 #if defined(NVCPU_PPC64LE)
3413 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu);
3414
3415 if (pKernelNvlink == NULL)
3416 {
3417 return NV_FALSE;
3418 }
3419
3420 // If it's the same GPU or GPUs belonging to same SLI group, return early
3421 if ((pGpu == pRemoteGpu) ||
3422 (pGpu->deviceInstance == pRemoteGpu->deviceInstance))
3423 {
3424 return NV_FALSE;
3425 }
3426
3427 // If we are not on P9 + NVLINK2 systems then we don't support indirect peers
3428 if ((GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu)->coherentCpuFbBase == 0) ||
3429 (GPU_GET_KERNEL_MEMORY_SYSTEM(pRemoteGpu)->coherentCpuFbBase == 0))
3430 {
3431 return NV_FALSE;
3432 }
3433
3434 return !(knvlinkGetNumLinksToPeer(pGpu, pKernelNvlink, pRemoteGpu));
3435 #else
3436 return NV_FALSE;
3437 #endif
3438 }
3439
3440 /*!
3441 * @brief Set NVLinks (mask) for which initialization is disabled.
3442 *
3443 * @param[in] gpuId Platform specific GPU Id.
3444 * @param[in] mask Mask representing the links to be disabled.
3445 *
3446 * @return NV_OK on success, appropriate error on failure.
3447 */
3448 NV_STATUS
gpumgrSetGpuInitDisabledNvlinks_IMPL(NvU32 gpuId,NvU32 mask,NvBool bSkipHwNvlinkDisable)3449 gpumgrSetGpuInitDisabledNvlinks_IMPL
3450 (
3451 NvU32 gpuId,
3452 NvU32 mask,
3453 NvBool bSkipHwNvlinkDisable
3454 )
3455 {
3456 OBJSYS *pSys = SYS_GET_INSTANCE();
3457 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
3458 NV_STATUS status = NV_ERR_INVALID_DEVICE;
3459 NvU32 i;
3460
3461 if (gpumgrGetGpuFromId(gpuId) != NULL)
3462 {
3463 return NV_ERR_IN_USE;
3464 }
3465
3466 portSyncMutexAcquire(pGpuMgr->probedGpusLock);
3467
3468 for (i = 0; i < NV_ARRAY_ELEMENTS(pGpuMgr->probedGpus); ++i)
3469 {
3470 if (pGpuMgr->probedGpus[i].gpuId == gpuId)
3471 {
3472 // Mask will be validated during Nvlink construct.
3473 pGpuMgr->probedGpus[i].initDisabledNvlinksMask = mask;
3474 pGpuMgr->probedGpus[i].bSkipHwNvlinkDisable = bSkipHwNvlinkDisable;
3475 status = NV_OK;
3476 break;
3477 }
3478 }
3479
3480 portSyncMutexRelease(pGpuMgr->probedGpusLock);
3481
3482 return status;
3483 }
3484
3485 /*!
3486 * @brief Get NVLinks (mask) for which initialization is disabled.
3487 *
3488 * @param[in] gpuId Platform specific GPU Id.
3489 * @param[out] mask Mask representing the links to be disabled.
3490 *
3491 * @return NV_OK on success, appropriate error on failure.
3492 */
3493 NV_STATUS
gpumgrGetGpuInitDisabledNvlinks_IMPL(NvU32 gpuId,NvU32 * pMask,NvBool * pbSkipHwNvlinkDisable)3494 gpumgrGetGpuInitDisabledNvlinks_IMPL
3495 (
3496 NvU32 gpuId,
3497 NvU32 *pMask,
3498 NvBool *pbSkipHwNvlinkDisable
3499 )
3500 {
3501 OBJSYS *pSys = SYS_GET_INSTANCE();
3502 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
3503 NV_STATUS status = NV_ERR_INVALID_DEVICE;
3504 NvU32 i;
3505
3506 portSyncMutexAcquire(pGpuMgr->probedGpusLock);
3507
3508 for (i = 0; i < NV_ARRAY_ELEMENTS(pGpuMgr->probedGpus); ++i)
3509 {
3510 if (pGpuMgr->probedGpus[i].gpuId == gpuId)
3511 {
3512 *pMask = pGpuMgr->probedGpus[i].initDisabledNvlinksMask;
3513 *pbSkipHwNvlinkDisable = pGpuMgr->probedGpus[i].bSkipHwNvlinkDisable;
3514 status = NV_OK;
3515 break;
3516 }
3517 }
3518
3519 portSyncMutexRelease(pGpuMgr->probedGpusLock);
3520
3521 return status;
3522 }
3523
3524 /*!
3525 * @brief Get nvlink bandwidth mode
3526 *
3527 * @return mode reduced bandwidth mode.
3528 */
3529 NvU8
gpumgrGetGpuNvlinkBwMode_IMPL(void)3530 gpumgrGetGpuNvlinkBwMode_IMPL(void)
3531 {
3532 OBJSYS *pSys = SYS_GET_INSTANCE();
3533 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
3534
3535 return pGpuMgr->nvlinkBwMode;
3536 }
3537
3538 /*!
3539 * @brief Set nvlink bandwidth mode from Registry
3540 *
3541 * @param[in] pGpu reference of OBJGPU
3542 *
3543 */
3544 void
gpumgrSetGpuNvlinkBwModeFromRegistry_IMPL(OBJGPU * pGpu)3545 gpumgrSetGpuNvlinkBwModeFromRegistry_IMPL
3546 (
3547 OBJGPU *pGpu
3548 )
3549 {
3550 OBJSYS *pSys = SYS_GET_INSTANCE();
3551 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
3552 const char *pStrChar;
3553 NvU32 strLength = 32;
3554 NvU8 pStr[32];
3555
3556 //
3557 // An RM client can set NVLink BW mode using
3558 // NV0000_CTRL_CMD_GPU_SET_NVLINK_BW_MODE control call.
3559 // If the value is not default i.e. `GPU_NVLINK_BW_MODE_FULL`, then skip.
3560 //
3561 if (pGpuMgr->nvlinkBwMode != GPU_NVLINK_BW_MODE_FULL)
3562 {
3563 return;
3564 }
3565
3566 // sysInitRegistryOverrides should pass in valid pGpu
3567 NV_ASSERT (pGpu != NULL);
3568
3569 if (osReadRegistryString(pGpu, NV_REG_STR_RM_NVLINK_BW, pStr, &strLength) != NV_OK)
3570 {
3571 goto out;
3572 }
3573
3574 pStrChar = (const char *)pStr;
3575 strLength = portStringLength(pStrChar);
3576 if (portStringCompare(pStrChar, "OFF", strLength) == 0)
3577 {
3578 pGpuMgr->nvlinkBwMode = GPU_NVLINK_BW_MODE_OFF;
3579 }
3580 else if (portStringCompare(pStrChar, "MIN", strLength) == 0)
3581 {
3582 pGpuMgr->nvlinkBwMode = GPU_NVLINK_BW_MODE_MIN;
3583 }
3584 else if (portStringCompare(pStrChar, "HALF", strLength) == 0)
3585 {
3586 pGpuMgr->nvlinkBwMode = GPU_NVLINK_BW_MODE_HALF;
3587 }
3588 else if (portStringCompare(pStrChar, "3QUARTER", strLength) == 0)
3589 {
3590 pGpuMgr->nvlinkBwMode = GPU_NVLINK_BW_MODE_3QUARTER;
3591 }
3592
3593 out:
3594 NV_PRINTF(LEVEL_INFO, "nvlinkBwMode=%d\n", pGpuMgr->nvlinkBwMode);
3595 }
3596
3597 static NvBool
_gpumgrIsP2PObjectPresent(void)3598 _gpumgrIsP2PObjectPresent(void)
3599 {
3600 NvU32 attachedGpuCount;
3601 NvU32 attachedGpuMask;
3602 KernelBus *pKernelBus;
3603 NV_STATUS rmStatus;
3604 NvU32 gpuIndex;
3605 OBJGPU *pGpu;
3606
3607 rmStatus = gpumgrGetGpuAttachInfo(&attachedGpuCount, &attachedGpuMask);
3608 if (rmStatus != NV_OK)
3609 {
3610 return NV_FALSE;
3611 }
3612
3613 gpuIndex = 0;
3614 for(pGpu = gpumgrGetNextGpu(attachedGpuMask, &gpuIndex);
3615 pGpu != NULL;
3616 pGpu = gpumgrGetNextGpu(attachedGpuMask, &gpuIndex))
3617 {
3618 pKernelBus = GPU_GET_KERNEL_BUS(pGpu);
3619 if (pKernelBus->totalP2pObjectsAliveRefCount > 0)
3620 {
3621 return NV_TRUE;
3622 }
3623 }
3624
3625 return NV_FALSE;
3626 }
3627
3628 /*!
3629 * @brief Set nvlink bandwidth mode
3630 *
3631 * @param[in] mode nvlink bandwidth mode
3632 *
3633 * @return NV_OK on success, appropriate error on failure.
3634 */
3635 NV_STATUS
gpumgrSetGpuNvlinkBwMode_IMPL(NvU8 mode)3636 gpumgrSetGpuNvlinkBwMode_IMPL
3637 (
3638 NvU8 mode
3639 )
3640 {
3641 OBJSYS *pSys = SYS_GET_INSTANCE();
3642 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
3643 NV_STATUS status;
3644
3645 if (mode == pGpuMgr->nvlinkBwMode)
3646 {
3647 return NV_OK;
3648 }
3649
3650 if (_gpumgrIsP2PObjectPresent())
3651 {
3652 return NV_ERR_IN_USE;
3653 }
3654
3655 if (mode > GPU_NVLINK_BW_MODE_3QUARTER)
3656 {
3657 return NV_ERR_INVALID_ARGUMENT;
3658 }
3659
3660 status = gpuFabricProbeSetBwMode(mode);
3661 if (status != NV_OK)
3662 {
3663 return status;
3664 }
3665
3666 pGpuMgr->nvlinkBwMode = mode;
3667 return NV_OK;
3668 }
3669
3670 /*!
3671 * @brief Adds an entry in the system partition topology save for the given GPU
3672 * ID. Note that this does not create any saved partition topology.
3673 *
3674 * @param[in] DomainBusDevice: the PCI DomainBusDevice for the gpu to be registered
3675 */
3676 void
gpumgrAddSystemMIGInstanceTopo_IMPL(NvU64 domainBusDevice)3677 gpumgrAddSystemMIGInstanceTopo_IMPL
3678 (
3679 NvU64 domainBusDevice
3680 )
3681 {
3682 OBJSYS *pSys = SYS_GET_INSTANCE();
3683 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
3684 NvU32 i;
3685
3686 if (gpumgrGetSystemMIGInstanceTopo(domainBusDevice, NULL))
3687 {
3688 // This gpu is already registered
3689 return;
3690 }
3691
3692 for (i = 0; i < NV_ARRAY_ELEMENTS(pGpuMgr->MIGTopologyInfo); i++)
3693 {
3694 // add new gpu entry into the first slot available
3695 if (!pGpuMgr->MIGTopologyInfo[i].bValid)
3696 {
3697 pGpuMgr->MIGTopologyInfo[i].bValid = NV_TRUE;
3698 pGpuMgr->MIGTopologyInfo[i].domainBusDevice = domainBusDevice;
3699
3700 // Set MIG enablement to disabled by default
3701 pGpuMgr->MIGTopologyInfo[i].bMIGEnabled = NV_FALSE;
3702 break;
3703 }
3704 }
3705
3706 // Shouldn't be possible to not find an open slot
3707 NV_ASSERT(i < NV_ARRAY_ELEMENTS(pGpuMgr->MIGTopologyInfo));
3708 }
3709
3710 /*!
3711 * @brief Retrieves the entry in the system partition topology save for the given GPU
3712 * ID. Note that retrieval of this entry does not constitute a guarantee
3713 * that there is any valid data saved.
3714 *
3715 * @param[in] DomainBusDevice: the PCI DomainBusDevice for the gpu to be registered
3716 * @param[out] ppTopology: Stores the saved MIG topology for the given GPU, if
3717 * found.
3718 *
3719 * @returns NV_TRUE if entry found
3720 * NV_FALSE otherwise
3721 */
3722 NvBool
gpumgrGetSystemMIGInstanceTopo_IMPL(NvU64 domainBusDevice,GPUMGR_SAVE_MIG_INSTANCE_TOPOLOGY ** ppTopology)3723 gpumgrGetSystemMIGInstanceTopo_IMPL
3724 (
3725 NvU64 domainBusDevice,
3726 GPUMGR_SAVE_MIG_INSTANCE_TOPOLOGY **ppTopology
3727 )
3728 {
3729 OBJSYS *pSys = SYS_GET_INSTANCE();
3730 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
3731 NvU32 i;
3732
3733 for (i = 0; i < NV_ARRAY_ELEMENTS(pGpuMgr->MIGTopologyInfo); i++)
3734 {
3735 //
3736 // Choose the correct GPU by comparing PCI BusDomainDevice
3737 // This ensures we are using the same GPU across gpu load/unload
3738 //
3739 if (!pGpuMgr->MIGTopologyInfo[i].bValid ||
3740 (pGpuMgr->MIGTopologyInfo[i].domainBusDevice != domainBusDevice))
3741 {
3742 continue;
3743 }
3744
3745 if (NULL != ppTopology)
3746 *ppTopology = &pGpuMgr->MIGTopologyInfo[i];
3747
3748 return NV_TRUE;
3749 }
3750
3751 return NV_FALSE;
3752 }
3753
3754
3755 /*!
3756 * @brief Retrieves the entry in the system partition topology save for the given GPU
3757 * ID and returns value of bMIGEnabled.
3758 *
3759 * @param[in] domainBusDevice: the PCI DomainBusDevice for the gpu to be registered
3760 *
3761 * @returns NV_TRUE if entry found
3762 * NV_FALSE otherwise
3763 */
3764 NvBool
gpumgrIsSystemMIGEnabled_IMPL(NvU64 domainBusDevice)3765 gpumgrIsSystemMIGEnabled_IMPL
3766 (
3767 NvU64 domainBusDevice
3768 )
3769 {
3770 OBJSYS *pSys = SYS_GET_INSTANCE();
3771 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
3772 NvU32 i;
3773
3774 for (i = 0; i < NV_ARRAY_ELEMENTS(pGpuMgr->MIGTopologyInfo); i++)
3775 {
3776 //
3777 // Choose the correct GPU by comparing PCI BusDomainDevice
3778 // This ensures we are using the same GPU across gpu load/unload
3779 //
3780 if (pGpuMgr->MIGTopologyInfo[i].bValid &&
3781 (pGpuMgr->MIGTopologyInfo[i].domainBusDevice == domainBusDevice))
3782 {
3783 return pGpuMgr->MIGTopologyInfo[i].bMIGEnabled;
3784 }
3785 }
3786
3787 return NV_FALSE;
3788 }
3789
3790
3791 /*!
3792 * @brief Retrieves the entry in the system partition topology save for the given GPU
3793 * ID and sets value of bMIGEnabled.
3794 *
3795 * @param[in] DomainBusDevice: the PCI DomainBusDevice for the gpu to be registered
3796 * @param[in] bMIGEnabled: The new MIG enablement state to be set
3797 *
3798 * @returns NV_TRUE if entry found
3799 * NV_FALSE otherwise
3800 */
3801 void
gpumgrSetSystemMIGEnabled_IMPL(NvU64 domainBusDevice,NvBool bMIGEnabled)3802 gpumgrSetSystemMIGEnabled_IMPL
3803 (
3804 NvU64 domainBusDevice,
3805 NvBool bMIGEnabled
3806 )
3807 {
3808 OBJSYS *pSys = SYS_GET_INSTANCE();
3809 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
3810 NvU32 i;
3811
3812 for (i = 0; i < NV_ARRAY_ELEMENTS(pGpuMgr->MIGTopologyInfo); i++)
3813 {
3814 //
3815 // Choose the correct GPU by comparing PCI BusDomainDevice
3816 // This ensures we are using the same GPU across gpu load/unload
3817 //
3818 if (pGpuMgr->MIGTopologyInfo[i].bValid &&
3819 (pGpuMgr->MIGTopologyInfo[i].domainBusDevice == domainBusDevice))
3820 {
3821 pGpuMgr->MIGTopologyInfo[i].bMIGEnabled = bMIGEnabled;
3822 break;
3823 }
3824 }
3825 }
3826
3827 static void
_gpumgrUnregisterRmCapsForMIGCI(GPUMGR_SAVE_GPU_INSTANCE * pGPUInstanceSave)3828 _gpumgrUnregisterRmCapsForMIGCI(GPUMGR_SAVE_GPU_INSTANCE *pGPUInstanceSave)
3829 {
3830 GPUMGR_SAVE_COMPUTE_INSTANCE *pComputeInstanceSave;
3831 NvU32 i;
3832
3833 for (i = 0; i < NV_ARRAY_ELEMENTS(pGPUInstanceSave->saveCI); i++)
3834 {
3835 pComputeInstanceSave = &pGPUInstanceSave->saveCI[i];
3836
3837 if (pComputeInstanceSave->bValid)
3838 {
3839 osRmCapUnregister(&pComputeInstanceSave->pOsRmCaps);
3840
3841 //
3842 // Mark invalid as the partition caps have been unregistered from RM
3843 // completely.
3844 //
3845 pComputeInstanceSave->bValid = NV_FALSE;
3846 }
3847 }
3848 }
3849
3850 void
gpumgrUnregisterRmCapsForMIGGI_IMPL(NvU64 gpuDomainBusDevice)3851 gpumgrUnregisterRmCapsForMIGGI_IMPL(NvU64 gpuDomainBusDevice)
3852 {
3853 GPUMGR_SAVE_MIG_INSTANCE_TOPOLOGY *pTopologySave;
3854 GPUMGR_SAVE_GPU_INSTANCE *pGPUInstanceSave;
3855 NvU32 i;
3856
3857 if (!gpumgrGetSystemMIGInstanceTopo(gpuDomainBusDevice, &pTopologySave))
3858 {
3859 return;
3860 }
3861
3862 for (i = 0; i < NV_ARRAY_ELEMENTS(pTopologySave->saveGI); i++)
3863 {
3864 pGPUInstanceSave = &pTopologySave->saveGI[i];
3865
3866 if (pGPUInstanceSave->bValid)
3867 {
3868 _gpumgrUnregisterRmCapsForMIGCI(pGPUInstanceSave);
3869
3870 osRmCapUnregister(&pGPUInstanceSave->pOsRmCaps);
3871
3872 //
3873 // Mark invalid as the partition caps have been unregistered from RM
3874 // completely.
3875 //
3876 pGPUInstanceSave->bValid = NV_FALSE;
3877 }
3878 }
3879 }
3880
3881 void
gpumgrCacheCreateGpuInstance_IMPL(OBJGPU * pGpu,NvU32 swizzId)3882 gpumgrCacheCreateGpuInstance_IMPL(OBJGPU *pGpu, NvU32 swizzId)
3883 {
3884 OBJSYS *pSys = SYS_GET_INSTANCE();
3885 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
3886 GPUMGR_CACHED_MIG_GPU_INSTANCE *pGpuInstances = NULL;
3887 NvU32 i;
3888
3889 portSyncRwLockAcquireWrite(pGpuMgr->cachedMIGInfoLock);
3890
3891 for (i = 0; i < NV_MAX_DEVICES; i++)
3892 {
3893 if (pGpuMgr->cachedMIGInfo[i].gpuId == pGpu->gpuId)
3894 {
3895 pGpuInstances = pGpuMgr->cachedMIGInfo[i].gpuInstances;
3896 break;
3897 }
3898 }
3899
3900 NV_ASSERT_OR_GOTO(pGpuInstances != NULL, done);
3901
3902 for (i = 0; i < GPUMGR_MAX_GPU_INSTANCES; i++)
3903 {
3904 if (!pGpuInstances[i].bValid)
3905 {
3906 pGpuInstances[i].bValid = NV_TRUE;
3907 pGpuInstances[i].swizzId = swizzId;
3908 goto done;
3909 }
3910 }
3911
3912 NV_ASSERT(!"Unreachable");
3913
3914 done:
3915 portSyncRwLockReleaseWrite(pGpuMgr->cachedMIGInfoLock);
3916 }
3917
3918 void
gpumgrCacheDestroyGpuInstance_IMPL(OBJGPU * pGpu,NvU32 swizzId)3919 gpumgrCacheDestroyGpuInstance_IMPL(OBJGPU *pGpu, NvU32 swizzId)
3920 {
3921 OBJSYS *pSys = SYS_GET_INSTANCE();
3922 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
3923 GPUMGR_CACHED_MIG_GPU_INSTANCE *pGpuInstances = NULL;
3924 NvU32 i;
3925
3926 portSyncRwLockAcquireWrite(pGpuMgr->cachedMIGInfoLock);
3927
3928 for (i = 0; i < NV_MAX_DEVICES; i++)
3929 {
3930 if (pGpuMgr->cachedMIGInfo[i].gpuId == pGpu->gpuId)
3931 {
3932 pGpuInstances = pGpuMgr->cachedMIGInfo[i].gpuInstances;
3933 break;
3934 }
3935 }
3936
3937 NV_ASSERT_OR_GOTO(pGpuInstances != NULL, done);
3938
3939 for (i = 0; i < GPUMGR_MAX_GPU_INSTANCES; i++)
3940 {
3941 if (pGpuInstances[i].bValid &&
3942 pGpuInstances[i].swizzId == swizzId)
3943 {
3944 pGpuInstances[i].bValid = NV_FALSE;
3945 goto done;
3946 }
3947 }
3948
3949 NV_ASSERT(!"Unreachable");
3950
3951 done:
3952 portSyncRwLockReleaseWrite(pGpuMgr->cachedMIGInfoLock);
3953 }
3954
3955 void
gpumgrCacheCreateComputeInstance_IMPL(OBJGPU * pGpu,NvU32 swizzId,NvU32 ciId)3956 gpumgrCacheCreateComputeInstance_IMPL(OBJGPU *pGpu, NvU32 swizzId, NvU32 ciId)
3957 {
3958 OBJSYS *pSys = SYS_GET_INSTANCE();
3959 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
3960 GPUMGR_CACHED_MIG_GPU_INSTANCE *pGpuInstances = NULL;
3961 NvU32 i;
3962
3963 portSyncRwLockAcquireWrite(pGpuMgr->cachedMIGInfoLock);
3964
3965 for (i = 0; i < NV_MAX_DEVICES; i++)
3966 {
3967 if (pGpuMgr->cachedMIGInfo[i].gpuId == pGpu->gpuId)
3968 {
3969 pGpuInstances = pGpuMgr->cachedMIGInfo[i].gpuInstances;
3970 break;
3971 }
3972 }
3973
3974 NV_ASSERT_OR_GOTO(pGpuInstances != NULL, done);
3975
3976 for (i = 0; i < GPUMGR_MAX_GPU_INSTANCES; i++)
3977 {
3978 if (pGpuInstances[i].bValid &&
3979 pGpuInstances[i].swizzId == swizzId)
3980 {
3981 pGpuInstances[i].bValidComputeInstances[ciId] = NV_TRUE;
3982 goto done;
3983 }
3984 }
3985
3986 NV_ASSERT(!"Unreachable");
3987
3988 done:
3989 portSyncRwLockReleaseWrite(pGpuMgr->cachedMIGInfoLock);
3990 }
3991
3992 void
gpumgrCacheDestroyComputeInstance_IMPL(OBJGPU * pGpu,NvU32 swizzId,NvU32 ciId)3993 gpumgrCacheDestroyComputeInstance_IMPL(OBJGPU *pGpu, NvU32 swizzId, NvU32 ciId)
3994 {
3995 OBJSYS *pSys = SYS_GET_INSTANCE();
3996 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
3997 GPUMGR_CACHED_MIG_GPU_INSTANCE *pGpuInstances = NULL;
3998 NvU32 i;
3999
4000 portSyncRwLockAcquireWrite(pGpuMgr->cachedMIGInfoLock);
4001
4002 for (i = 0; i < NV_MAX_DEVICES; i++)
4003 {
4004 if (pGpuMgr->cachedMIGInfo[i].gpuId == pGpu->gpuId)
4005 {
4006 pGpuInstances = pGpuMgr->cachedMIGInfo[i].gpuInstances;
4007 break;
4008 }
4009 }
4010
4011 NV_ASSERT_OR_GOTO(pGpuInstances != NULL, done);
4012
4013 for (i = 0; i < GPUMGR_MAX_GPU_INSTANCES; i++)
4014 {
4015 if (pGpuInstances[i].bValid &&
4016 pGpuInstances[i].swizzId == swizzId)
4017 {
4018 pGpuInstances[i].bValidComputeInstances[ciId] = NV_FALSE;
4019 goto done;
4020 }
4021 }
4022
4023 NV_ASSERT(!"Unreachable");
4024
4025 done:
4026 portSyncRwLockReleaseWrite(pGpuMgr->cachedMIGInfoLock);
4027 }
4028
4029 void
gpumgrCacheSetMIGEnabled_IMPL(OBJGPU * pGpu,NvBool bMIGEnabled)4030 gpumgrCacheSetMIGEnabled_IMPL(OBJGPU *pGpu, NvBool bMIGEnabled)
4031 {
4032 OBJSYS *pSys = SYS_GET_INSTANCE();
4033 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
4034 GPUMGR_CACHED_MIG_STATE *pMIGInfo = NULL;
4035 NvU32 gpuId = pGpu->gpuId;
4036 NvU32 i;
4037
4038 portSyncRwLockAcquireWrite(pGpuMgr->cachedMIGInfoLock);
4039
4040 for (i = 0; i < NV_MAX_DEVICES; i++)
4041 {
4042 if (pGpuMgr->cachedMIGInfo[i].bValid &&
4043 (pGpuMgr->cachedMIGInfo[i].gpuId == gpuId))
4044 {
4045 pMIGInfo = &pGpuMgr->cachedMIGInfo[i];
4046 break;
4047 }
4048 }
4049
4050 if (pMIGInfo == NULL)
4051 {
4052 // Get first invalid entry, for not yet seen gpuId.
4053 for (i = 0; i < NV_MAX_DEVICES; i++)
4054 {
4055 if (!pGpuMgr->cachedMIGInfo[i].bValid)
4056 {
4057 pMIGInfo = &pGpuMgr->cachedMIGInfo[i];
4058 pMIGInfo->bValid = NV_TRUE;
4059 pMIGInfo->gpuId = gpuId;
4060 break;
4061 }
4062 }
4063 }
4064
4065 NV_ASSERT_OR_GOTO(pMIGInfo != NULL, done);
4066
4067 pMIGInfo->bMIGEnabled = bMIGEnabled;
4068 if (!bMIGEnabled)
4069 {
4070 portMemSet(&pMIGInfo->gpuInstances, 0x0, sizeof(pMIGInfo->gpuInstances));
4071 }
4072
4073 done:
4074 portSyncRwLockReleaseWrite(pGpuMgr->cachedMIGInfoLock);
4075 }
4076
4077 NV_STATUS
gpumgrCacheGetActiveDeviceIds_IMPL(NV0000_CTRL_GPU_GET_ACTIVE_DEVICE_IDS_PARAMS * pActiveDeviceIdsParams)4078 gpumgrCacheGetActiveDeviceIds_IMPL
4079 (
4080 NV0000_CTRL_GPU_GET_ACTIVE_DEVICE_IDS_PARAMS *pActiveDeviceIdsParams
4081 )
4082 {
4083 NV0000_CTRL_GPU_GET_PROBED_IDS_PARAMS probedGpuIds;
4084 NV0000_CTRL_GPU_ACTIVE_DEVICE *pDevices = pActiveDeviceIdsParams->devices;
4085 NvU32 *pNumDevices = &pActiveDeviceIdsParams->numDevices;
4086 NvU32 total = 0;
4087 NvU32 i;
4088
4089 NV_ASSERT_OK_OR_RETURN(gpumgrGetProbedGpuIds(&probedGpuIds));
4090
4091 OBJSYS *pSys = SYS_GET_INSTANCE();
4092 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
4093 NV_STATUS status = NV_OK;
4094
4095 portSyncRwLockAcquireRead(pGpuMgr->cachedMIGInfoLock);
4096
4097 // Walk probed gpus.
4098 for (i = 0;
4099 (i < NV0000_CTRL_GPU_MAX_PROBED_GPUS) &&
4100 (probedGpuIds.gpuIds[i] != NV0000_CTRL_GPU_INVALID_ID); i++)
4101 {
4102 NvU32 gpuId = probedGpuIds.gpuIds[i];
4103 NvBool bGpuHandled = NV_FALSE;
4104 NvU32 gpuIdx;
4105
4106 for (gpuIdx = 0; gpuIdx < NV_MAX_DEVICES; gpuIdx++)
4107 {
4108 NvU32 giIdx;
4109 GPUMGR_CACHED_MIG_STATE *pMIGState = &pGpuMgr->cachedMIGInfo[gpuIdx];
4110 GPUMGR_CACHED_MIG_GPU_INSTANCE *pGpuInstances = pMIGState->gpuInstances;
4111
4112 if (!pMIGState->bValid || (pMIGState->gpuId != gpuId))
4113 {
4114 continue;
4115 }
4116
4117 // MIG not enabled, add device to list and carry on
4118 if (!pMIGState->bMIGEnabled)
4119 {
4120 break;
4121 }
4122
4123 bGpuHandled = NV_TRUE;
4124
4125 for (giIdx = 0; giIdx < GPUMGR_MAX_GPU_INSTANCES; giIdx++)
4126 {
4127 NvU32 ciIdx;
4128 NvU32 swizzId = pGpuInstances[giIdx].swizzId;
4129 NvBool *pbValidComputeInstances =
4130 pGpuInstances[giIdx].bValidComputeInstances;
4131
4132 if (!pGpuInstances[giIdx].bValid)
4133 continue;
4134
4135 for (ciIdx = 0; ciIdx < GPUMGR_MAX_COMPUTE_INSTANCES; ciIdx++)
4136 {
4137 if (pbValidComputeInstances[ciIdx])
4138 {
4139 NV_ASSERT_OR_ELSE(total < NV0000_CTRL_GPU_MAX_ACTIVE_DEVICES,
4140 status = NV_ERR_INVALID_STATE; goto done; );
4141
4142 pDevices[total].gpuId = gpuId;
4143 pDevices[total].gpuInstanceId = swizzId;
4144 pDevices[total].computeInstanceId = ciIdx;
4145 total++;
4146 }
4147 }
4148 }
4149 }
4150
4151 // Not in MIG mode, or we never had it in cache.
4152 if (!bGpuHandled)
4153 {
4154 NV_ASSERT_OR_ELSE(total < NV0000_CTRL_GPU_MAX_ACTIVE_DEVICES,
4155 status = NV_ERR_INVALID_STATE; goto done; );
4156
4157 pDevices[total].gpuId = gpuId;
4158 pDevices[total].gpuInstanceId = NV0000_CTRL_GPU_INVALID_ID;
4159 pDevices[total].computeInstanceId = NV0000_CTRL_GPU_INVALID_ID;
4160 total++;
4161 }
4162 }
4163
4164 *pNumDevices = total;
4165
4166 done:
4167 portSyncRwLockReleaseRead(pGpuMgr->cachedMIGInfoLock);
4168
4169 return status;
4170 }
4171
4172 /**
4173 * @brief Saves a pointer to the current GPU instance in thread local storage,
4174 * to be logged by NVLOG, until gpumgrSetGpuRelease is called.
4175 * Returns a pointer to tls entry (to be passed to gpumgrSetGpuRelease)
4176 *
4177 * @param[in] pGpu
4178 */
4179 NvBool
gpumgrSetGpuAcquire(OBJGPU * pGpu)4180 gpumgrSetGpuAcquire(OBJGPU *pGpu)
4181 {
4182 NvU32 **ppGpuInstance;
4183 ppGpuInstance = (NvU32 **)tlsEntryAcquire
4184 (TLS_ENTRY_ID_CURRENT_GPU_INSTANCE);
4185 if (ppGpuInstance)
4186 {
4187 *ppGpuInstance = &(pGpu->gpuInstance);
4188 return NV_TRUE;
4189 }
4190 return NV_FALSE;
4191 }
4192
4193 /**
4194 * @brief Releases the thread local storage for GPU ID.
4195 */
4196 void
gpumgrSetGpuRelease(void)4197 gpumgrSetGpuRelease(void)
4198 {
4199 tlsEntryRelease(TLS_ENTRY_ID_CURRENT_GPU_INSTANCE);
4200 }
4201
4202 /**
4203 * @brief Returns the type of bridge SLI_BT_*
4204 */
4205 NvU8
gpumgrGetGpuBridgeType(void)4206 gpumgrGetGpuBridgeType(void)
4207 {
4208 OBJSYS *pSys = SYS_GET_INSTANCE();
4209 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
4210
4211 return pGpuMgr->gpuBridgeType;
4212 }
4213
4214 /**
4215 * @brief Init the PCIE P2P info cache
4216 */
4217 NV_STATUS
gpumgrInitPcieP2PCapsCache_IMPL(OBJGPUMGR * pGpuMgr)4218 gpumgrInitPcieP2PCapsCache_IMPL(OBJGPUMGR* pGpuMgr)
4219 {
4220 listInitIntrusive(&pGpuMgr->pcieP2PCapsInfoCache);
4221 pGpuMgr->pcieP2PCapsInfoLock = portSyncMutexCreate(portMemAllocatorGetGlobalNonPaged());
4222 if (pGpuMgr->pcieP2PCapsInfoLock == NULL)
4223 {
4224 return NV_ERR_INSUFFICIENT_RESOURCES;
4225 }
4226 return NV_OK;
4227 }
4228
4229 /**
4230 * @brief Destroy the PCIE P2P info cache
4231 */
4232 void
gpumgrDestroyPcieP2PCapsCache_IMPL(OBJGPUMGR * pGpuMgr)4233 gpumgrDestroyPcieP2PCapsCache_IMPL(OBJGPUMGR* pGpuMgr)
4234 {
4235 PCIEP2PCAPSINFO *pPcieCapsInfo, *pPcieCapsInfoNext;
4236
4237 portSyncMutexAcquire(pGpuMgr->pcieP2PCapsInfoLock);
4238
4239 // Remove and free all entries that have this GPU
4240 for (pPcieCapsInfo = listHead(&(pGpuMgr->pcieP2PCapsInfoCache));
4241 pPcieCapsInfo != NULL;
4242 pPcieCapsInfo = pPcieCapsInfoNext)
4243 {
4244 pPcieCapsInfoNext = listNext(&(pGpuMgr->pcieP2PCapsInfoCache), pPcieCapsInfo);
4245 portMemFree(pPcieCapsInfo);
4246 }
4247
4248 listDestroy(&pGpuMgr->pcieP2PCapsInfoCache);
4249 portSyncMutexRelease(pGpuMgr->pcieP2PCapsInfoLock);
4250
4251 portSyncMutexDestroy(pGpuMgr->pcieP2PCapsInfoLock);
4252 }
4253
4254 /**
4255 * @brief Add an entry in the PCIE P2P info cache
4256 * @param[in] gpuMask NvU32 value
4257 * @param[in] p2pWriteCapsStatus NvU8 value
4258 * @param[in] pP2PReadCapsStatus NvU8 value
4259 *
4260 * @return NV_OK or NV_ERR_NO_MEMORY
4261 */
4262 NV_STATUS
gpumgrStorePcieP2PCapsCache_IMPL(NvU32 gpuMask,NvU8 p2pWriteCapsStatus,NvU8 p2pReadCapsStatus)4263 gpumgrStorePcieP2PCapsCache_IMPL
4264 (
4265 NvU32 gpuMask,
4266 NvU8 p2pWriteCapsStatus,
4267 NvU8 p2pReadCapsStatus
4268 )
4269 {
4270 OBJSYS *pSys = SYS_GET_INSTANCE();
4271 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
4272 PCIEP2PCAPSINFO *pPcieCapsInfo;
4273 NvU32 gpuInstance;
4274 OBJGPU *pGpu;
4275 NvU32 gpuCount = 0;
4276 NvU32 status = NV_OK;
4277
4278 portSyncMutexAcquire(pGpuMgr->pcieP2PCapsInfoLock);
4279 if (_gpumgrGetPcieP2PCapsFromCache(gpuMask, NULL, NULL))
4280 {
4281 // Entry already present in cache
4282 goto exit;
4283 }
4284
4285 pPcieCapsInfo = portMemAllocNonPaged(sizeof(PCIEP2PCAPSINFO));
4286 if (pPcieCapsInfo == NULL)
4287 {
4288 status = NV_ERR_NO_MEMORY;
4289 goto exit;
4290 }
4291 listAppendExisting(&(pGpuMgr->pcieP2PCapsInfoCache), pPcieCapsInfo);
4292
4293 gpuInstance = 0;
4294 while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
4295 {
4296 pPcieCapsInfo->gpuId[gpuCount] = pGpu->gpuId;
4297 gpuCount++;
4298 }
4299
4300 pPcieCapsInfo->gpuCount = gpuCount;
4301 pPcieCapsInfo->p2pWriteCapsStatus = p2pWriteCapsStatus;
4302 pPcieCapsInfo->p2pReadCapsStatus = p2pReadCapsStatus;
4303
4304 exit:
4305 portSyncMutexRelease(pGpuMgr->pcieP2PCapsInfoLock);
4306 return status;
4307 }
4308
4309 /**
4310 * @brief Get the PCIE P2P info from cache if present
4311 * - Helper function
4312 *
4313 * @param[in] gpuMask NvU32 value
4314 * @param[out] pP2PWriteCapsStatus NvU8* pointer
4315 * Can be NULL
4316 * @param[out] pP2PReadCapsStatus NvU8* pointer
4317 * Can be NULL
4318 * Return bFound NvBool
4319 */
4320 static NvBool
_gpumgrGetPcieP2PCapsFromCache(NvU32 gpuMask,NvU8 * pP2PWriteCapsStatus,NvU8 * pP2PReadCapsStatus)4321 _gpumgrGetPcieP2PCapsFromCache
4322 (
4323 NvU32 gpuMask,
4324 NvU8 *pP2PWriteCapsStatus,
4325 NvU8 *pP2PReadCapsStatus)
4326 {
4327 OBJSYS *pSys = SYS_GET_INSTANCE();
4328 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
4329 PCIEP2PCAPSINFO *pPcieCapsInfo;
4330 pcieP2PCapsInfoListIter it;
4331 NvU32 gpuInstance;
4332 NvU32 gpuCount;
4333 NvU32 remainingGpuCount;
4334 OBJGPU *pGpu;
4335 NvU32 gpuIdLoop;
4336 NvBool bFound = NV_FALSE;
4337
4338 gpuCount = gpumgrGetSubDeviceCount(gpuMask);
4339
4340 it = listIterAll(&pGpuMgr->pcieP2PCapsInfoCache);
4341 while (listIterNext(&it))
4342 {
4343 pPcieCapsInfo = it.pValue;
4344 if (gpuCount != pPcieCapsInfo->gpuCount)
4345 {
4346 continue;
4347 }
4348
4349 //
4350 // Same count of GPUs in gpuId array and GPU mask.
4351 // All GPU in the gpuMask must have a match in gpuId[]
4352 //
4353 gpuInstance = 0;
4354 remainingGpuCount = gpuCount;
4355 while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
4356 {
4357 gpuIdLoop = 0;
4358 while (gpuIdLoop < gpuCount)
4359 {
4360 if (pPcieCapsInfo->gpuId[gpuIdLoop] == pGpu->gpuId)
4361 {
4362 remainingGpuCount--;
4363 break;
4364 }
4365 gpuIdLoop++;
4366 }
4367 if (remainingGpuCount == 0)
4368 {
4369 break;
4370 }
4371 }
4372
4373 if (remainingGpuCount == 0)
4374 {
4375 if (pP2PWriteCapsStatus != NULL)
4376 *pP2PWriteCapsStatus = pPcieCapsInfo->p2pWriteCapsStatus;
4377 if (pP2PReadCapsStatus != NULL)
4378 *pP2PReadCapsStatus = pPcieCapsInfo->p2pReadCapsStatus;
4379 bFound = NV_TRUE;
4380 break;
4381 }
4382 }
4383 return bFound;
4384 }
4385
4386 /**
4387 * @brief Get the PCIE P2P info from cache if present
4388 * - Take cache locks
4389 *
4390 * @param[in] gpuMask NvU32 value
4391 * @param[out] pP2PWriteCapsStatus NvU8* pointer
4392 * Can be NULL
4393 * @param[out] pP2PReadCapsStatus NvU8* pointer
4394 * Can be NULL
4395 *
4396 * return bFound NvBool
4397 */
4398 NvBool
gpumgrGetPcieP2PCapsFromCache_IMPL(NvU32 gpuMask,NvU8 * pP2PWriteCapsStatus,NvU8 * pP2PReadCapsStatus)4399 gpumgrGetPcieP2PCapsFromCache_IMPL
4400 (
4401 NvU32 gpuMask,
4402 NvU8 *pP2PWriteCapsStatus,
4403 NvU8 *pP2PReadCapsStatus
4404 )
4405 {
4406 OBJSYS *pSys = SYS_GET_INSTANCE();
4407 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
4408 NvBool bFound;
4409
4410 portSyncMutexAcquire(pGpuMgr->pcieP2PCapsInfoLock);
4411
4412 bFound = _gpumgrGetPcieP2PCapsFromCache(gpuMask, pP2PWriteCapsStatus, pP2PReadCapsStatus);
4413
4414 portSyncMutexRelease(pGpuMgr->pcieP2PCapsInfoLock);
4415
4416 return bFound;
4417 }
4418
4419
4420 /**
4421 * @brief Remove the PCIE P2P info from cache if present
4422 *
4423 * @param[in] gpuId NvU32 value
4424 */
4425 void
gpumgrRemovePcieP2PCapsFromCache_IMPL(NvU32 gpuId)4426 gpumgrRemovePcieP2PCapsFromCache_IMPL
4427 (
4428 NvU32 gpuId
4429 )
4430 {
4431 OBJSYS *pSys = SYS_GET_INSTANCE();
4432 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
4433 PCIEP2PCAPSINFO *pPcieCapsInfo, *pPcieCapsInfoNext;
4434 NvU32 gpuIdLoop;
4435
4436 portSyncMutexAcquire(pGpuMgr->pcieP2PCapsInfoLock);
4437
4438 // Remove and free all entries that have this GPU
4439 for (pPcieCapsInfo = listHead(&(pGpuMgr->pcieP2PCapsInfoCache));
4440 pPcieCapsInfo != NULL;
4441 pPcieCapsInfo = pPcieCapsInfoNext)
4442 {
4443 // As we potentially remove an entry we need to save off the next one.
4444 pPcieCapsInfoNext = listNext(&(pGpuMgr->pcieP2PCapsInfoCache), pPcieCapsInfo);
4445 gpuIdLoop = 0;
4446 while (gpuIdLoop < pPcieCapsInfo->gpuCount)
4447 {
4448 if (pPcieCapsInfo->gpuId[gpuIdLoop] == gpuId)
4449 {
4450 listRemove(&pGpuMgr->pcieP2PCapsInfoCache, pPcieCapsInfo);
4451 portMemFree(pPcieCapsInfo);
4452 // Go to next entry (for loop)
4453 break;
4454 }
4455 gpuIdLoop++;
4456 }
4457 }
4458 portSyncMutexRelease(pGpuMgr->pcieP2PCapsInfoLock);
4459 }
4460
gpumgrAreAllGpusInOffloadMode(void)4461 NvBool gpumgrAreAllGpusInOffloadMode(void)
4462 {
4463 OBJSYS *pSys = SYS_GET_INSTANCE();
4464 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
4465
4466 return pGpuMgr->gpuMonolithicRmMask == 0;
4467 }
4468
gpumgrIsSafeToReadGpuInfo(void)4469 NvBool gpumgrIsSafeToReadGpuInfo(void)
4470 {
4471 //
4472 // A thread that tears down the GPU must own both the API lock for WRITE
4473 // and all GPU locks.
4474 //
4475 // Conversely, if you hold the API lock (either READ or WRITE), or hold
4476 // any GPU locks, you know that no GPUs will be freed from under you.
4477 //
4478
4479 //
4480 // NOTE: Currently rmapiLockIsOwner() returns TRUE if you own the lock in
4481 // either READ or WRITE modes
4482 //
4483 return rmapiLockIsOwner() || (rmGpuLocksGetOwnedMask() != 0);
4484 }
4485
4486 //
4487 // Workaround for Bug 3809777. This is a HW bug happening in Ampere and
4488 // Ada GPU's. For these GPU's, after device reset, CRS (Configuration Request
4489 // Retry Status) is being released without waiting for GFW boot completion.
4490 // MSI-X capability in the config space may be inconsistent when GFW boot
4491 // is in progress, so this function checks if MSI-X is allowed.
4492 // For Hopper and above, the CRS will be released after
4493 // GFW boot completion, so the WAR is not needed.
4494 // The bug will be exposed only when GPU is running inside guest in
4495 // pass-through mode.
4496 //
gpumgrIsDeviceMsixAllowed(RmPhysAddr bar0BaseAddr,NvU32 pmcBoot1,NvU32 pmcBoot42)4497 NvBool gpumgrIsDeviceMsixAllowed
4498 (
4499 RmPhysAddr bar0BaseAddr,
4500 NvU32 pmcBoot1,
4501 NvU32 pmcBoot42
4502 )
4503 {
4504 OBJSYS *pSys = SYS_GET_INSTANCE();
4505 OBJHYPERVISOR *pHypervisor = SYS_GET_HYPERVISOR(pSys);
4506 NvU32 chipArch;
4507
4508 if ((hypervisorGetHypervisorType(pHypervisor) == OS_HYPERVISOR_UNKNOWN) ||
4509 !FLD_TEST_DRF(_PMC, _BOOT_1, _VGPU, _REAL, pmcBoot1))
4510 {
4511 return NV_TRUE;
4512 }
4513
4514 chipArch = DRF_VAL(_PMC, _BOOT_42, _ARCHITECTURE, pmcBoot42);
4515 if ((chipArch != NV_PMC_BOOT_42_ARCHITECTURE_AD100) &&
4516 (chipArch != NV_PMC_BOOT_42_ARCHITECTURE_GA100))
4517 {
4518 return NV_TRUE;
4519 }
4520
4521 return gpuIsMsixAllowed_TU102(bar0BaseAddr);
4522 }
4523