1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "core/system.h"
25 #include "gpu_mgr/gpu_mgr.h"
26 #include "kernel/gpu/mig_mgr/kernel_mig_manager.h"
27 #include "kernel/gpu/nvlink/kernel_nvlink.h"
28 #include "kernel/gpu/bif/kernel_bif.h"
29 #include "gpu/subdevice/subdevice.h"
30 #include "gpu/gpu.h"
31 #include "virtualization/hypervisor/hypervisor.h"
32 #include "vgpu/rpc.h"
33 #include "vgpu/vgpu_events.h"
34 #include "platform/chipset/chipset.h"
35 #include "platform/p2p/p2p_caps.h"
36 #include "nvrm_registry.h"
37 #include "nvlimits.h"
38 #include "nvdevid.h"
39
40 ct_assert(NV2080_GET_P2P_CAPS_UUID_LEN == NV_GPU_UUID_LEN);
41
42 /**
43 * @brief Determines if the GPUs are P2P compatible
44 *
45 * @param[in] pGpu0
46 * @param[in] pGpu1
47 *
48 * @return NV_TRUE if the GPUs are P2P compatible
49 */
50 static NvBool
areGpusP2PCompatible(OBJGPU * pGpu0,OBJGPU * pGpu1)51 areGpusP2PCompatible(OBJGPU *pGpu0, OBJGPU *pGpu1)
52 {
53 // Mark GPUs of different arch or impl incapable of P2P over pcie
54 if ((gpuGetChipArch(pGpu0) != gpuGetChipArch(pGpu1)) ||
55 (gpuGetChipImpl(pGpu0) != gpuGetChipImpl(pGpu1)))
56 {
57 return NV_FALSE;
58 }
59
60 // Mark GPUs of different notebook implementation incapable of P2P over pcie
61 if (IsMobile(pGpu0) != IsMobile(pGpu1))
62 {
63 return NV_FALSE;
64 }
65
66 return NV_TRUE;
67 }
68
69 NV_STATUS
p2pGetCaps(NvU32 gpuMask,NvBool * pP2PWriteCapable,NvBool * pP2PReadCapable,P2P_CONNECTIVITY * pConnectivity)70 p2pGetCaps
71 (
72 NvU32 gpuMask,
73 NvBool *pP2PWriteCapable,
74 NvBool *pP2PReadCapable,
75 P2P_CONNECTIVITY *pConnectivity
76 )
77 {
78 NvU8 p2PWriteCapsStatus;
79 NvU8 p2PReadCapsStatus;
80 NV_STATUS status;
81 P2P_CONNECTIVITY connectivity;
82
83 if ((pP2PWriteCapable == NULL) || (pP2PReadCapable == NULL))
84 {
85 return NV_ERR_INVALID_ARGUMENT;
86 }
87
88 status = p2pGetCapsStatus(gpuMask, &p2PWriteCapsStatus,
89 &p2PReadCapsStatus, &connectivity
90 );
91 if (status != NV_OK)
92 {
93 return status;
94 }
95
96 //
97 // The classes like NV50_P2P, NV50_THIRD_PARTY_P2P depends on direct P2P
98 // connectivity, hence the check.
99 //
100 if (!((connectivity == P2P_CONNECTIVITY_PCIE_PROPRIETARY) ||
101 (connectivity == P2P_CONNECTIVITY_PCIE_BAR1) ||
102 (connectivity == P2P_CONNECTIVITY_NVLINK) ||
103 (connectivity == P2P_CONNECTIVITY_C2C)))
104 {
105 return NV_ERR_NOT_SUPPORTED;
106 }
107
108 *pP2PWriteCapable = (p2PWriteCapsStatus == NV0000_P2P_CAPS_STATUS_OK);
109 *pP2PReadCapable = (p2PReadCapsStatus == NV0000_P2P_CAPS_STATUS_OK);
110
111 if (pConnectivity != NULL)
112 {
113 *pConnectivity = connectivity;
114 }
115
116 return status;
117 }
118
119 static NV_STATUS
_kp2pCapsGetStatusIndirectOverNvLink(NvU32 gpuMask,NvU8 * pP2PWriteCapStatus,NvU8 * pP2PReadCapStatus)120 _kp2pCapsGetStatusIndirectOverNvLink
121 (
122 NvU32 gpuMask,
123 NvU8 *pP2PWriteCapStatus,
124 NvU8 *pP2PReadCapStatus
125 )
126 {
127 OBJGPU *pGpu = NULL;
128 NvU32 gpuInstance = 0;
129 OBJGPU *pFirstGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance);
130 NvBool bIndirectPeers = NV_FALSE;
131 KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pFirstGpu);
132
133 if ((pKernelBif->forceP2PType != NV_REG_STR_RM_FORCE_P2P_TYPE_DEFAULT) &&
134 (pKernelBif->forceP2PType != NV_REG_STR_RM_FORCE_P2P_TYPE_NVLINK))
135 {
136 *pP2PReadCapStatus = NV0000_P2P_CAPS_STATUS_NOT_SUPPORTED;
137 *pP2PWriteCapStatus = NV0000_P2P_CAPS_STATUS_NOT_SUPPORTED;
138 return NV_OK;
139 }
140
141 while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
142 {
143 bIndirectPeers = gpumgrCheckIndirectPeer(pFirstGpu, pGpu);
144 if (!bIndirectPeers)
145 {
146 break;
147 }
148 }
149
150 if (bIndirectPeers)
151 {
152 *pP2PReadCapStatus = NV0000_P2P_CAPS_STATUS_OK;
153 *pP2PWriteCapStatus = NV0000_P2P_CAPS_STATUS_OK;
154 }
155 else
156 {
157 *pP2PReadCapStatus = NV0000_P2P_CAPS_STATUS_NOT_SUPPORTED;
158 *pP2PWriteCapStatus = NV0000_P2P_CAPS_STATUS_NOT_SUPPORTED;
159 }
160
161 return NV_OK;
162 }
163
164 static NV_STATUS
_gpumgrGetP2PCapsStatusOverNvLink(NvU32 gpuMask,NvU8 * pP2PWriteCapStatus,NvU8 * pP2PReadCapStatus)165 _gpumgrGetP2PCapsStatusOverNvLink
166 (
167 NvU32 gpuMask,
168 NvU8 *pP2PWriteCapStatus,
169 NvU8 *pP2PReadCapStatus
170 )
171 {
172 OBJGPU *pGpu = NULL;
173 NvU32 gpuInstance = 0;
174 NV_STATUS status;
175 OBJGPU *pFirstGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance);
176 RMTIMEOUT timeout;
177 NvU32 linkTrainingTimeout = 10000000;
178 KernelBif *pKernelBif = NULL;
179 KernelNvlink *pKernelNvlink = NULL;
180
181 NV_ASSERT_OR_RETURN(pFirstGpu != NULL, NV_ERR_INVALID_ARGUMENT);
182 pKernelNvlink = GPU_GET_KERNEL_NVLINK(pFirstGpu);
183 pKernelBif = GPU_GET_KERNEL_BIF(pFirstGpu);
184
185 if ((pKernelBif->forceP2PType != NV_REG_STR_RM_FORCE_P2P_TYPE_DEFAULT) &&
186 (pKernelBif->forceP2PType != NV_REG_STR_RM_FORCE_P2P_TYPE_NVLINK))
187 {
188 *pP2PReadCapStatus = NV0000_P2P_CAPS_STATUS_NOT_SUPPORTED;
189 *pP2PWriteCapStatus = NV0000_P2P_CAPS_STATUS_NOT_SUPPORTED;
190 return NV_OK;
191 }
192
193 //
194 // Re-initialize to check loop back configuration if only single GPU in
195 // requested mask.
196 //
197 gpuInstance = (gpumgrGetSubDeviceCount(gpuMask) > 1) ? gpuInstance : 0;
198
199 // Check NvLink P2P connectivity
200 while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
201 {
202 //
203 // If ALI is enabled then poll to make sure that the links have
204 // finished training on the two given gpus. If timeout occurs then
205 // log an error, but continue on as there could be another gpu pair
206 // that could have connectivity
207 //
208
209 if ((pKernelNvlink != NULL) &&
210 knvlinkDiscoverPostRxDetLinks_HAL(pFirstGpu, pKernelNvlink, pGpu) == NV_OK)
211 {
212 // Check to make sure that the links are active
213
214 gpuSetTimeout(pGpu, linkTrainingTimeout, &timeout, IS_SILICON(pGpu) ?
215 (GPU_TIMEOUT_FLAGS_BYPASS_THREAD_STATE | GPU_TIMEOUT_FLAGS_DEFAULT) : 0);
216 do
217 {
218 status = gpuCheckTimeout(pGpu, &timeout);
219
220 if (knvlinkCheckTrainingIsComplete(pFirstGpu, pGpu, pKernelNvlink) == NV_OK)
221 {
222 break;
223 }
224
225 if (status == NV_ERR_TIMEOUT)
226 {
227 NV_PRINTF(LEVEL_ERROR,
228 "Links failed to train for the given gpu pairs!\n");
229 knvlinkLogAliDebugMessages(pFirstGpu, pKernelNvlink, NV_TRUE);
230 return status;
231 }
232 }
233 while(status != NV_ERR_TIMEOUT);
234 }
235
236 // Ensure that we can create a NvLink P2P object between the two object
237 if ((pKernelNvlink != NULL) &&
238 knvlinkIsNvlinkP2pSupported(pFirstGpu, pKernelNvlink, pGpu))
239 {
240 // Ensure training completes on legacy nvlink devices
241 status = knvlinkTrainP2pLinksToActive(pFirstGpu, pGpu, pKernelNvlink);
242 NV_ASSERT(status == NV_OK);
243
244 if (status != NV_OK)
245 {
246 *pP2PReadCapStatus = NV0000_P2P_CAPS_STATUS_NOT_SUPPORTED;
247 *pP2PWriteCapStatus = NV0000_P2P_CAPS_STATUS_NOT_SUPPORTED;
248 return NV_OK;
249 }
250 }
251 else
252 {
253 *pP2PReadCapStatus = NV0000_P2P_CAPS_STATUS_NOT_SUPPORTED;
254 *pP2PWriteCapStatus = NV0000_P2P_CAPS_STATUS_NOT_SUPPORTED;
255 return NV_OK;
256 }
257 }
258
259 *pP2PReadCapStatus = NV0000_P2P_CAPS_STATUS_OK;
260 *pP2PWriteCapStatus = NV0000_P2P_CAPS_STATUS_OK;
261 return NV_OK;
262 }
263
264 // Returns true if overrides are enabled for PCI-E.
265 static NvBool
_kp2pCapsCheckStatusOverridesForPcie(NvU32 gpuMask,NvU8 * pP2PWriteCapStatus,NvU8 * pP2PReadCapStatus)266 _kp2pCapsCheckStatusOverridesForPcie
267 (
268 NvU32 gpuMask,
269 NvU8 *pP2PWriteCapStatus,
270 NvU8 *pP2PReadCapStatus
271 )
272 {
273 KernelBif *pKernelBif = NULL;
274 NvU32 gpuInstance = 0;
275 OBJGPU *pGpu = NULL;
276
277 // Check overrides for all GPUs in the mask.
278 while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
279 {
280 pKernelBif = GPU_GET_KERNEL_BIF(pGpu);
281 if (pKernelBif->p2pOverride != BIF_P2P_NOT_OVERRIDEN)
282 {
283 switch(DRF_VAL(_REG_STR, _CL_FORCE_P2P, _READ, pKernelBif->p2pOverride))
284 {
285 case NV_REG_STR_CL_FORCE_P2P_READ_DISABLE:
286 *pP2PReadCapStatus = NV0000_P2P_CAPS_STATUS_DISABLED_BY_REGKEY;
287 break;
288 case NV_REG_STR_CL_FORCE_P2P_READ_ENABLE:
289 *pP2PReadCapStatus = NV0000_P2P_CAPS_STATUS_OK;
290 break;
291 default:
292 break;
293 }
294
295 switch(DRF_VAL(_REG_STR, _CL_FORCE_P2P, _WRITE, pKernelBif->p2pOverride))
296 {
297 case NV_REG_STR_CL_FORCE_P2P_WRITE_DISABLE:
298 *pP2PWriteCapStatus = NV0000_P2P_CAPS_STATUS_DISABLED_BY_REGKEY;
299 break;
300 case NV_REG_STR_CL_FORCE_P2P_WRITE_ENABLE:
301 *pP2PWriteCapStatus = NV0000_P2P_CAPS_STATUS_OK;
302 break;
303 default:
304 break;
305 }
306
307 return NV_TRUE;
308 }
309
310 }
311
312 return NV_FALSE;
313 }
314
315 /**
316 * @brief Check GPU Pcie mailbox P2P capability
317 *
318 * @param[in] pGpu OBJGPU pointer
319 * @param[out] pP2PWriteCapStatus Pointer to get the P2P write capability
320 * @param[out] pP2PReadCapStatus Pointer to get the P2P read capability
321 * @param[out] pbCommonPciSwitch To return if GPUs are on a common PCIE switch
322 *
323 * @returns NV_OK, if successfully
324 * The write and read capability status are in the pP2PWriteCapStatus and pP2PReadCapStatus
325 */
326 static NV_STATUS
_kp2pCapsGetStatusOverPcie(NvU32 gpuMask,NvU8 * pP2PWriteCapStatus,NvU8 * pP2PReadCapStatus,NvBool * pbCommonPciSwitch)327 _kp2pCapsGetStatusOverPcie
328 (
329 NvU32 gpuMask,
330 NvU8 *pP2PWriteCapStatus,
331 NvU8 *pP2PReadCapStatus,
332 NvBool *pbCommonPciSwitch
333 )
334 {
335 OBJGPU *pGpu = NULL;
336 OBJGPU *pFirstGpu = NULL;
337 NvU32 gpuInstance = 0;
338 KernelBif *pKernelBif = NULL;
339 OBJSYS *pSys = SYS_GET_INSTANCE();
340 OBJCL *pCl = SYS_GET_CL(pSys);
341 NvU32 iohDomain_ref = 0xFFFFFFFF;
342 NvU8 iohBus_ref = 0xFF;
343 NvU8 pciSwitchBus = 0, pciSwitchBus_ref = 0xFF;
344 NvBool bCommonPciSwitchFound = NV_TRUE;
345 NvU16 deviceID;
346 NvU8 gpuP2PReadCapsStatus = NV0000_P2P_CAPS_STATUS_OK;
347 NvU8 gpuP2PWriteCapsStatus = NV0000_P2P_CAPS_STATUS_OK;
348 NvU32 lockedGpuMask = 0;
349 NV_STATUS status = NV_OK;
350 OBJHYPERVISOR *pHypervisor = SYS_GET_HYPERVISOR(pSys);
351
352 // Check if any overrides are enabled.
353 if (_kp2pCapsCheckStatusOverridesForPcie(gpuMask, pP2PWriteCapStatus,
354 pP2PReadCapStatus))
355 {
356 return NV_OK;
357 }
358
359 pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance);
360 if (IS_GSP_CLIENT(pGpu))
361 {
362 if (gpumgrGetPcieP2PCapsFromCache(gpuMask, pP2PWriteCapStatus, pP2PReadCapStatus))
363 {
364 return NV_OK;
365 }
366 }
367
368 // Check for hypervisor oriented PCIe P2P overrides
369 if (pHypervisor &&
370 pHypervisor->bDetected &&
371 hypervisorPcieP2pDetection(pHypervisor, gpuMask))
372 {
373 *pP2PReadCapStatus = NV0000_P2P_CAPS_STATUS_OK;
374 *pP2PWriteCapStatus = NV0000_P2P_CAPS_STATUS_OK;
375 goto done;
376 }
377
378 // If the PCIE topology cannot be accessed do not allow P2P
379 if (!pCl->ChipsetInitialized)
380 {
381 *pP2PReadCapStatus = NV0000_P2P_CAPS_STATUS_NOT_SUPPORTED;
382 *pP2PWriteCapStatus= NV0000_P2P_CAPS_STATUS_NOT_SUPPORTED;
383 goto done;
384 }
385
386 // PCI-E topology checks
387 gpuInstance = 0;
388 while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
389 {
390 //
391 // While PCI-E P2P transactions are forwarded between different
392 // root ports implemented within a given Intel I/O hub, they
393 // are not forwarded between any two I/O hubs. We must therefore
394 // complement the table-driven chipset validation check below
395 // with an IOH-specific topology check.
396 //
397 if (pGpu->gpuClData.rootPort.addr.valid &&
398 (pGpu->gpuClData.rootPort.VendorID == PCI_VENDOR_ID_INTEL))
399 {
400 deviceID = pGpu->gpuClData.rootPort.DeviceID;
401
402 if (((deviceID >= DEVICE_ID_INTEL_3408_ROOT_PORT) &&
403 (deviceID <= DEVICE_ID_INTEL_3411_ROOT_PORT)) ||
404 ((deviceID >= DEVICE_ID_INTEL_3C02_ROOT_PORT) &&
405 (deviceID <= DEVICE_ID_INTEL_3C0B_ROOT_PORT)) ||
406 ((deviceID >= DEVICE_ID_INTEL_0E02_ROOT_PORT) &&
407 (deviceID <= DEVICE_ID_INTEL_0E0B_ROOT_PORT)) ||
408 ((deviceID >= DEVICE_ID_INTEL_2F01_ROOT_PORT) &&
409 (deviceID <= DEVICE_ID_INTEL_2F0B_ROOT_PORT)) ||
410 ((deviceID >= DEVICE_ID_INTEL_6F01_ROOT_PORT) &&
411 (deviceID <= DEVICE_ID_INTEL_6F0B_ROOT_PORT)) ||
412 (deviceID == DEVICE_ID_INTEL_3420_ROOT_PORT) ||
413 (deviceID == DEVICE_ID_INTEL_3421_ROOT_PORT))
414 {
415 if (iohDomain_ref == 0xFFFFFFFF)
416 {
417 iohDomain_ref = pGpu->gpuClData.rootPort.addr.domain;
418 iohBus_ref = pGpu->gpuClData.rootPort.addr.bus;
419 }
420 else if ((iohDomain_ref != pGpu->gpuClData.rootPort.addr.domain) ||
421 (iohBus_ref != pGpu->gpuClData.rootPort.addr.bus))
422 {
423 *pP2PReadCapStatus = NV0000_P2P_CAPS_STATUS_IOH_TOPOLOGY_NOT_SUPPORTED;
424 *pP2PWriteCapStatus = NV0000_P2P_CAPS_STATUS_IOH_TOPOLOGY_NOT_SUPPORTED;
425 goto done;
426 }
427 }
428 }
429
430 // Test common bridges. Skip first GPU
431 if (pFirstGpu == NULL)
432 {
433 pFirstGpu = pGpu;
434 continue;
435 }
436
437 if (!areGpusP2PCompatible(pFirstGpu, pGpu))
438 {
439 *pP2PReadCapStatus = NV0000_P2P_CAPS_STATUS_NOT_SUPPORTED;
440 *pP2PWriteCapStatus = NV0000_P2P_CAPS_STATUS_NOT_SUPPORTED;
441 goto done;
442 }
443
444 // This call returns the most upper bridge
445 clFindCommonDownstreamBR(pFirstGpu, pGpu, pCl, &pciSwitchBus);
446
447 if (pciSwitchBus_ref == 0xFF)
448 {
449 pciSwitchBus_ref = pciSwitchBus;
450 }
451
452 // If no bridge found or different to the one previously found
453 if ((pciSwitchBus == 0xFF) || (pciSwitchBus_ref != pciSwitchBus))
454 {
455 bCommonPciSwitchFound = NV_FALSE;
456 }
457 }
458
459 // Check if GPUs have the HW P2P implementation
460
461 // Only lock for GSP_CLIENT. Get one GPU.
462 if (pFirstGpu == NULL)
463 {
464 gpuInstance = 0;
465 pFirstGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance);
466 }
467
468 if (IS_GSP_CLIENT(pFirstGpu))
469 {
470 // Lock GPUs
471 lockedGpuMask = gpuMask;
472 status = rmGpuGroupLockAcquire(0, GPU_LOCK_GRP_MASK,
473 GPU_LOCK_FLAGS_SAFE_LOCK_UPGRADE, RM_LOCK_MODULES_P2P, &lockedGpuMask);
474
475 // If we get NOTHING_TO_DO, we already have the needed locks, so don't free them
476 if (status == NV_WARN_NOTHING_TO_DO)
477 lockedGpuMask = 0;
478 else if (status != NV_OK)
479 {
480 lockedGpuMask = 0;
481 goto done;
482 }
483 }
484
485 // Reset P2P caps as statuses will be accumulated below.
486 *pP2PReadCapStatus = *pP2PWriteCapStatus = NV0000_P2P_CAPS_STATUS_OK;
487
488 gpuInstance = 0;
489 while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
490 {
491 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
492 NV2080_CTRL_INTERNAL_GET_PCIE_P2P_CAPS_PARAMS p2pCapsParams = {0};
493
494 p2pCapsParams.bCommonPciSwitchFound = bCommonPciSwitchFound;
495
496 NV_ASSERT_OK_OR_GOTO(status, pRmApi->Control(pRmApi,
497 pGpu->hInternalClient,
498 pGpu->hInternalSubdevice,
499 NV2080_CTRL_CMD_INTERNAL_GET_PCIE_P2P_CAPS,
500 &p2pCapsParams,
501 sizeof(NV2080_CTRL_INTERNAL_GET_PCIE_P2P_CAPS_PARAMS)),
502 done);
503
504 // GPU specific P2P caps
505 pKernelBif = GPU_GET_KERNEL_BIF(pGpu);
506 if (pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_P2P_WRITES_DISABLED))
507 gpuP2PWriteCapsStatus = NV0000_P2P_CAPS_STATUS_GPU_NOT_SUPPORTED;
508 if (pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_P2P_READS_DISABLED))
509 gpuP2PReadCapsStatus = NV0000_P2P_CAPS_STATUS_GPU_NOT_SUPPORTED;
510
511 //
512 // Reconcile the system and GPU specific P2P information
513 // The system P2P status takes precedence
514 // Do not override status from not OK to OK
515 //
516 if (*pP2PReadCapStatus == NV0000_P2P_CAPS_STATUS_OK)
517 *pP2PReadCapStatus = (p2pCapsParams.p2pReadCapsStatus == NV0000_P2P_CAPS_STATUS_OK ? gpuP2PReadCapsStatus : p2pCapsParams.p2pReadCapsStatus);
518 if (*pP2PWriteCapStatus == NV0000_P2P_CAPS_STATUS_OK)
519 *pP2PWriteCapStatus = (p2pCapsParams.p2pWriteCapsStatus == NV0000_P2P_CAPS_STATUS_OK ? gpuP2PWriteCapsStatus : p2pCapsParams.p2pWriteCapsStatus);
520
521 // No need to continue if P2P is not supported
522 if ((*pP2PReadCapStatus != NV0000_P2P_CAPS_STATUS_OK) &&
523 (*pP2PWriteCapStatus != NV0000_P2P_CAPS_STATUS_OK))
524 {
525 break;
526 }
527 }
528
529 done:
530 if (lockedGpuMask != 0)
531 {
532 rmGpuGroupLockRelease(lockedGpuMask, GPUS_LOCK_FLAGS_NONE);
533 }
534
535 if (status != NV_OK)
536 {
537 if (*pP2PReadCapStatus == NV0000_P2P_CAPS_STATUS_OK)
538 {
539 *pP2PReadCapStatus = NV0000_P2P_CAPS_STATUS_NOT_SUPPORTED;
540 }
541 if (*pP2PWriteCapStatus == NV0000_P2P_CAPS_STATUS_OK)
542 {
543 *pP2PWriteCapStatus = NV0000_P2P_CAPS_STATUS_NOT_SUPPORTED;
544 }
545 }
546
547 if (pbCommonPciSwitch != NULL)
548 {
549 *pbCommonPciSwitch = bCommonPciSwitchFound;
550 }
551
552 //
553 // Not fatal if failing, effect would be perf degradation as we would not hit the cache.
554 // So just assert.
555 //
556 gpuInstance = 0;
557 pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance);
558 if (IS_GSP_CLIENT(pGpu))
559 {
560 NV_ASSERT_OK(gpumgrStorePcieP2PCapsCache(gpuMask, *pP2PWriteCapStatus, *pP2PReadCapStatus));
561 }
562 return status;
563 }
564
565 /**
566 * @brief Check the host system BAR1 P2P capability
567 *
568 * @param[in] pGpu OBJGPU pointer
569 * @param[out] pP2PWriteCapStatus Pointer to get the P2P write capability
570 * @param[out] pP2PReadCapStatus Pointer to get the P2P read capability
571 * @param[out] bCommonPciSwitchFound To indicate if GPUs are on a common PCIE switch
572 *
573 * @returns NV_OK
574 */
575 static NV_STATUS
_p2pCapsGetHostSystemStatusOverPcieBar1(OBJGPU * pGpu,NvU8 * pP2PWriteCapStatus,NvU8 * pP2PReadCapStatus,NvBool bCommonPciSwitchFound)576 _p2pCapsGetHostSystemStatusOverPcieBar1
577 (
578 OBJGPU *pGpu,
579 NvU8 *pP2PWriteCapStatus,
580 NvU8 *pP2PReadCapStatus,
581 NvBool bCommonPciSwitchFound
582 )
583 {
584 OBJSYS *pSys = SYS_GET_INSTANCE();
585
586 *pP2PWriteCapStatus = NV0000_P2P_CAPS_STATUS_OK;
587
588 if (bCommonPciSwitchFound ||
589 (pSys->cpuInfo.type == NV0000_CTRL_SYSTEM_CPU_TYPE_RYZEN) ||
590 (pSys->cpuInfo.type == NV0000_CTRL_SYSTEM_CPU_TYPE_XEON_SPR))
591 {
592 *pP2PReadCapStatus = NV0000_P2P_CAPS_STATUS_OK;
593 }
594 else
595 {
596 *pP2PReadCapStatus = NV0000_P2P_CAPS_STATUS_CHIPSET_NOT_SUPPORTED;
597 NV_PRINTF(LEVEL_INFO, "Unrecognized CPU. Read Cap is disabled\n");
598 }
599
600 return NV_OK;
601 }
602
603 /**
604 * @brief Check GPU Pcie BAR1 P2P capability
605 *
606 * @param[in] pGpu OBJGPU pointer
607 * @param[out] pP2PWriteCapStatus Pointer to get the P2P write capability
608 * @param[out] pP2PReadCapStatus Pointer to get the P2P read capability
609 * @param[out] bCommonPciSwitchFound To indicate if GPUs are on a common PCIE switch
610 *
611 * @returns NV_OK, if GPUs at least support read or write
612 * NV_ERR_NOT_SUPPORTED, if GPUs does not support read and write
613 */
614 static NV_STATUS
_kp2pCapsGetStatusOverPcieBar1(NvU32 gpuMask,NvU8 * pP2PWriteCapStatus,NvU8 * pP2PReadCapStatus,NvBool bCommonPciSwitchFound)615 _kp2pCapsGetStatusOverPcieBar1
616 (
617 NvU32 gpuMask,
618 NvU8 *pP2PWriteCapStatus,
619 NvU8 *pP2PReadCapStatus,
620 NvBool bCommonPciSwitchFound
621 )
622 {
623 OBJGPU *pGpuPeer = NULL;
624 NvU32 gpuInstance = 0;
625 OBJGPU *pFirstGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance);
626 KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pFirstGpu);
627 NvU8 writeCapStatus = *pP2PWriteCapStatus;
628 NvU8 readCapStatus = *pP2PReadCapStatus;
629
630 if ((pKernelBif->forceP2PType != NV_REG_STR_RM_FORCE_P2P_TYPE_BAR1P2P))
631 {
632 return NV_ERR_NOT_SUPPORTED;
633 }
634
635 // Check if any overrides are enabled.
636 if (_kp2pCapsCheckStatusOverridesForPcie(gpuMask, pP2PWriteCapStatus,
637 pP2PReadCapStatus))
638 {
639 return NV_OK;
640 }
641
642 //
643 // Re-initialize to check loop back configuration if only single GPU in
644 // requested mask.
645 //
646 gpuInstance = (gpumgrGetSubDeviceCount(gpuMask) > 1) ? gpuInstance : 0;
647
648 while ((pGpuPeer = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
649 {
650 if (!kbusIsPcieBar1P2PMappingSupported_HAL(pFirstGpu, GPU_GET_KERNEL_BUS(pFirstGpu),
651 pGpuPeer, GPU_GET_KERNEL_BUS(pGpuPeer)))
652 {
653 return NV_ERR_NOT_SUPPORTED;
654 }
655 }
656
657 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
658 _p2pCapsGetHostSystemStatusOverPcieBar1(pFirstGpu, &writeCapStatus,
659 &readCapStatus, bCommonPciSwitchFound));
660
661 if (*pP2PWriteCapStatus == NV0000_P2P_CAPS_STATUS_OK)
662 *pP2PWriteCapStatus = writeCapStatus;
663 if (*pP2PReadCapStatus == NV0000_P2P_CAPS_STATUS_OK)
664 *pP2PReadCapStatus = readCapStatus;
665
666 if ((*pP2PReadCapStatus != NV0000_P2P_CAPS_STATUS_OK) ||
667 (*pP2PWriteCapStatus != NV0000_P2P_CAPS_STATUS_OK))
668 {
669 // return not supported if it does not support both operations
670 return NV_ERR_NOT_SUPPORTED;
671 }
672
673 return NV_OK;
674 }
675
676 NV_STATUS
p2pGetCapsStatus(NvU32 gpuMask,NvU8 * pP2PWriteCapStatus,NvU8 * pP2PReadCapStatus,P2P_CONNECTIVITY * pConnectivity)677 p2pGetCapsStatus
678 (
679 NvU32 gpuMask,
680 NvU8 *pP2PWriteCapStatus,
681 NvU8 *pP2PReadCapStatus,
682 P2P_CONNECTIVITY *pConnectivity
683 )
684 {
685 OBJSYS *pSys = SYS_GET_INSTANCE();
686 KernelNvlink *pKernelNvlink = NULL;
687 OBJGPU *pGpu = NULL;
688 NvU32 gpuInstance = 0;
689 NvBool bCommonSwitchFound = NV_FALSE;
690
691 if ((pP2PWriteCapStatus == NULL) ||
692 (pP2PReadCapStatus == NULL) ||
693 (pConnectivity == NULL)
694 )
695 {
696 return NV_ERR_INVALID_ARGUMENT;
697 }
698
699 // Default values
700 *pP2PWriteCapStatus = NV0000_P2P_CAPS_STATUS_NOT_SUPPORTED;
701 *pP2PReadCapStatus = NV0000_P2P_CAPS_STATUS_NOT_SUPPORTED;
702 *pConnectivity = P2P_CONNECTIVITY_UNKNOWN;
703
704 // MIG-Nvlink-P2P can be incompatible, so check compatibility for all GPUs
705 while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
706 {
707 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu);
708 NvBool bSmcNvLinkP2PSupported = ((pKernelMIGManager != NULL) &&
709 kmigmgrIsMIGNvlinkP2PSupported(pGpu, pKernelMIGManager));
710
711 // If any of the GPU has MIG enabled, return with no P2P support
712 if (!bSmcNvLinkP2PSupported)
713 {
714 NV_PRINTF(LEVEL_NOTICE,
715 "P2P is marked unsupported with MIG for GPU instance = 0x%x\n",
716 gpuInstance);
717 return NV_OK;
718 }
719 }
720
721 gpuInstance = 0;
722
723 // Check NvLink P2P connectivity.
724 if (_gpumgrGetP2PCapsStatusOverNvLink(gpuMask, pP2PWriteCapStatus,
725 pP2PReadCapStatus) == NV_OK)
726 {
727 if (*pP2PWriteCapStatus == NV0000_P2P_CAPS_STATUS_OK &&
728 *pP2PReadCapStatus == NV0000_P2P_CAPS_STATUS_OK)
729 {
730 *pConnectivity = P2P_CONNECTIVITY_NVLINK;
731 return NV_OK;
732 }
733 }
734
735 //
736 // On NVSwitch systems, if the NVLink P2P path fails, don't fall back to
737 // other P2P paths. To ensure that, check if any GPU in the mask has NVLink
738 // support. If supported, enforce NVSwitch/NVLink connectivity by returning
739 // NV0000_P2P_CAPS_STATUS_NOT_SUPPORTED.
740 //
741 gpuInstance = 0;
742 while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
743 {
744 pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu);
745 if (pKernelNvlink != NULL && knvlinkGetDiscoveredLinkMask(pGpu, pKernelNvlink) != 0 &&
746 (pSys->getProperty(pSys, PDB_PROP_SYS_NVSWITCH_IS_PRESENT) ||
747 knvlinkIsNvswitchProxyPresent(pGpu, pKernelNvlink) ||
748 GPU_IS_NVSWITCH_DETECTED(pGpu)))
749 {
750 *pP2PReadCapStatus = NV0000_P2P_CAPS_STATUS_NOT_SUPPORTED;
751 *pP2PWriteCapStatus = NV0000_P2P_CAPS_STATUS_NOT_SUPPORTED;
752 return NV_OK;
753 }
754 }
755
756 // We didn't find direct P2P, so check for indirect P2P.
757 if (_kp2pCapsGetStatusIndirectOverNvLink(gpuMask, pP2PWriteCapStatus,
758 pP2PReadCapStatus) == NV_OK)
759 {
760 if ((*pP2PWriteCapStatus == NV0000_P2P_CAPS_STATUS_OK) &&
761 (*pP2PReadCapStatus == NV0000_P2P_CAPS_STATUS_OK))
762 {
763 *pConnectivity = P2P_CONNECTIVITY_NVLINK_INDIRECT;
764 return NV_OK;
765 }
766 }
767
768 //
769 // Check PCIE P2P connectivity.
770 //
771 // We can control P2P connectivity for PCI-E peers using regkeys, hence
772 // if either read or write is supported, return success. See
773 // _kp2pCapsCheckStatusOverridesForPcie for details.
774 //
775 if (_kp2pCapsGetStatusOverPcie(gpuMask, pP2PWriteCapStatus,
776 pP2PReadCapStatus, &bCommonSwitchFound) == NV_OK)
777 {
778 if ((*pP2PWriteCapStatus == NV0000_P2P_CAPS_STATUS_OK) ||
779 (*pP2PReadCapStatus == NV0000_P2P_CAPS_STATUS_OK))
780 {
781 NvU8 bar1P2PWriteCapStatus = *pP2PWriteCapStatus;
782 NvU8 bar1P2PReadCapStatus = *pP2PReadCapStatus;
783
784 *pConnectivity = P2P_CONNECTIVITY_PCIE_PROPRIETARY;
785
786 if (_kp2pCapsGetStatusOverPcieBar1(gpuMask, &bar1P2PWriteCapStatus,
787 &bar1P2PReadCapStatus, bCommonSwitchFound) == NV_OK)
788 {
789 *pP2PWriteCapStatus = bar1P2PWriteCapStatus;
790 *pP2PReadCapStatus = bar1P2PReadCapStatus;
791 *pConnectivity = P2P_CONNECTIVITY_PCIE_BAR1;
792 }
793
794 return NV_OK;
795 }
796 }
797
798 return NV_OK;
799 }
800
801 static NV_STATUS
subdeviceGetP2pCaps_VIRTUAL(OBJGPU * pGpu,NV2080_CTRL_GET_P2P_CAPS_PARAMS * pParams)802 subdeviceGetP2pCaps_VIRTUAL
803 (
804 OBJGPU *pGpu,
805 NV2080_CTRL_GET_P2P_CAPS_PARAMS *pParams
806 )
807 {
808 NvU32 i;
809 NV_STATUS status = NV_OK;
810 CALL_CONTEXT *pCallContext = resservGetTlsCallContext();
811 RS_RES_CONTROL_PARAMS_INTERNAL *pControlParams = pCallContext->pControlParams;
812 NV2080_CTRL_GET_P2P_CAPS_PARAMS *pShimParams =
813 portMemAllocNonPaged(sizeof *pShimParams);
814
815 NV_CHECK_OR_RETURN(LEVEL_INFO, pShimParams != NULL, NV_ERR_NO_MEMORY);
816
817 portMemCopy(pShimParams, sizeof *pShimParams, pParams, sizeof *pParams);
818
819 if (!pShimParams->bAllCaps)
820 {
821 // Must translate Guest GpuIds to Guest UUIDs
822 for (i = 0; i < pShimParams->peerGpuCount; i++)
823 {
824 NV2080_CTRL_GPU_P2P_PEER_CAPS_PEER_INFO *pParamsPeerInfo = &pShimParams->peerGpuCaps[i];
825 OBJGPU *pRemoteGpu = gpumgrGetGpuFromId(pParamsPeerInfo->gpuId);
826
827 NV_CHECK_OR_ELSE(LEVEL_INFO, pRemoteGpu != NULL,
828 status = NV_ERR_INVALID_ARGUMENT; goto done);
829
830 portMemCopy(pParamsPeerInfo->gpuUuid,
831 sizeof(pParamsPeerInfo->gpuUuid),
832 pRemoteGpu->gpuUuid.uuid,
833 sizeof(pRemoteGpu->gpuUuid.uuid));
834 }
835 }
836
837 pShimParams->bUseUuid = 1;
838
839 NV_RM_RPC_CONTROL(pGpu,
840 pControlParams->hClient,
841 pControlParams->hObject,
842 pControlParams->cmd,
843 pShimParams,
844 sizeof *pShimParams,
845 status);
846 if (status != NV_OK)
847 {
848 goto done;
849 }
850
851 // If bAllCaps, transfer additional output gpuIds and peerGpuCount
852 if (pParams->bAllCaps)
853 {
854 pParams->peerGpuCount = pShimParams->peerGpuCount;
855
856 for (i = 0; i < pParams->peerGpuCount; ++i)
857 {
858 // Use UUID to compute corresponding Guest GPU ID
859 OBJGPU *pRemoteGpu = gpumgrGetGpuFromUuid(pShimParams->peerGpuCaps[i].gpuUuid,
860 DRF_DEF(2080_GPU_CMD, _GPU_GET_GID_FLAGS, _TYPE, _SHA1) |
861 DRF_DEF(2080_GPU_CMD, _GPU_GET_GID_FLAGS, _FORMAT, _BINARY));
862
863 NV_CHECK_OR_ELSE(LEVEL_INFO, pRemoteGpu != NULL,
864 status = NV_ERR_GPU_UUID_NOT_FOUND; goto done);
865
866 pParams->peerGpuCaps[i].gpuId = pRemoteGpu->gpuId;
867 }
868 }
869
870 // Transfer output values from shimParams to user params.
871 for (i = 0; i < pParams->peerGpuCount; ++i)
872 {
873 pParams->peerGpuCaps[i].p2pCaps = pShimParams->peerGpuCaps[i].p2pCaps;
874 pParams->peerGpuCaps[i].p2pOptimalReadCEs = pShimParams->peerGpuCaps[i].p2pOptimalReadCEs;
875 pParams->peerGpuCaps[i].p2pOptimalWriteCEs = pShimParams->peerGpuCaps[i].p2pOptimalWriteCEs;
876 portMemCopy(pParams->peerGpuCaps[i].p2pCapsStatus,
877 sizeof(pParams->peerGpuCaps[i].p2pCapsStatus),
878 pShimParams->peerGpuCaps[i].p2pCapsStatus,
879 sizeof(pShimParams->peerGpuCaps[i].p2pCapsStatus));
880 pParams->peerGpuCaps[i].busPeerId = pShimParams->peerGpuCaps[i].busPeerId;
881 }
882
883 done:
884 portMemFree(pShimParams);
885
886 return status;
887 }
888
889 NV_STATUS
subdeviceCtrlCmdGetP2pCaps_IMPL(Subdevice * pSubdevice,NV2080_CTRL_GET_P2P_CAPS_PARAMS * pParams)890 subdeviceCtrlCmdGetP2pCaps_IMPL
891 (
892 Subdevice *pSubdevice,
893 NV2080_CTRL_GET_P2P_CAPS_PARAMS *pParams
894 )
895 {
896 NvU32 i;
897 NvU32 gfid = GPU_GFID_PF;
898 OBJGPU *pGpu = gpumgrGetGpuFromSubDeviceInst(pSubdevice->deviceInst,
899 pSubdevice->subDeviceInst);
900
901 // TODO: GPUSWSEC-1433 remove this check to enable this control for baremetal
902 if (!IS_VGPU_GSP_PLUGIN_OFFLOAD_ENABLED(pGpu))
903 return NV_ERR_NOT_SUPPORTED;
904
905 NV_CHECK_OR_RETURN(LEVEL_ERROR, pGpu != NULL, NV_ERR_INVALID_STATE);
906 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, vgpuGetCallingContextGfid(pGpu, &gfid));
907
908 NV_CHECK_OR_RETURN(LEVEL_ERROR,
909 (pParams->bAllCaps ||
910 (pParams->peerGpuCount <= NV_ARRAY_ELEMENTS(pGpu->P2PPeerGpuCaps))),
911 NV_ERR_INVALID_ARGUMENT);
912
913 NV_CHECK_OR_RETURN(LEVEL_ERROR, (pParams->bUseUuid == NV_FALSE), NV_ERR_NOT_SUPPORTED);
914
915 if (IS_VIRTUAL(pGpu))
916 {
917 return subdeviceGetP2pCaps_VIRTUAL(pGpu, pParams);
918 }
919
920 NV_CHECK_OR_RETURN(LEVEL_ERROR, RMCFG_FEATURE_PLATFORM_GSP, NV_ERR_NOT_SUPPORTED);
921
922 if (pParams->bAllCaps)
923 {
924 pParams->peerGpuCount = pGpu->P2PPeerGpuCount;
925
926 for (i = 0; i < pParams->peerGpuCount; i++)
927 {
928 pParams->peerGpuCaps[i].gpuId = pGpu->P2PPeerGpuCaps[i].peerGpuId;
929 }
930 }
931
932 for (i = 0; i < pParams->peerGpuCount; i++)
933 {
934 NV2080_CTRL_GPU_P2P_PEER_CAPS_PEER_INFO *pParamsPeerInfo = &pParams->peerGpuCaps[i];
935 GPU_P2P_PEER_GPU_CAPS *pLocalPeerCaps = NULL;
936
937 pLocalPeerCaps = gpuFindP2PPeerGpuCapsByGpuId(pGpu, pParamsPeerInfo->gpuId);
938 NV_CHECK_OR_RETURN(LEVEL_ERROR, pLocalPeerCaps != NULL, NV_ERR_OBJECT_NOT_FOUND);
939
940 //
941 // TODO: Currently, vGPU-GSP Guest only supports NVLINK DIRECT so unset caps for other modes.
942 // May remove once vGPU adds support for other modes.
943 //
944 if (IS_GFID_VF(gfid) &&
945 !REF_VAL(NV0000_CTRL_SYSTEM_GET_P2P_CAPS_NVLINK_SUPPORTED, pLocalPeerCaps->p2pCaps))
946 {
947 pParamsPeerInfo->p2pCaps = 0;
948 pParamsPeerInfo->p2pOptimalReadCEs = 0;
949 pParamsPeerInfo->p2pOptimalWriteCEs = 0;
950 pParamsPeerInfo->busPeerId = NV0000_CTRL_SYSTEM_GET_P2P_CAPS_INVALID_PEER;
951 pParamsPeerInfo->busEgmPeerId = NV0000_CTRL_SYSTEM_GET_P2P_CAPS_INVALID_PEER;
952
953 portMemSet(pParamsPeerInfo->p2pCapsStatus,
954 NV0000_P2P_CAPS_STATUS_NOT_SUPPORTED,
955 sizeof(pParamsPeerInfo->p2pCapsStatus));
956 }
957 else
958 {
959 pParamsPeerInfo->p2pCaps = pLocalPeerCaps->p2pCaps;
960 pParamsPeerInfo->p2pOptimalReadCEs = pLocalPeerCaps->p2pOptimalReadCEs;
961 pParamsPeerInfo->p2pOptimalWriteCEs = pLocalPeerCaps->p2pOptimalWriteCEs;
962 pParamsPeerInfo->busPeerId = pLocalPeerCaps->busPeerId;
963 pParamsPeerInfo->busEgmPeerId = pLocalPeerCaps->busEgmPeerId;
964
965 portMemCopy(pParamsPeerInfo->p2pCapsStatus,
966 sizeof(pParamsPeerInfo->p2pCapsStatus),
967 pLocalPeerCaps->p2pCapsStatus,
968 sizeof(pLocalPeerCaps->p2pCapsStatus));
969 }
970 }
971
972 return NV_OK;
973 }
974