1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2009-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "core/core.h"
25 #include "gpu/gpu.h"
26 #include "gpu/device/device.h"
27 #include "gpu/subdevice/subdevice.h"
28 #include "gpu/bus/kern_bus.h"
29 #include "gpu/bus/p2p_api.h"
30 #include "gpu/bus/third_party_p2p.h"
31 #include "gpu/mem_mgr/mem_mgr.h"
32 #include "platform/p2p/p2p_caps.h"
33 #include "kernel/gpu/nvlink/kernel_nvlink.h"
34 #include "nvRmReg.h"
35 #include "rmapi/rs_utils.h"
36 #include "vgpu/rpc.h"
37 #include "vgpu/vgpu_events.h"
38
39 #include "class/cl503b.h"
40 #include <class/cl90f1.h> //FERMI_VASPACE_A
41
42 /*!
43 * @brief Helper function to reserve peer ids in non-GSP offload vGPU case.
44 */
45 static
46 NV_STATUS
_p2papiReservePeerID(OBJGPU * pLocalGpu,KernelBus * pLocalKernelBus,OBJGPU * pRemoteGpu,KernelBus * pRemoteKernelBus,NV503B_ALLOC_PARAMETERS * pNv503bAllocParams,P2PApi * pP2PApi,NvU32 * peer1,NvU32 * peer2,NvBool bEgmPeer,NvBool bSpaAccessOnly)47 _p2papiReservePeerID
48 (
49 OBJGPU *pLocalGpu,
50 KernelBus *pLocalKernelBus,
51 OBJGPU *pRemoteGpu,
52 KernelBus *pRemoteKernelBus,
53 NV503B_ALLOC_PARAMETERS *pNv503bAllocParams,
54 P2PApi *pP2PApi,
55 NvU32 *peer1,
56 NvU32 *peer2,
57 NvBool bEgmPeer,
58 NvBool bSpaAccessOnly
59 )
60 {
61 NvU32 gpu0Instance = gpuGetInstance(pLocalGpu);
62 NvU32 gpu1Instance = gpuGetInstance(pRemoteGpu);
63
64 // loopback request
65 if (
66 !bEgmPeer &&
67 (pNv503bAllocParams->hSubDevice == pNv503bAllocParams->hPeerSubDevice))
68 {
69 *peer1 = *peer2 = 0;
70
71 // Reserve the peer1 ID for NVLink use
72 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
73 kbusReserveP2PPeerIds_HAL(pLocalGpu, pLocalKernelBus, NVBIT(*peer1)));
74 }
75 else
76 {
77 if (bEgmPeer)
78 {
79 // Check if a peer ID is already allocated for P2P from pLocalGpu to pRemoteGpu
80 *peer1 = kbusGetEgmPeerId_HAL(pLocalGpu, pLocalKernelBus, pRemoteGpu);
81
82 // Check if a peer ID is already allocated for P2P from pRemoteGpu to pLocalGpu
83 *peer2 = kbusGetEgmPeerId_HAL(pRemoteGpu, pRemoteKernelBus, pLocalGpu);
84 }
85 else
86 {
87 // Check if a peer ID is already allocated for P2P from pLocalGpu to pRemoteGpu
88 *peer1 = kbusGetPeerId_HAL(pLocalGpu, pLocalKernelBus, pRemoteGpu);
89
90 // Check if a peer ID is already allocated for P2P from pRemoteGpu to pLocalGpu
91 *peer2 = kbusGetPeerId_HAL(pRemoteGpu, pRemoteKernelBus, pLocalGpu);
92 }
93 }
94
95 if (*peer1 != BUS_INVALID_PEER && *peer2 != BUS_INVALID_PEER)
96 {
97 goto update_mask;
98 }
99 else if (*peer1 == BUS_INVALID_PEER && *peer2 == BUS_INVALID_PEER)
100 {
101 // Get the peer ID pGpu0 should use for P2P over NVLINK to pGpu1i
102 *peer1 = kbusGetUnusedPeerId_HAL(pLocalGpu, pLocalKernelBus);
103 // If could not find a free peer ID, return error
104 if (*peer1 == BUS_INVALID_PEER)
105 {
106 NV_PRINTF(LEVEL_ERROR,
107 "GPU%d: peerID not available for NVLink P2P\n",
108 gpu0Instance);
109 return NV_ERR_GENERIC;
110 }
111
112 // Reserve the peer ID for NVLink use
113 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
114 kbusReserveP2PPeerIds_HAL(pLocalGpu, pLocalKernelBus, NVBIT(*peer1)));
115
116 if (pNv503bAllocParams->hSubDevice == pNv503bAllocParams->hPeerSubDevice)
117 {
118 // The loopback check here becomes true only in the EGM case
119 NV_ASSERT_OR_RETURN(bEgmPeer, NV_ERR_INVALID_STATE);
120 *peer2 = *peer1;
121 }
122 else
123 {
124 // Get the peer ID pGpu1 should use for P2P over NVLINK to pGpu0
125 *peer2 = kbusGetUnusedPeerId_HAL(pRemoteGpu, pRemoteKernelBus);
126 // If could not find a free peer ID, return error
127 if (*peer2 == BUS_INVALID_PEER)
128 {
129 NV_PRINTF(LEVEL_ERROR,
130 "GPU%d: peerID not available for NVLink P2P\n",
131 gpu1Instance);
132 return NV_ERR_GENERIC;
133 }
134
135 // Reserve the peer ID for NVLink use
136 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
137 kbusReserveP2PPeerIds_HAL(pRemoteGpu, pRemoteKernelBus, NVBIT(*peer2)));
138 }
139 }
140 else
141 {
142 NV_PRINTF(LEVEL_ERROR,
143 "Unexpected state, either of the peer ID is invalid \n");
144 return NV_ERR_GENERIC;
145 }
146
147 update_mask:
148 if (bEgmPeer)
149 {
150 NV_PRINTF(LEVEL_INFO, "EGM peer\n");
151 }
152 //
153 // Does the mapping already exist between the given pair of GPUs using the peerIDs
154 // peer1 and peer2 respectively ?
155 //
156 if ((pLocalKernelBus->p2p.busNvlinkPeerNumberMask[gpu1Instance] & NVBIT(*peer1)) &&
157 (pRemoteKernelBus->p2p.busNvlinkPeerNumberMask[gpu0Instance] & NVBIT(*peer2)))
158 {
159 //
160 // Increment the mapping refcount per peerID - since there is another usage
161 // of a mapping that is using this peerID
162 //
163 pLocalKernelBus->p2p.busNvlinkMappingRefcountPerPeerId[*peer1]++;
164 pRemoteKernelBus->p2p.busNvlinkMappingRefcountPerPeerId[*peer2]++;
165
166 //
167 // Increment the mapping refcount per GPU - since there is another usage of
168 // the mapping to the given remote GPU
169 //
170 pLocalKernelBus->p2p.busNvlinkMappingRefcountPerGpu[gpu1Instance]++;
171 pRemoteKernelBus->p2p.busNvlinkMappingRefcountPerGpu[gpu0Instance]++;
172
173 if (bSpaAccessOnly)
174 {
175 pLocalKernelBus->p2p.busNvlinkMappingRefcountPerPeerIdSpa[*peer1]++;
176 pRemoteKernelBus->p2p.busNvlinkMappingRefcountPerPeerId[*peer2]++;
177 }
178
179 NV_PRINTF(LEVEL_INFO,
180 "- P2P: Peer mapping is already in use for gpu instances %x and %x "
181 "with peer id's %d and %d. Increasing the mapping refcounts for the"
182 " peer IDs to %d and %d respectively.\n",
183 gpu0Instance, gpu1Instance, *peer1, *peer2,
184 pLocalKernelBus->p2p.busNvlinkMappingRefcountPerPeerId[*peer1],
185 pRemoteKernelBus->p2p.busNvlinkMappingRefcountPerPeerId[*peer2]);
186 goto update_params;
187 }
188
189 //
190 // Reached here implies the mapping between the given pair of GPUs using the peerIDs
191 // peer1 and peer2 does not exist. Create the mapping
192 //
193
194 // Set the peer IDs in the corresponding peer number masks
195 pLocalKernelBus->p2p.busNvlinkPeerNumberMask[gpu1Instance] |= NVBIT(*peer1);
196 pRemoteKernelBus->p2p.busNvlinkPeerNumberMask[gpu0Instance] |= NVBIT(*peer2);
197
198 pLocalKernelBus->p2p.bEgmPeer[*peer1] = bEgmPeer;
199 pRemoteKernelBus->p2p.bEgmPeer[*peer2] = bEgmPeer;
200
201 //
202 // Increment the mapping refcount per peerID - since there is a new mapping that
203 // will use this peerID
204 //
205 pLocalKernelBus->p2p.busNvlinkMappingRefcountPerPeerId[*peer1]++;
206 pRemoteKernelBus->p2p.busNvlinkMappingRefcountPerPeerId[*peer2]++;
207
208 if (bSpaAccessOnly)
209 {
210 pLocalKernelBus->p2p.busNvlinkMappingRefcountPerPeerIdSpa[*peer1]++;
211 pRemoteKernelBus->p2p.busNvlinkMappingRefcountPerPeerId[*peer2]++;
212 }
213
214 //
215 // Increment the mapping refcount per GPU - since there a new mapping now to the
216 // given remote GPU
217 //
218 pLocalKernelBus->p2p.busNvlinkMappingRefcountPerGpu[gpu1Instance]++;
219 pRemoteKernelBus->p2p.busNvlinkMappingRefcountPerGpu[gpu0Instance]++;
220
221 NV_PRINTF(LEVEL_INFO,
222 "added NVLink P2P mapping between GPU%u (peer %u) and GPU%u (peer %u)\n",
223 gpu0Instance, *peer1, gpu1Instance, *peer2);
224
225 update_params:
226 if (bEgmPeer)
227 {
228 pNv503bAllocParams->subDeviceEgmPeerIdMask = NVBIT(*peer1);
229 pNv503bAllocParams->peerSubDeviceEgmPeerIdMask = NVBIT(*peer2);
230 }
231 else
232 {
233 pNv503bAllocParams->subDevicePeerIdMask = NVBIT(*peer1);
234 pNv503bAllocParams->peerSubDevicePeerIdMask = NVBIT(*peer2);
235 }
236
237 // Update connection type for SRIOV.
238 pP2PApi->attributes = FLD_SET_DRF(_P2PAPI, _ATTRIBUTES, _CONNECTION_TYPE, _NVLINK, pP2PApi->attributes);
239
240 return NV_OK;
241 }
242
243
244 NV_STATUS
p2papiConstruct_IMPL(P2PApi * pP2PApi,CALL_CONTEXT * pCallContext,RS_RES_ALLOC_PARAMS_INTERNAL * pParams)245 p2papiConstruct_IMPL
246 (
247 P2PApi *pP2PApi,
248 CALL_CONTEXT *pCallContext,
249 RS_RES_ALLOC_PARAMS_INTERNAL *pParams
250 )
251 {
252 Subdevice *pSubDevice;
253 Subdevice *pPeerSubDevice;
254 NvU32 subDevicePeerIdMask;
255 NvU32 peerSubDevicePeerIdMask;
256 RsClient *pClient;
257 RmClient *pRmClient;
258 NvU32 peer1;
259 NvU32 peer2;
260 NvHandle hDevice;
261 NvHandle hPeerDevice;
262 NvHandle hSubDevice;
263 NvHandle hPeerSubDevice;
264 OBJGPU *pGpu;
265 OBJGPU *pLocalGpu;
266 KernelBus *pLocalKernelBus;
267 KernelNvlink *pLocalKernelNvlink;
268 OBJGPU *pRemoteGpu;
269 KernelBus *pRemoteKernelBus;
270 KernelNvlink *pRemoteKernelNvlink;
271 NV_STATUS status;
272 NvBool bP2PWriteCapable = NV_FALSE;
273 NvBool bP2PReadCapable = NV_FALSE;
274 NV503B_ALLOC_PARAMETERS *pNv503bAllocParams = pParams->pAllocParams;
275 NvU32 flags = pNv503bAllocParams->flags;
276 NvBool bSpaAccessOnly = FLD_TEST_DRF(503B, _FLAGS, _P2P_TYPE, _SPA, flags);
277 P2P_CONNECTIVITY p2pConnectionType = P2P_CONNECTIVITY_UNKNOWN;
278 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
279 NV0000_CTRL_SYSTEM_GET_P2P_CAPS_V2_PARAMS *pP2pCapsParams;
280 NvU32 p2pCaps;
281
282 subDevicePeerIdMask = pNv503bAllocParams->subDevicePeerIdMask;
283 peerSubDevicePeerIdMask = pNv503bAllocParams->peerSubDevicePeerIdMask;
284 NvU32 egmPeer1;
285 NvU32 egmPeer2;
286 NvU32 subDeviceEgmPeerIdMask;
287 NvU32 peerSubDeviceEgmPeerIdMask;
288 NvBool bEgmPeer;
289
290 subDeviceEgmPeerIdMask = pNv503bAllocParams->subDeviceEgmPeerIdMask;
291 peerSubDeviceEgmPeerIdMask = pNv503bAllocParams->peerSubDeviceEgmPeerIdMask;
292
293 pRmClient = dynamicCast(pCallContext->pClient, RmClient);
294 if (pRmClient == NULL)
295 return NV_ERR_INVALID_ARGUMENT;
296
297 pClient = staticCast(pRmClient, RsClient);
298
299 status = subdeviceGetByHandle(pClient, pNv503bAllocParams->hSubDevice, &pSubDevice);
300 if (status != NV_OK)
301 return NV_ERR_INVALID_ARGUMENT;
302
303 status = subdeviceGetByHandle(pClient, pNv503bAllocParams->hPeerSubDevice, &pPeerSubDevice);
304 if (status != NV_OK)
305 return NV_ERR_INVALID_ARGUMENT;
306
307 if (pNv503bAllocParams->subDevicePeerIdMask)
308 {
309 if (!ONEBITSET(pNv503bAllocParams->subDevicePeerIdMask))
310 {
311 return NV_ERR_INVALID_ARGUMENT;
312 }
313 }
314
315 if (pNv503bAllocParams->peerSubDevicePeerIdMask)
316 {
317 if (!ONEBITSET(pNv503bAllocParams->peerSubDevicePeerIdMask))
318 {
319 return NV_ERR_INVALID_ARGUMENT;
320 }
321 }
322
323 if (pNv503bAllocParams->subDeviceEgmPeerIdMask)
324 {
325 if (!ONEBITSET(pNv503bAllocParams->subDeviceEgmPeerIdMask))
326 {
327 return NV_ERR_INVALID_ARGUMENT;
328 }
329 }
330
331 if (pNv503bAllocParams->peerSubDeviceEgmPeerIdMask)
332 {
333 if (!ONEBITSET(pNv503bAllocParams->peerSubDeviceEgmPeerIdMask))
334 {
335 return NV_ERR_INVALID_ARGUMENT;
336 }
337 }
338
339 // Ensure any loopback requests match
340 if (pNv503bAllocParams->hSubDevice == pNv503bAllocParams->hPeerSubDevice)
341 {
342 if (pNv503bAllocParams->subDevicePeerIdMask != pNv503bAllocParams->peerSubDevicePeerIdMask)
343 {
344 return NV_ERR_INVALID_ARGUMENT;
345 }
346 if (pNv503bAllocParams->subDeviceEgmPeerIdMask != pNv503bAllocParams->peerSubDeviceEgmPeerIdMask)
347 {
348 return NV_ERR_INVALID_ARGUMENT;
349 }
350 }
351
352 hSubDevice = RES_GET_HANDLE(pSubDevice);
353 hPeerSubDevice = RES_GET_HANDLE(pPeerSubDevice);
354
355 // Find the gpu for the subdevices passed to us
356 pLocalGpu = GPU_RES_GET_GPU(pSubDevice);
357 pRemoteGpu = GPU_RES_GET_GPU(pPeerSubDevice);
358
359 // Get device handles
360 hDevice = RES_GET_HANDLE(pSubDevice->pDevice);
361 hPeerDevice = RES_GET_HANDLE(pPeerSubDevice->pDevice);
362
363 API_GPU_FULL_POWER_SANITY_CHECK(pLocalGpu, NV_TRUE, NV_FALSE);
364 API_GPU_FULL_POWER_SANITY_CHECK(pRemoteGpu, NV_TRUE, NV_FALSE);
365
366 if (gpuIsApmFeatureEnabled(pLocalGpu))
367 {
368 return NV_ERR_NOT_SUPPORTED;
369 }
370
371 // SPA peer only supported when we support ATS
372 if (bSpaAccessOnly && (!pLocalGpu->getProperty(pLocalGpu, PDB_PROP_GPU_ATS_SUPPORTED)))
373 {
374 return NV_ERR_INVALID_ARGUMENT;
375 }
376
377 pLocalKernelBus = GPU_GET_KERNEL_BUS(pLocalGpu);
378 pRemoteKernelBus = GPU_GET_KERNEL_BUS(pRemoteGpu);
379
380 pP2pCapsParams = portMemAllocStackOrHeap(sizeof(*pP2pCapsParams));
381 if (pP2pCapsParams == NULL)
382 {
383 return NV_ERR_NO_MEMORY;
384 }
385
386 portMemSet(pP2pCapsParams, 0, sizeof(*pP2pCapsParams));
387
388 pP2pCapsParams->gpuCount = 2;
389 pP2pCapsParams->gpuIds[0] = pLocalGpu->gpuId;
390 pP2pCapsParams->gpuIds[1] = pRemoteGpu->gpuId;
391
392 NV_CHECK_OK_OR_ELSE(status, LEVEL_ERROR,
393 pRmApi->Control(pRmApi, pClient->hClient, pClient->hClient,
394 NV0000_CTRL_CMD_SYSTEM_GET_P2P_CAPS_V2,
395 pP2pCapsParams, sizeof(*pP2pCapsParams)),
396 portMemFreeStackOrHeap(pP2pCapsParams);
397 return status);
398
399 bP2PWriteCapable = REF_VAL(NV0000_CTRL_SYSTEM_GET_P2P_CAPS_WRITES_SUPPORTED, pP2pCapsParams->p2pCaps);
400 bP2PReadCapable = REF_VAL(NV0000_CTRL_SYSTEM_GET_P2P_CAPS_READS_SUPPORTED, pP2pCapsParams->p2pCaps);
401
402 p2pCaps = pP2pCapsParams->p2pCaps;
403
404 portMemFreeStackOrHeap(pP2pCapsParams);
405
406 if (REF_VAL(NV0000_CTRL_SYSTEM_GET_P2P_CAPS_C2C_SUPPORTED, p2pCaps))
407 p2pConnectionType = P2P_CONNECTIVITY_C2C;
408 // It's impossible to detect P2P_CONNECTIVITY_NVLINK_INDIRECT connectivity.
409 // There is no difference between NVLINK and NVLINK_INDIRECT in P2PApi creation path,
410 // so always use P2P_CONNECTIVITY_NVLINK.
411 else if (REF_VAL(NV0000_CTRL_SYSTEM_GET_P2P_CAPS_NVLINK_SUPPORTED, p2pCaps))
412 p2pConnectionType = P2P_CONNECTIVITY_NVLINK;
413 else if (REF_VAL(NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PCI_BAR1_SUPPORTED, p2pCaps))
414 p2pConnectionType = P2P_CONNECTIVITY_PCIE_BAR1;
415 else if (REF_VAL(NV0000_CTRL_SYSTEM_GET_P2P_CAPS_PROP_SUPPORTED, p2pCaps))
416 p2pConnectionType = P2P_CONNECTIVITY_PCIE_PROPRIETARY;
417 else
418 {
419 NV_PRINTF(LEVEL_ERROR, "Unknown connection type\n");
420 return NV_ERR_NOT_SUPPORTED;
421 }
422
423 //
424 // Allocate P2P PCIE Mailbox areas if all of the following conditions occur:
425 // - P2P reads or/and writes are supported
426 // - The P2P connection is PCIE Mailbox based
427 //
428 if ((bP2PWriteCapable || bP2PReadCapable) &&
429 p2pConnectionType == P2P_CONNECTIVITY_PCIE_PROPRIETARY)
430 {
431 status = kbusSetP2PMailboxBar1Area_HAL(pLocalGpu, pLocalKernelBus,
432 pNv503bAllocParams->mailboxBar1Addr,
433 pNv503bAllocParams->mailboxTotalSize);
434 NV_ASSERT_OK_OR_RETURN(status);
435
436 status = kbusSetP2PMailboxBar1Area_HAL(pRemoteGpu, pRemoteKernelBus,
437 pNv503bAllocParams->mailboxBar1Addr,
438 pNv503bAllocParams->mailboxTotalSize);
439 NV_ASSERT_OK_OR_RETURN(status);
440 }
441
442 // Process any specific peer id requests for peer 1
443 if (subDevicePeerIdMask)
444 {
445 peer1 = BIT_IDX_32(subDevicePeerIdMask);
446 }
447 else
448 {
449 peer1 = BUS_INVALID_PEER;
450 }
451
452 // Process any specific peer id requests for peer 2
453 if (peerSubDevicePeerIdMask)
454 {
455 peer2 = BIT_IDX_32(peerSubDevicePeerIdMask);
456 }
457 else
458 {
459 peer2 = BUS_INVALID_PEER;
460 }
461
462 // Process any specific peer id requests for EGM peer 1
463 if (subDeviceEgmPeerIdMask)
464 {
465 egmPeer1 = BIT_IDX_32(subDeviceEgmPeerIdMask);
466 }
467 else
468 {
469 egmPeer1 = BUS_INVALID_PEER;
470 }
471
472 // Process any specific peer id requests for EGM peer 2
473 if (peerSubDeviceEgmPeerIdMask)
474 {
475 egmPeer2 = BIT_IDX_32(peerSubDeviceEgmPeerIdMask);
476 }
477 else
478 {
479 egmPeer2 = BUS_INVALID_PEER;
480 }
481
482 if (!IS_VIRTUAL(pLocalGpu))
483 {
484 if (!(bP2PWriteCapable || bP2PReadCapable))
485 {
486 NV_PRINTF(LEVEL_ERROR,
487 "ERROR: P2P is Disabled, cannot create mappings\n");
488 return NV_ERR_NOT_SUPPORTED;
489 }
490
491 // Train links to high speed.
492 pLocalKernelNvlink = GPU_GET_KERNEL_NVLINK(pLocalGpu);
493 pRemoteKernelNvlink = GPU_GET_KERNEL_NVLINK(pRemoteGpu);
494
495 if (pLocalKernelNvlink && pRemoteKernelNvlink)
496 {
497 status = knvlinkTrainFabricLinksToActive(pLocalGpu, pLocalKernelNvlink);
498 if (status != NV_OK)
499 {
500 NV_PRINTF(LEVEL_ERROR,
501 "link training between GPU%u and SWITCH failed with status %x\n",
502 pLocalGpu->gpuInstance, status);
503 return status;
504 }
505
506 status = knvlinkTrainFabricLinksToActive(pRemoteGpu, pRemoteKernelNvlink);
507 if (status != NV_OK)
508 {
509 NV_PRINTF(LEVEL_ERROR,
510 "link training between GPU%u and SWITCH failed with status %x\n",
511 pRemoteGpu->gpuInstance, status);
512 return status;
513 }
514 }
515 }
516
517 // check to see if a p2p mapping between these two subdevices already exist
518 if (!IsGP100orBetter(pLocalGpu))
519 {
520 RS_ORDERED_ITERATOR iter;
521 P2PApi *pOtherP2PApi = NULL;
522
523 iter = clientRefOrderedIter(pCallContext->pClient, NULL /*pScoreRef*/,
524 classId(P2PApi), NV_TRUE /*bExactMatch*/);
525 while (clientRefOrderedIterNext(iter.pClient, &iter))
526 {
527 pOtherP2PApi = dynamicCast(iter.pResourceRef->pResource, P2PApi);
528 if (pOtherP2PApi == NULL)
529 return NV_ERR_INVALID_OBJECT_HANDLE;
530
531 if (pP2PApi != pOtherP2PApi &&
532 ((pLocalGpu == pOtherP2PApi->peer1 && pRemoteGpu == pOtherP2PApi->peer2) ||
533 (pLocalGpu == pOtherP2PApi->peer2 && pRemoteGpu == pOtherP2PApi->peer1)))
534 {
535 NV_PRINTF(LEVEL_INFO,
536 "Mapping already exists between the two subdevices (0x%08x), (0x%08x). "
537 "Multiple mappings not supported on pre-PASCAL GPUs\n",
538 hSubDevice, hPeerSubDevice);
539 return NV_ERR_INVALID_ARGUMENT;
540 }
541 }
542 }
543
544 pP2PApi->peer1 = pLocalGpu;
545 pP2PApi->peer2 = pRemoteGpu;
546 pP2PApi->attributes = DRF_NUM(_P2PAPI, _ATTRIBUTES, _CONNECTION_TYPE, p2pConnectionType);
547 pP2PApi->attributes |= bSpaAccessOnly ? DRF_DEF(_P2PAPI, _ATTRIBUTES, _LINK_TYPE, _SPA) :
548 DRF_DEF(_P2PAPI, _ATTRIBUTES, _LINK_TYPE, _GPA);
549
550 //
551 // For Nvswitch connected systems, AAS(Alternate Address Space) is set by Nvswitch itself
552 // based on the EGM fabric address range and so there is no need for a separate peer id
553 // in the Nvswitch case.
554 //
555 bEgmPeer = (!bSpaAccessOnly &&
556 memmgrIsLocalEgmEnabled(GPU_GET_MEMORY_MANAGER(pLocalGpu)) &&
557 memmgrIsLocalEgmEnabled(GPU_GET_MEMORY_MANAGER(pRemoteGpu)) &&
558 !GPU_IS_NVSWITCH_DETECTED(pLocalGpu));
559
560 if (bSpaAccessOnly &&
561 memmgrIsLocalEgmEnabled(GPU_GET_MEMORY_MANAGER(pLocalGpu)) &&
562 memmgrIsLocalEgmEnabled(GPU_GET_MEMORY_MANAGER(pRemoteGpu)))
563 {
564 NV_PRINTF(LEVEL_INFO, "EGM P2P not setup because of SPA only P2P flag!\n");
565 }
566
567 // Set the default Bar1 P2P DMA Info
568 pNv503bAllocParams->l2pBar1P2PDmaInfo.dma_address = \
569 pNv503bAllocParams->p2lBar1P2PDmaInfo.dma_address = NV_U64_MAX;
570 pNv503bAllocParams->l2pBar1P2PDmaInfo.dma_size = \
571 pNv503bAllocParams->p2lBar1P2PDmaInfo.dma_size = 0;
572
573 if (IS_VGPU_GSP_PLUGIN_OFFLOAD_ENABLED(pLocalGpu) || !IS_VIRTUAL(pLocalGpu))
574 {
575 //
576 // TODO: This function need to have a cleanup path when this function
577 // fails after kbusCreateP2PMaping(), busBindLocalGfidForP2P()
578 // and busBindRemoteGfidForP2P(). The current state, the
579 // function just returns an error. Bug 4016670 filed to track
580 // the effort.
581 //
582
583 // setup the p2p resources
584 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
585 kbusCreateP2PMapping_HAL(pLocalGpu, pLocalKernelBus, pRemoteGpu,
586 pRemoteKernelBus, &peer1, &peer2,
587 pP2PApi->attributes));
588 if (bEgmPeer)
589 {
590 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
591 kbusCreateP2PMapping_HAL(pLocalGpu, pLocalKernelBus, pRemoteGpu,
592 pRemoteKernelBus, &egmPeer1, &egmPeer2,
593 pP2PApi->attributes |
594 DRF_DEF(_P2PAPI, _ATTRIBUTES, _REMOTE_EGM, _YES)));
595 }
596
597 if ((p2pConnectionType == P2P_CONNECTIVITY_PCIE_BAR1) &&
598 (pCallContext->secInfo.privLevel >= RS_PRIV_LEVEL_KERNEL))
599 {
600 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
601 kbusGetBar1P2PDmaInfo_HAL(pLocalGpu, pRemoteGpu,
602 pRemoteKernelBus,
603 &pNv503bAllocParams->l2pBar1P2PDmaInfo.dma_address,
604 &pNv503bAllocParams->l2pBar1P2PDmaInfo.dma_size));
605
606 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
607 kbusGetBar1P2PDmaInfo_HAL(pRemoteGpu, pLocalGpu,
608 pLocalKernelBus,
609 &pNv503bAllocParams->p2lBar1P2PDmaInfo.dma_address,
610 &pNv503bAllocParams->p2lBar1P2PDmaInfo.dma_size));
611 }
612 }
613
614 pGpu = pLocalGpu;
615
616 if (!IS_VGPU_GSP_PLUGIN_OFFLOAD_ENABLED(pGpu) &&
617 IS_VIRTUAL_WITH_SRIOV(pGpu) &&
618 gpuIsSplitVasManagementServerClientRmEnabled(pGpu))
619 {
620 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
621 _p2papiReservePeerID(pLocalGpu, pLocalKernelBus, pRemoteGpu,
622 pRemoteKernelBus, pNv503bAllocParams, pP2PApi,
623 &peer1, &peer2, NV_FALSE, bSpaAccessOnly));
624 if (bEgmPeer)
625 {
626 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
627 _p2papiReservePeerID(pLocalGpu, pLocalKernelBus, pRemoteGpu,
628 pRemoteKernelBus, pNv503bAllocParams, pP2PApi,
629 &egmPeer1, &egmPeer2, NV_TRUE, bSpaAccessOnly));
630 }
631 }
632
633 pP2PApi->peerId1 = peer1;
634 pP2PApi->peerId2 = peer2;
635 pP2PApi->egmPeerId1 = egmPeer1;
636 pP2PApi->egmPeerId2 = egmPeer2;
637 pP2PApi->localGfid = GPU_GFID_PF;
638 pP2PApi->remoteGfid = GPU_GFID_PF;
639
640 if (!IS_VGPU_GSP_PLUGIN_OFFLOAD_ENABLED(pLocalGpu) && IS_VIRTUAL(pLocalGpu))
641 {
642 NV_RM_RPC_ALLOC_OBJECT(pLocalGpu,
643 pParams->hClient,
644 pParams->hParent,
645 pParams->hResource,
646 pParams->externalClassId,
647 pNv503bAllocParams,
648 sizeof(*pNv503bAllocParams),
649 status);
650 if (status != NV_OK)
651 return status;
652 }
653
654 //
655 // For SRIOV system, always check for P2P allocation to determine whether
656 // this function is allowed to bind FLA
657 //
658 if (pLocalKernelBus->flaInfo.bFlaAllocated && !pLocalKernelBus->flaInfo.bFlaBind)
659 {
660 if (!IS_VIRTUAL(pLocalGpu))
661 {
662 goto remote_fla_bind;
663 }
664 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
665 kbusSetupBindFla_HAL(pLocalGpu, pLocalKernelBus, pP2PApi->localGfid));
666 }
667
668 remote_fla_bind:
669
670 if (hDevice != hPeerDevice)
671 {
672 if (pRemoteKernelBus->flaInfo.bFlaAllocated && !pRemoteKernelBus->flaInfo.bFlaBind)
673 {
674 if (!IS_VIRTUAL(pRemoteGpu))
675 {
676 return status;
677 }
678 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
679 kbusSetupBindFla_HAL(pRemoteGpu, pRemoteKernelBus, pP2PApi->remoteGfid));
680 }
681 }
682
683 // Adding P2PApi as a dependant of 2 Subdevices, P2PApi must be destroyed before OBJGPU destruction
684 NV_ASSERT_OK_OR_RETURN(refAddDependant(RES_GET_REF(pSubDevice), pCallContext->pResourceRef));
685 if (hDevice != hPeerDevice)
686 {
687 NV_ASSERT_OK_OR_RETURN(refAddDependant(RES_GET_REF(pPeerSubDevice), pCallContext->pResourceRef));
688 }
689
690 if (status == NV_OK)
691 {
692 NV_CHECK_OR_RETURN(LEVEL_ERROR, pLocalKernelBus->totalP2pObjectsAliveRefCount < NV_U32_MAX, NV_ERR_INSUFFICIENT_RESOURCES);
693 NV_CHECK_OR_RETURN(LEVEL_ERROR, pRemoteKernelBus->totalP2pObjectsAliveRefCount < NV_U32_MAX, NV_ERR_INSUFFICIENT_RESOURCES);
694 pLocalKernelBus->totalP2pObjectsAliveRefCount++;
695 pRemoteKernelBus->totalP2pObjectsAliveRefCount++;
696 }
697 return status;
698 }
699
700 void
p2papiDestruct_IMPL(P2PApi * pP2PApi)701 p2papiDestruct_IMPL
702 (
703 P2PApi *pP2PApi
704 )
705 {
706 CALL_CONTEXT *pCallContext;
707 RS_RES_FREE_PARAMS_INTERNAL *pParams;
708 NvHandle hClient;
709 OBJGPU *pLocalGpu;
710 KernelBus *pLocalKernelBus;
711 OBJGPU *pRemoteGpu;
712 KernelBus *pRemoteKernelBus;
713 RsClient *pClient;
714 NV_STATUS status = NV_OK;
715
716 resGetFreeParams(staticCast(pP2PApi, RsResource), &pCallContext, &pParams);
717 hClient = pParams->hClient;
718
719 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR,
720 serverGetClientUnderLock(&g_resServ, hClient, &pClient), end);
721
722 if (NULL == pP2PApi->peer1 || NULL == pP2PApi->peer2)
723 return;
724
725 pLocalGpu = pP2PApi->peer1;
726 pRemoteGpu = pP2PApi->peer2;
727
728 pLocalKernelBus = GPU_GET_KERNEL_BUS(pLocalGpu);
729 pRemoteKernelBus = GPU_GET_KERNEL_BUS(pRemoteGpu);
730
731 NV_ASSERT(pLocalKernelBus->totalP2pObjectsAliveRefCount > 0);
732 NV_ASSERT(pRemoteKernelBus->totalP2pObjectsAliveRefCount > 0);
733 if (pLocalKernelBus->totalP2pObjectsAliveRefCount > 0)
734 pLocalKernelBus->totalP2pObjectsAliveRefCount--;
735
736 if (pRemoteKernelBus->totalP2pObjectsAliveRefCount > 0)
737 pRemoteKernelBus->totalP2pObjectsAliveRefCount--;
738
739 if (IS_VGPU_GSP_PLUGIN_OFFLOAD_ENABLED(pLocalGpu) || !IS_VIRTUAL(pLocalGpu))
740 {
741 // remove any resources associated with this mapping
742 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR,
743 kbusRemoveP2PMapping_HAL(pLocalGpu, pLocalKernelBus,
744 pRemoteGpu, pRemoteKernelBus,
745 pP2PApi->peerId1, pP2PApi->peerId2,
746 pP2PApi->attributes), end);
747 if (!FLD_TEST_DRF(_P2PAPI, _ATTRIBUTES, _LINK_TYPE, _SPA, pP2PApi->attributes) &&
748 memmgrIsLocalEgmEnabled(GPU_GET_MEMORY_MANAGER(pLocalGpu)) &&
749 memmgrIsLocalEgmEnabled(GPU_GET_MEMORY_MANAGER(pRemoteGpu)) &&
750 !GPU_IS_NVSWITCH_DETECTED(pLocalGpu))
751 {
752 status = kbusRemoveP2PMapping_HAL(pLocalGpu, pLocalKernelBus,
753 pRemoteGpu, pRemoteKernelBus,
754 pP2PApi->egmPeerId1, pP2PApi->egmPeerId2,
755 pP2PApi->attributes |
756 DRF_DEF(_P2PAPI, _ATTRIBUTES, _REMOTE_EGM, _YES));
757 }
758 }
759
760 pP2PApi->peer1 = NULL;
761 pP2PApi->peer2 = NULL;
762
763 kbusUnsetP2PMailboxBar1Area_HAL(pLocalGpu, pLocalKernelBus);
764 kbusUnsetP2PMailboxBar1Area_HAL(pRemoteGpu, pRemoteKernelBus);
765
766 end:
767 pParams->status = status;
768 }
769