1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2006-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "core/core.h"
25 #include "platform/chipset/chipset.h"
26 #include "mem_mgr/vaspace.h"
27 #include "gpu_mgr/gpu_mgr.h"
28 #include "gpu/gpu.h"
29 #include "gpu/bus/kern_bus.h"
30 #include "gpu/bus/p2p_api.h"
31 #include "gpu/bif/kernel_bif.h"
32
33 #include "published/maxwell/gm200/dev_nv_p2p.h"
34
35 // Defines for PCIE P2P
36 #define PCIE_P2P_MAX_WRITE_MAILBOX_ADDR \
37 ((PCIE_P2P_WRITE_MAILBOX_SIZE << DRF_SIZE(NV_P2P_WMBOX_ADDR_ADDR)) - \
38 PCIE_P2P_WRITE_MAILBOX_SIZE)
39
40 /*!
41 * @brief Setup the mailboxes of 2 GPUs so that the local GPU can access remote GPU.
42 *
43 * @param[in] pGpu0 (local GPU)
44 * @param[in] pKernelBus0 (local GPU)
45 * @param[in] pGpu1 (remote GPU)
46 * @param[in] pKernelBus1 (remote GPU)
47 * @param[in] local2Remote Local peer ID of pRemoteGpu on pLocalGpu
48 * @param[in] remote2Local Remote peer ID of pLocalGpu on pRemoteGpu
49 *
50 * @return void
51 */
52 void
kbusSetupMailboxes_GM200(OBJGPU * pGpu0,KernelBus * pKernelBus0,OBJGPU * pGpu1,KernelBus * pKernelBus1,NvU32 local2Remote,NvU32 remote2Local)53 kbusSetupMailboxes_GM200
54 (
55 OBJGPU *pGpu0,
56 KernelBus *pKernelBus0,
57 OBJGPU *pGpu1,
58 KernelBus *pKernelBus1,
59 NvU32 local2Remote,
60 NvU32 remote2Local
61 )
62 {
63 PMEMORY_DESCRIPTOR *ppMemDesc = NULL;
64 RmPhysAddr localP2PDomainRemoteAddr;
65 RmPhysAddr remoteP2PDomainLocalAddr;
66 RmPhysAddr remoteWMBoxLocalAddr;
67 NvU64 remoteWMBoxAddrU64;
68 NvBool bNeedWarBug999673 = kbusNeedWarForBug999673_HAL(pGpu0, pKernelBus0, pGpu1) ||
69 kbusNeedWarForBug999673_HAL(pGpu1, pKernelBus1, pGpu0);
70 RM_API *pRmApi0 = GPU_GET_PHYSICAL_RMAPI(pGpu0);
71 RM_API *pRmApi1 = GPU_GET_PHYSICAL_RMAPI(pGpu1);
72 NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_LOCAL_PARAMS params0 = {0};
73 NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_REMOTE_PARAMS params1 = {0};
74 NV_STATUS status;
75
76 NV_ASSERT_OR_RETURN_VOID(local2Remote < P2P_MAX_NUM_PEERS);
77 NV_ASSERT_OR_RETURN_VOID(remote2Local < P2P_MAX_NUM_PEERS);
78
79 // Ensure we have the correct bidirectional peer mapping
80 NV_ASSERT_OR_RETURN_VOID(pKernelBus1->p2pPcie.busPeer[remote2Local].remotePeerId ==
81 local2Remote);
82 NV_ASSERT_OR_RETURN_VOID(pKernelBus0->p2pPcie.busPeer[local2Remote].remotePeerId ==
83 remote2Local);
84
85 ppMemDesc = &pKernelBus0->p2pPcie.busPeer[local2Remote].pRemoteWMBoxMemDesc;
86 remoteWMBoxLocalAddr = kbusSetupMailboxAccess_HAL(pGpu1, pKernelBus1,
87 pGpu0, remote2Local,
88 ppMemDesc);
89 NV_ASSERT_OR_RETURN_VOID(remoteWMBoxLocalAddr != ~0ULL);
90
91 ppMemDesc = &pKernelBus1->p2pPcie.busPeer[remote2Local].pRemoteP2PDomMemDesc;
92 localP2PDomainRemoteAddr = kbusSetupP2PDomainAccess_HAL(pGpu0,
93 pKernelBus0,
94 pGpu1,
95 ppMemDesc);
96 NV_ASSERT_OR_RETURN_VOID(localP2PDomainRemoteAddr != ~0ULL);
97
98 ppMemDesc = &pKernelBus0->p2pPcie.busPeer[local2Remote].pRemoteP2PDomMemDesc;
99 remoteP2PDomainLocalAddr = kbusSetupP2PDomainAccess_HAL(pGpu1,
100 pKernelBus1,
101 pGpu0,
102 ppMemDesc);
103 NV_ASSERT_OR_RETURN_VOID(remoteP2PDomainLocalAddr != ~0ULL);
104
105 // Setup the local GPU to access remote GPU's FB.
106
107 // 0. Set write mail box data window on remote visible GPU to be
108 // used by local GPU.
109 remoteWMBoxAddrU64 = pKernelBus1->p2pPcie.writeMailboxBar1Addr +
110 PCIE_P2P_WRITE_MAILBOX_SIZE * remote2Local;
111
112 // Write mailbox data window needs to be 64KB aligned.
113 NV_ASSERT((remoteWMBoxAddrU64 & 0xFFFF) == 0);
114
115 // Setup PCIE P2P Mailbox on local GPU
116 params0.local2Remote = local2Remote;
117 params0.remote2Local = remote2Local;
118 params0.localP2PDomainRemoteAddr = localP2PDomainRemoteAddr;
119 params0.remoteP2PDomainLocalAddr = remoteP2PDomainLocalAddr;
120 params0.remoteWMBoxLocalAddr = remoteWMBoxLocalAddr;
121 params0.p2pWmbTag = 0;
122 params0.bNeedWarBug999673 = bNeedWarBug999673;
123
124 status = pRmApi0->Control(pRmApi0,
125 pGpu0->hInternalClient,
126 pGpu0->hInternalSubdevice,
127 NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_LOCAL,
128 ¶ms0,
129 sizeof(NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_LOCAL_PARAMS));
130 NV_ASSERT(status == NV_OK);
131
132 // Setup PCIE P2P Mailbox on remote GPU
133 params1.local2Remote = local2Remote;
134 params1.remote2Local = remote2Local;
135 params1.localP2PDomainRemoteAddr = localP2PDomainRemoteAddr;
136 params1.remoteP2PDomainLocalAddr = remoteP2PDomainLocalAddr;
137 params1.remoteWMBoxAddrU64 = remoteWMBoxAddrU64;
138 params1.p2pWmbTag = params0.p2pWmbTag;
139
140 status = pRmApi1->Control(pRmApi1,
141 pGpu1->hInternalClient,
142 pGpu1->hInternalSubdevice,
143 NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_REMOTE,
144 ¶ms1,
145 sizeof(NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_REMOTE_PARAMS));
146 NV_ASSERT(status == NV_OK);
147
148 kbusWriteP2PWmbTag_HAL(pGpu1, pKernelBus1, remote2Local, params0.p2pWmbTag);
149 }
150
151 void
kbusWriteP2PWmbTag_GM200(OBJGPU * pGpu,KernelBus * pKernelBus,NvU32 remote2Local,NvU64 p2pWmbTag)152 kbusWriteP2PWmbTag_GM200
153 (
154 OBJGPU *pGpu,
155 KernelBus *pKernelBus,
156 NvU32 remote2Local,
157 NvU64 p2pWmbTag
158 )
159 {
160 // See bug 3558208 comment 34 and 50
161 GPU_REG_RD32(pGpu, NV_P2P_WREQMB_L(remote2Local));
162 GPU_REG_WR32(pGpu, NV_P2P_WREQMB_L(remote2Local), NvU64_LO32(p2pWmbTag));
163 GPU_REG_WR32(pGpu, NV_P2P_WREQMB_H(remote2Local), NvU64_HI32(p2pWmbTag));
164 }
165
166 RmPhysAddr
kbusSetupP2PDomainAccess_GM200(OBJGPU * pGpu0,KernelBus * pKernelBus0,OBJGPU * pGpu1,PMEMORY_DESCRIPTOR * ppP2PDomMemDesc)167 kbusSetupP2PDomainAccess_GM200
168 (
169 OBJGPU *pGpu0,
170 KernelBus *pKernelBus0,
171 OBJGPU *pGpu1,
172 PMEMORY_DESCRIPTOR *ppP2PDomMemDesc
173 )
174 {
175 return kbusSetupPeerBarAccess(pGpu0, pGpu1,
176 pGpu0->busInfo.gpuPhysAddr + DRF_BASE(NV_P2P),
177 DRF_SIZE(NV_P2P), ppP2PDomMemDesc);
178 }
179
180 /*!
181 * @brief Creates a mapping for the remote peer to access its mailbox in
182 * the local GPU's BAR1
183 *
184 * @param[in] pGpu0 (local GPU)
185 * @param[in] pKernelBus0 (local GPU)
186 * @param[in] pGpu1 (remote GPU)
187 * @param[in] local2Remote Peer ID (local to remote)
188 * @param[out] ppWMBoxMemDesc
189 *
190 */
191 RmPhysAddr
kbusSetupMailboxAccess_GM200(OBJGPU * pGpu0,KernelBus * pKernelBus0,OBJGPU * pGpu1,NvU32 local2Remote,PMEMORY_DESCRIPTOR * ppWMBoxMemDesc)192 kbusSetupMailboxAccess_GM200
193 (
194 OBJGPU *pGpu0,
195 KernelBus *pKernelBus0,
196 OBJGPU *pGpu1,
197 NvU32 local2Remote,
198 PMEMORY_DESCRIPTOR *ppWMBoxMemDesc
199 )
200 {
201 return kbusSetupPeerBarAccess(pGpu0, pGpu1,
202 gpumgrGetGpuPhysFbAddr(pGpu0) +
203 pKernelBus0->p2pPcie.writeMailboxBar1Addr +
204 PCIE_P2P_WRITE_MAILBOX_SIZE * local2Remote,
205 PCIE_P2P_WRITE_MAILBOX_SIZE, ppWMBoxMemDesc);
206 }
207
208 void
kbusDestroyPeerAccess_GM200(OBJGPU * pGpu,KernelBus * pKernelBus,NvU32 peerNum)209 kbusDestroyPeerAccess_GM200
210 (
211 OBJGPU *pGpu,
212 KernelBus *pKernelBus,
213 NvU32 peerNum
214 )
215 {
216 memdescDestroy(pKernelBus->p2pPcie.busPeer[peerNum].pRemoteWMBoxMemDesc);
217 pKernelBus->p2pPcie.busPeer[peerNum].pRemoteWMBoxMemDesc = NULL;
218
219 memdescDestroy(pKernelBus->p2pPcie.busPeer[peerNum].pRemoteP2PDomMemDesc);
220 pKernelBus->p2pPcie.busPeer[peerNum].pRemoteP2PDomMemDesc = NULL;
221 }
222
223 /*!
224 * @brief Returns the P2P mailbox attributes such as size, aligment, max offset.
225 *
226 * @param[in] pGpu
227 * @param[in] pKernelBus
228 * @param[out] pMailboxAreaSize NvU32 pointer. Can be NULL
229 * @param[out] pMailboxAlignment NvU32 pointer. Can be NULL
230 * @param[out] pMailboxMaxOffset64KB NvU32 pointer. Can be NULL
231 *
232 * Returns the P2P mailbox attributes such as:
233 * - pMailboxAreaSize: total size
234 * - pMailboxAlignment: aligment
235 * - pMailboxMaxOffset: max supported offset
236 *
237 * return void
238 */
239 void
kbusGetP2PMailboxAttributes_GM200(OBJGPU * pGpu,KernelBus * pKernelBus,NvU32 * pMailboxAreaSize,NvU32 * pMailboxAlignmentSize,NvU32 * pMailboxBar1MaxOffset64KB)240 kbusGetP2PMailboxAttributes_GM200
241 (
242 OBJGPU *pGpu,
243 KernelBus *pKernelBus,
244 NvU32* pMailboxAreaSize,
245 NvU32* pMailboxAlignmentSize,
246 NvU32* pMailboxBar1MaxOffset64KB
247 )
248 {
249 KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu);
250
251 // Initialize null values by default
252 if (pMailboxAreaSize != NULL)
253 {
254 *pMailboxAreaSize = 0;
255 }
256 if (pMailboxAlignmentSize != NULL)
257 {
258 *pMailboxAlignmentSize = 0;
259 }
260 if (pMailboxBar1MaxOffset64KB != NULL)
261 {
262 *pMailboxBar1MaxOffset64KB = 0;
263 }
264
265 if (pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_P2P_READS_DISABLED) &&
266 pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_P2P_WRITES_DISABLED))
267 {
268 // Return null values
269 return;
270 }
271
272 // Retrieve attributes
273 if (pMailboxAreaSize != NULL)
274 {
275 *pMailboxAreaSize = PCIE_P2P_WRITE_MAILBOX_SIZE * P2P_MAX_NUM_PEERS;
276 }
277
278 if (pMailboxAlignmentSize != NULL)
279 {
280 // Write mailbox data window needs to be 64KB aligned.
281 *pMailboxAlignmentSize = 0x10000;
282 }
283
284 if (pMailboxBar1MaxOffset64KB != NULL)
285 {
286 // Max offset, exclusive
287 *pMailboxBar1MaxOffset64KB =
288 NvU64_LO32(
289 (PCIE_P2P_MAX_WRITE_MAILBOX_ADDR + PCIE_P2P_WRITE_MAILBOX_SIZE) >> 16
290 );
291 }
292
293 return;
294 }
295
296 /*!
297 * @brief Create PCIE Mailbox P2P mapping between 2 GPUs
298 *
299 * @param[in] pGpu0 (local GPU)
300 * @param[in] pKernelBus0 (local GPU)
301 * @param[in] pGpu1 (remote GPU)
302 * @param[in] pKernelBus1 (remote GPU)
303 * @param[out] peer0 NvU32 pointer, peerId on pGpu0
304 * @param[out] peer1 NvU32 pointer, peerId on pGpu1
305 * @param[in] attributes Sepcial attributes for the mapping
306 *
307 * @return NV_STATUS
308 */
309 NV_STATUS
kbusCreateP2PMapping_GM200(OBJGPU * pGpu0,KernelBus * pKernelBus0,OBJGPU * pGpu1,KernelBus * pKernelBus1,NvU32 * peer0,NvU32 * peer1,NvU32 attributes)310 kbusCreateP2PMapping_GM200
311 (
312 OBJGPU *pGpu0,
313 KernelBus *pKernelBus0,
314 OBJGPU *pGpu1,
315 KernelBus *pKernelBus1,
316 NvU32 *peer0,
317 NvU32 *peer1,
318 NvU32 attributes
319 )
320 {
321 if (FLD_TEST_DRF(_P2PAPI, _ATTRIBUTES, _CONNECTION_TYPE, _PCIE, attributes))
322 {
323 return kbusCreateP2PMappingForMailbox_HAL(pGpu0, pKernelBus0, pGpu1, pKernelBus1, peer0, peer1, attributes);
324 }
325
326 NV_PRINTF(LEVEL_ERROR, "P2P type %d is not supported\n", DRF_VAL(_P2PAPI, _ATTRIBUTES, _CONNECTION_TYPE, attributes));
327
328 return NV_ERR_NOT_SUPPORTED;
329 }
330
331 /*!
332 * @brief Create PCIE (not NVLINK) P2P mapping between 2 GPUs
333 *
334 * @param[in] pGpu0 (local GPU)
335 * @param[in] pKernelBus0 (local GPU)
336 * @param[in] pGpu1 (remote GPU)
337 * @param[in] pKernelBus1 (remote GPU)
338 * @param[out] peer0 Peer ID (local to remote)
339 * @param[out] peer1 Peer ID (remote to local)
340 * @param[in] attributes Unused
341 *
342 * @return NV_STATUS
343 */
344 NV_STATUS
kbusCreateP2PMappingForMailbox_GM200(OBJGPU * pGpu0,KernelBus * pKernelBus0,OBJGPU * pGpu1,KernelBus * pKernelBus1,NvU32 * peer0,NvU32 * peer1,NvU32 attributes)345 kbusCreateP2PMappingForMailbox_GM200
346 (
347 OBJGPU *pGpu0,
348 KernelBus *pKernelBus0,
349 OBJGPU *pGpu1,
350 KernelBus *pKernelBus1,
351 NvU32 *peer0,
352 NvU32 *peer1,
353 NvU32 attributes
354 )
355 {
356 RM_API *pRmApi;
357 NV2080_CTRL_INTERNAL_HSHUB_PEER_CONN_CONFIG_PARAMS params;
358 NvU32 gpuInst0, gpuInst1;
359
360 if (IS_VIRTUAL(pGpu0) || IS_VIRTUAL(pGpu1))
361 {
362 return NV_ERR_NOT_SUPPORTED;
363 }
364
365 if (peer0 == NULL || peer1 == NULL)
366 {
367 return NV_ERR_INVALID_ARGUMENT;
368 }
369
370 gpuInst0 = gpuGetInstance(pGpu0);
371 gpuInst1 = gpuGetInstance(pGpu1);
372
373 // Is a specific peer ID mapping requested?
374 if ((*peer0 != BUS_INVALID_PEER) && (*peer1 != BUS_INVALID_PEER))
375 {
376 NV_ASSERT_OR_RETURN(*peer0 < P2P_MAX_NUM_PEERS, NV_ERR_INVALID_ARGUMENT);
377 NV_ASSERT_OR_RETURN(*peer1 < P2P_MAX_NUM_PEERS, NV_ERR_INVALID_ARGUMENT);
378 //
379 // Ensure that if the requested peer ID is already in use, it
380 // corresponds to the requested remote GPU.
381 //
382 if (!pKernelBus0->p2pPcie.busPeer[*peer0].bReserved &&
383 !pKernelBus1->p2pPcie.busPeer[*peer1].bReserved)
384 {
385 if ((pKernelBus0->p2pPcie.busPeer[*peer0].refCount == 0) &&
386 (pKernelBus1->p2pPcie.busPeer[*peer1].refCount == 0))
387 {
388 goto busCreateP2PMapping_setupMapping;
389 }
390
391 if (((pKernelBus0->p2pPcie.peerNumberMask[gpuInst1] & NVBIT(*peer0)) != 0) &&
392 ((pKernelBus1->p2pPcie.peerNumberMask[gpuInst0] & NVBIT(*peer1)) != 0))
393 {
394 pKernelBus0->p2pPcie.busPeer[*peer0].refCount++;
395 pKernelBus1->p2pPcie.busPeer[*peer1].refCount++;
396
397 NV_ASSERT(pKernelBus0->p2pPcie.busPeer[*peer0].remotePeerId == *peer1);
398 NV_ASSERT(pKernelBus1->p2pPcie.busPeer[*peer1].remotePeerId == *peer0);
399
400 pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu0);
401 portMemSet(¶ms, 0, sizeof(params));
402 params.programPciePeerMask = NVBIT32(*peer0);
403 NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi,
404 pGpu0->hInternalClient,
405 pGpu0->hInternalSubdevice,
406 NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG,
407 ¶ms,
408 sizeof(params)));
409
410 pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu1);
411 portMemSet(¶ms, 0, sizeof(params));
412 params.programPciePeerMask = NVBIT32(*peer1);
413 NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi,
414 pGpu1->hInternalClient,
415 pGpu1->hInternalSubdevice,
416 NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG,
417 ¶ms,
418 sizeof(params)));
419
420 return NV_OK;
421 }
422 }
423
424 NV_PRINTF(LEVEL_WARNING,
425 "explicit peer IDs %u and %u requested for GPU%u and GPU%u are not "
426 "available, will assign dynamically\n", *peer0, *peer1,
427 gpuInst0, gpuInst1);
428 }
429
430 // Does a mapping already exist between these GPUs?
431 if ((pKernelBus0->p2pPcie.peerNumberMask[gpuInst1] != 0) &&
432 (pKernelBus1->p2pPcie.peerNumberMask[gpuInst0] != 0))
433 {
434 *peer0 = pKernelBus0->p2pPcie.peerNumberMask[gpuInst1];
435 LOWESTBITIDX_32(*peer0);
436 NV_ASSERT_OR_RETURN(*peer0 < P2P_MAX_NUM_PEERS,
437 NV_ERR_INVALID_STATE);
438
439 *peer1 = pKernelBus0->p2pPcie.busPeer[*peer0].remotePeerId;
440 NV_ASSERT_OR_RETURN(*peer1 < P2P_MAX_NUM_PEERS,
441 NV_ERR_INVALID_STATE);
442
443 NV_ASSERT_OR_RETURN(pKernelBus1->p2pPcie.busPeer[*peer1].remotePeerId == *peer0,
444 NV_ERR_INVALID_STATE);
445
446 pKernelBus0->p2pPcie.busPeer[*peer0].refCount++;
447 pKernelBus1->p2pPcie.busPeer[*peer1].refCount++;
448
449 NV_ASSERT(!pKernelBus0->p2pPcie.busPeer[*peer0].bReserved);
450 NV_ASSERT(!pKernelBus1->p2pPcie.busPeer[*peer1].bReserved);
451
452 pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu0);
453 portMemSet(¶ms, 0, sizeof(params));
454 params.programPciePeerMask = NVBIT32(*peer0);
455 NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi,
456 pGpu0->hInternalClient,
457 pGpu0->hInternalSubdevice,
458 NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG,
459 ¶ms,
460 sizeof(params)));
461
462 pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu1);
463 portMemSet(¶ms, 0, sizeof(params));
464 params.programPciePeerMask = NVBIT32(*peer1);
465 NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi,
466 pGpu1->hInternalClient,
467 pGpu1->hInternalSubdevice,
468 NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG,
469 ¶ms,
470 sizeof(params)));
471
472 return NV_OK;
473 }
474
475 // We'd better not hit this case (one gpu has mapping and other doesn't).
476 NV_ASSERT((pKernelBus0->p2pPcie.peerNumberMask[gpuInst1] == 0) &&
477 (pKernelBus1->p2pPcie.peerNumberMask[gpuInst0] == 0));
478
479 *peer0 = BUS_INVALID_PEER;
480 *peer1 = BUS_INVALID_PEER;
481
482 // If we're in loopback mode, check for specified peer ID
483 if ((pGpu0 == pGpu1) && pKernelBus0->p2pMapSpecifyId)
484 {
485 if ((pKernelBus0->p2pPcie.busPeer[pKernelBus0->p2pMapPeerId].refCount == 0) &&
486 (!pKernelBus0->p2pPcie.busPeer[pKernelBus0->p2pMapPeerId].bReserved) &&
487 (pKernelBus1->p2pPcie.busPeer[pKernelBus1->p2pMapPeerId].refCount == 0))
488 {
489 *peer0 = *peer1 = pKernelBus0->p2pMapPeerId;
490 }
491 else
492 {
493 NV_PRINTF(LEVEL_ERROR,
494 "- ERROR: Peer ID %d is already in use. Default RM P2P mapping will be used.\n",
495 pKernelBus0->p2pMapPeerId);
496 }
497 }
498
499 //
500 // These loops will handle loop back (pGpu0 == pGpu1) since they will find
501 // the same free peer twice on the same GPU.
502 //
503 if (*peer0 == BUS_INVALID_PEER)
504 {
505 *peer0 = kbusGetUnusedPciePeerId_HAL(pGpu0, pKernelBus0);
506 }
507
508 if (*peer1 == BUS_INVALID_PEER)
509 {
510 *peer1 = kbusGetUnusedPciePeerId_HAL(pGpu1, pKernelBus1);
511 }
512
513 // couldn't find an available peer on both gpus
514 if (*peer0 == BUS_INVALID_PEER ||
515 *peer1 == BUS_INVALID_PEER)
516 {
517 *peer0 = BUS_INVALID_PEER;
518 *peer1 = BUS_INVALID_PEER;
519 NV_PRINTF(LEVEL_ERROR, "no peer IDs available\n");
520 return NV_ERR_INSUFFICIENT_RESOURCES;
521 }
522
523 busCreateP2PMapping_setupMapping:
524 pKernelBus0->p2pPcie.busPeer[*peer0].remotePeerId = *peer1;
525 pKernelBus0->p2pPcie.peerNumberMask[gpuInst1] |= NVBIT(*peer0);
526 pKernelBus1->p2pPcie.busPeer[*peer1].remotePeerId = *peer0;
527 pKernelBus1->p2pPcie.peerNumberMask[gpuInst0] |= NVBIT(*peer1);
528
529 NV_ASSERT(pKernelBus0->p2pPcie.busPeer[*peer0].refCount == 0);
530 NV_ASSERT(!pKernelBus0->p2pPcie.busPeer[*peer0].bReserved);
531 NV_ASSERT(pKernelBus1->p2pPcie.busPeer[*peer1].refCount == 0);
532 NV_ASSERT(!pKernelBus1->p2pPcie.busPeer[*peer1].bReserved);
533
534 //
535 // Note if this is loopback we will have a refCount of 2. This will be
536 // accounted for in the free.
537 //
538 pKernelBus0->p2pPcie.busPeer[*peer0].refCount++;
539 pKernelBus1->p2pPcie.busPeer[*peer1].refCount++;
540
541 pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu0);
542 portMemSet(¶ms, 0, sizeof(params));
543 params.programPciePeerMask = NVBIT32(*peer0);
544 NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi,
545 pGpu0->hInternalClient,
546 pGpu0->hInternalSubdevice,
547 NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG,
548 ¶ms,
549 sizeof(params)));
550
551 pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu1);
552 portMemSet(¶ms, 0, sizeof(params));
553 params.programPciePeerMask = NVBIT32(*peer1);
554 NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi,
555 pGpu1->hInternalClient,
556 pGpu1->hInternalSubdevice,
557 NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG,
558 ¶ms,
559 sizeof(params)));
560
561 NV_PRINTF(LEVEL_INFO,
562 "added PCIe P2P mapping between GPU%u (peer %u) and GPU%u (peer %u)\n",
563 gpuInst0, *peer0, gpuInst1, *peer1);
564
565 kbusSetupMailboxes_HAL(pGpu0, pKernelBus0, pGpu1, pKernelBus1, *peer0, *peer1);
566 kbusSetupMailboxes_HAL(pGpu1, pKernelBus1, pGpu0, pKernelBus0, *peer1, *peer0);
567
568 return NV_OK;
569 }
570
571 /*!
572 * Does it need P2P WAR for bug 999673?
573 *
574 * @param[in] pGpu
575 * @param[in] pKernelBus
576 * @param[in] pRemoteGpu
577 *
578 * @return NvBool
579 *
580 */
581 NvBool
kbusNeedWarForBug999673_GM200(OBJGPU * pGpu,KernelBus * pKernelBus,OBJGPU * pRemoteGpu)582 kbusNeedWarForBug999673_GM200
583 (
584 OBJGPU *pGpu,
585 KernelBus *pKernelBus,
586 OBJGPU *pRemoteGpu
587 )
588 {
589 OBJCL *pCl = SYS_GET_CL(SYS_GET_INSTANCE());
590 NvU8 pciSwitchBus = 0;
591
592 // Return if WAR is not needed
593 if (!pCl->getProperty(pCl, PDB_PROP_CL_BUG_999673_P2P_ARBITRARY_SPLIT_WAR))
594 {
595 return NV_FALSE;
596 }
597
598 // See if we have a known bridge
599 clFindCommonDownstreamBR(pGpu, pRemoteGpu, pCl, &pciSwitchBus);
600 if (pciSwitchBus != 0xFF)
601 {
602 // P2P does not go through the chipset needing the WAR.
603 return NV_FALSE;
604 }
605
606 return NV_TRUE;
607 }
608
609 /*!
610 * @brief Remove the P2P mapping to a given peer GPU
611 *
612 * @param[in] pGpu0 (local GPU)
613 * @param[in] pKernelBus0 (local GPU)
614 * @param[in] pGpu1 (remote GPU)
615 * @param[in] pKernelBus1 (remote GPU)
616 * @param[in] peer0 Peer ID (local to remote)
617 * @param[in] peer1 Peer ID (remote to local)
618 * @param[in] attributes Sepcial attributes for the mapping
619 *
620 * return NV_OK on success
621 */
622 NV_STATUS
kbusRemoveP2PMapping_GM200(OBJGPU * pGpu0,KernelBus * pKernelBus0,OBJGPU * pGpu1,KernelBus * pKernelBus1,NvU32 peer0,NvU32 peer1,NvU32 attributes)623 kbusRemoveP2PMapping_GM200
624 (
625 OBJGPU *pGpu0,
626 KernelBus *pKernelBus0,
627 OBJGPU *pGpu1,
628 KernelBus *pKernelBus1,
629 NvU32 peer0,
630 NvU32 peer1,
631 NvU32 attributes
632 )
633 {
634 if (FLD_TEST_DRF(_P2PAPI, _ATTRIBUTES, _CONNECTION_TYPE, _PCIE, attributes))
635 {
636 return kbusRemoveP2PMappingForMailbox_HAL(pGpu0, pKernelBus0, pGpu1, pKernelBus1, peer0, peer1, attributes);
637 }
638
639 NV_PRINTF(LEVEL_ERROR, "P2P type %d is not supported\n", DRF_VAL(_P2PAPI, _ATTRIBUTES, _CONNECTION_TYPE, attributes));
640
641 return NV_ERR_NOT_SUPPORTED;
642 }
643
644 /*!
645 * @brief Create a P2P mapping to a given peer GPU
646 *
647 * @param[in] pGpu0 (local GPU)
648 * @param[in] pKernelBus0 (local GPU)
649 * @param[in] pGpu1 (remote GPU)
650 * @param[in] pKernelBus1 (remote GPU)
651 * @param[out] peer0 Peer ID (local to remote)
652 * @param[out] peer1 Peer ID (remote to local)
653 * @param[in] attributes Sepcial attributes for the mapping
654 *
655 * return NV_OK on success
656 */
657 NV_STATUS
kbusRemoveP2PMappingForMailbox_GM200(OBJGPU * pGpu0,KernelBus * pKernelBus0,OBJGPU * pGpu1,KernelBus * pKernelBus1,NvU32 peer0,NvU32 peer1,NvU32 attributes)658 kbusRemoveP2PMappingForMailbox_GM200
659 (
660 OBJGPU *pGpu0,
661 KernelBus *pKernelBus0,
662 OBJGPU *pGpu1,
663 KernelBus *pKernelBus1,
664 NvU32 peer0,
665 NvU32 peer1,
666 NvU32 attributes
667 )
668 {
669 NvU32 gpuInst0 = gpuGetInstance(pGpu0);
670 NvU32 gpuInst1 = gpuGetInstance(pGpu1);
671
672 if (IS_VIRTUAL(pGpu0) || IS_VIRTUAL(pGpu1))
673 {
674 return NV_ERR_NOT_SUPPORTED;
675 }
676
677 // a non-existent mapping
678 if(peer0 == BUS_INVALID_PEER ||
679 peer1 == BUS_INVALID_PEER)
680 {
681 return NV_ERR_INVALID_ARGUMENT;
682 }
683
684 // Do the peer IDs correspond to the expected GPUs?
685 if (((pKernelBus0->p2pPcie.peerNumberMask[gpuInst1] & NVBIT(peer0)) == 0) ||
686 ((pKernelBus1->p2pPcie.peerNumberMask[gpuInst0] & NVBIT(peer1)) == 0))
687 {
688 return NV_ERR_INVALID_ARGUMENT;
689 }
690
691 //
692 // a programming error somewhere in RM.
693 // A mapping exists with a refCount == 0
694 //
695 if (pKernelBus0->p2pPcie.busPeer[peer0].refCount == 0 ||
696 pKernelBus1->p2pPcie.busPeer[peer1].refCount == 0)
697 {
698 DBG_BREAKPOINT();
699 return NV_ERR_GENERIC;
700 }
701
702 //
703 // Again a programming error. The mapping should have the same refCount on
704 // both GPUs.
705 //
706 if (pKernelBus0->p2pPcie.busPeer[peer0].refCount !=
707 pKernelBus1->p2pPcie.busPeer[peer1].refCount)
708 {
709 DBG_BREAKPOINT();
710 return NV_ERR_GENERIC;
711 }
712
713
714 --pKernelBus1->p2pPcie.busPeer[peer1].refCount;
715 if (--pKernelBus0->p2pPcie.busPeer[peer0].refCount == 0)
716 {
717 NV_PRINTF(LEVEL_INFO,
718 "Removing mapping GPU %d Peer %d <-> GPU %d Peer %d\n",
719 gpuInst0, peer0, gpuInst1, peer1);
720
721 pKernelBus0->p2pPcie.peerNumberMask[gpuInst1] &= ~NVBIT(peer0);
722 pKernelBus1->p2pPcie.peerNumberMask[gpuInst0] &= ~NVBIT(peer1);
723
724 kbusDestroyMailbox(pGpu0, pKernelBus0, pGpu1, peer0);
725 kbusDestroyMailbox(pGpu1, pKernelBus1, pGpu0, peer1);
726 }
727 else
728 {
729 NV_PRINTF(LEVEL_INFO,
730 "Decremented refCount for Mapping GPU %d Peer %d <-> GPU %d Peer %d "
731 "New Count: %d\n", gpuInst0, peer0, gpuInst1, peer1,
732 pKernelBus0->p2pPcie.busPeer[peer0].refCount);
733 }
734
735 return NV_OK;
736 }
737
738 /*!
739 * @brief Reserve peer IDs for nvlink usage
740 *
741 * @param[in] pGpu
742 * @param[in] pKernelBus
743 * @param[in] peerMask Mask of peer IDs to reserve
744 *
745 * return NV_OK on success
746 */
747 NV_STATUS
kbusReserveP2PPeerIds_GM200(OBJGPU * pGpu,KernelBus * pKernelBus,NvU32 peerMask)748 kbusReserveP2PPeerIds_GM200
749 (
750 OBJGPU *pGpu,
751 KernelBus *pKernelBus,
752 NvU32 peerMask
753 )
754 {
755 NvU32 peerId = 0;
756
757 FOR_EACH_INDEX_IN_MASK(32, peerId, peerMask)
758 {
759 NV_PRINTF(LEVEL_INFO,
760 "reserving peer ID %u on GPU%u for NVLINK/C2C use\n", peerId,
761 gpuGetInstance(pGpu));
762 if (pKernelBus->p2pPcie.busPeer[peerId].refCount != 0)
763 {
764 return NV_ERR_IN_USE;
765 }
766
767 pKernelBus->p2pPcie.busPeer[peerId].bReserved = NV_TRUE;
768 }
769 FOR_EACH_INDEX_IN_MASK_END;
770
771 return NV_OK;
772 }
773
774 /*!
775 * @brief Sets the BAR1 P2P mailbox address and size
776 *
777 * @param[in] pGpu
778 * @param[in] pKernelBus
779 * @param[in] mailboxBar1Addr NvU64
780 * @param[in] mailboxTotalSize NvU32
781 *
782 * @returns NV_STATUS
783 */
784 NV_STATUS
kbusSetP2PMailboxBar1Area_GM200(OBJGPU * pGpu,KernelBus * pKernelBus,NvU64 mailboxBar1Addr,NvU32 mailboxTotalSize)785 kbusSetP2PMailboxBar1Area_GM200
786 (
787 OBJGPU *pGpu,
788 KernelBus *pKernelBus,
789 NvU64 mailboxBar1Addr,
790 NvU32 mailboxTotalSize
791 )
792 {
793 NvU32 mailboxAreaSizeReq;
794 NvU32 mailboxAlignmentSizeReq;
795 NvU32 mailboxBar1MaxOffset64KBReq;
796
797
798 if (!kbusIsP2pMailboxClientAllocated(pKernelBus))
799 {
800 // P2P mailbox area already allocated by RM. Nothing to do.
801 return NV_OK;
802 }
803
804 if (mailboxTotalSize == 0)
805 {
806 NV_PRINTF(LEVEL_ERROR, "P2P mailbox area size is not set\n");
807 return NV_ERR_INVALID_ARGUMENT;
808 }
809
810 kbusGetP2PMailboxAttributes_HAL(pGpu, pKernelBus, &mailboxAreaSizeReq, &mailboxAlignmentSizeReq, &mailboxBar1MaxOffset64KBReq);
811
812 // Mailbox size
813 NV_ASSERT_OR_RETURN(mailboxTotalSize == mailboxAreaSizeReq, NV_ERR_INVALID_ARGUMENT);
814 // Mailbox alignment
815 NV_ASSERT_OR_RETURN((mailboxBar1Addr & (mailboxAlignmentSizeReq - 1)) == 0, NV_ERR_INVALID_ARGUMENT);
816 // Mailbox offset limit
817 NV_ASSERT_OR_RETURN((mailboxBar1Addr + mailboxTotalSize) < (((NvU64)mailboxBar1MaxOffset64KBReq) << 16),
818 NV_ERR_INVALID_ARGUMENT);
819
820 if (pKernelBus->p2pPcie.writeMailboxBar1Addr != PCIE_P2P_INVALID_WRITE_MAILBOX_ADDR)
821 {
822 NV_ASSERT_OR_RETURN(mailboxBar1Addr == pKernelBus->p2pPcie.writeMailboxBar1Addr, NV_ERR_INVALID_ARGUMENT);
823 NV_ASSERT_OR_RETURN(mailboxTotalSize == pKernelBus->p2pPcie.writeMailboxTotalSize, NV_ERR_INVALID_ARGUMENT);
824 return NV_OK;
825 }
826
827 pKernelBus->p2pPcie.writeMailboxBar1Addr = mailboxBar1Addr;
828 pKernelBus->p2pPcie.writeMailboxTotalSize = mailboxTotalSize;
829 pKernelBus->bP2pInitialized = NV_TRUE;
830
831 return NV_OK;
832 }
833
834
835 /*!
836 * @brief Unset the BAR1 P2P mailbox address and size
837 *
838 * @param[in] pGpu
839 * @param[in] pKernelBus
840 *
841 * @returns void
842 */
843 void
kbusUnsetP2PMailboxBar1Area_GM200(OBJGPU * pGpu,KernelBus * pKernelBus)844 kbusUnsetP2PMailboxBar1Area_GM200
845 (
846 OBJGPU *pGpu,
847 KernelBus *pKernelBus
848 )
849 {
850 NvU32 i;
851
852 if (!kbusIsP2pMailboxClientAllocated(pKernelBus))
853 {
854 // P2P mailbox area already allocated by RM. Nothing to do.
855 return;
856 }
857
858 for (i = 0; i < P2P_MAX_NUM_PEERS; ++i)
859 {
860 if (pKernelBus->p2pPcie.busPeer[i].refCount)
861 break;
862 }
863
864 if (i == P2P_MAX_NUM_PEERS)
865 {
866 pKernelBus->p2pPcie.writeMailboxBar1Addr = PCIE_P2P_INVALID_WRITE_MAILBOX_ADDR;
867 pKernelBus->p2pPcie.writeMailboxTotalSize = 0;
868 pKernelBus->bP2pInitialized = NV_FALSE;
869 }
870
871 return;
872 }
873
874 NV_STATUS
kbusAllocP2PMailboxBar1_GM200(OBJGPU * pGpu,KernelBus * pKernelBus,NvU32 gfid,NvU64 vaRangeMax)875 kbusAllocP2PMailboxBar1_GM200
876 (
877 OBJGPU *pGpu,
878 KernelBus *pKernelBus,
879 NvU32 gfid,
880 NvU64 vaRangeMax
881 )
882 {
883 OBJGPU *pParentGpu;
884 NvU64 vaAllocMax;
885 NV_STATUS status = NV_OK;
886
887 VAS_ALLOC_FLAGS flags = {0};
888
889 pParentGpu = gpumgrGetParentGPU(pGpu);
890
891 if (!gpumgrIsParentGPU(pGpu))
892 {
893 flags.bFixedAddressAllocate = NV_TRUE;
894 pKernelBus->p2pPcie.writeMailboxBar1Addr = GPU_GET_KERNEL_BUS(pParentGpu)->p2pPcie.writeMailboxBar1Addr;
895 }
896
897 pKernelBus->p2pPcie.writeMailboxTotalSize =
898 PCIE_P2P_WRITE_MAILBOX_SIZE * P2P_MAX_NUM_PEERS;
899 vaAllocMax = NV_MIN(vaRangeMax,
900 PCIE_P2P_MAX_WRITE_MAILBOX_ADDR + PCIE_P2P_WRITE_MAILBOX_SIZE - 1);
901
902 status = vaspaceAlloc(pKernelBus->bar1[gfid].pVAS,
903 pKernelBus->p2pPcie.writeMailboxTotalSize,
904 PCIE_P2P_WRITE_MAILBOX_SIZE,
905 0, vaAllocMax,
906 0,
907 flags,
908 &pKernelBus->p2pPcie.writeMailboxBar1Addr);
909 if (status != NV_OK)
910 {
911 NV_PRINTF(LEVEL_ERROR,
912 "cannot allocate vaspace for P2P write mailboxes (0x%x)\n",
913 status);
914 goto kbusAllocP2PMailboxBar1_failed;
915 }
916
917 NV_ASSERT(GPU_GET_KERNEL_BUS(pParentGpu)->p2pPcie.writeMailboxBar1Addr == pKernelBus->p2pPcie.writeMailboxBar1Addr);
918
919 NV_PRINTF(LEVEL_INFO,
920 "[GPU%u] P2P write mailboxes allocated at BAR1 addr = 0x%llx\n",
921 gpuGetInstance(pGpu), pKernelBus->p2pPcie.writeMailboxBar1Addr);
922
923 kbusAllocP2PMailboxBar1_failed:
924 if (status != NV_OK)
925 {
926 pKernelBus->p2pPcie.writeMailboxBar1Addr = PCIE_P2P_INVALID_WRITE_MAILBOX_ADDR;
927 pKernelBus->p2pPcie.writeMailboxTotalSize = 0;
928 }
929
930 return status;
931 }
932