1 /*
2  * SPDX-FileCopyrightText: Copyright (c) 2006-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3  * SPDX-License-Identifier: MIT
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21  * DEALINGS IN THE SOFTWARE.
22  */
23 
24 #include "core/core.h"
25 #include "platform/chipset/chipset.h"
26 #include "mem_mgr/vaspace.h"
27 #include "gpu_mgr/gpu_mgr.h"
28 #include "gpu/gpu.h"
29 #include "gpu/bus/kern_bus.h"
30 #include "gpu/bus/p2p_api.h"
31 #include "gpu/bif/kernel_bif.h"
32 
33 #include "published/maxwell/gm200/dev_nv_p2p.h"
34 
35 // Defines for PCIE P2P
36 #define PCIE_P2P_MAX_WRITE_MAILBOX_ADDR                                  \
37     ((PCIE_P2P_WRITE_MAILBOX_SIZE << DRF_SIZE(NV_P2P_WMBOX_ADDR_ADDR)) - \
38      PCIE_P2P_WRITE_MAILBOX_SIZE)
39 
40 /*!
41  * @brief Setup the mailboxes of 2 GPUs so that the local GPU can access remote GPU.
42  *
43  * @param[in]   pGpu0          (local GPU)
44  * @param[in]   pKernelBus0    (local GPU)
45  * @param[in]   pGpu1          (remote GPU)
46  * @param[in]   pKernelBus1    (remote GPU)
47  * @param[in]   local2Remote   Local peer ID of pRemoteGpu on pLocalGpu
48  * @param[in]   remote2Local   Remote peer ID of pLocalGpu on pRemoteGpu
49  *
50  * @return void
51  */
52 void
kbusSetupMailboxes_GM200(OBJGPU * pGpu0,KernelBus * pKernelBus0,OBJGPU * pGpu1,KernelBus * pKernelBus1,NvU32 local2Remote,NvU32 remote2Local)53 kbusSetupMailboxes_GM200
54 (
55     OBJGPU    *pGpu0,
56     KernelBus *pKernelBus0,
57     OBJGPU    *pGpu1,
58     KernelBus *pKernelBus1,
59     NvU32      local2Remote,
60     NvU32      remote2Local
61 )
62 {
63     PMEMORY_DESCRIPTOR *ppMemDesc    = NULL;
64     RmPhysAddr          localP2PDomainRemoteAddr;
65     RmPhysAddr          remoteP2PDomainLocalAddr;
66     RmPhysAddr          remoteWMBoxLocalAddr;
67     NvU64               remoteWMBoxAddrU64;
68     NvBool              bNeedWarBug999673 = kbusNeedWarForBug999673_HAL(pGpu0, pKernelBus0, pGpu1) ||
69                                             kbusNeedWarForBug999673_HAL(pGpu1, pKernelBus1, pGpu0);
70     RM_API *pRmApi0     = GPU_GET_PHYSICAL_RMAPI(pGpu0);
71     RM_API *pRmApi1     = GPU_GET_PHYSICAL_RMAPI(pGpu1);
72     NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_LOCAL_PARAMS  params0 = {0};
73     NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_REMOTE_PARAMS params1 = {0};
74     NV_STATUS status;
75 
76     NV_ASSERT_OR_RETURN_VOID(local2Remote < P2P_MAX_NUM_PEERS);
77     NV_ASSERT_OR_RETURN_VOID(remote2Local < P2P_MAX_NUM_PEERS);
78 
79     // Ensure we have the correct bidirectional peer mapping
80     NV_ASSERT_OR_RETURN_VOID(pKernelBus1->p2pPcie.busPeer[remote2Local].remotePeerId ==
81                           local2Remote);
82     NV_ASSERT_OR_RETURN_VOID(pKernelBus0->p2pPcie.busPeer[local2Remote].remotePeerId ==
83                           remote2Local);
84 
85     ppMemDesc = &pKernelBus0->p2pPcie.busPeer[local2Remote].pRemoteWMBoxMemDesc;
86     remoteWMBoxLocalAddr = kbusSetupMailboxAccess_HAL(pGpu1, pKernelBus1,
87                                                       pGpu0, remote2Local,
88                                                       ppMemDesc);
89     NV_ASSERT_OR_RETURN_VOID(remoteWMBoxLocalAddr != ~0ULL);
90 
91     ppMemDesc = &pKernelBus1->p2pPcie.busPeer[remote2Local].pRemoteP2PDomMemDesc;
92     localP2PDomainRemoteAddr = kbusSetupP2PDomainAccess_HAL(pGpu0,
93                                                             pKernelBus0,
94                                                             pGpu1,
95                                                             ppMemDesc);
96     NV_ASSERT_OR_RETURN_VOID(localP2PDomainRemoteAddr != ~0ULL);
97 
98     ppMemDesc = &pKernelBus0->p2pPcie.busPeer[local2Remote].pRemoteP2PDomMemDesc;
99     remoteP2PDomainLocalAddr = kbusSetupP2PDomainAccess_HAL(pGpu1,
100                                                             pKernelBus1,
101                                                             pGpu0,
102                                                             ppMemDesc);
103     NV_ASSERT_OR_RETURN_VOID(remoteP2PDomainLocalAddr != ~0ULL);
104 
105     // Setup the local GPU to access remote GPU's FB.
106 
107     // 0. Set write mail box data window on remote visible GPU to be
108     // used by local GPU.
109     remoteWMBoxAddrU64 = pKernelBus1->p2pPcie.writeMailboxBar1Addr +
110       PCIE_P2P_WRITE_MAILBOX_SIZE * remote2Local;
111 
112     // Write mailbox data window needs to be 64KB aligned.
113     NV_ASSERT((remoteWMBoxAddrU64 & 0xFFFF) == 0);
114 
115     // Setup PCIE P2P Mailbox on local GPU
116     params0.local2Remote                = local2Remote;
117     params0.remote2Local                = remote2Local;
118     params0.localP2PDomainRemoteAddr    = localP2PDomainRemoteAddr;
119     params0.remoteP2PDomainLocalAddr    = remoteP2PDomainLocalAddr;
120     params0.remoteWMBoxLocalAddr        = remoteWMBoxLocalAddr;
121     params0.p2pWmbTag                   = 0;
122     params0.bNeedWarBug999673           = bNeedWarBug999673;
123 
124     status = pRmApi0->Control(pRmApi0,
125                               pGpu0->hInternalClient,
126                               pGpu0->hInternalSubdevice,
127                               NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_LOCAL,
128                               &params0,
129                               sizeof(NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_LOCAL_PARAMS));
130     NV_ASSERT(status == NV_OK);
131 
132     // Setup PCIE P2P Mailbox on remote GPU
133     params1.local2Remote                = local2Remote;
134     params1.remote2Local                = remote2Local;
135     params1.localP2PDomainRemoteAddr    = localP2PDomainRemoteAddr;
136     params1.remoteP2PDomainLocalAddr    = remoteP2PDomainLocalAddr;
137     params1.remoteWMBoxAddrU64          = remoteWMBoxAddrU64;
138     params1.p2pWmbTag                   = params0.p2pWmbTag;
139 
140     status = pRmApi1->Control(pRmApi1,
141                               pGpu1->hInternalClient,
142                               pGpu1->hInternalSubdevice,
143                               NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_REMOTE,
144                               &params1,
145                               sizeof(NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_REMOTE_PARAMS));
146     NV_ASSERT(status == NV_OK);
147 
148     kbusWriteP2PWmbTag_HAL(pGpu1, pKernelBus1, remote2Local, params0.p2pWmbTag);
149 }
150 
151 void
kbusWriteP2PWmbTag_GM200(OBJGPU * pGpu,KernelBus * pKernelBus,NvU32 remote2Local,NvU64 p2pWmbTag)152 kbusWriteP2PWmbTag_GM200
153 (
154     OBJGPU    *pGpu,
155     KernelBus *pKernelBus,
156     NvU32      remote2Local,
157     NvU64      p2pWmbTag
158 )
159 {
160     // See bug 3558208 comment 34 and 50
161     GPU_REG_RD32(pGpu, NV_P2P_WREQMB_L(remote2Local));
162     GPU_REG_WR32(pGpu, NV_P2P_WREQMB_L(remote2Local), NvU64_LO32(p2pWmbTag));
163     GPU_REG_WR32(pGpu, NV_P2P_WREQMB_H(remote2Local), NvU64_HI32(p2pWmbTag));
164 }
165 
166 RmPhysAddr
kbusSetupP2PDomainAccess_GM200(OBJGPU * pGpu0,KernelBus * pKernelBus0,OBJGPU * pGpu1,PMEMORY_DESCRIPTOR * ppP2PDomMemDesc)167 kbusSetupP2PDomainAccess_GM200
168 (
169     OBJGPU    *pGpu0,
170     KernelBus *pKernelBus0,
171     OBJGPU    *pGpu1,
172     PMEMORY_DESCRIPTOR *ppP2PDomMemDesc
173 )
174 {
175     return kbusSetupPeerBarAccess(pGpu0, pGpu1,
176                 pGpu0->busInfo.gpuPhysAddr + DRF_BASE(NV_P2P),
177                 DRF_SIZE(NV_P2P), ppP2PDomMemDesc);
178 }
179 
180 /*!
181  * @brief Creates a mapping for the remote peer to access its mailbox in
182  *        the local GPU's BAR1
183  *
184  * @param[in]   pGpu0          (local GPU)
185  * @param[in]   pKernelBus0    (local GPU)
186  * @param[in]   pGpu1          (remote GPU)
187  * @param[in]   local2Remote   Peer ID (local to remote)
188  * @param[out]  ppWMBoxMemDesc
189  *
190  */
191 RmPhysAddr
kbusSetupMailboxAccess_GM200(OBJGPU * pGpu0,KernelBus * pKernelBus0,OBJGPU * pGpu1,NvU32 local2Remote,PMEMORY_DESCRIPTOR * ppWMBoxMemDesc)192 kbusSetupMailboxAccess_GM200
193 (
194     OBJGPU    *pGpu0,
195     KernelBus *pKernelBus0,
196     OBJGPU    *pGpu1,
197     NvU32      local2Remote,
198     PMEMORY_DESCRIPTOR *ppWMBoxMemDesc
199 )
200 {
201     return kbusSetupPeerBarAccess(pGpu0, pGpu1,
202                 gpumgrGetGpuPhysFbAddr(pGpu0) +
203                     pKernelBus0->p2pPcie.writeMailboxBar1Addr +
204                     PCIE_P2P_WRITE_MAILBOX_SIZE * local2Remote,
205                 PCIE_P2P_WRITE_MAILBOX_SIZE, ppWMBoxMemDesc);
206 }
207 
208 void
kbusDestroyPeerAccess_GM200(OBJGPU * pGpu,KernelBus * pKernelBus,NvU32 peerNum)209 kbusDestroyPeerAccess_GM200
210 (
211     OBJGPU    *pGpu,
212     KernelBus *pKernelBus,
213     NvU32      peerNum
214 )
215 {
216     memdescDestroy(pKernelBus->p2pPcie.busPeer[peerNum].pRemoteWMBoxMemDesc);
217     pKernelBus->p2pPcie.busPeer[peerNum].pRemoteWMBoxMemDesc = NULL;
218 
219     memdescDestroy(pKernelBus->p2pPcie.busPeer[peerNum].pRemoteP2PDomMemDesc);
220     pKernelBus->p2pPcie.busPeer[peerNum].pRemoteP2PDomMemDesc = NULL;
221 }
222 
223 /*!
224  * @brief  Returns the P2P mailbox attributes such as size, aligment, max offset.
225  *
226  * @param[in]   pGpu
227  * @param[in]   pKernelBus
228  * @param[out]  pMailboxAreaSize       NvU32 pointer. Can be NULL
229  * @param[out]  pMailboxAlignment      NvU32 pointer. Can be NULL
230  * @param[out]  pMailboxMaxOffset64KB  NvU32 pointer. Can be NULL
231  *
232  *  Returns the P2P mailbox attributes such as:
233  *  - pMailboxAreaSize: total size
234  *  - pMailboxAlignment: aligment
235  *  - pMailboxMaxOffset: max supported offset
236  *
237  * return   void
238  */
239 void
kbusGetP2PMailboxAttributes_GM200(OBJGPU * pGpu,KernelBus * pKernelBus,NvU32 * pMailboxAreaSize,NvU32 * pMailboxAlignmentSize,NvU32 * pMailboxBar1MaxOffset64KB)240 kbusGetP2PMailboxAttributes_GM200
241 (
242     OBJGPU    *pGpu,
243     KernelBus *pKernelBus,
244     NvU32*     pMailboxAreaSize,
245     NvU32*     pMailboxAlignmentSize,
246     NvU32*     pMailboxBar1MaxOffset64KB
247 )
248 {
249     KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu);
250 
251     // Initialize null values by default
252     if (pMailboxAreaSize != NULL)
253     {
254         *pMailboxAreaSize = 0;
255     }
256     if (pMailboxAlignmentSize != NULL)
257     {
258         *pMailboxAlignmentSize = 0;
259     }
260     if (pMailboxBar1MaxOffset64KB != NULL)
261     {
262         *pMailboxBar1MaxOffset64KB = 0;
263     }
264 
265     if (pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_P2P_READS_DISABLED) &&
266         pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_P2P_WRITES_DISABLED))
267     {
268         // Return null values
269         return;
270     }
271 
272     // Retrieve attributes
273     if (pMailboxAreaSize != NULL)
274     {
275         *pMailboxAreaSize = PCIE_P2P_WRITE_MAILBOX_SIZE * P2P_MAX_NUM_PEERS;
276     }
277 
278     if (pMailboxAlignmentSize != NULL)
279     {
280         // Write mailbox data window needs to be 64KB aligned.
281         *pMailboxAlignmentSize = 0x10000;
282     }
283 
284     if (pMailboxBar1MaxOffset64KB != NULL)
285     {
286         // Max offset, exclusive
287         *pMailboxBar1MaxOffset64KB =
288             NvU64_LO32(
289                 (PCIE_P2P_MAX_WRITE_MAILBOX_ADDR + PCIE_P2P_WRITE_MAILBOX_SIZE) >> 16
290             );
291     }
292 
293     return;
294 }
295 
296 /*!
297  * @brief  Create PCIE Mailbox P2P mapping between 2 GPUs
298  *
299  * @param[in]   pGpu0          (local GPU)
300  * @param[in]   pKernelBus0    (local GPU)
301  * @param[in]   pGpu1          (remote GPU)
302  * @param[in]   pKernelBus1    (remote GPU)
303  * @param[out]  peer0          NvU32 pointer, peerId on pGpu0
304  * @param[out]  peer1          NvU32 pointer, peerId on pGpu1
305  * @param[in]   attributes     Sepcial attributes for the mapping
306  *
307  * @return NV_STATUS
308  */
309 NV_STATUS
kbusCreateP2PMapping_GM200(OBJGPU * pGpu0,KernelBus * pKernelBus0,OBJGPU * pGpu1,KernelBus * pKernelBus1,NvU32 * peer0,NvU32 * peer1,NvU32 attributes)310 kbusCreateP2PMapping_GM200
311 (
312     OBJGPU    *pGpu0,
313     KernelBus *pKernelBus0,
314     OBJGPU    *pGpu1,
315     KernelBus *pKernelBus1,
316     NvU32     *peer0,
317     NvU32     *peer1,
318     NvU32      attributes
319 )
320 {
321     if (FLD_TEST_DRF(_P2PAPI, _ATTRIBUTES, _CONNECTION_TYPE, _PCIE, attributes))
322     {
323         return kbusCreateP2PMappingForMailbox_HAL(pGpu0, pKernelBus0, pGpu1, pKernelBus1, peer0, peer1, attributes);
324     }
325 
326     NV_PRINTF(LEVEL_ERROR, "P2P type %d is not supported\n", DRF_VAL(_P2PAPI, _ATTRIBUTES, _CONNECTION_TYPE, attributes));
327 
328     return NV_ERR_NOT_SUPPORTED;
329 }
330 
331 /*!
332  * @brief  Create PCIE (not NVLINK) P2P mapping between 2 GPUs
333  *
334  * @param[in]   pGpu0          (local GPU)
335  * @param[in]   pKernelBus0    (local GPU)
336  * @param[in]   pGpu1          (remote GPU)
337  * @param[in]   pKernelBus1    (remote GPU)
338  * @param[out]  peer0  Peer ID (local to remote)
339  * @param[out]  peer1  Peer ID (remote to local)
340  * @param[in]   attributes Unused
341  *
342  * @return NV_STATUS
343  */
344 NV_STATUS
kbusCreateP2PMappingForMailbox_GM200(OBJGPU * pGpu0,KernelBus * pKernelBus0,OBJGPU * pGpu1,KernelBus * pKernelBus1,NvU32 * peer0,NvU32 * peer1,NvU32 attributes)345 kbusCreateP2PMappingForMailbox_GM200
346 (
347     OBJGPU    *pGpu0,
348     KernelBus *pKernelBus0,
349     OBJGPU    *pGpu1,
350     KernelBus *pKernelBus1,
351     NvU32     *peer0,
352     NvU32     *peer1,
353     NvU32      attributes
354 )
355 {
356     RM_API *pRmApi;
357     NV2080_CTRL_INTERNAL_HSHUB_PEER_CONN_CONFIG_PARAMS params;
358     NvU32 gpuInst0, gpuInst1;
359 
360     if (IS_VIRTUAL(pGpu0) || IS_VIRTUAL(pGpu1))
361     {
362         return NV_ERR_NOT_SUPPORTED;
363     }
364 
365     if (peer0 == NULL || peer1 == NULL)
366     {
367         return NV_ERR_INVALID_ARGUMENT;
368     }
369 
370     gpuInst0 = gpuGetInstance(pGpu0);
371     gpuInst1 = gpuGetInstance(pGpu1);
372 
373     // Is a specific peer ID mapping requested?
374     if ((*peer0 != BUS_INVALID_PEER) && (*peer1 != BUS_INVALID_PEER))
375     {
376         NV_ASSERT_OR_RETURN(*peer0 < P2P_MAX_NUM_PEERS, NV_ERR_INVALID_ARGUMENT);
377         NV_ASSERT_OR_RETURN(*peer1 < P2P_MAX_NUM_PEERS, NV_ERR_INVALID_ARGUMENT);
378         //
379         // Ensure that if the requested peer ID is already in use, it
380         // corresponds to the requested remote GPU.
381         //
382         if (!pKernelBus0->p2pPcie.busPeer[*peer0].bReserved &&
383             !pKernelBus1->p2pPcie.busPeer[*peer1].bReserved)
384         {
385             if ((pKernelBus0->p2pPcie.busPeer[*peer0].refCount == 0) &&
386                 (pKernelBus1->p2pPcie.busPeer[*peer1].refCount == 0))
387             {
388                 goto busCreateP2PMapping_setupMapping;
389             }
390 
391             if (((pKernelBus0->p2pPcie.peerNumberMask[gpuInst1] & NVBIT(*peer0)) != 0) &&
392                 ((pKernelBus1->p2pPcie.peerNumberMask[gpuInst0] & NVBIT(*peer1)) != 0))
393             {
394                 pKernelBus0->p2pPcie.busPeer[*peer0].refCount++;
395                 pKernelBus1->p2pPcie.busPeer[*peer1].refCount++;
396 
397                 NV_ASSERT(pKernelBus0->p2pPcie.busPeer[*peer0].remotePeerId == *peer1);
398                 NV_ASSERT(pKernelBus1->p2pPcie.busPeer[*peer1].remotePeerId == *peer0);
399 
400                 pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu0);
401                 portMemSet(&params, 0, sizeof(params));
402                 params.programPciePeerMask = NVBIT32(*peer0);
403                 NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi,
404                                        pGpu0->hInternalClient,
405                                        pGpu0->hInternalSubdevice,
406                                        NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG,
407                                        &params,
408                                        sizeof(params)));
409 
410                 pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu1);
411                 portMemSet(&params, 0, sizeof(params));
412                 params.programPciePeerMask = NVBIT32(*peer1);
413                 NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi,
414                                        pGpu1->hInternalClient,
415                                        pGpu1->hInternalSubdevice,
416                                        NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG,
417                                        &params,
418                                        sizeof(params)));
419 
420                 return NV_OK;
421             }
422         }
423 
424         NV_PRINTF(LEVEL_WARNING,
425                   "explicit peer IDs %u and %u requested for GPU%u and GPU%u are not "
426                   "available, will assign dynamically\n", *peer0, *peer1,
427                   gpuInst0, gpuInst1);
428     }
429 
430     // Does a mapping already exist between these GPUs?
431     if ((pKernelBus0->p2pPcie.peerNumberMask[gpuInst1] != 0) &&
432         (pKernelBus1->p2pPcie.peerNumberMask[gpuInst0] != 0))
433     {
434         *peer0 = pKernelBus0->p2pPcie.peerNumberMask[gpuInst1];
435         LOWESTBITIDX_32(*peer0);
436         NV_ASSERT_OR_RETURN(*peer0 < P2P_MAX_NUM_PEERS,
437                           NV_ERR_INVALID_STATE);
438 
439         *peer1 = pKernelBus0->p2pPcie.busPeer[*peer0].remotePeerId;
440         NV_ASSERT_OR_RETURN(*peer1 < P2P_MAX_NUM_PEERS,
441                           NV_ERR_INVALID_STATE);
442 
443         NV_ASSERT_OR_RETURN(pKernelBus1->p2pPcie.busPeer[*peer1].remotePeerId == *peer0,
444                           NV_ERR_INVALID_STATE);
445 
446         pKernelBus0->p2pPcie.busPeer[*peer0].refCount++;
447         pKernelBus1->p2pPcie.busPeer[*peer1].refCount++;
448 
449         NV_ASSERT(!pKernelBus0->p2pPcie.busPeer[*peer0].bReserved);
450         NV_ASSERT(!pKernelBus1->p2pPcie.busPeer[*peer1].bReserved);
451 
452         pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu0);
453         portMemSet(&params, 0, sizeof(params));
454         params.programPciePeerMask = NVBIT32(*peer0);
455         NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi,
456                                pGpu0->hInternalClient,
457                                pGpu0->hInternalSubdevice,
458                                NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG,
459                                &params,
460                                sizeof(params)));
461 
462         pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu1);
463         portMemSet(&params, 0, sizeof(params));
464         params.programPciePeerMask = NVBIT32(*peer1);
465         NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi,
466                                pGpu1->hInternalClient,
467                                pGpu1->hInternalSubdevice,
468                                NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG,
469                                &params,
470                                sizeof(params)));
471 
472         return NV_OK;
473     }
474 
475     // We'd better not hit this case (one gpu has mapping and other doesn't).
476     NV_ASSERT((pKernelBus0->p2pPcie.peerNumberMask[gpuInst1] == 0) &&
477               (pKernelBus1->p2pPcie.peerNumberMask[gpuInst0] == 0));
478 
479     *peer0 = BUS_INVALID_PEER;
480     *peer1 = BUS_INVALID_PEER;
481 
482     // If we're in loopback mode, check for specified peer ID
483     if ((pGpu0 == pGpu1) && pKernelBus0->p2pMapSpecifyId)
484     {
485         if ((pKernelBus0->p2pPcie.busPeer[pKernelBus0->p2pMapPeerId].refCount == 0) &&
486             (!pKernelBus0->p2pPcie.busPeer[pKernelBus0->p2pMapPeerId].bReserved) &&
487             (pKernelBus1->p2pPcie.busPeer[pKernelBus1->p2pMapPeerId].refCount == 0))
488         {
489             *peer0 = *peer1 = pKernelBus0->p2pMapPeerId;
490         }
491         else
492         {
493            NV_PRINTF(LEVEL_ERROR,
494                      "- ERROR: Peer ID %d is already in use. Default RM P2P mapping will be used.\n",
495                      pKernelBus0->p2pMapPeerId);
496         }
497     }
498 
499     //
500     // These loops will handle loop back (pGpu0 == pGpu1) since they will find
501     // the same free peer twice on the same GPU.
502     //
503     if (*peer0 == BUS_INVALID_PEER)
504     {
505         *peer0 = kbusGetUnusedPciePeerId_HAL(pGpu0, pKernelBus0);
506     }
507 
508     if (*peer1 == BUS_INVALID_PEER)
509     {
510         *peer1 = kbusGetUnusedPciePeerId_HAL(pGpu1, pKernelBus1);
511     }
512 
513     // couldn't find an available peer on both gpus
514     if (*peer0 == BUS_INVALID_PEER ||
515             *peer1 == BUS_INVALID_PEER)
516     {
517         *peer0 = BUS_INVALID_PEER;
518         *peer1 = BUS_INVALID_PEER;
519         NV_PRINTF(LEVEL_ERROR, "no peer IDs available\n");
520         return NV_ERR_INSUFFICIENT_RESOURCES;
521     }
522 
523 busCreateP2PMapping_setupMapping:
524     pKernelBus0->p2pPcie.busPeer[*peer0].remotePeerId = *peer1;
525     pKernelBus0->p2pPcie.peerNumberMask[gpuInst1] |= NVBIT(*peer0);
526     pKernelBus1->p2pPcie.busPeer[*peer1].remotePeerId = *peer0;
527     pKernelBus1->p2pPcie.peerNumberMask[gpuInst0] |= NVBIT(*peer1);
528 
529     NV_ASSERT(pKernelBus0->p2pPcie.busPeer[*peer0].refCount == 0);
530     NV_ASSERT(!pKernelBus0->p2pPcie.busPeer[*peer0].bReserved);
531     NV_ASSERT(pKernelBus1->p2pPcie.busPeer[*peer1].refCount == 0);
532     NV_ASSERT(!pKernelBus1->p2pPcie.busPeer[*peer1].bReserved);
533 
534     //
535     // Note if this is loopback we will have a refCount of 2.  This will be
536     // accounted for in the free.
537     //
538     pKernelBus0->p2pPcie.busPeer[*peer0].refCount++;
539     pKernelBus1->p2pPcie.busPeer[*peer1].refCount++;
540 
541     pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu0);
542     portMemSet(&params, 0, sizeof(params));
543     params.programPciePeerMask = NVBIT32(*peer0);
544     NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi,
545                            pGpu0->hInternalClient,
546                            pGpu0->hInternalSubdevice,
547                            NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG,
548                            &params,
549                            sizeof(params)));
550 
551     pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu1);
552     portMemSet(&params, 0, sizeof(params));
553     params.programPciePeerMask = NVBIT32(*peer1);
554     NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi,
555                            pGpu1->hInternalClient,
556                            pGpu1->hInternalSubdevice,
557                            NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG,
558                            &params,
559                            sizeof(params)));
560 
561     NV_PRINTF(LEVEL_INFO,
562               "added PCIe P2P mapping between GPU%u (peer %u) and GPU%u (peer %u)\n",
563               gpuInst0, *peer0, gpuInst1, *peer1);
564 
565     kbusSetupMailboxes_HAL(pGpu0, pKernelBus0, pGpu1, pKernelBus1, *peer0, *peer1);
566     kbusSetupMailboxes_HAL(pGpu1, pKernelBus1, pGpu0, pKernelBus0, *peer1, *peer0);
567 
568     return NV_OK;
569 }
570 
571 /*!
572  * Does it need P2P WAR for bug 999673?
573  *
574  * @param[in] pGpu
575  * @param[in] pKernelBus
576  * @param[in] pRemoteGpu
577  *
578  * @return NvBool
579  *
580  */
581 NvBool
kbusNeedWarForBug999673_GM200(OBJGPU * pGpu,KernelBus * pKernelBus,OBJGPU * pRemoteGpu)582 kbusNeedWarForBug999673_GM200
583 (
584     OBJGPU    *pGpu,
585     KernelBus *pKernelBus,
586     OBJGPU    *pRemoteGpu
587 )
588 {
589     OBJCL  *pCl = SYS_GET_CL(SYS_GET_INSTANCE());
590     NvU8    pciSwitchBus = 0;
591 
592     // Return if WAR is not needed
593     if (!pCl->getProperty(pCl, PDB_PROP_CL_BUG_999673_P2P_ARBITRARY_SPLIT_WAR))
594     {
595         return NV_FALSE;
596     }
597 
598     // See if we have a known bridge
599     clFindCommonDownstreamBR(pGpu, pRemoteGpu, pCl, &pciSwitchBus);
600     if (pciSwitchBus != 0xFF)
601     {
602         // P2P does not go through the chipset needing the WAR.
603         return NV_FALSE;
604     }
605 
606     return NV_TRUE;
607 }
608 
609 /*!
610  * @brief Remove the P2P mapping to a given peer GPU
611  *
612  * @param[in]   pGpu0          (local GPU)
613  * @param[in]   pKernelBus0    (local GPU)
614  * @param[in]   pGpu1          (remote GPU)
615  * @param[in]   pKernelBus1    (remote GPU)
616  * @param[in]   peer0  Peer ID (local to remote)
617  * @param[in]   peer1  Peer ID (remote to local)
618  * @param[in]   attributes Sepcial attributes for the mapping
619  *
620  * return NV_OK on success
621  */
622 NV_STATUS
kbusRemoveP2PMapping_GM200(OBJGPU * pGpu0,KernelBus * pKernelBus0,OBJGPU * pGpu1,KernelBus * pKernelBus1,NvU32 peer0,NvU32 peer1,NvU32 attributes)623 kbusRemoveP2PMapping_GM200
624 (
625     OBJGPU    *pGpu0,
626     KernelBus *pKernelBus0,
627     OBJGPU    *pGpu1,
628     KernelBus *pKernelBus1,
629     NvU32      peer0,
630     NvU32      peer1,
631     NvU32      attributes
632 )
633 {
634     if (FLD_TEST_DRF(_P2PAPI, _ATTRIBUTES, _CONNECTION_TYPE, _PCIE, attributes))
635     {
636         return kbusRemoveP2PMappingForMailbox_HAL(pGpu0, pKernelBus0, pGpu1, pKernelBus1, peer0, peer1, attributes);
637     }
638 
639     NV_PRINTF(LEVEL_ERROR, "P2P type %d is not supported\n", DRF_VAL(_P2PAPI, _ATTRIBUTES, _CONNECTION_TYPE, attributes));
640 
641     return NV_ERR_NOT_SUPPORTED;
642 }
643 
644 /*!
645  * @brief Create a P2P mapping to a given peer GPU
646  *
647  * @param[in]   pGpu0          (local GPU)
648  * @param[in]   pKernelBus0    (local GPU)
649  * @param[in]   pGpu1          (remote GPU)
650  * @param[in]   pKernelBus1    (remote GPU)
651  * @param[out]  peer0  Peer ID (local to remote)
652  * @param[out]  peer1  Peer ID (remote to local)
653  * @param[in]   attributes Sepcial attributes for the mapping
654  *
655  * return NV_OK on success
656  */
657 NV_STATUS
kbusRemoveP2PMappingForMailbox_GM200(OBJGPU * pGpu0,KernelBus * pKernelBus0,OBJGPU * pGpu1,KernelBus * pKernelBus1,NvU32 peer0,NvU32 peer1,NvU32 attributes)658 kbusRemoveP2PMappingForMailbox_GM200
659 (
660     OBJGPU    *pGpu0,
661     KernelBus *pKernelBus0,
662     OBJGPU    *pGpu1,
663     KernelBus *pKernelBus1,
664     NvU32      peer0,
665     NvU32      peer1,
666     NvU32      attributes
667 )
668 {
669     NvU32 gpuInst0 = gpuGetInstance(pGpu0);
670     NvU32 gpuInst1 = gpuGetInstance(pGpu1);
671 
672     if (IS_VIRTUAL(pGpu0) || IS_VIRTUAL(pGpu1))
673     {
674         return NV_ERR_NOT_SUPPORTED;
675     }
676 
677     // a non-existent mapping
678     if(peer0 == BUS_INVALID_PEER ||
679        peer1 == BUS_INVALID_PEER)
680     {
681         return NV_ERR_INVALID_ARGUMENT;
682     }
683 
684     // Do the peer IDs correspond to the expected GPUs?
685     if (((pKernelBus0->p2pPcie.peerNumberMask[gpuInst1] & NVBIT(peer0)) == 0) ||
686         ((pKernelBus1->p2pPcie.peerNumberMask[gpuInst0] & NVBIT(peer1)) == 0))
687     {
688         return NV_ERR_INVALID_ARGUMENT;
689     }
690 
691     //
692     // a programming error somewhere in RM.
693     // A mapping exists with a refCount == 0
694     //
695     if (pKernelBus0->p2pPcie.busPeer[peer0].refCount == 0 ||
696         pKernelBus1->p2pPcie.busPeer[peer1].refCount == 0)
697     {
698         DBG_BREAKPOINT();
699         return NV_ERR_GENERIC;
700     }
701 
702     //
703     // Again a programming error.  The mapping should have the same refCount on
704     // both GPUs.
705     //
706     if (pKernelBus0->p2pPcie.busPeer[peer0].refCount !=
707         pKernelBus1->p2pPcie.busPeer[peer1].refCount)
708     {
709         DBG_BREAKPOINT();
710         return NV_ERR_GENERIC;
711     }
712 
713 
714     --pKernelBus1->p2pPcie.busPeer[peer1].refCount;
715     if (--pKernelBus0->p2pPcie.busPeer[peer0].refCount == 0)
716     {
717         NV_PRINTF(LEVEL_INFO,
718                   "Removing mapping GPU %d Peer %d <-> GPU %d Peer %d\n",
719                   gpuInst0, peer0, gpuInst1, peer1);
720 
721         pKernelBus0->p2pPcie.peerNumberMask[gpuInst1] &= ~NVBIT(peer0);
722         pKernelBus1->p2pPcie.peerNumberMask[gpuInst0] &= ~NVBIT(peer1);
723 
724         kbusDestroyMailbox(pGpu0, pKernelBus0, pGpu1, peer0);
725         kbusDestroyMailbox(pGpu1, pKernelBus1, pGpu0, peer1);
726     }
727     else
728     {
729         NV_PRINTF(LEVEL_INFO,
730                   "Decremented refCount for Mapping GPU %d Peer %d  <-> GPU %d Peer %d "
731                   "New Count: %d\n", gpuInst0, peer0, gpuInst1, peer1,
732                   pKernelBus0->p2pPcie.busPeer[peer0].refCount);
733     }
734 
735     return NV_OK;
736 }
737 
738 /*!
739  * @brief Reserve peer IDs for nvlink usage
740  *
741  * @param[in]   pGpu
742  * @param[in]   pKernelBus
743  * @param[in]   peerMask    Mask of peer IDs to reserve
744  *
745  * return   NV_OK on success
746  */
747 NV_STATUS
kbusReserveP2PPeerIds_GM200(OBJGPU * pGpu,KernelBus * pKernelBus,NvU32 peerMask)748 kbusReserveP2PPeerIds_GM200
749 (
750     OBJGPU    *pGpu,
751     KernelBus *pKernelBus,
752     NvU32      peerMask
753 )
754 {
755     NvU32 peerId = 0;
756 
757     FOR_EACH_INDEX_IN_MASK(32, peerId, peerMask)
758     {
759         NV_PRINTF(LEVEL_INFO,
760                   "reserving peer ID %u on GPU%u for NVLINK/C2C use\n", peerId,
761                   gpuGetInstance(pGpu));
762         if (pKernelBus->p2pPcie.busPeer[peerId].refCount != 0)
763         {
764             return NV_ERR_IN_USE;
765         }
766 
767         pKernelBus->p2pPcie.busPeer[peerId].bReserved = NV_TRUE;
768     }
769     FOR_EACH_INDEX_IN_MASK_END;
770 
771     return NV_OK;
772 }
773 
774 /*!
775  * @brief Sets the BAR1 P2P mailbox address and size
776  *
777  * @param[in] pGpu
778  * @param[in] pKernelBus
779  * @param[in] mailboxBar1Addr   NvU64
780  * @param[in] mailboxTotalSize  NvU32
781  *
782  * @returns NV_STATUS
783  */
784 NV_STATUS
kbusSetP2PMailboxBar1Area_GM200(OBJGPU * pGpu,KernelBus * pKernelBus,NvU64 mailboxBar1Addr,NvU32 mailboxTotalSize)785 kbusSetP2PMailboxBar1Area_GM200
786 (
787     OBJGPU    *pGpu,
788     KernelBus *pKernelBus,
789     NvU64      mailboxBar1Addr,
790     NvU32      mailboxTotalSize
791 )
792 {
793     NvU32 mailboxAreaSizeReq;
794     NvU32 mailboxAlignmentSizeReq;
795     NvU32 mailboxBar1MaxOffset64KBReq;
796 
797 
798     if (!kbusIsP2pMailboxClientAllocated(pKernelBus))
799     {
800         // P2P mailbox area already allocated by RM. Nothing to do.
801         return NV_OK;
802     }
803 
804     if (mailboxTotalSize == 0)
805     {
806         NV_PRINTF(LEVEL_ERROR, "P2P mailbox area size is not set\n");
807         return NV_ERR_INVALID_ARGUMENT;
808     }
809 
810     kbusGetP2PMailboxAttributes_HAL(pGpu, pKernelBus, &mailboxAreaSizeReq, &mailboxAlignmentSizeReq, &mailboxBar1MaxOffset64KBReq);
811 
812     // Mailbox size
813     NV_ASSERT_OR_RETURN(mailboxTotalSize == mailboxAreaSizeReq,                NV_ERR_INVALID_ARGUMENT);
814     // Mailbox alignment
815     NV_ASSERT_OR_RETURN((mailboxBar1Addr & (mailboxAlignmentSizeReq - 1)) == 0, NV_ERR_INVALID_ARGUMENT);
816     // Mailbox  offset limit
817     NV_ASSERT_OR_RETURN((mailboxBar1Addr + mailboxTotalSize) < (((NvU64)mailboxBar1MaxOffset64KBReq) << 16),
818         NV_ERR_INVALID_ARGUMENT);
819 
820     if (pKernelBus->p2pPcie.writeMailboxBar1Addr != PCIE_P2P_INVALID_WRITE_MAILBOX_ADDR)
821     {
822         NV_ASSERT_OR_RETURN(mailboxBar1Addr == pKernelBus->p2pPcie.writeMailboxBar1Addr, NV_ERR_INVALID_ARGUMENT);
823         NV_ASSERT_OR_RETURN(mailboxTotalSize == pKernelBus->p2pPcie.writeMailboxTotalSize, NV_ERR_INVALID_ARGUMENT);
824         return NV_OK;
825     }
826 
827     pKernelBus->p2pPcie.writeMailboxBar1Addr = mailboxBar1Addr;
828     pKernelBus->p2pPcie.writeMailboxTotalSize = mailboxTotalSize;
829     pKernelBus->bP2pInitialized = NV_TRUE;
830 
831     return NV_OK;
832 }
833 
834 
835 /*!
836  * @brief Unset the BAR1 P2P mailbox address and size
837  *
838  * @param[in] pGpu
839  * @param[in] pKernelBus
840  *
841  * @returns void
842  */
843 void
kbusUnsetP2PMailboxBar1Area_GM200(OBJGPU * pGpu,KernelBus * pKernelBus)844 kbusUnsetP2PMailboxBar1Area_GM200
845 (
846     OBJGPU    *pGpu,
847     KernelBus *pKernelBus
848 )
849 {
850     NvU32 i;
851 
852     if (!kbusIsP2pMailboxClientAllocated(pKernelBus))
853     {
854         // P2P mailbox area already allocated by RM. Nothing to do.
855         return;
856     }
857 
858     for (i = 0; i < P2P_MAX_NUM_PEERS; ++i)
859     {
860         if (pKernelBus->p2pPcie.busPeer[i].refCount)
861             break;
862     }
863 
864     if (i == P2P_MAX_NUM_PEERS)
865     {
866         pKernelBus->p2pPcie.writeMailboxBar1Addr  = PCIE_P2P_INVALID_WRITE_MAILBOX_ADDR;
867         pKernelBus->p2pPcie.writeMailboxTotalSize = 0;
868         pKernelBus->bP2pInitialized = NV_FALSE;
869     }
870 
871     return;
872 }
873 
874 NV_STATUS
kbusAllocP2PMailboxBar1_GM200(OBJGPU * pGpu,KernelBus * pKernelBus,NvU32 gfid,NvU64 vaRangeMax)875 kbusAllocP2PMailboxBar1_GM200
876 (
877     OBJGPU    *pGpu,
878     KernelBus *pKernelBus,
879     NvU32      gfid,
880     NvU64      vaRangeMax
881 )
882 {
883     OBJGPU           *pParentGpu;
884     NvU64             vaAllocMax;
885     NV_STATUS         status = NV_OK;
886 
887     VAS_ALLOC_FLAGS flags = {0};
888 
889     pParentGpu  = gpumgrGetParentGPU(pGpu);
890 
891     if (!gpumgrIsParentGPU(pGpu))
892     {
893         flags.bFixedAddressAllocate = NV_TRUE;
894         pKernelBus->p2pPcie.writeMailboxBar1Addr = GPU_GET_KERNEL_BUS(pParentGpu)->p2pPcie.writeMailboxBar1Addr;
895     }
896 
897     pKernelBus->p2pPcie.writeMailboxTotalSize =
898         PCIE_P2P_WRITE_MAILBOX_SIZE * P2P_MAX_NUM_PEERS;
899     vaAllocMax = NV_MIN(vaRangeMax,
900         PCIE_P2P_MAX_WRITE_MAILBOX_ADDR + PCIE_P2P_WRITE_MAILBOX_SIZE - 1);
901 
902     status = vaspaceAlloc(pKernelBus->bar1[gfid].pVAS,
903                           pKernelBus->p2pPcie.writeMailboxTotalSize,
904                           PCIE_P2P_WRITE_MAILBOX_SIZE,
905                           0, vaAllocMax,
906                           0,
907                           flags,
908                           &pKernelBus->p2pPcie.writeMailboxBar1Addr);
909     if (status != NV_OK)
910     {
911         NV_PRINTF(LEVEL_ERROR,
912                   "cannot allocate vaspace for P2P write mailboxes (0x%x)\n",
913                   status);
914         goto kbusAllocP2PMailboxBar1_failed;
915     }
916 
917     NV_ASSERT(GPU_GET_KERNEL_BUS(pParentGpu)->p2pPcie.writeMailboxBar1Addr == pKernelBus->p2pPcie.writeMailboxBar1Addr);
918 
919     NV_PRINTF(LEVEL_INFO,
920               "[GPU%u] P2P write mailboxes allocated at BAR1 addr = 0x%llx\n",
921               gpuGetInstance(pGpu), pKernelBus->p2pPcie.writeMailboxBar1Addr);
922 
923 kbusAllocP2PMailboxBar1_failed:
924     if (status != NV_OK)
925     {
926         pKernelBus->p2pPcie.writeMailboxBar1Addr  = PCIE_P2P_INVALID_WRITE_MAILBOX_ADDR;
927         pKernelBus->p2pPcie.writeMailboxTotalSize = 0;
928     }
929 
930     return status;
931 }
932