1 /*
2  * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3  * SPDX-License-Identifier: MIT
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21  * DEALINGS IN THE SOFTWARE.
22  */
23 
24 /*!
25  * @file
26  * @brief GPU Fabric Probe handling
27  */
28 
29 
30 #include "gpu/gpu.h"
31 #include "os/os.h"
32 #include "nvport/atomic.h"
33 #include "utils/nvprintf.h"
34 #include "kernel/gpu/nvlink/kernel_nvlink.h"
35 #include "gpu/gpu_fabric_probe.h"
36 #include "nvlink_inband_msg.h"
37 #include "kernel/mem_mgr/fabric_vaspace.h"
38 #include "ctrl/ctrl2080/ctrl2080internal.h"
39 
40 #include "compute/fabric.h"
41 #include "class/cl00f1.h"
42 #include "vgpu/rpc.h"
43 
44 // Structure to hold gpu probe information
45 typedef struct GPU_FABRIC_PROBE_INFO_KERNEL
46 {
47     volatile NvU32 probeRespRcvd;
48     NvU8 bwMode;
49 
50     OBJGPU *pGpu;
51 
52     nvlink_inband_gpu_probe_rsp_msg_t probeResponseMsg;
53 
54 } GPU_FABRIC_PROBE_INFO_KERNEL;
55 
56 static NV_STATUS
_gpuFabricProbeFullSanityCheck(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel)57 _gpuFabricProbeFullSanityCheck
58 (
59     GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel
60 )
61 {
62     if (pGpuFabricProbeInfoKernel == NULL)
63     {
64         return NV_ERR_NOT_SUPPORTED;
65     }
66 
67     if (!gpuFabricProbeIsReceived(pGpuFabricProbeInfoKernel))
68     {
69         return NV_ERR_NOT_READY;
70     }
71 
72     if (!gpuFabricProbeIsSuccess(pGpuFabricProbeInfoKernel))
73     {
74         NV_PRINTF(LEVEL_ERROR, "Fabric Probe failed: 0x%x\n",
75                   pGpuFabricProbeInfoKernel->probeResponseMsg.msgHdr.status);
76 
77         return pGpuFabricProbeInfoKernel->probeResponseMsg.msgHdr.status;
78     }
79 
80     return NV_OK;
81 }
82 
83 NV_STATUS
gpuFabricProbeGetGpuFabricHandle(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel,NvU64 * pHandle)84 gpuFabricProbeGetGpuFabricHandle
85 (
86     GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel,
87     NvU64 *pHandle
88 )
89 {
90     NV_STATUS status;
91 
92     status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
93 
94     NV_CHECK_OR_RETURN(LEVEL_SILENT, status == NV_OK, status);
95 
96     *pHandle = pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.gpuHandle;
97 
98     return status;
99 }
100 
101 NV_STATUS
gpuFabricProbeGetGfId(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel,NvU32 * pGfId)102 gpuFabricProbeGetGfId
103 (
104     GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel,
105     NvU32 *pGfId
106 )
107 {
108     NV_STATUS status;
109 
110     status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
111 
112     NV_CHECK_OR_RETURN(LEVEL_SILENT, status == NV_OK, status);
113 
114     *pGfId = pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.gfId;
115 
116     return status;
117 }
118 
119 NV_STATUS
gpuFabricProbeGetfmCaps(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel,NvU64 * pFmCaps)120 gpuFabricProbeGetfmCaps
121 (
122     GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel,
123     NvU64 *pFmCaps
124 )
125 {
126     NV_STATUS status;
127 
128     status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
129 
130     NV_CHECK_OR_RETURN(LEVEL_SILENT, status == NV_OK, status);
131 
132     *pFmCaps = pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.fmCaps;
133 
134     return status;
135 }
136 
137 NV_STATUS
gpuFabricProbeGetClusterUuid(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel,NvUuid * pClusterUuid)138 gpuFabricProbeGetClusterUuid
139 (
140     GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel,
141     NvUuid *pClusterUuid
142 )
143 {
144     NV_STATUS status;
145 
146     status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
147 
148     NV_CHECK_OR_RETURN(LEVEL_SILENT, status == NV_OK, status);
149 
150     portMemCopy(&pClusterUuid->uuid[0],
151                 sizeof(pClusterUuid->uuid),
152                 &pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.clusterUuid.uuid[0],
153                 sizeof(pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.clusterUuid.uuid));
154 
155     return status;
156 }
157 
158 NV_STATUS
gpuFabricProbeGetFabricPartitionId(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel,NvU16 * pFabricPartitionId)159 gpuFabricProbeGetFabricPartitionId
160 (
161     GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel,
162     NvU16 *pFabricPartitionId
163 )
164 {
165     NV_STATUS status;
166 
167     status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
168 
169     NV_CHECK_OR_RETURN(LEVEL_SILENT, status == NV_OK, status);
170 
171     *pFabricPartitionId = pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.fabricPartitionId;
172 
173     return status;
174 }
175 
176 NV_STATUS
gpuFabricProbeGetGpaAddress(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel,NvU64 * pGpaAddress)177 gpuFabricProbeGetGpaAddress
178 (
179     GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel,
180     NvU64 *pGpaAddress
181 )
182 {
183     NV_STATUS status;
184 
185     status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
186 
187     NV_CHECK_OR_RETURN(LEVEL_SILENT, status == NV_OK, status);
188 
189     *pGpaAddress = pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.gpaAddress;
190 
191     return status;
192 }
193 
194 NV_STATUS
gpuFabricProbeGetGpaAddressRange(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel,NvU64 * pGpaAddressRange)195 gpuFabricProbeGetGpaAddressRange
196 (
197     GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel,
198     NvU64 *pGpaAddressRange
199 )
200 {
201     NV_STATUS status;
202 
203     status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
204 
205     NV_CHECK_OR_RETURN(LEVEL_SILENT, status == NV_OK, status);
206 
207     *pGpaAddressRange = pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.gpaAddressRange;
208 
209     return status;
210 }
211 
212 NV_STATUS
gpuFabricProbeGetFlaAddress(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel,NvU64 * pFlaAddress)213 gpuFabricProbeGetFlaAddress
214 (
215     GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel,
216     NvU64 *pFlaAddress
217 )
218 {
219     NV_STATUS status;
220 
221     status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
222 
223     NV_CHECK_OR_RETURN(LEVEL_SILENT, status == NV_OK, status);
224 
225     *pFlaAddress = pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.flaAddress;
226 
227     return status;
228 }
229 
230 NV_STATUS
gpuFabricProbeGetFlaAddressRange(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel,NvU64 * pFlaAddressRange)231 gpuFabricProbeGetFlaAddressRange
232 (
233     GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel,
234     NvU64 *pFlaAddressRange
235 )
236 {
237     NV_STATUS status;
238 
239     status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
240 
241     NV_CHECK_OR_RETURN(LEVEL_SILENT, status == NV_OK, status);
242 
243     *pFlaAddressRange = pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.flaAddressRange;
244 
245     return status;
246 }
247 
248 /*
249  * This function is used to get the peer GPU EGM address from FM to RM.
250  * FM passes only the upper 32 bits of the address.
251  */
252 NV_STATUS
gpuFabricProbeGetEgmGpaAddress(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel,NvU64 * pEgmGpaAddress)253 gpuFabricProbeGetEgmGpaAddress
254 (
255     GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel,
256     NvU64 *pEgmGpaAddress
257 )
258 {
259     NV_STATUS status;
260 
261     status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
262 
263     NV_CHECK_OR_RETURN(LEVEL_SILENT, status == NV_OK, status);
264 
265     *pEgmGpaAddress = (NvU64)pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.gpaAddressEGMHi << 32ULL;
266 
267     return status;
268 }
269 
270 NV_STATUS
gpuFabricProbeGetNumProbeReqs(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel,NvU64 * numProbes)271 gpuFabricProbeGetNumProbeReqs
272 (
273     GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel,
274     NvU64 *numProbes
275 )
276 {
277     NV2080_CTRL_CMD_INTERNAL_GET_GPU_FABRIC_PROBE_INFO_PARAMS params = { 0 };
278     RM_API *pRmApi;
279     OBJGPU *pGpu;
280 
281     if (pGpuFabricProbeInfoKernel == NULL)
282     {
283         return NV_ERR_NOT_SUPPORTED;
284     }
285 
286     pGpu = pGpuFabricProbeInfoKernel->pGpu;
287     pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
288 
289     LOCK_ASSERT_AND_RETURN(rmDeviceGpuLockIsOwner(
290                            gpuGetInstance(pGpuFabricProbeInfoKernel->pGpu)));
291 
292     if (IS_VIRTUAL(pGpu))
293     {
294         *numProbes = 1;
295         return NV_OK;
296     }
297 
298     NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
299           pRmApi->Control(pRmApi,
300                           pGpu->hInternalClient,
301                           pGpu->hInternalSubdevice,
302                           NV2080_CTRL_CMD_INTERNAL_GPU_GET_FABRIC_PROBE_INFO,
303                           &params,
304                           sizeof(params)));
305 
306     *numProbes = params.numProbes;
307 
308     return NV_OK;
309 }
310 
311 NV_STATUS
gpuFabricProbeGetFabricCliqueId(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel,NvU32 * pFabricCliqueId)312 gpuFabricProbeGetFabricCliqueId
313 (
314     GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel,
315     NvU32 *pFabricCliqueId
316 )
317 {
318     NV_STATUS status;
319 
320     status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
321 
322     NV_CHECK_OR_RETURN(LEVEL_SILENT, status == NV_OK, status);
323 
324     *pFabricCliqueId =
325         pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.cliqueId;
326 
327     return NV_OK;
328 }
329 
330 NV_STATUS
gpuFabricProbeGetFabricHealthStatus(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel,NvU32 * pFabricHealthStatusMask)331 gpuFabricProbeGetFabricHealthStatus
332 (
333     GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel,
334     NvU32 *pFabricHealthStatusMask
335 )
336 {
337     NV_STATUS status;
338 
339     status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
340 
341     NV_CHECK_OR_RETURN(LEVEL_SILENT, status == NV_OK, status);
342 
343     *pFabricHealthStatusMask =
344         pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.fabricHealthMask;
345 
346     return NV_OK;
347 }
348 
349 NvBool
gpuFabricProbeIsReceived(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel)350 gpuFabricProbeIsReceived
351 (
352     GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel
353 )
354 {
355     if (pGpuFabricProbeInfoKernel == NULL)
356     {
357         return NV_FALSE;
358     }
359 
360     return !!portAtomicOrU32(&pGpuFabricProbeInfoKernel->probeRespRcvd, 0);
361 }
362 
363 NvBool
gpuFabricProbeIsSuccess(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel)364 gpuFabricProbeIsSuccess
365 (
366     GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel
367 )
368 {
369     nvlink_inband_gpu_probe_rsp_msg_t *pProbeResponseMsg;
370     nvlink_inband_msg_header_t *pProbeRespMsgHdr;
371 
372     if (pGpuFabricProbeInfoKernel == NULL)
373     {
374         return NV_FALSE;
375     }
376 
377     pProbeResponseMsg = &pGpuFabricProbeInfoKernel->probeResponseMsg;
378     pProbeRespMsgHdr = &pProbeResponseMsg->msgHdr;
379 
380     return pProbeRespMsgHdr->status == NV_OK;
381 }
382 
383 NV_STATUS
gpuFabricProbeGetFmStatus(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel)384 gpuFabricProbeGetFmStatus
385 (
386     GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel
387 )
388 {
389     if (pGpuFabricProbeInfoKernel == NULL)
390     {
391         return NV_ERR_NOT_SUPPORTED;
392     }
393 
394     return pGpuFabricProbeInfoKernel->probeResponseMsg.msgHdr.status;
395 }
396 
397 static void
_gpuFabricProbeSetupGpaRange(OBJGPU * pGpu,GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel)398 _gpuFabricProbeSetupGpaRange
399 (
400     OBJGPU                *pGpu,
401     GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel
402 )
403 {
404     KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu);
405     // setup GPA based system
406     if (pKernelNvlink != NULL)
407     {
408         NvU64 gpaAddress;
409         NvU64 gpaAddressSize;
410         NvU64 egmGpaAddress;
411 
412         NV_CHECK_OR_RETURN_VOID(LEVEL_ERROR,
413                     gpuFabricProbeGetGpaAddress(pGpuFabricProbeInfoKernel,
414                                                 &gpaAddress) == NV_OK);
415 
416         NV_CHECK_OR_RETURN_VOID(LEVEL_ERROR,
417                     gpuFabricProbeGetGpaAddressRange(pGpuFabricProbeInfoKernel,
418                                                     &gpaAddressSize) == NV_OK);
419 
420         NV_CHECK_OR_RETURN_VOID(LEVEL_ERROR,
421                     knvlinkSetUniqueFabricBaseAddress_HAL(pGpu, pKernelNvlink,
422                                                         gpaAddress) == NV_OK);
423 
424         NV_CHECK_OR_RETURN_VOID(LEVEL_ERROR,
425                     gpuFabricProbeGetEgmGpaAddress(pGpuFabricProbeInfoKernel,
426                                                 &egmGpaAddress) == NV_OK);
427 
428         NV_CHECK_OR_RETURN_VOID(LEVEL_ERROR,
429                     knvlinkSetUniqueFabricEgmBaseAddress_HAL(pGpu, pKernelNvlink,
430                                                         egmGpaAddress) == NV_OK);
431     }
432 }
433 
434 static void
_gpuFabricProbeSetupFlaRange(OBJGPU * pGpu,GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel)435 _gpuFabricProbeSetupFlaRange
436 (
437     OBJGPU                  *pGpu,
438     GPU_FABRIC_PROBE_INFO_KERNEL   *pGpuFabricProbeInfoKernel
439 )
440 {
441     if (pGpu->pFabricVAS != NULL)
442     {
443         NvU64 flaBaseAddress;
444         NvU64 flaSize;
445 
446         NV_CHECK_OR_RETURN_VOID(LEVEL_ERROR,
447             gpuFabricProbeGetFlaAddress(pGpuFabricProbeInfoKernel,
448                                         &flaBaseAddress) == NV_OK);
449 
450         NV_CHECK_OR_RETURN_VOID(LEVEL_ERROR,
451             gpuFabricProbeGetFlaAddressRange(pGpuFabricProbeInfoKernel,
452                                              &flaSize) == NV_OK);
453 
454         if (IS_VIRTUAL(pGpu))
455         {
456             fabricvaspaceClearUCRange(dynamicCast(pGpu->pFabricVAS, FABRIC_VASPACE));
457         }
458 
459         NV_CHECK_OR_RETURN_VOID(LEVEL_ERROR,
460             fabricvaspaceInitUCRange(dynamicCast(pGpu->pFabricVAS, FABRIC_VASPACE),
461                                      pGpu, flaBaseAddress, flaSize) == NV_OK);
462     }
463 }
464 
465 static void
_gpuFabricProbeSendCliqueIdChangeEvent(OBJGPU * pGpu,NvU32 cliqueId)466 _gpuFabricProbeSendCliqueIdChangeEvent
467 (
468     OBJGPU *pGpu,
469     NvU32 cliqueId
470 )
471 {
472     NV_STATUS status;
473     Fabric *pFabric = SYS_GET_FABRIC(SYS_GET_INSTANCE());
474     NV00F1_CTRL_FABRIC_EVENT event;
475     event.id = fabricGenerateEventId_IMPL(pFabric);
476     event.type = NV00F1_CTRL_FABRIC_EVENT_TYPE_CLIQUE_ID_CHANGE;
477     event.imexChannel = 0;
478     event.data.cliqueIdChange.gpuId = pGpu->gpuId;
479     event.data.cliqueIdChange.cliqueId = cliqueId;
480     status = fabricPostEventsV2(pFabric, &event, 1);
481     if (status != NV_OK)
482     {
483         NV_PRINTF(LEVEL_ERROR, "GPU%u Notifying cliqueId change failed\n",
484               gpuGetInstance(pGpu));
485     }
486 }
487 
488 NV_STATUS
gpuFabricProbeReceiveKernelCallback(NvU32 gpuInstance,NvU64 * pNotifyGfidMask,NV2080_CTRL_NVLINK_INBAND_RECEIVED_DATA_PARAMS * pInbandRcvParams)489 gpuFabricProbeReceiveKernelCallback
490 (
491     NvU32 gpuInstance,
492     NvU64 *pNotifyGfidMask,
493     NV2080_CTRL_NVLINK_INBAND_RECEIVED_DATA_PARAMS *pInbandRcvParams
494 )
495 {
496     OBJGPU *pGpu;
497     NvU32 gpuMaskUnused;
498     nvlink_inband_gpu_probe_rsp_msg_t *pProbeRespMsg;
499     GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel;
500     NV_STATUS status;
501 
502     if ((pGpu = gpumgrGetGpu(gpuInstance)) == NULL)
503     {
504         NV_ASSERT_FAILED("Invalid GPU instance");
505         return NV_ERR_INVALID_ARGUMENT;
506     }
507 
508     //
509     // There is a scenario where _gpuFabricProbeStart fails in the GSP
510     // and returns failure to kernel ctrl call to start probe.
511     // This will set the pGpuFabricProbeInfoKernel to NULL.
512     // GSP also sends a probe response with failure error code.
513     // Handling this response causes kernel driver to crash since
514     // pGpuFabricProbeInfoKernel is already cleared in the kernel.
515     // This check is added to handle this scenario.
516     //
517     NV_CHECK_OR_RETURN(LEVEL_ERROR, pGpu->pGpuFabricProbeInfoKernel != NULL, NV_OK);
518 
519     NV_ASSERT(rmGpuGroupLockIsOwner(gpuInstance, GPU_LOCK_GRP_SUBDEVICE,
520                                     &gpuMaskUnused));
521 
522     NV_ASSERT(pInbandRcvParams != NULL);
523 
524     pGpuFabricProbeInfoKernel = pGpu->pGpuFabricProbeInfoKernel;
525 
526     pProbeRespMsg = \
527         (nvlink_inband_gpu_probe_rsp_msg_t *)&pInbandRcvParams->data[0];
528 
529     portMemCopy(&pGpuFabricProbeInfoKernel->probeResponseMsg,
530                 sizeof(pGpuFabricProbeInfoKernel->probeResponseMsg),
531                 pProbeRespMsg,
532                 sizeof(*pProbeRespMsg));
533 
534     portAtomicMemoryFenceFull();
535     //
536     // TODO - Add additional check with versioning to continue with the
537     // timer and send lower version requests
538     //
539     portAtomicSetU32(&pGpuFabricProbeInfoKernel->probeRespRcvd, 1);
540 
541     status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
542     NV_CHECK_OR_RETURN(LEVEL_ERROR, status == NV_OK, status);
543 
544     _gpuFabricProbeSetupGpaRange(pGpu, pGpuFabricProbeInfoKernel);
545     _gpuFabricProbeSetupFlaRange(pGpu, pGpuFabricProbeInfoKernel);
546 
547     return NV_OK;
548 }
549 
550 NV_STATUS
gpuFabricProbeReceiveUpdateKernelCallback(NvU32 gpuInstance,NvU64 * pNotifyGfidMask,NV2080_CTRL_NVLINK_INBAND_RECEIVED_DATA_PARAMS * pInbandRcvParams)551 gpuFabricProbeReceiveUpdateKernelCallback
552 (
553     NvU32 gpuInstance,
554     NvU64 *pNotifyGfidMask,
555     NV2080_CTRL_NVLINK_INBAND_RECEIVED_DATA_PARAMS *pInbandRcvParams
556 )
557 {
558     OBJGPU *pGpu;
559     NvU32 gpuMaskUnused;
560     nvlink_inband_gpu_probe_update_req_msg_t *pProbeUpdateReqMsg;
561     nvlink_inband_gpu_probe_rsp_msg_t *pProbeRespMsg;
562     GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel;
563     NV_STATUS status;
564 
565     if ((pGpu = gpumgrGetGpu(gpuInstance)) == NULL)
566     {
567         NV_ASSERT_FAILED("Invalid GPU instance");
568         return NV_ERR_INVALID_ARGUMENT;
569     }
570 
571     NV_CHECK_OR_RETURN(LEVEL_ERROR, pGpu->pGpuFabricProbeInfoKernel != NULL, NV_OK);
572 
573     NV_ASSERT(rmGpuGroupLockIsOwner(gpuInstance, GPU_LOCK_GRP_SUBDEVICE,
574                                     &gpuMaskUnused));
575 
576     NV_ASSERT(pInbandRcvParams != NULL);
577 
578     pGpuFabricProbeInfoKernel = pGpu->pGpuFabricProbeInfoKernel;
579 
580     status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
581     NV_CHECK_OR_RETURN(LEVEL_ERROR, status == NV_OK, status);
582 
583     pProbeRespMsg = &pGpuFabricProbeInfoKernel->probeResponseMsg;
584 
585     pProbeUpdateReqMsg = \
586         (nvlink_inband_gpu_probe_update_req_msg_t *)&pInbandRcvParams->data[0];
587 
588     pProbeRespMsg->probeRsp.fabricHealthMask =
589         pProbeUpdateReqMsg->probeUpdate.fabricHealthMask;
590 
591     if (pProbeRespMsg->probeRsp.cliqueId !=
592             pProbeUpdateReqMsg->probeUpdate.cliqueId)
593     {
594         pProbeRespMsg->probeRsp.cliqueId =
595             pProbeUpdateReqMsg->probeUpdate.cliqueId;
596         _gpuFabricProbeSendCliqueIdChangeEvent(pGpu,
597                                                pProbeRespMsg->probeRsp.cliqueId);
598     }
599 
600     return NV_OK;
601 }
602 
603 void
gpuFabricProbeSuspend(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel)604 gpuFabricProbeSuspend
605 (
606     GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel
607 )
608 {
609     OBJGPU *pGpu;
610     RM_API *pRmApi;
611     NV_STATUS status;
612 
613     if (pGpuFabricProbeInfoKernel == NULL)
614     {
615         return;
616     }
617 
618     pGpu = pGpuFabricProbeInfoKernel->pGpu;
619     pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
620 
621     NV_ASSERT(rmDeviceGpuLockIsOwner(gpuGetInstance(pGpu)));
622 
623     NV_CHECK_OK(status, LEVEL_ERROR,
624             pRmApi->Control(pRmApi,
625                             pGpu->hInternalClient,
626                             pGpu->hInternalSubdevice,
627                             NV2080_CTRL_CMD_INTERNAL_GPU_SUSPEND_FABRIC_PROBE,
628                             NULL, 0));
629 }
630 
631 NV_STATUS
gpuFabricProbeResume(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel)632 gpuFabricProbeResume
633 (
634     GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel
635 )
636 {
637     OBJGPU *pGpu;
638     RM_API *pRmApi;
639     NV2080_CTRL_CMD_INTERNAL_RESUME_GPU_FABRIC_PROBE_INFO_PARAMS params = { 0 };
640 
641     if (pGpuFabricProbeInfoKernel == NULL)
642     {
643         return NV_ERR_NOT_SUPPORTED;
644     }
645 
646     pGpu = pGpuFabricProbeInfoKernel->pGpu;
647     pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
648 
649     NV_ASSERT(rmDeviceGpuLockIsOwner(gpuGetInstance(pGpu)));
650 
651     params.bwMode = pGpuFabricProbeInfoKernel->bwMode;
652 
653     NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
654               pRmApi->Control(pRmApi,
655                               pGpu->hInternalClient,
656                               pGpu->hInternalSubdevice,
657                               NV2080_CTRL_CMD_INTERNAL_GPU_RESUME_FABRIC_PROBE,
658                               &params, sizeof(params)));
659 
660     return NV_OK;
661 }
662 
663 NV_STATUS
gpuFabricProbeStart(OBJGPU * pGpu,GPU_FABRIC_PROBE_INFO_KERNEL ** ppGpuFabricProbeInfoKernel)664 gpuFabricProbeStart
665 (
666     OBJGPU *pGpu,
667     GPU_FABRIC_PROBE_INFO_KERNEL **ppGpuFabricProbeInfoKernel
668 )
669 {
670     NV_STATUS status = NV_OK;
671     GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel;
672     RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
673     NV2080_CTRL_CMD_INTERNAL_START_GPU_FABRIC_PROBE_INFO_PARAMS params = { 0 };
674     MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
675 
676     LOCK_ASSERT_AND_RETURN(rmDeviceGpuLockIsOwner(gpuGetInstance(pGpu)));
677 
678     // Check if NVSwitch based system. If not return without doing anything
679     if (!gpuFabricProbeIsSupported(pGpu))
680     {
681         return NV_OK;
682     }
683 
684     *ppGpuFabricProbeInfoKernel =
685                 portMemAllocNonPaged(sizeof(*pGpuFabricProbeInfoKernel));
686     NV_ASSERT_OR_RETURN(*ppGpuFabricProbeInfoKernel != NULL, NV_ERR_NO_MEMORY);
687 
688     pGpuFabricProbeInfoKernel = *ppGpuFabricProbeInfoKernel;
689 
690     portMemSet(pGpuFabricProbeInfoKernel, 0, sizeof(*pGpuFabricProbeInfoKernel));
691 
692     pGpuFabricProbeInfoKernel->pGpu = pGpu;
693     pGpuFabricProbeInfoKernel->bwMode = gpumgrGetGpuNvlinkBwMode();
694     params.bwMode = pGpuFabricProbeInfoKernel->bwMode;
695     params.bLocalEgmEnabled = pMemoryManager->bLocalEgmEnabled;
696 
697     if (IS_VIRTUAL(pGpu))
698     {
699         NV_RM_RPC_CONTROL(pGpu, NV01_NULL_OBJECT, NV01_NULL_OBJECT,
700                            NV2080_CTRL_CMD_INTERNAL_GPU_START_FABRIC_PROBE,
701                            &params, sizeof(params),
702                            status);
703 
704         if (status != NV_OK)
705             goto fail;
706     }
707     else
708     {
709         // Send IOCTL to start probe
710         NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR,
711                 pRmApi->Control(pRmApi,
712                                 pGpu->hInternalClient,
713                                 pGpu->hInternalSubdevice,
714                                 NV2080_CTRL_CMD_INTERNAL_GPU_START_FABRIC_PROBE,
715                                 &params, sizeof(params)),
716                 fail);
717     }
718 
719     return NV_OK;
720 
721 fail:
722     portMemFree(pGpuFabricProbeInfoKernel);
723     pGpu->pGpuFabricProbeInfoKernel = NULL;
724 
725     return status;
726 }
727 
728 void
gpuFabricProbeStop(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel)729 gpuFabricProbeStop
730 (
731     GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel
732 )
733 {
734     OBJGPU *pGpu;
735     RM_API *pRmApi;
736 
737     if (pGpuFabricProbeInfoKernel == NULL)
738     {
739         return;
740     }
741 
742     pGpu = pGpuFabricProbeInfoKernel->pGpu;
743 
744     NV_ASSERT_OR_RETURN_VOID(rmDeviceGpuLockIsOwner(gpuGetInstance(pGpu)));
745 
746     if (!IS_VIRTUAL(pGpu))
747     {
748         //
749         // On VGPU, we have to stop the probe for guest driver unload as well as for guest clean/forced shutdown/reboot.
750         // An RPC from this point will not be triggered for forced shutdown/reboot.
751         // vmioplugin already has callbacks for guest driver unload, guest shutdown and guest reboot.
752         // Hence it is simpler to handle the probe stop directly from these callbacks in plugin instead of a separate
753         // RPC for this RM control.
754         //
755 
756         pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
757 
758         NV_ASSERT_OK(pRmApi->Control(pRmApi,
759                                     pGpu->hInternalClient,
760                                     pGpu->hInternalSubdevice,
761                                     NV2080_CTRL_CMD_INTERNAL_GPU_STOP_FABRIC_PROBE,
762                                     NULL, 0));
763     }
764 
765     portMemFree(pGpuFabricProbeInfoKernel);
766     pGpu->pGpuFabricProbeInfoKernel = NULL;
767 }
768 
769 NvBool
gpuFabricProbeIsSupported(OBJGPU * pGpu)770 gpuFabricProbeIsSupported
771 (
772     OBJGPU *pGpu
773 )
774 {
775     if (pGpu->fabricProbeRetryDelay == 0)
776     {
777         NV_PRINTF(LEVEL_INFO, "GPU%u Probe handling is disabled\n",
778                   gpuGetInstance(pGpu));
779         return NV_FALSE;
780     }
781 
782     if (!IS_VIRTUAL(pGpu) && (GPU_GET_KERNEL_NVLINK(pGpu) == NULL))
783     {
784         return NV_FALSE;
785     }
786 
787     return NV_TRUE;
788 }
789 
790 static void
_gpuFabricProbeInvalidate(OBJGPU * pGpu)791 _gpuFabricProbeInvalidate
792 (
793     OBJGPU *pGpu
794 )
795 {
796     GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel =
797                                     pGpu->pGpuFabricProbeInfoKernel;
798     KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu);
799     FABRIC_VASPACE *pFabricVAS = dynamicCast(pGpu->pFabricVAS, FABRIC_VASPACE);
800 
801     portAtomicSetU32(&pGpuFabricProbeInfoKernel->probeRespRcvd, 0);
802 
803     if (pKernelNvlink != NULL)
804     {
805         knvlinkClearUniqueFabricBaseAddress_HAL(pGpu, pKernelNvlink);
806         knvlinkClearUniqueFabricEgmBaseAddress_HAL(pGpu, pKernelNvlink);
807     }
808 
809     if (pFabricVAS != NULL)
810         fabricvaspaceClearUCRange(pFabricVAS);
811 }
812 
813 #define GPU_FABRIC_CHECK_BW_MODE(fmCaps, mode)                    \
814     do                                                            \
815     {                                                             \
816         if ((fmCaps & NVLINK_INBAND_FM_CAPS_BW_MODE_##mode) == 0) \
817             return NV_ERR_NOT_SUPPORTED;                          \
818     } while (0)
819 
820 static NV_STATUS
_gpuFabricProbeUpdateBwMode(OBJGPU * pGpu,NvU8 mode)821 _gpuFabricProbeUpdateBwMode
822 (
823     OBJGPU *pGpu,
824     NvU8 mode
825 )
826 {
827     GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel =
828                                                 pGpu->pGpuFabricProbeInfoKernel;
829     RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
830 
831     pGpuFabricProbeInfoKernel->bwMode = mode;
832 
833     gpuFabricProbeSuspend(pGpuFabricProbeInfoKernel);
834 
835     NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
836           pRmApi->Control(pRmApi,
837                           pGpu->hInternalClient,
838                           pGpu->hInternalSubdevice,
839                           NV2080_CTRL_CMD_INTERNAL_GPU_INVALIDATE_FABRIC_PROBE,
840                           NULL, 0));
841 
842     _gpuFabricProbeInvalidate(pGpu);
843 
844     NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, gpuFabricProbeResume(pGpuFabricProbeInfoKernel));
845 
846     return NV_OK;
847 }
848 
849 NV_STATUS
gpuFabricProbeSetBwMode(NvU8 mode)850 gpuFabricProbeSetBwMode
851 (
852     NvU8 mode
853 )
854 {
855     NvU32 attachedGpuCount;
856     NvU32 attachedGpuMask;
857     NV_STATUS status;
858     NvU32 gpuIndex;
859     OBJGPU *pGpu;
860 
861     status = gpumgrGetGpuAttachInfo(&attachedGpuCount, &attachedGpuMask);
862     if (status != NV_OK)
863     {
864         return NV_ERR_INVALID_STATE;
865     }
866 
867     // Check if all GPUs belong to NvSwitch
868     gpuIndex = 0;
869     for(pGpu = gpumgrGetNextGpu(attachedGpuMask, &gpuIndex);
870         pGpu != NULL;
871         pGpu = gpumgrGetNextGpu(attachedGpuMask, &gpuIndex))
872     {
873         if (!gpuFabricProbeIsSupported(pGpu))
874         {
875             // For directed connected system
876             return NV_OK;
877         }
878     }
879 
880     //
881     // Check if all GPUs received fabric probe and
882     //       if the mode is supported on all GPUs.
883     //
884     gpuIndex = 0;
885     for(pGpu = gpumgrGetNextGpu(attachedGpuMask, &gpuIndex);
886         pGpu != NULL;
887         pGpu = gpumgrGetNextGpu(attachedGpuMask, &gpuIndex))
888     {
889         if (!gpuFabricProbeIsReceived(pGpu->pGpuFabricProbeInfoKernel) ||
890             !gpuFabricProbeIsSuccess(pGpu->pGpuFabricProbeInfoKernel))
891         {
892             return NV_ERR_NOT_READY;
893         }
894 
895         NvU64 fmCaps = pGpu->pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.fmCaps;
896         switch(mode)
897         {
898             case GPU_NVLINK_BW_MODE_MIN:
899                 GPU_FABRIC_CHECK_BW_MODE(fmCaps, MIN);
900                 break;
901             case GPU_NVLINK_BW_MODE_HALF:
902                 GPU_FABRIC_CHECK_BW_MODE(fmCaps, HALF);
903                 break;
904             case GPU_NVLINK_BW_MODE_3QUARTER:
905                 GPU_FABRIC_CHECK_BW_MODE(fmCaps, 3QUARTER);
906                 break;
907             case GPU_NVLINK_BW_MODE_OFF:
908                 return NV_OK; // Don't need to ask FM
909             default:
910                 break;
911         }
912     }
913 
914     gpuIndex = 0;
915     for(pGpu = gpumgrGetNextGpu(attachedGpuMask, &gpuIndex);
916         pGpu != NULL;
917         pGpu = gpumgrGetNextGpu(attachedGpuMask, &gpuIndex))
918     {
919         FABRIC_VASPACE *pFabricVAS = dynamicCast(pGpu->pFabricVAS,
920                                                  FABRIC_VASPACE);
921         if (pFabricVAS == NULL)
922         {
923             continue;
924         }
925 
926         if (fabricvaspaceIsInUse(pFabricVAS))
927         {
928             return NV_ERR_STATE_IN_USE;
929         }
930     }
931 
932     gpuIndex = 0;
933     for(pGpu = gpumgrGetNextGpu(attachedGpuMask, &gpuIndex);
934         pGpu != NULL;
935         pGpu = gpumgrGetNextGpu(attachedGpuMask, &gpuIndex))
936     {
937         status = _gpuFabricProbeUpdateBwMode(pGpu, mode);
938         if (status != NV_OK)
939         {
940             return status;
941         }
942     }
943 
944     return NV_OK;
945 }
946 
947 NV_STATUS
gpuFabricProbeGetlinkMaskToBeReduced(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel,NvU32 * linkMaskToBeReduced)948 gpuFabricProbeGetlinkMaskToBeReduced
949 (
950     GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel,
951     NvU32 *linkMaskToBeReduced
952 )
953 {
954     NV_STATUS status;
955 
956     status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
957 
958     NV_CHECK_OR_RETURN(LEVEL_SILENT, status == NV_OK, status);
959 
960     *linkMaskToBeReduced =
961         pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.linkMaskToBeReduced;
962 
963     return NV_OK;
964 }
965