1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 /*!
25 * @file
26 * @brief GPU Fabric Probe handling
27 */
28
29
30 #include "gpu/gpu.h"
31 #include "os/os.h"
32 #include "nvport/atomic.h"
33 #include "utils/nvprintf.h"
34 #include "kernel/gpu/nvlink/kernel_nvlink.h"
35 #include "gpu/gpu_fabric_probe.h"
36 #include "nvlink_inband_msg.h"
37 #include "kernel/mem_mgr/fabric_vaspace.h"
38 #include "ctrl/ctrl2080/ctrl2080internal.h"
39
40 #include "compute/fabric.h"
41 #include "class/cl00f1.h"
42 #include "vgpu/rpc.h"
43
44 // Structure to hold gpu probe information
45 typedef struct GPU_FABRIC_PROBE_INFO_KERNEL
46 {
47 volatile NvU32 probeRespRcvd;
48 NvU8 bwMode;
49
50 OBJGPU *pGpu;
51
52 nvlink_inband_gpu_probe_rsp_msg_t probeResponseMsg;
53
54 } GPU_FABRIC_PROBE_INFO_KERNEL;
55
56 static NV_STATUS
_gpuFabricProbeFullSanityCheck(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel)57 _gpuFabricProbeFullSanityCheck
58 (
59 GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel
60 )
61 {
62 if (pGpuFabricProbeInfoKernel == NULL)
63 {
64 return NV_ERR_NOT_SUPPORTED;
65 }
66
67 if (!gpuFabricProbeIsReceived(pGpuFabricProbeInfoKernel))
68 {
69 return NV_ERR_NOT_READY;
70 }
71
72 if (!gpuFabricProbeIsSuccess(pGpuFabricProbeInfoKernel))
73 {
74 NV_PRINTF(LEVEL_ERROR, "Fabric Probe failed: 0x%x\n",
75 pGpuFabricProbeInfoKernel->probeResponseMsg.msgHdr.status);
76
77 return pGpuFabricProbeInfoKernel->probeResponseMsg.msgHdr.status;
78 }
79
80 return NV_OK;
81 }
82
83 NV_STATUS
gpuFabricProbeGetGpuFabricHandle(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel,NvU64 * pHandle)84 gpuFabricProbeGetGpuFabricHandle
85 (
86 GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel,
87 NvU64 *pHandle
88 )
89 {
90 NV_STATUS status;
91
92 status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
93
94 NV_CHECK_OR_RETURN(LEVEL_SILENT, status == NV_OK, status);
95
96 *pHandle = pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.gpuHandle;
97
98 return status;
99 }
100
101 NV_STATUS
gpuFabricProbeGetGfId(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel,NvU32 * pGfId)102 gpuFabricProbeGetGfId
103 (
104 GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel,
105 NvU32 *pGfId
106 )
107 {
108 NV_STATUS status;
109
110 status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
111
112 NV_CHECK_OR_RETURN(LEVEL_SILENT, status == NV_OK, status);
113
114 *pGfId = pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.gfId;
115
116 return status;
117 }
118
119 NV_STATUS
gpuFabricProbeGetfmCaps(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel,NvU64 * pFmCaps)120 gpuFabricProbeGetfmCaps
121 (
122 GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel,
123 NvU64 *pFmCaps
124 )
125 {
126 NV_STATUS status;
127
128 status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
129
130 NV_CHECK_OR_RETURN(LEVEL_SILENT, status == NV_OK, status);
131
132 *pFmCaps = pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.fmCaps;
133
134 return status;
135 }
136
137 NV_STATUS
gpuFabricProbeGetClusterUuid(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel,NvUuid * pClusterUuid)138 gpuFabricProbeGetClusterUuid
139 (
140 GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel,
141 NvUuid *pClusterUuid
142 )
143 {
144 NV_STATUS status;
145
146 status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
147
148 NV_CHECK_OR_RETURN(LEVEL_SILENT, status == NV_OK, status);
149
150 portMemCopy(&pClusterUuid->uuid[0],
151 sizeof(pClusterUuid->uuid),
152 &pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.clusterUuid.uuid[0],
153 sizeof(pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.clusterUuid.uuid));
154
155 return status;
156 }
157
158 NV_STATUS
gpuFabricProbeGetFabricPartitionId(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel,NvU16 * pFabricPartitionId)159 gpuFabricProbeGetFabricPartitionId
160 (
161 GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel,
162 NvU16 *pFabricPartitionId
163 )
164 {
165 NV_STATUS status;
166
167 status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
168
169 NV_CHECK_OR_RETURN(LEVEL_SILENT, status == NV_OK, status);
170
171 *pFabricPartitionId = pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.fabricPartitionId;
172
173 return status;
174 }
175
176 NV_STATUS
gpuFabricProbeGetGpaAddress(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel,NvU64 * pGpaAddress)177 gpuFabricProbeGetGpaAddress
178 (
179 GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel,
180 NvU64 *pGpaAddress
181 )
182 {
183 NV_STATUS status;
184
185 status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
186
187 NV_CHECK_OR_RETURN(LEVEL_SILENT, status == NV_OK, status);
188
189 *pGpaAddress = pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.gpaAddress;
190
191 return status;
192 }
193
194 NV_STATUS
gpuFabricProbeGetGpaAddressRange(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel,NvU64 * pGpaAddressRange)195 gpuFabricProbeGetGpaAddressRange
196 (
197 GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel,
198 NvU64 *pGpaAddressRange
199 )
200 {
201 NV_STATUS status;
202
203 status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
204
205 NV_CHECK_OR_RETURN(LEVEL_SILENT, status == NV_OK, status);
206
207 *pGpaAddressRange = pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.gpaAddressRange;
208
209 return status;
210 }
211
212 NV_STATUS
gpuFabricProbeGetFlaAddress(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel,NvU64 * pFlaAddress)213 gpuFabricProbeGetFlaAddress
214 (
215 GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel,
216 NvU64 *pFlaAddress
217 )
218 {
219 NV_STATUS status;
220
221 status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
222
223 NV_CHECK_OR_RETURN(LEVEL_SILENT, status == NV_OK, status);
224
225 *pFlaAddress = pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.flaAddress;
226
227 return status;
228 }
229
230 NV_STATUS
gpuFabricProbeGetFlaAddressRange(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel,NvU64 * pFlaAddressRange)231 gpuFabricProbeGetFlaAddressRange
232 (
233 GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel,
234 NvU64 *pFlaAddressRange
235 )
236 {
237 NV_STATUS status;
238
239 status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
240
241 NV_CHECK_OR_RETURN(LEVEL_SILENT, status == NV_OK, status);
242
243 *pFlaAddressRange = pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.flaAddressRange;
244
245 return status;
246 }
247
248 /*
249 * This function is used to get the peer GPU EGM address from FM to RM.
250 * FM passes only the upper 32 bits of the address.
251 */
252 NV_STATUS
gpuFabricProbeGetEgmGpaAddress(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel,NvU64 * pEgmGpaAddress)253 gpuFabricProbeGetEgmGpaAddress
254 (
255 GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel,
256 NvU64 *pEgmGpaAddress
257 )
258 {
259 NV_STATUS status;
260
261 status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
262
263 NV_CHECK_OR_RETURN(LEVEL_SILENT, status == NV_OK, status);
264
265 *pEgmGpaAddress = (NvU64)pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.gpaAddressEGMHi << 32ULL;
266
267 return status;
268 }
269
270 NV_STATUS
gpuFabricProbeGetNumProbeReqs(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel,NvU64 * numProbes)271 gpuFabricProbeGetNumProbeReqs
272 (
273 GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel,
274 NvU64 *numProbes
275 )
276 {
277 NV2080_CTRL_CMD_INTERNAL_GET_GPU_FABRIC_PROBE_INFO_PARAMS params = { 0 };
278 RM_API *pRmApi;
279 OBJGPU *pGpu;
280
281 if (pGpuFabricProbeInfoKernel == NULL)
282 {
283 return NV_ERR_NOT_SUPPORTED;
284 }
285
286 pGpu = pGpuFabricProbeInfoKernel->pGpu;
287 pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
288
289 LOCK_ASSERT_AND_RETURN(rmDeviceGpuLockIsOwner(
290 gpuGetInstance(pGpuFabricProbeInfoKernel->pGpu)));
291
292 if (IS_VIRTUAL(pGpu))
293 {
294 *numProbes = 1;
295 return NV_OK;
296 }
297
298 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
299 pRmApi->Control(pRmApi,
300 pGpu->hInternalClient,
301 pGpu->hInternalSubdevice,
302 NV2080_CTRL_CMD_INTERNAL_GPU_GET_FABRIC_PROBE_INFO,
303 ¶ms,
304 sizeof(params)));
305
306 *numProbes = params.numProbes;
307
308 return NV_OK;
309 }
310
311 NV_STATUS
gpuFabricProbeGetFabricCliqueId(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel,NvU32 * pFabricCliqueId)312 gpuFabricProbeGetFabricCliqueId
313 (
314 GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel,
315 NvU32 *pFabricCliqueId
316 )
317 {
318 NV_STATUS status;
319
320 status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
321
322 NV_CHECK_OR_RETURN(LEVEL_SILENT, status == NV_OK, status);
323
324 *pFabricCliqueId =
325 pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.cliqueId;
326
327 return NV_OK;
328 }
329
330 NV_STATUS
gpuFabricProbeGetFabricHealthStatus(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel,NvU32 * pFabricHealthStatusMask)331 gpuFabricProbeGetFabricHealthStatus
332 (
333 GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel,
334 NvU32 *pFabricHealthStatusMask
335 )
336 {
337 NV_STATUS status;
338
339 status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
340
341 NV_CHECK_OR_RETURN(LEVEL_SILENT, status == NV_OK, status);
342
343 *pFabricHealthStatusMask =
344 pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.fabricHealthMask;
345
346 return NV_OK;
347 }
348
349 NvBool
gpuFabricProbeIsReceived(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel)350 gpuFabricProbeIsReceived
351 (
352 GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel
353 )
354 {
355 if (pGpuFabricProbeInfoKernel == NULL)
356 {
357 return NV_FALSE;
358 }
359
360 return !!portAtomicOrU32(&pGpuFabricProbeInfoKernel->probeRespRcvd, 0);
361 }
362
363 NvBool
gpuFabricProbeIsSuccess(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel)364 gpuFabricProbeIsSuccess
365 (
366 GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel
367 )
368 {
369 nvlink_inband_gpu_probe_rsp_msg_t *pProbeResponseMsg;
370 nvlink_inband_msg_header_t *pProbeRespMsgHdr;
371
372 if (pGpuFabricProbeInfoKernel == NULL)
373 {
374 return NV_FALSE;
375 }
376
377 pProbeResponseMsg = &pGpuFabricProbeInfoKernel->probeResponseMsg;
378 pProbeRespMsgHdr = &pProbeResponseMsg->msgHdr;
379
380 return pProbeRespMsgHdr->status == NV_OK;
381 }
382
383 NV_STATUS
gpuFabricProbeGetFmStatus(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel)384 gpuFabricProbeGetFmStatus
385 (
386 GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel
387 )
388 {
389 if (pGpuFabricProbeInfoKernel == NULL)
390 {
391 return NV_ERR_NOT_SUPPORTED;
392 }
393
394 return pGpuFabricProbeInfoKernel->probeResponseMsg.msgHdr.status;
395 }
396
397 static void
_gpuFabricProbeSetupGpaRange(OBJGPU * pGpu,GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel)398 _gpuFabricProbeSetupGpaRange
399 (
400 OBJGPU *pGpu,
401 GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel
402 )
403 {
404 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu);
405 // setup GPA based system
406 if (pKernelNvlink != NULL)
407 {
408 NvU64 gpaAddress;
409 NvU64 gpaAddressSize;
410 NvU64 egmGpaAddress;
411
412 NV_CHECK_OR_RETURN_VOID(LEVEL_ERROR,
413 gpuFabricProbeGetGpaAddress(pGpuFabricProbeInfoKernel,
414 &gpaAddress) == NV_OK);
415
416 NV_CHECK_OR_RETURN_VOID(LEVEL_ERROR,
417 gpuFabricProbeGetGpaAddressRange(pGpuFabricProbeInfoKernel,
418 &gpaAddressSize) == NV_OK);
419
420 NV_CHECK_OR_RETURN_VOID(LEVEL_ERROR,
421 knvlinkSetUniqueFabricBaseAddress_HAL(pGpu, pKernelNvlink,
422 gpaAddress) == NV_OK);
423
424 NV_CHECK_OR_RETURN_VOID(LEVEL_ERROR,
425 gpuFabricProbeGetEgmGpaAddress(pGpuFabricProbeInfoKernel,
426 &egmGpaAddress) == NV_OK);
427
428 NV_CHECK_OR_RETURN_VOID(LEVEL_ERROR,
429 knvlinkSetUniqueFabricEgmBaseAddress_HAL(pGpu, pKernelNvlink,
430 egmGpaAddress) == NV_OK);
431 }
432 }
433
434 static void
_gpuFabricProbeSetupFlaRange(OBJGPU * pGpu,GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel)435 _gpuFabricProbeSetupFlaRange
436 (
437 OBJGPU *pGpu,
438 GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel
439 )
440 {
441 if (pGpu->pFabricVAS != NULL)
442 {
443 NvU64 flaBaseAddress;
444 NvU64 flaSize;
445
446 NV_CHECK_OR_RETURN_VOID(LEVEL_ERROR,
447 gpuFabricProbeGetFlaAddress(pGpuFabricProbeInfoKernel,
448 &flaBaseAddress) == NV_OK);
449
450 NV_CHECK_OR_RETURN_VOID(LEVEL_ERROR,
451 gpuFabricProbeGetFlaAddressRange(pGpuFabricProbeInfoKernel,
452 &flaSize) == NV_OK);
453
454 if (IS_VIRTUAL(pGpu))
455 {
456 fabricvaspaceClearUCRange(dynamicCast(pGpu->pFabricVAS, FABRIC_VASPACE));
457 }
458
459 NV_CHECK_OR_RETURN_VOID(LEVEL_ERROR,
460 fabricvaspaceInitUCRange(dynamicCast(pGpu->pFabricVAS, FABRIC_VASPACE),
461 pGpu, flaBaseAddress, flaSize) == NV_OK);
462 }
463 }
464
465 static void
_gpuFabricProbeSendCliqueIdChangeEvent(OBJGPU * pGpu,NvU32 cliqueId)466 _gpuFabricProbeSendCliqueIdChangeEvent
467 (
468 OBJGPU *pGpu,
469 NvU32 cliqueId
470 )
471 {
472 NV_STATUS status;
473 Fabric *pFabric = SYS_GET_FABRIC(SYS_GET_INSTANCE());
474 NV00F1_CTRL_FABRIC_EVENT event;
475 event.id = fabricGenerateEventId_IMPL(pFabric);
476 event.type = NV00F1_CTRL_FABRIC_EVENT_TYPE_CLIQUE_ID_CHANGE;
477 event.imexChannel = 0;
478 event.data.cliqueIdChange.gpuId = pGpu->gpuId;
479 event.data.cliqueIdChange.cliqueId = cliqueId;
480 status = fabricPostEventsV2(pFabric, &event, 1);
481 if (status != NV_OK)
482 {
483 NV_PRINTF(LEVEL_ERROR, "GPU%u Notifying cliqueId change failed\n",
484 gpuGetInstance(pGpu));
485 }
486 }
487
488 NV_STATUS
gpuFabricProbeReceiveKernelCallback(NvU32 gpuInstance,NvU64 * pNotifyGfidMask,NV2080_CTRL_NVLINK_INBAND_RECEIVED_DATA_PARAMS * pInbandRcvParams)489 gpuFabricProbeReceiveKernelCallback
490 (
491 NvU32 gpuInstance,
492 NvU64 *pNotifyGfidMask,
493 NV2080_CTRL_NVLINK_INBAND_RECEIVED_DATA_PARAMS *pInbandRcvParams
494 )
495 {
496 OBJGPU *pGpu;
497 NvU32 gpuMaskUnused;
498 nvlink_inband_gpu_probe_rsp_msg_t *pProbeRespMsg;
499 GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel;
500 NV_STATUS status;
501
502 if ((pGpu = gpumgrGetGpu(gpuInstance)) == NULL)
503 {
504 NV_ASSERT_FAILED("Invalid GPU instance");
505 return NV_ERR_INVALID_ARGUMENT;
506 }
507
508 //
509 // There is a scenario where _gpuFabricProbeStart fails in the GSP
510 // and returns failure to kernel ctrl call to start probe.
511 // This will set the pGpuFabricProbeInfoKernel to NULL.
512 // GSP also sends a probe response with failure error code.
513 // Handling this response causes kernel driver to crash since
514 // pGpuFabricProbeInfoKernel is already cleared in the kernel.
515 // This check is added to handle this scenario.
516 //
517 NV_CHECK_OR_RETURN(LEVEL_ERROR, pGpu->pGpuFabricProbeInfoKernel != NULL, NV_OK);
518
519 NV_ASSERT(rmGpuGroupLockIsOwner(gpuInstance, GPU_LOCK_GRP_SUBDEVICE,
520 &gpuMaskUnused));
521
522 NV_ASSERT(pInbandRcvParams != NULL);
523
524 pGpuFabricProbeInfoKernel = pGpu->pGpuFabricProbeInfoKernel;
525
526 pProbeRespMsg = \
527 (nvlink_inband_gpu_probe_rsp_msg_t *)&pInbandRcvParams->data[0];
528
529 portMemCopy(&pGpuFabricProbeInfoKernel->probeResponseMsg,
530 sizeof(pGpuFabricProbeInfoKernel->probeResponseMsg),
531 pProbeRespMsg,
532 sizeof(*pProbeRespMsg));
533
534 portAtomicMemoryFenceFull();
535 //
536 // TODO - Add additional check with versioning to continue with the
537 // timer and send lower version requests
538 //
539 portAtomicSetU32(&pGpuFabricProbeInfoKernel->probeRespRcvd, 1);
540
541 status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
542 NV_CHECK_OR_RETURN(LEVEL_ERROR, status == NV_OK, status);
543
544 _gpuFabricProbeSetupGpaRange(pGpu, pGpuFabricProbeInfoKernel);
545 _gpuFabricProbeSetupFlaRange(pGpu, pGpuFabricProbeInfoKernel);
546
547 return NV_OK;
548 }
549
550 NV_STATUS
gpuFabricProbeReceiveUpdateKernelCallback(NvU32 gpuInstance,NvU64 * pNotifyGfidMask,NV2080_CTRL_NVLINK_INBAND_RECEIVED_DATA_PARAMS * pInbandRcvParams)551 gpuFabricProbeReceiveUpdateKernelCallback
552 (
553 NvU32 gpuInstance,
554 NvU64 *pNotifyGfidMask,
555 NV2080_CTRL_NVLINK_INBAND_RECEIVED_DATA_PARAMS *pInbandRcvParams
556 )
557 {
558 OBJGPU *pGpu;
559 NvU32 gpuMaskUnused;
560 nvlink_inband_gpu_probe_update_req_msg_t *pProbeUpdateReqMsg;
561 nvlink_inband_gpu_probe_rsp_msg_t *pProbeRespMsg;
562 GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel;
563 NV_STATUS status;
564
565 if ((pGpu = gpumgrGetGpu(gpuInstance)) == NULL)
566 {
567 NV_ASSERT_FAILED("Invalid GPU instance");
568 return NV_ERR_INVALID_ARGUMENT;
569 }
570
571 NV_CHECK_OR_RETURN(LEVEL_ERROR, pGpu->pGpuFabricProbeInfoKernel != NULL, NV_OK);
572
573 NV_ASSERT(rmGpuGroupLockIsOwner(gpuInstance, GPU_LOCK_GRP_SUBDEVICE,
574 &gpuMaskUnused));
575
576 NV_ASSERT(pInbandRcvParams != NULL);
577
578 pGpuFabricProbeInfoKernel = pGpu->pGpuFabricProbeInfoKernel;
579
580 status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
581 NV_CHECK_OR_RETURN(LEVEL_ERROR, status == NV_OK, status);
582
583 pProbeRespMsg = &pGpuFabricProbeInfoKernel->probeResponseMsg;
584
585 pProbeUpdateReqMsg = \
586 (nvlink_inband_gpu_probe_update_req_msg_t *)&pInbandRcvParams->data[0];
587
588 pProbeRespMsg->probeRsp.fabricHealthMask =
589 pProbeUpdateReqMsg->probeUpdate.fabricHealthMask;
590
591 if (pProbeRespMsg->probeRsp.cliqueId !=
592 pProbeUpdateReqMsg->probeUpdate.cliqueId)
593 {
594 pProbeRespMsg->probeRsp.cliqueId =
595 pProbeUpdateReqMsg->probeUpdate.cliqueId;
596 _gpuFabricProbeSendCliqueIdChangeEvent(pGpu,
597 pProbeRespMsg->probeRsp.cliqueId);
598 }
599
600 return NV_OK;
601 }
602
603 void
gpuFabricProbeSuspend(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel)604 gpuFabricProbeSuspend
605 (
606 GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel
607 )
608 {
609 OBJGPU *pGpu;
610 RM_API *pRmApi;
611 NV_STATUS status;
612
613 if (pGpuFabricProbeInfoKernel == NULL)
614 {
615 return;
616 }
617
618 pGpu = pGpuFabricProbeInfoKernel->pGpu;
619 pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
620
621 NV_ASSERT(rmDeviceGpuLockIsOwner(gpuGetInstance(pGpu)));
622
623 NV_CHECK_OK(status, LEVEL_ERROR,
624 pRmApi->Control(pRmApi,
625 pGpu->hInternalClient,
626 pGpu->hInternalSubdevice,
627 NV2080_CTRL_CMD_INTERNAL_GPU_SUSPEND_FABRIC_PROBE,
628 NULL, 0));
629 }
630
631 NV_STATUS
gpuFabricProbeResume(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel)632 gpuFabricProbeResume
633 (
634 GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel
635 )
636 {
637 OBJGPU *pGpu;
638 RM_API *pRmApi;
639 NV2080_CTRL_CMD_INTERNAL_RESUME_GPU_FABRIC_PROBE_INFO_PARAMS params = { 0 };
640
641 if (pGpuFabricProbeInfoKernel == NULL)
642 {
643 return NV_ERR_NOT_SUPPORTED;
644 }
645
646 pGpu = pGpuFabricProbeInfoKernel->pGpu;
647 pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
648
649 NV_ASSERT(rmDeviceGpuLockIsOwner(gpuGetInstance(pGpu)));
650
651 params.bwMode = pGpuFabricProbeInfoKernel->bwMode;
652
653 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
654 pRmApi->Control(pRmApi,
655 pGpu->hInternalClient,
656 pGpu->hInternalSubdevice,
657 NV2080_CTRL_CMD_INTERNAL_GPU_RESUME_FABRIC_PROBE,
658 ¶ms, sizeof(params)));
659
660 return NV_OK;
661 }
662
663 NV_STATUS
gpuFabricProbeStart(OBJGPU * pGpu,GPU_FABRIC_PROBE_INFO_KERNEL ** ppGpuFabricProbeInfoKernel)664 gpuFabricProbeStart
665 (
666 OBJGPU *pGpu,
667 GPU_FABRIC_PROBE_INFO_KERNEL **ppGpuFabricProbeInfoKernel
668 )
669 {
670 NV_STATUS status = NV_OK;
671 GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel;
672 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
673 NV2080_CTRL_CMD_INTERNAL_START_GPU_FABRIC_PROBE_INFO_PARAMS params = { 0 };
674 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
675
676 LOCK_ASSERT_AND_RETURN(rmDeviceGpuLockIsOwner(gpuGetInstance(pGpu)));
677
678 // Check if NVSwitch based system. If not return without doing anything
679 if (!gpuFabricProbeIsSupported(pGpu))
680 {
681 return NV_OK;
682 }
683
684 *ppGpuFabricProbeInfoKernel =
685 portMemAllocNonPaged(sizeof(*pGpuFabricProbeInfoKernel));
686 NV_ASSERT_OR_RETURN(*ppGpuFabricProbeInfoKernel != NULL, NV_ERR_NO_MEMORY);
687
688 pGpuFabricProbeInfoKernel = *ppGpuFabricProbeInfoKernel;
689
690 portMemSet(pGpuFabricProbeInfoKernel, 0, sizeof(*pGpuFabricProbeInfoKernel));
691
692 pGpuFabricProbeInfoKernel->pGpu = pGpu;
693 pGpuFabricProbeInfoKernel->bwMode = gpumgrGetGpuNvlinkBwMode();
694 params.bwMode = pGpuFabricProbeInfoKernel->bwMode;
695 params.bLocalEgmEnabled = pMemoryManager->bLocalEgmEnabled;
696
697 if (IS_VIRTUAL(pGpu))
698 {
699 NV_RM_RPC_CONTROL(pGpu, NV01_NULL_OBJECT, NV01_NULL_OBJECT,
700 NV2080_CTRL_CMD_INTERNAL_GPU_START_FABRIC_PROBE,
701 ¶ms, sizeof(params),
702 status);
703
704 if (status != NV_OK)
705 goto fail;
706 }
707 else
708 {
709 // Send IOCTL to start probe
710 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR,
711 pRmApi->Control(pRmApi,
712 pGpu->hInternalClient,
713 pGpu->hInternalSubdevice,
714 NV2080_CTRL_CMD_INTERNAL_GPU_START_FABRIC_PROBE,
715 ¶ms, sizeof(params)),
716 fail);
717 }
718
719 return NV_OK;
720
721 fail:
722 portMemFree(pGpuFabricProbeInfoKernel);
723 pGpu->pGpuFabricProbeInfoKernel = NULL;
724
725 return status;
726 }
727
728 void
gpuFabricProbeStop(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel)729 gpuFabricProbeStop
730 (
731 GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel
732 )
733 {
734 OBJGPU *pGpu;
735 RM_API *pRmApi;
736
737 if (pGpuFabricProbeInfoKernel == NULL)
738 {
739 return;
740 }
741
742 pGpu = pGpuFabricProbeInfoKernel->pGpu;
743
744 NV_ASSERT_OR_RETURN_VOID(rmDeviceGpuLockIsOwner(gpuGetInstance(pGpu)));
745
746 if (!IS_VIRTUAL(pGpu))
747 {
748 //
749 // On VGPU, we have to stop the probe for guest driver unload as well as for guest clean/forced shutdown/reboot.
750 // An RPC from this point will not be triggered for forced shutdown/reboot.
751 // vmioplugin already has callbacks for guest driver unload, guest shutdown and guest reboot.
752 // Hence it is simpler to handle the probe stop directly from these callbacks in plugin instead of a separate
753 // RPC for this RM control.
754 //
755
756 pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
757
758 NV_ASSERT_OK(pRmApi->Control(pRmApi,
759 pGpu->hInternalClient,
760 pGpu->hInternalSubdevice,
761 NV2080_CTRL_CMD_INTERNAL_GPU_STOP_FABRIC_PROBE,
762 NULL, 0));
763 }
764
765 portMemFree(pGpuFabricProbeInfoKernel);
766 pGpu->pGpuFabricProbeInfoKernel = NULL;
767 }
768
769 NvBool
gpuFabricProbeIsSupported(OBJGPU * pGpu)770 gpuFabricProbeIsSupported
771 (
772 OBJGPU *pGpu
773 )
774 {
775 if (pGpu->fabricProbeRetryDelay == 0)
776 {
777 NV_PRINTF(LEVEL_INFO, "GPU%u Probe handling is disabled\n",
778 gpuGetInstance(pGpu));
779 return NV_FALSE;
780 }
781
782 if (!IS_VIRTUAL(pGpu) && (GPU_GET_KERNEL_NVLINK(pGpu) == NULL))
783 {
784 return NV_FALSE;
785 }
786
787 return NV_TRUE;
788 }
789
790 static void
_gpuFabricProbeInvalidate(OBJGPU * pGpu)791 _gpuFabricProbeInvalidate
792 (
793 OBJGPU *pGpu
794 )
795 {
796 GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel =
797 pGpu->pGpuFabricProbeInfoKernel;
798 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu);
799 FABRIC_VASPACE *pFabricVAS = dynamicCast(pGpu->pFabricVAS, FABRIC_VASPACE);
800
801 portAtomicSetU32(&pGpuFabricProbeInfoKernel->probeRespRcvd, 0);
802
803 if (pKernelNvlink != NULL)
804 {
805 knvlinkClearUniqueFabricBaseAddress_HAL(pGpu, pKernelNvlink);
806 knvlinkClearUniqueFabricEgmBaseAddress_HAL(pGpu, pKernelNvlink);
807 }
808
809 if (pFabricVAS != NULL)
810 fabricvaspaceClearUCRange(pFabricVAS);
811 }
812
813 #define GPU_FABRIC_CHECK_BW_MODE(fmCaps, mode) \
814 do \
815 { \
816 if ((fmCaps & NVLINK_INBAND_FM_CAPS_BW_MODE_##mode) == 0) \
817 return NV_ERR_NOT_SUPPORTED; \
818 } while (0)
819
820 static NV_STATUS
_gpuFabricProbeUpdateBwMode(OBJGPU * pGpu,NvU8 mode)821 _gpuFabricProbeUpdateBwMode
822 (
823 OBJGPU *pGpu,
824 NvU8 mode
825 )
826 {
827 GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel =
828 pGpu->pGpuFabricProbeInfoKernel;
829 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
830
831 pGpuFabricProbeInfoKernel->bwMode = mode;
832
833 gpuFabricProbeSuspend(pGpuFabricProbeInfoKernel);
834
835 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
836 pRmApi->Control(pRmApi,
837 pGpu->hInternalClient,
838 pGpu->hInternalSubdevice,
839 NV2080_CTRL_CMD_INTERNAL_GPU_INVALIDATE_FABRIC_PROBE,
840 NULL, 0));
841
842 _gpuFabricProbeInvalidate(pGpu);
843
844 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, gpuFabricProbeResume(pGpuFabricProbeInfoKernel));
845
846 return NV_OK;
847 }
848
849 NV_STATUS
gpuFabricProbeSetBwMode(NvU8 mode)850 gpuFabricProbeSetBwMode
851 (
852 NvU8 mode
853 )
854 {
855 NvU32 attachedGpuCount;
856 NvU32 attachedGpuMask;
857 NV_STATUS status;
858 NvU32 gpuIndex;
859 OBJGPU *pGpu;
860
861 status = gpumgrGetGpuAttachInfo(&attachedGpuCount, &attachedGpuMask);
862 if (status != NV_OK)
863 {
864 return NV_ERR_INVALID_STATE;
865 }
866
867 // Check if all GPUs belong to NvSwitch
868 gpuIndex = 0;
869 for(pGpu = gpumgrGetNextGpu(attachedGpuMask, &gpuIndex);
870 pGpu != NULL;
871 pGpu = gpumgrGetNextGpu(attachedGpuMask, &gpuIndex))
872 {
873 if (!gpuFabricProbeIsSupported(pGpu))
874 {
875 // For directed connected system
876 return NV_OK;
877 }
878 }
879
880 //
881 // Check if all GPUs received fabric probe and
882 // if the mode is supported on all GPUs.
883 //
884 gpuIndex = 0;
885 for(pGpu = gpumgrGetNextGpu(attachedGpuMask, &gpuIndex);
886 pGpu != NULL;
887 pGpu = gpumgrGetNextGpu(attachedGpuMask, &gpuIndex))
888 {
889 if (!gpuFabricProbeIsReceived(pGpu->pGpuFabricProbeInfoKernel) ||
890 !gpuFabricProbeIsSuccess(pGpu->pGpuFabricProbeInfoKernel))
891 {
892 return NV_ERR_NOT_READY;
893 }
894
895 NvU64 fmCaps = pGpu->pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.fmCaps;
896 switch(mode)
897 {
898 case GPU_NVLINK_BW_MODE_MIN:
899 GPU_FABRIC_CHECK_BW_MODE(fmCaps, MIN);
900 break;
901 case GPU_NVLINK_BW_MODE_HALF:
902 GPU_FABRIC_CHECK_BW_MODE(fmCaps, HALF);
903 break;
904 case GPU_NVLINK_BW_MODE_3QUARTER:
905 GPU_FABRIC_CHECK_BW_MODE(fmCaps, 3QUARTER);
906 break;
907 case GPU_NVLINK_BW_MODE_OFF:
908 return NV_OK; // Don't need to ask FM
909 default:
910 break;
911 }
912 }
913
914 gpuIndex = 0;
915 for(pGpu = gpumgrGetNextGpu(attachedGpuMask, &gpuIndex);
916 pGpu != NULL;
917 pGpu = gpumgrGetNextGpu(attachedGpuMask, &gpuIndex))
918 {
919 FABRIC_VASPACE *pFabricVAS = dynamicCast(pGpu->pFabricVAS,
920 FABRIC_VASPACE);
921 if (pFabricVAS == NULL)
922 {
923 continue;
924 }
925
926 if (fabricvaspaceIsInUse(pFabricVAS))
927 {
928 return NV_ERR_STATE_IN_USE;
929 }
930 }
931
932 gpuIndex = 0;
933 for(pGpu = gpumgrGetNextGpu(attachedGpuMask, &gpuIndex);
934 pGpu != NULL;
935 pGpu = gpumgrGetNextGpu(attachedGpuMask, &gpuIndex))
936 {
937 status = _gpuFabricProbeUpdateBwMode(pGpu, mode);
938 if (status != NV_OK)
939 {
940 return status;
941 }
942 }
943
944 return NV_OK;
945 }
946
947 NV_STATUS
gpuFabricProbeGetlinkMaskToBeReduced(GPU_FABRIC_PROBE_INFO_KERNEL * pGpuFabricProbeInfoKernel,NvU32 * linkMaskToBeReduced)948 gpuFabricProbeGetlinkMaskToBeReduced
949 (
950 GPU_FABRIC_PROBE_INFO_KERNEL *pGpuFabricProbeInfoKernel,
951 NvU32 *linkMaskToBeReduced
952 )
953 {
954 NV_STATUS status;
955
956 status = _gpuFabricProbeFullSanityCheck(pGpuFabricProbeInfoKernel);
957
958 NV_CHECK_OR_RETURN(LEVEL_SILENT, status == NV_OK, status);
959
960 *linkMaskToBeReduced =
961 pGpuFabricProbeInfoKernel->probeResponseMsg.probeRsp.linkMaskToBeReduced;
962
963 return NV_OK;
964 }
965