1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "ctrl/ctrl2080/ctrl2080ce.h"
25 #include "ctrl/ctrl2080/ctrl2080nvlink.h"
26 #include "gpu/ce/kernel_ce.h"
27 #include "gpu/nvlink/kernel_nvlink.h"
28 #include "gpu/ce/kernel_ce_private.h"
29 #include "gpu/gpu.h"
30 #include "gpu/bif/kernel_bif.h"
31 #include "platform/chipset/chipset.h"
32
33 #include "published/ampere/ga100/dev_ce.h"
34 #include "published/ampere/ga100/dev_nv_xve.h"
35 #include "published/ampere/ga100/dev_nv_xve_addendum.h"
36
37 #define NV_CE_INVALID_TOPO_IDX 0xFFFF
38
39 // Ampere +
40 #define NV_CE_MIN_PCE_PER_SYS_LINK 2
41 #define NV_CE_MIN_PCE_PER_PEER_LINK 1
42
43 // Defines for PCE-LCE mapping algorithm
44 #define NV_CE_LCE_MASK_INIT 0xFFFFFFFF
45 #define NV_CE_SYS_ALLOWED_LCE_MASK 0x0C
46 #define NV_CE_GRCE_ALLOWED_LCE_MASK 0x03
47 #define NV_CE_EVEN_ASYNC_LCE_MASK 0x55555550
48 #define NV_CE_ODD_ASYNC_LCE_MASK 0xAAAAAAA0
49 #define NV_CE_MAX_LCE_MASK 0x3FF
50 #define NV_CE_MAX_GRCE 2
51 #define NV_CE_NUM_HSHUB_PCES 16
52 #define NV_CE_PCE_STRIDE 3
53 #define NV_CE_SYS_LCE_ALLOWED_HSPCE_CONFIG 0x8
54 #define NV_CE_NUM_DEFAULT_PCES 2
55
56 #define NV_CE_SYS_ALLOWED_LCE_MASK 0x0C
57 #define NV_CE_GRCE_ALLOWED_LCE_MASK 0x03
58 #define NV_CE_EVEN_ASYNC_LCE_MASK 0x55555550
59 #define NV_CE_ODD_ASYNC_LCE_MASK 0xAAAAAAA0
60 #define NV_CE_MAX_LCE_MASK 0x3FF
61
62 static void _ceGetAlgorithmPceIndex(OBJGPU *, KernelCE*, NvU32 *, NvU32 *, NvBool *, NvU8 *);
63
64 /*
65 * Table for setting the PCE2LCE mapping for WAR configs that cannot be implemented
66 * using the algorithm because the config does not conform to the algorithm's set
67 * of requirements/assumptions
68 */
69 static NVLINK_CE_AUTO_CONFIG_TABLE nvLinkCeAutoConfigTable_GA100[] =
70 {
71
72 //
73 // #systmem #max #peers Symmetric Switch PCE-LCE GRCE exposeCe
74 // links (links/peer) Config? Config Map Config Mask
75 //
76
77 // Default minimal configuration - NOTE: do not add entrys before this
78 {0x0, 0x0, 0x0, NV_FALSE, NV_FALSE, {0xF,0xF,0xF,0xF,0xF,0xF,0xF,0xF,0xF,
79 0xF,0xF,0xF,0xF,0xF,0xF,0xF,0x2,0x3}, {0xF,0xF}, 0xF}
80 };
81
82 /*!
83 * @brief Returns the size of the PCE2LCE register array
84 *
85 *
86 * @param[in] pGpu OBJGPU pointer
87 * @param[in] pKCe KernelCE pointer
88 *
89 * @return NV_CE_PCE2LCE_CONFIG__SIZE_1
90 *
91 */
92 NvU32
kceGetPce2lceConfigSize1_GA100(KernelCE * pKCe)93 kceGetPce2lceConfigSize1_GA100
94 (
95 KernelCE * pKCe
96 )
97 {
98 return NV_CE_PCE2LCE_CONFIG__SIZE_1;
99 }
100
101 /**
102 * Return the pce-lce mappings and grce config
103 * reg values when nvlink topology is NOT forced
104 *
105 * @param[in] pGpu OBJGPU pointer
106 * @param[in] pKCe KernelCE pointer
107 * @param[out] pPceLceMap Stores the pce-lce mappings
108 * @param[out] pGrceConfig Stores the grce configuration
109 * @param[out] pExposeCeMask Mask of CEs to expose to clients
110 *
111 * @return NV_OK on success
112 */
113 NV_STATUS
kceGetNvlinkAutoConfigCeValues_GA100(OBJGPU * pGpu,KernelCE * pKCe,NvU32 * pPceLceMap,NvU32 * pGrceConfig,NvU32 * pExposeCeMask)114 kceGetNvlinkAutoConfigCeValues_GA100
115 (
116 OBJGPU *pGpu,
117 KernelCE *pKCe,
118 NvU32 *pPceLceMap,
119 NvU32 *pGrceConfig,
120 NvU32 *pExposeCeMask
121 )
122 {
123 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu);
124 OBJGPU *pRemoteGpu = NULL;
125 NV_STATUS status = NV_OK;
126 NvU32 gpuMask = 0;
127 NvU32 sysmemLinks = 0;
128 NvU32 numPeers = 0;
129 NvBool bSymmetric = NV_TRUE;
130 NvBool bCurrentTopoMax = NV_FALSE;
131 NvU32 maxLinksPerPeer = 0;
132 NvU32 gpuInstance = 0;
133 NvU32 topoIdx = NV_CE_INVALID_TOPO_IDX;
134 NvU32 pce2lceConfigSize1 = kceGetPce2lceConfigSize1_HAL(pKCe);
135 NvU32 grceConfigSize1 = kceGetGrceConfigSize1_HAL(pKCe);
136 NvBool bEntryExists;
137 NvU32 pceIdx, grceIdx, i;
138 NVLINK_TOPOLOGY_PARAMS *pCurrentTopo = portMemAllocNonPaged(sizeof(*pCurrentTopo));
139 NvU32 *pLocalPceLceMap = NULL, *pLocalGrceConfig = NULL;
140
141 NV_ASSERT_OR_RETURN(pCurrentTopo != NULL, NV_ERR_NO_MEMORY);
142
143 if ((pPceLceMap == NULL) || (pGrceConfig == NULL) || (pExposeCeMask == NULL))
144 {
145 status = NV_ERR_INVALID_ARGUMENT;
146 goto done;
147 }
148
149 if (pKernelNvlink == NULL && pGpu->getProperty(pGpu, PDB_PROP_GPU_SKIP_CE_MAPPINGS_NO_NVLINK))
150 {
151 status = NV_ERR_NOT_SUPPORTED;
152 goto done;
153 }
154
155 portMemSet(pCurrentTopo, 0, sizeof(*pCurrentTopo));
156
157 // Bug 200283711: Use the largest of all chips in allocating these arrays
158 pLocalPceLceMap = portMemAllocNonPaged(sizeof(NvU32[NV2080_CTRL_MAX_PCES]));
159 pLocalGrceConfig = portMemAllocNonPaged(sizeof(NvU32[NV2080_CTRL_MAX_GRCES]));
160 NvU32 localExposeCeMask = 0;
161
162 if (pLocalPceLceMap == NULL || pLocalGrceConfig == NULL)
163 {
164 status = NV_ERR_NO_MEMORY;
165 goto done;
166 }
167
168 for (i = 0; i < NV2080_CTRL_MAX_PCES; i++)
169 {
170 pLocalPceLceMap[i] = NV2080_CTRL_CE_UPDATE_PCE_LCE_MAPPINGS_INVALID_LCE;
171 }
172
173 for (i = 0; i < NV2080_CTRL_MAX_GRCES; i++)
174 {
175 pLocalGrceConfig[i] = NV2080_CTRL_CE_UPDATE_PCE_LCE_MAPPINGS_INVALID_LCE;
176 }
177
178 sysmemLinks = pKernelNvlink ? knvlinkGetNumLinksToSystem(pGpu, pKernelNvlink) : 0;
179
180 if (gpuGetNumCEs(pGpu) == 0)
181 {
182 status = NV_ERR_NOT_SUPPORTED;
183 goto done;
184 }
185
186 (void)gpumgrGetGpuAttachInfo(NULL, &gpuMask);
187
188 // Get the max{nvlinks/peer, for all connected peers}
189 while ((pKernelNvlink != NULL) && ((pRemoteGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL))
190 {
191 NvU32 numLinksToPeer = knvlinkGetNumLinksToPeer(pGpu, pKernelNvlink,
192 pRemoteGpu);
193 if (numLinksToPeer == 0)
194 {
195 continue;
196 }
197
198 numPeers++;
199
200 //
201 // The topology remains symmetric if this is either the first GPU we've
202 // seen connected over NVLINK, or the number of links connected to this
203 // peer is the same as the maximum number of links connected to any peer
204 // seen so far.
205 //
206 bSymmetric = (bSymmetric &&
207 ((maxLinksPerPeer == 0) ||
208 (maxLinksPerPeer == numLinksToPeer)));
209
210 if (numLinksToPeer > maxLinksPerPeer)
211 {
212 maxLinksPerPeer = numLinksToPeer;
213 }
214 }
215
216 pCurrentTopo->sysmemLinks = sysmemLinks;
217 pCurrentTopo->maxLinksPerPeer = maxLinksPerPeer;
218 pCurrentTopo->numPeers = numPeers;
219 pCurrentTopo->bSymmetric = bSymmetric;
220 pCurrentTopo->bSwitchConfig = pKernelNvlink ? knvlinkIsGpuConnectedToNvswitch(pGpu, pKernelNvlink) : NV_FALSE;
221
222 //
223 // Check if the current config exists in the table
224 // Here, we only fill exposeCeMask.
225 //
226 bEntryExists = kceGetAutoConfigTableEntry_HAL(pGpu, pKCe, pCurrentTopo, nvLinkCeAutoConfigTable_GA100,
227 NV_ARRAY_ELEMENTS(nvLinkCeAutoConfigTable_GA100),
228 &topoIdx, &localExposeCeMask);
229 if (bEntryExists)
230 {
231 // Since entry exists, fill local variables with the associated table entry
232 for (pceIdx = 0; pceIdx < pce2lceConfigSize1; pceIdx++)
233 {
234 pLocalPceLceMap[pceIdx] = nvLinkCeAutoConfigTable_GA100[topoIdx].pceLceMap[pceIdx];
235 }
236 for (grceIdx = 0; grceIdx < grceConfigSize1; grceIdx++)
237 {
238 pLocalGrceConfig[grceIdx] = nvLinkCeAutoConfigTable_GA100[topoIdx].grceConfig[grceIdx];
239 }
240 }
241 else
242 {
243 //
244 // There is no table entry - use algorithm to determine mapping
245 // Here the currentTopo struct comes with pce-lce & grce mappings & exposeCeMask
246 //
247
248 status = kceGetMappings_HAL(pGpu, pKCe, pCurrentTopo,
249 pLocalPceLceMap, pLocalGrceConfig, &localExposeCeMask);
250 }
251
252 // Get the largest topology that has been cached
253 bEntryExists = gpumgrGetSystemNvlinkTopo(gpuGetDBDF(pGpu), pCurrentTopo);
254
255 // Is this the largest topology that we've ever seen compared to the cached one?
256 bCurrentTopoMax = kceIsCurrentMaxTopology_HAL(pGpu, pKCe, pCurrentTopo, &localExposeCeMask, &topoIdx);
257
258 if (bCurrentTopoMax)
259 {
260 //
261 // Replace cached state with current config
262 // Store the state globally in gpumgr so that we can preserve the topology
263 // info across GPU loads.
264 // Preserving across GPU loads enables UVM to optimize perf
265 //
266 for (pceIdx = 0; pceIdx < pce2lceConfigSize1; pceIdx++)
267 {
268 pCurrentTopo->maxPceLceMap[pceIdx] = pLocalPceLceMap[pceIdx];
269 }
270 for (grceIdx = 0; grceIdx < grceConfigSize1; grceIdx++)
271 {
272 pCurrentTopo->maxGrceConfig[grceIdx] = pLocalGrceConfig[grceIdx];
273 }
274 pCurrentTopo->maxExposeCeMask = localExposeCeMask;
275
276 if (topoIdx != NV_CE_INVALID_TOPO_IDX)
277 {
278 // Only if we used table to determine config, store this value
279 pCurrentTopo->maxTopoIdx = topoIdx;
280 pCurrentTopo->sysmemLinks = nvLinkCeAutoConfigTable_GA100[topoIdx].sysmemLinks;
281 pCurrentTopo->maxLinksPerPeer = nvLinkCeAutoConfigTable_GA100[topoIdx].maxLinksPerPeer;
282 pCurrentTopo->numPeers = nvLinkCeAutoConfigTable_GA100[topoIdx].numPeers;
283 pCurrentTopo->bSymmetric = nvLinkCeAutoConfigTable_GA100[topoIdx].bSymmetric;
284 pCurrentTopo->bSwitchConfig = nvLinkCeAutoConfigTable_GA100[topoIdx].bSwitchConfig;
285 }
286 gpumgrUpdateSystemNvlinkTopo(gpuGetDBDF(pGpu), pCurrentTopo);
287 }
288
289 NV_PRINTF(LEVEL_INFO, "GPU%d : RM Configured Values for CE Config\n", gpuGetInstance(pGpu));
290
291 // Now, fill up the information to return. We'll always return max config information.
292 for (pceIdx = 0; pceIdx < pce2lceConfigSize1; pceIdx++)
293 {
294 pPceLceMap[pceIdx] = pCurrentTopo->maxPceLceMap[pceIdx];
295 NV_PRINTF(LEVEL_INFO, "PCE-LCE map: PCE %d LCE 0x%x\n", pceIdx, pPceLceMap[pceIdx]);
296 }
297
298 for (grceIdx = 0; grceIdx < grceConfigSize1; grceIdx++)
299 {
300 NvU32 grceSharedLce = pCurrentTopo->maxGrceConfig[grceIdx];
301
302 if (grceSharedLce != 0xF)
303 {
304 // GRCE is shared
305 pGrceConfig[grceIdx] = DRF_NUM(_CE, _GRCE_CONFIG, _SHARED, 1) |
306 DRF_NUM(_CE, _GRCE_CONFIG, _SHARED_LCE, grceSharedLce);
307 }
308 else
309 {
310 // GRCE got its own PCE
311 pGrceConfig[grceIdx] = DRF_NUM(_CE, _GRCE_CONFIG, _SHARED, 0);
312 }
313 NV_PRINTF(LEVEL_INFO, "GRCE Config: GRCE %d LCE 0x%x\n", grceIdx, pGrceConfig[grceIdx]);
314 }
315
316 *pExposeCeMask = pCurrentTopo->maxExposeCeMask;
317 NV_PRINTF(LEVEL_INFO, "exposeCeMask = 0x%x\n", *pExposeCeMask);
318
319 done:
320 portMemFree(pCurrentTopo);
321 portMemFree(pLocalPceLceMap);
322 portMemFree(pLocalGrceConfig);
323
324 return status;
325 }
326
327 /**
328 * @brief Check if current config's topology is larger than cached one
329 * Return NV_TRUE if yes, else return NV_FALSE
330 *
331 * @param[in] pGpu OBJGPU pointer
332 * @param[in] pKCe KernelCE pointer
333 * @param[in] pCurrentTopo NVLINK_TOPOLOGY_INFO pointer
334 * @param[in] pLocalExposeCeMask Pointer to caller HSHUB ID
335 * @param[in/out] pTopoIdx NvU32 pointer to topology index, if it exists
336 */
337 NvBool
kceIsCurrentMaxTopology_GA100(OBJGPU * pGpu,KernelCE * pKCe,NVLINK_TOPOLOGY_PARAMS * pCurrentTopo,NvU32 * pLocalExposeCeMask,NvU32 * pTopoIdx)338 kceIsCurrentMaxTopology_GA100
339 (
340 OBJGPU *pGpu,
341 KernelCE * pKCe,
342 NVLINK_TOPOLOGY_PARAMS * pCurrentTopo,
343 NvU32 *pLocalExposeCeMask,
344 NvU32 *pTopoIdx
345 )
346 {
347 if (pCurrentTopo->maxExposeCeMask & ~(*pLocalExposeCeMask))
348 {
349 //
350 // Current config's exposeCeMask is a subset of cached maxExposeCeMask
351 // Hence, we will return NV_FALSE and use cached state config
352 //
353 if (*pTopoIdx != NV_CE_INVALID_TOPO_IDX)
354 {
355 *pTopoIdx = pCurrentTopo->maxTopoIdx;
356 }
357 *pLocalExposeCeMask = pCurrentTopo->maxExposeCeMask;
358 return NV_FALSE;
359 }
360
361 //
362 // Current config is equal or a superset of cached maxExposeCeMask
363 // This means that the topology has increased and hence we should
364 // cache the current config as the max config. Return NV_TRUE to do so
365 //
366 return NV_TRUE;
367 }
368
369 /**
370 * @brief This function returns the pceIndex for a particular link ID
371 * Must always be called with the hshub ID for the calling link ID
372 *
373 * @param[in] pGpu OBJGPU pointer
374 * @param[in] pKCe KernelCE pointer
375 * @param[in] pceAvailableMaskPerHshub Pointer to CEs available per HSHUB
376 * @param[out] pceIndex Pointer to caller pceIndex
377 * @param[out] pHshubId Pointer to caller HSHUB ID
378 * @param[out] pFirstIter Pointer to iteration value
379 */
380 static void
_ceGetAlgorithmPceIndex(OBJGPU * pGpu,KernelCE * pKCe,NvU32 * pceAvailableMaskPerHshub,NvU32 * pceIndex,NvBool * pBFirstIter,NvU8 * pHshubId)381 _ceGetAlgorithmPceIndex
382 (
383 OBJGPU *pGpu,
384 KernelCE* pKCe,
385 NvU32 *pceAvailableMaskPerHshub,
386 NvU32 *pceIndex,
387 NvBool *pBFirstIter,
388 NvU8 *pHshubId
389 )
390 {
391 NV_STATUS status = NV_OK;
392
393 // 1. Apply PCE striding
394 if ((*pBFirstIter) != NV_TRUE)
395 {
396 *pceIndex += NV_CE_PCE_STRIDE;
397 }
398 *pBFirstIter = NV_FALSE;
399
400 if(!(NVBIT32(*pceIndex) & pceAvailableMaskPerHshub[*pHshubId]))
401 {
402 //
403 // 2. We couldn't find an applicable strided PCE in given HSHUB
404 // So, we'll assign the next consecutive PCE on the same HSHUB
405 //
406 *pceIndex = CE_GET_LOWEST_AVAILABLE_IDX(pceAvailableMaskPerHshub[*pHshubId]);
407 if(!(NVBIT32(*pceIndex) & pceAvailableMaskPerHshub[*pHshubId]))
408 {
409 // 3. If this is not a valid PCE on given HSHUB, assign PCE from alternative HSHUB
410 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu);
411
412 if (pKernelNvlink != NULL)
413 {
414 NV2080_CTRL_INTERNAL_HSHUB_NEXT_HSHUB_ID_PARAMS params;
415
416 portMemSet(¶ms, 0, sizeof(params));
417 params.hshubId = *pHshubId;
418
419 status = knvlinkExecGspRmRpc(pGpu, pKernelNvlink,
420 NV2080_CTRL_CMD_INTERNAL_HSHUB_NEXT_HSHUB_ID,
421 (void *)¶ms, sizeof(params));
422
423 NV_ASSERT_OK(status);
424 if (status == NV_OK)
425 {
426 *pHshubId = params.hshubId;
427 }
428 }
429
430 *pceIndex = CE_GET_LOWEST_AVAILABLE_IDX(pceAvailableMaskPerHshub[*pHshubId]);
431 if(!(NVBIT32(*pceIndex) & pceAvailableMaskPerHshub[*pHshubId]))
432 {
433 // If we've reached this point, then we have no more available PCEs to assign
434 NV_PRINTF(LEVEL_ERROR, "No more available PCEs to assign!\n");
435 NV_ASSERT(0);
436 }
437 }
438 }
439 return;
440 }
441
442 /**
443 * @brief Use the algorithm to determine all the mappings for
444 * the given GPU.
445 *
446 * @param[in] pGpu OBJGPU pointer
447 * @param[in] pKCe KernelCE pointer
448 * @param[out] pLocalPceLceMap Pointer to PCE-LCE array
449 * @param[out] pLocalGrceMap Pointer to GRCE array
450 * @param[out] pLocalExposeCeMask Pointer to LCE Mask
451 *
452 * Returns NV_TRUE if algorithm ran to completion with no erros
453 */
454
455 NV_STATUS
kceGetMappings_GA100(OBJGPU * pGpu,KernelCE * pKCe,NVLINK_TOPOLOGY_PARAMS * pTopoParams,NvU32 * pLocalPceLceMap,NvU32 * pLocalGrceMap,NvU32 * pExposeCeMask)456 kceGetMappings_GA100
457 (
458 OBJGPU *pGpu,
459 KernelCE *pKCe,
460 NVLINK_TOPOLOGY_PARAMS *pTopoParams,
461 NvU32 *pLocalPceLceMap,
462 NvU32 *pLocalGrceMap,
463 NvU32 *pExposeCeMask
464 )
465 {
466 NvU32 lceMask = 0;
467 NvU32 fbPceMask = 0;
468 NV_STATUS status = NV_OK;
469 NvU32 pceIndex, lceIndex, grceIdx;
470 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu);
471
472 if (!pKernelNvlink || knvlinkIsForcedConfig(pGpu, pKernelNvlink))
473 {
474 return NV_OK;
475 }
476
477 // Prepare the per-HSHUB/FBHUB available PCE mask
478 status = kceGetAvailableHubPceMask(pGpu, pKCe, pTopoParams);
479
480 // A. Start with assigning PCEs for "SYSMEM"
481 status = kceMapPceLceForSysmemLinks_HAL(pGpu, pKCe,
482 pTopoParams->pceAvailableMaskPerConnectingHub,
483 pLocalPceLceMap,
484 pExposeCeMask,
485 pTopoParams->fbhubPceMask);
486 if (status == NV_ERR_NOT_SUPPORTED)
487 {
488 NV_PRINTF(LEVEL_ERROR,
489 "No sysmem connections on this chip (PCIe or NVLink)!\n");
490 }
491
492 // B. Assign PCEs to "PEER"s
493 status = kceMapPceLceForNvlinkPeers_HAL(pGpu, pKCe,
494 pTopoParams->pceAvailableMaskPerConnectingHub,
495 pLocalPceLceMap,
496 pExposeCeMask);
497 if (status == NV_WARN_NOTHING_TO_DO)
498 {
499 // If there's no NVLink peers available, still expose an additional async LCE
500 status = kceMapAsyncLceDefault_HAL(pGpu, pKCe,
501 pTopoParams->pceAvailableMaskPerConnectingHub,
502 pLocalPceLceMap,
503 pExposeCeMask,
504 NV_CE_NUM_DEFAULT_PCES);
505 }
506
507 // C. Lastly, do the assignment for "GRCE"s
508 lceMask = kceGetGrceSupportedLceMask_HAL(pGpu, pKCe);
509
510 // Get the FBHUB PCE mask
511 fbPceMask = pTopoParams->fbhubPceMask;
512
513 // Store lceMask in the exposeCeMask before moving on
514 *pExposeCeMask |= lceMask;
515
516 for (grceIdx = 0; grceIdx < NV_CE_MAX_GRCE; grceIdx++)
517 {
518 // We should have fbhub PCEs left to assign, but avoid an invalid pceIndex if we do not
519 NV_ASSERT_OR_ELSE(fbPceMask != 0, break);
520
521 //
522 // Check if we are sharing GRCEs
523 // On Ampere, GRCEs can only use FBHUB PCEs
524 // So, we need to check if the FBHUB PCEs have already been assigned.
525 //
526 pceIndex = CE_GET_LOWEST_AVAILABLE_IDX(fbPceMask);
527 fbPceMask &= (~(NVBIT32(pceIndex)));
528
529 if ((NVBIT32(pLocalPceLceMap[pceIndex])) & *pExposeCeMask)
530 {
531 // GRCE is shared - set the status and shared LCE # in register field
532 lceIndex = pLocalPceLceMap[pceIndex];
533 pLocalGrceMap[grceIdx] = DRF_NUM(_CE, _GRCE_CONFIG, _SHARED, 1) |
534 DRF_NUM(_CE, _GRCE_CONFIG, _SHARED_LCE, lceIndex);
535
536 if ((kceIsGenXorHigherSupported_HAL(pGpu, pKCe, 4)) || (pKCe->bUseGen4Mapping == NV_TRUE))
537 {
538 kceApplyGen4orHigherMapping_HAL(pGpu, pKCe,
539 &pLocalPceLceMap[0],
540 &pTopoParams->pceAvailableMaskPerConnectingHub[0],
541 lceIndex,
542 pceIndex);
543 }
544 }
545 else
546 {
547 // GRCE got its own FBHUB PCE
548 // Store the LCE in associated PCE for GRCE
549 lceIndex = CE_GET_LOWEST_AVAILABLE_IDX(lceMask);
550 pLocalPceLceMap[pceIndex] = lceIndex;
551 lceMask &= (~(NVBIT32(lceIndex)));
552 // Reflect non-sharing status in register field
553 pLocalGrceMap[grceIdx] = DRF_NUM(_CE, _GRCE_CONFIG, _SHARED, 0) |
554 DRF_DEF(_CE, _GRCE_CONFIG, _SHARED_LCE, _NONE);
555 }
556 }
557
558 return NV_OK;
559 }
560
561 /**
562 * @brief This function assigns PCE-LCE mappings for sysmem
563 * for the following two cases -
564 * 1. PCIe links - assign FBHUB PCEs
565 * 2. NVLinks - assign HSHUB PCEs
566 * If sysLinkMask is 0, then we assume that sysmem is over PCIe.
567 * Else, follow step 2 as above.
568 *
569 * @param[in] pGpu OBJGPU pointer
570 * @param[in] pKCe KernelCE pointer
571 * @param[in] pceAvailableMaskPerHshub Pointer to CEs available per HSHUB
572 * @param[out] pLocalPceLceMap Pointer to PCE-LCE array
573 * @param[out] pLocalExposeCeMask Pointer to LCE Mask
574 *
575 * Returns NV_OK if successful in assigning PCEs and LCEs for sysmem links
576 */
577 NV_STATUS
kceMapPceLceForSysmemLinks_GA100(OBJGPU * pGpu,KernelCE * pKCe,NvU32 * pceAvailableMaskPerHshub,NvU32 * pLocalPceLceMap,NvU32 * pLocalExposeCeMask,NvU32 fbPceMask)578 kceMapPceLceForSysmemLinks_GA100
579 (
580 OBJGPU *pGpu,
581 KernelCE *pKCe,
582 NvU32 *pceAvailableMaskPerHshub,
583 NvU32 *pLocalPceLceMap,
584 NvU32 *pLocalExposeCeMask,
585 NvU32 fbPceMask
586 )
587 {
588 NvU32 lceMask = 0;
589 NvU32 pceMask = 0;
590 NvU32 numTotalPces = 0;
591 NvBool bFirstIter = NV_FALSE;
592 NvU32 numPcePerLink, tempFbPceMask;
593 NvU32 lceIndex, pceIndex;
594 NvU32 linkId, i;
595 NvU8 hshubId;
596 NV_STATUS status = NV_OK;
597
598 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu);
599
600 NV2080_CTRL_NVLINK_HSHUB_GET_SYSMEM_NVLINK_MASK_PARAMS paramsNvlinkMask;
601 NV2080_CTRL_INTERNAL_HSHUB_GET_HSHUB_ID_FOR_LINKS_PARAMS *pParamsHshubId = NULL;
602
603 NV_ASSERT_OR_RETURN(pKernelNvlink != NULL, NV_ERR_NOT_SUPPORTED);
604
605 portMemSet(¶msNvlinkMask, 0, sizeof(paramsNvlinkMask));
606 status = knvlinkExecGspRmRpc(pGpu, pKernelNvlink,
607 NV2080_CTRL_CMD_NVLINK_HSHUB_GET_SYSMEM_NVLINK_MASK,
608 (void *)¶msNvlinkMask, sizeof(paramsNvlinkMask));
609 if (status != NV_OK)
610 {
611 NV_PRINTF(LEVEL_ERROR, "Unable to determine PCEs and LCEs for sysmem links\n");
612 return status;
613 }
614
615 lceMask = kceGetSysmemSupportedLceMask_HAL(pGpu, pKCe);
616
617 //
618 // Assign FBHUB PCEs when sysmem is over PCIE because PCIE
619 // accesses are not supported over HSHUB PCEs
620 //
621 if (paramsNvlinkMask.sysmemLinkMask == 0)
622 {
623 // Store lceMask in the exposeCeMask before moving on
624 *pLocalExposeCeMask |= lceMask;
625
626 tempFbPceMask = fbPceMask;
627 while(tempFbPceMask)
628 {
629 lceIndex = CE_GET_LOWEST_AVAILABLE_IDX(lceMask);
630 pceIndex = CE_GET_LOWEST_AVAILABLE_IDX(tempFbPceMask);
631 pLocalPceLceMap[pceIndex] = lceIndex;
632 // Clear the lowest set bits to get to the next index
633 tempFbPceMask &= (tempFbPceMask - 1);
634 lceMask &= (lceMask - 1);
635 }
636
637 return NV_OK;
638 }
639
640 // If sysmem is over NVlink, assign HSHUB PCEs
641 numPcePerLink = NV_CE_MIN_PCE_PER_SYS_LINK;
642
643 pParamsHshubId = portMemAllocNonPaged(sizeof(*pParamsHshubId));
644
645 NV_ASSERT_OR_RETURN(pParamsHshubId != NULL, NV_ERR_NO_MEMORY);
646
647 portMemSet(pParamsHshubId, 0, sizeof(*pParamsHshubId));
648 pParamsHshubId->linkMask = paramsNvlinkMask.sysmemLinkMask;
649
650 status = knvlinkExecGspRmRpc(pGpu, pKernelNvlink,
651 NV2080_CTRL_CMD_INTERNAL_HSHUB_GET_HSHUB_ID_FOR_LINKS,
652 pParamsHshubId, sizeof(*pParamsHshubId));
653 if (status != NV_OK)
654 {
655 NV_PRINTF(LEVEL_ERROR, "Unable to determine Hshub Id for sysmem links");
656 goto done;
657 }
658
659 FOR_EACH_INDEX_IN_MASK(32, linkId, paramsNvlinkMask.sysmemLinkMask)
660 {
661 hshubId = pParamsHshubId->hshubIds[linkId];
662 pceIndex = CE_GET_LOWEST_AVAILABLE_IDX(pceAvailableMaskPerHshub[hshubId]);
663 bFirstIter = NV_TRUE;
664 for (i = 0; i < numPcePerLink; i++)
665 {
666 _ceGetAlgorithmPceIndex(pGpu, pKCe, pceAvailableMaskPerHshub, &pceIndex, &bFirstIter, &hshubId);
667 pceMask |= NVBIT32(pceIndex);
668 numTotalPces++;
669 // Clear out the assigned PCE
670 pceAvailableMaskPerHshub[hshubId] &= (~(NVBIT32(pceIndex)));
671 }
672 }
673 FOR_EACH_INDEX_IN_MASK_END;
674
675 //
676 // Now, enter the PCE-LCE assignment - alternatively assign PCEs
677 // to each of the 2 LCEs for sysmem
678 //
679 for (i = 0; i < (numTotalPces/NV_CE_MIN_PCE_PER_SYS_LINK); i++)
680 {
681 NvU32 tempLceMask = lceMask;
682 while(tempLceMask)
683 {
684 pceIndex = CE_GET_LOWEST_AVAILABLE_IDX(pceMask);
685 lceIndex = CE_GET_LOWEST_AVAILABLE_IDX(tempLceMask);
686
687 pLocalPceLceMap[pceIndex] = lceIndex;
688
689 pceMask &= (pceMask - 1);
690 tempLceMask &= (tempLceMask - 1);
691 }
692
693 // Store lceMask in the exposeCeMask before moving on
694 *pLocalExposeCeMask |= lceMask;
695 }
696
697 done:
698 portMemFree(pParamsHshubId);
699
700 return status;
701 }
702
703 /**
704 * @brief Returns mask of LCEs that can be assigned to sysmem connection
705 * where the index of corresponding set bit indicates the LCE index
706 *
707 * @param[in] pGpu OBJGPU pointer
708 * @param[in] pKCe KernelCE pointer
709 *
710 * Returns the mask of LCEs valid for SYSMEM connections
711 */
712 NvU32
kceGetSysmemSupportedLceMask_GA100(OBJGPU * pGpu,KernelCE * pKCe)713 kceGetSysmemSupportedLceMask_GA100
714 (
715 OBJGPU *pGpu,
716 KernelCE *pKCe
717 )
718 {
719 return (NV_CE_SYS_ALLOWED_LCE_MASK & NV_CE_MAX_LCE_MASK);
720 }
721
722 /**
723 * @brief This function assigns PCE-LCE mappings for NVLink peers
724 * Based on HSHUBs that the links associated with a peer connect to,
725 * algorithm will attempt to assign a PCE from associated HSHUB taking into
726 * account striding as well.
727 *
728 * @param[in] pGpu OBJGPU pointer
729 * @param[in] pKCe KernelCE pointer
730 * @param[in] pceAvailableMaskPerHshub Pointer to CEs available per HSHUB
731 * @param[out] pLocalPceLceMap Pointer to PCE-LCE array
732 * @param[out] pLocalExposeCeMask Pointer to LCE Mask
733 *
734 * Returns NV_OK if successful in assigning PCEs and LCEs for each of the NVLink peers
735 */
736 NV_STATUS
kceMapPceLceForNvlinkPeers_GA100(OBJGPU * pGpu,KernelCE * pKCe,NvU32 * pceAvailableMaskPerHshub,NvU32 * pLocalPceLceMap,NvU32 * pLocalExposeCeMask)737 kceMapPceLceForNvlinkPeers_GA100
738 (
739 OBJGPU *pGpu,
740 KernelCE *pKCe,
741 NvU32 *pceAvailableMaskPerHshub,
742 NvU32 *pLocalPceLceMap,
743 NvU32 *pLocalExposeCeMask
744 )
745 {
746 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu);
747 OBJSYS *pSys = SYS_GET_INSTANCE();
748 NV_STATUS status = NV_OK;
749 NvU32 lceMask = 0;
750 NvU32 pceMask = 0;
751 NvU32 peerLinkMask = 0;
752 NvBool bFirstIter = NV_FALSE;
753 NvBool bPeerAssigned = NV_FALSE;
754 NvU32 peerAvailableLceMask = NV_CE_LCE_MASK_INIT;
755 OBJGPU *pRemoteGpu;
756 NvU32 numPcePerLink;
757 NvU32 lceIndex, pceIndex;
758 NvU32 linkId, gpuMask, gpuInstance = 0, i;
759 NvU8 hshubId, prevHshubId;
760 NV2080_CTRL_INTERNAL_HSHUB_GET_HSHUB_ID_FOR_LINKS_PARAMS *pParams = NULL;
761
762 if (pKernelNvlink == NULL)
763 {
764 return NV_WARN_NOTHING_TO_DO;
765 }
766
767 pParams = portMemAllocNonPaged(sizeof(*pParams));
768
769 NV_ASSERT_OR_RETURN(pParams != NULL, NV_ERR_NO_MEMORY);
770
771 peerAvailableLceMask = kceGetNvlinkPeerSupportedLceMask_HAL(pGpu, pKCe, peerAvailableLceMask);
772
773 if (knvlinkIsGpuConnectedToNvswitch(pGpu, pKernelNvlink))
774 {
775 //
776 // On NVSwitch systems, we only create 1 aperture for all p2p connections.
777 // For PCE2LCE mapping, we should only assign 1 LCE for this connection.
778 //
779 // Since we mark the loopback connections in peerLinkMasks with the appropriate
780 // links (see _nvlinkUpdateSwitchLinkMasks), we can use that to calculate
781 // the PCE2LCE config.
782 //
783 gpuMask = NVBIT32(pGpu->gpuInstance);
784 }
785 else
786 {
787 // On direct connected systems, we'll loop over each GPU in the system
788 // and assign a peer LCE for each connection
789 (void)gpumgrGetGpuAttachInfo(NULL, &gpuMask);
790 }
791 while ((pRemoteGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
792 {
793 NvU32 numLinksToPeer = knvlinkGetNumLinksToPeer(pGpu, pKernelNvlink,
794 pRemoteGpu);
795 if (numLinksToPeer == 0)
796 {
797 continue;
798 }
799
800 pceMask = 0;
801 lceMask = 0;
802
803 if (peerAvailableLceMask == 0)
804 {
805 //
806 // peerAvailableLceMask is initialized to even async LCEs at the
807 // top of the function.
808 // As a result, if at any point in the loop, this mask == 0,
809 // it implies we have used up all even async LCEs and should move to
810 // using odd async LCEs.
811 //
812 peerAvailableLceMask = kceGetNvlinkPeerSupportedLceMask_HAL(pGpu, pKCe, peerAvailableLceMask);
813 }
814 // Each peer gets 1 LCE
815 lceIndex = CE_GET_LOWEST_AVAILABLE_IDX(peerAvailableLceMask);
816 lceMask |= NVBIT32(lceIndex);
817
818 // Clear out the chosen LCE
819 peerAvailableLceMask &= (~(NVBIT32(lceIndex)));
820
821 peerLinkMask = knvlinkGetLinkMaskToPeer(pGpu, pKernelNvlink, pRemoteGpu);
822
823 if (peerLinkMask == 0)
824 {
825 NV_PRINTF(LEVEL_INFO, "GPU%d has nvlink disabled. Skip programming\n", pRemoteGpu->gpuInstance);
826 continue;
827 }
828
829 numPcePerLink = NV_CE_MIN_PCE_PER_PEER_LINK;
830 prevHshubId = 0xFF;
831
832 portMemSet(pParams, 0, sizeof(*pParams));
833 pParams->linkMask = peerLinkMask;
834
835 status = knvlinkExecGspRmRpc(pGpu, pKernelNvlink,
836 NV2080_CTRL_CMD_INTERNAL_HSHUB_GET_HSHUB_ID_FOR_LINKS,
837 pParams, sizeof(*pParams));
838 NV_ASSERT_OK_OR_RETURN(status);
839
840 FOR_EACH_INDEX_IN_MASK(32, linkId, peerLinkMask)
841 {
842 hshubId = pParams->hshubIds[linkId];
843 if (hshubId != prevHshubId)
844 {
845 pceIndex = CE_GET_LOWEST_AVAILABLE_IDX(pceAvailableMaskPerHshub[hshubId]);
846 bFirstIter = NV_TRUE;
847 }
848 for (i = 0; i < numPcePerLink; i++)
849 {
850 _ceGetAlgorithmPceIndex(pGpu, pKCe, pceAvailableMaskPerHshub, &pceIndex, &bFirstIter, &hshubId);
851 pceMask |= NVBIT32(pceIndex);
852 // Clear out the assigned PCE
853 pceAvailableMaskPerHshub[hshubId] &= (~(NVBIT32(pceIndex)));
854 prevHshubId = hshubId;
855 }
856 }
857 FOR_EACH_INDEX_IN_MASK_END;
858
859 // Now, assign the PCE-LCE association for the current peer
860 if (pceMask != 0)
861 {
862 // We just need atleast one peer to set this to TRUE
863 bPeerAssigned = NV_TRUE;
864
865 FOR_EACH_INDEX_IN_MASK(32, pceIndex, pceMask)
866 {
867 pLocalPceLceMap[pceIndex] = lceIndex;
868 NV_PRINTF(LEVEL_INFO, "GPU%d <-> GPU%d PCE Index: %d LCE Index: %d\n",
869 pGpu->gpuInstance, pRemoteGpu->gpuInstance, pceIndex, lceIndex);
870 }
871 FOR_EACH_INDEX_IN_MASK_END;
872
873 // Store lceMask in the exposeCeMask before moving on
874 *pLocalExposeCeMask |= lceMask;
875 }
876
877 //
878 // Bug 200659256 - Looping over GPUs rather than peers (CL 28776130)
879 // does not handle multi-GPUs/Peer as is the case on switch systems.
880 // We must only take this loop once on switch systems to account for this.
881 // If we need to support multiple peer connections with switch systems
882 // in the future, this code must be revisited
883 //
884 if (pSys->getProperty(pSys, PDB_PROP_SYS_FABRIC_IS_EXTERNALLY_MANAGED))
885 {
886 break;
887 }
888 }
889
890 if (bPeerAssigned == NV_FALSE)
891 {
892 status = NV_WARN_NOTHING_TO_DO;
893 }
894
895 portMemFree(pParams);
896 return status;
897 }
898
899 /**
900 * @brief This function assigns 2 PCEs to an additional LCE over the GRCEs and async LCEs
901 * for sysmem, since some clients rely on LCE 4 also being turned on when there
902 * are no NVLink peers
903 *
904 * @param[in] pGpu OBJGPU pointer
905 * @param[in] pKCe KernelCE pointer
906 * @param[in] pceAvailableMaskPerHshub Pointer to CEs available per HSHUB
907 * @param[out] pLocalPceLceMap Pointer to PCE-LCE array
908 * @param[out] pLocalExposeCeMask Pointer to LCE Mask
909 *
910 * Returns NV_OK if successful in assigning PCEs to a default async LCE (>= 4)
911 */
912 NV_STATUS
kceMapAsyncLceDefault_GA100(OBJGPU * pGpu,KernelCE * pKCe,NvU32 * pceAvailableMaskPerHshub,NvU32 * pLocalPceLceMap,NvU32 * pLocalExposeCeMask,NvU32 numDefaultPces)913 kceMapAsyncLceDefault_GA100
914 (
915 OBJGPU *pGpu,
916 KernelCE *pKCe,
917 NvU32 *pceAvailableMaskPerHshub,
918 NvU32 *pLocalPceLceMap,
919 NvU32 *pLocalExposeCeMask,
920 NvU32 numDefaultPces
921 )
922 {
923 NvU32 peerAvailableLceMask = NV_CE_LCE_MASK_INIT;
924 NvU32 lceMask = 0;
925 NvU32 pceMask = 0;
926 NvU32 lceIndex, pceIndex, hshubId, i;
927
928 peerAvailableLceMask = kceGetNvlinkPeerSupportedLceMask_HAL(pGpu, pKCe, peerAvailableLceMask);
929
930 // Pick from the 1st HSHUB - HSHUB 0 will not be floorswept
931 hshubId = 0;
932
933 //
934 // If no peers were found, then no async LCEs (>= 4) will be turned on.
935 // However, some clients rely on LCE 4 being present even without any
936 // NVLink peers being found. So, turn on the 1st available async LCE (>= 4)
937 // Reference bug 3042556
938 //
939 lceIndex = CE_GET_LOWEST_AVAILABLE_IDX(peerAvailableLceMask);
940 lceMask |= NVBIT32(lceIndex);
941 // Clear out the chosen LCE
942 peerAvailableLceMask &= (~(NVBIT32(lceIndex)));
943
944 // Assign PCEs to this LCE based on input request
945 for (i = 0; i < numDefaultPces; i++)
946 {
947 pceIndex = CE_GET_LOWEST_AVAILABLE_IDX(pceAvailableMaskPerHshub[hshubId]);
948 pceMask |= NVBIT32(pceIndex);
949 pceAvailableMaskPerHshub[hshubId] &= (~(NVBIT32(pceIndex)));
950 }
951
952 FOR_EACH_INDEX_IN_MASK(32, pceIndex, pceMask)
953 {
954 pLocalPceLceMap[pceIndex] = lceIndex;
955 NV_PRINTF(LEVEL_INFO, "GPU%d <-> GPU%d PCE Index: %d LCE Index: %d\n",
956 pGpu->gpuInstance, pGpu->gpuInstance, pceIndex, lceIndex);
957 }
958 FOR_EACH_INDEX_IN_MASK_END;
959
960 // Store lceMask in the exposeCeMask before moving on
961 *pLocalExposeCeMask |= lceMask;
962
963 return NV_OK;
964 }
965
966 /**
967 * @brief Returns mask of LCEs that can be assigned to NVLink peers
968 * where the index of corresponding set bit indicates the LCE index
969 *
970 * @param[in] pGpu OBJGPU pointer
971 * @param[in] pKCe KernelCE pointer
972 *
973 * Returns the mask of LCEs valid for NVLink peers
974 */
975 NvU32
kceGetNvlinkPeerSupportedLceMask_GA100(OBJGPU * pGpu,KernelCE * pKCe,NvU32 peerAvailableLceMask)976 kceGetNvlinkPeerSupportedLceMask_GA100
977 (
978 OBJGPU *pGpu,
979 KernelCE *pKCe,
980 NvU32 peerAvailableLceMask
981 )
982 {
983 //
984 // Start with assigning even async LCEs first as they are sized to accommodate
985 // more no. of PCEs versus odd async LCEs
986 // Hence, if caller is using this call to get 1st set of async LCEs for NVLink
987 // peers, then caller should initialize peerAvailableLceMask to NV_CE_LCE_MASK_INIT.
988 // Else we will run out of async LCEs since we will directly assign the odd async LCEs
989 // and there's no wraparound or detection mechanism is place.
990 //
991 if (peerAvailableLceMask == NV_CE_LCE_MASK_INIT)
992 {
993 return (NV_CE_EVEN_ASYNC_LCE_MASK & NV_CE_MAX_LCE_MASK);
994 }
995
996 return NV_CE_ODD_ASYNC_LCE_MASK & NV_CE_MAX_LCE_MASK;
997 }
998
999 /**
1000 * @brief Returns mask of LCEs that can be assigned for GRCEs
1001 * where the index of corresponding set bit indicates the LCE index
1002 *
1003 * @param[in] pGpu OBJGPU pointer
1004 * @param[in] pKCe KernelCE pointer
1005 *
1006 * Returns the mask of LCEs valid for GRCEs
1007 */
1008 NvU32
kceGetGrceSupportedLceMask_GA100(OBJGPU * pGpu,KernelCE * pKCe)1009 kceGetGrceSupportedLceMask_GA100
1010 (
1011 OBJGPU *pGpu,
1012 KernelCE *pKCe
1013 )
1014 {
1015 return (NV_CE_GRCE_ALLOWED_LCE_MASK & NV_CE_MAX_LCE_MASK);
1016 }
1017
1018 /**
1019 * @brief This function checks for root port gen speed or GPU
1020 * gen speed to determine if we should apply genX+ mapping
1021 *
1022 * @param[in] pGpu OBJGPU pointer
1023 * @param[in] pKCe KernelCE pointer
1024 * @param[in] checkGen genX for query
1025 */
1026 NvBool
kceIsGenXorHigherSupported_GA100(OBJGPU * pGpu,KernelCE * pKCe,NvU32 checkGen)1027 kceIsGenXorHigherSupported_GA100
1028 (
1029 OBJGPU *pGpu,
1030 KernelCE *pKCe,
1031 NvU32 checkGen
1032 )
1033 {
1034 OBJSYS *pSys = SYS_GET_INSTANCE();
1035 KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu);
1036 OBJCL *pCl = SYS_GET_CL(pSys);
1037 NvU8 genSpeed = 0;
1038 NV_STATUS status = NV_OK;
1039 NvBool bIsGenXorHigher = NV_FALSE;
1040 NvU32 regVal;
1041
1042 if (IS_PASSTHRU(pGpu) && (pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_PCIE_GEN4_CAPABLE)))
1043 {
1044 //
1045 // On passthrough the root port is commonly not accessible or fake. To
1046 // handle this case, we support the hypervisor explicitly communicating
1047 // the speed to us through emulated config space. See
1048 // bug 2927491 for more details.
1049 //
1050 NvU32 passthroughEmulatedConfig = osPciReadDword(osPciInitHandle(gpuGetDomain(pGpu),
1051 gpuGetBus(pGpu),
1052 gpuGetDevice(pGpu),
1053 0, NULL, NULL),
1054 NV_XVE_PASSTHROUGH_EMULATED_CONFIG);
1055 NvU32 rootPortSpeed = DRF_VAL(_XVE, _PASSTHROUGH_EMULATED_CONFIG, _ROOT_PORT_SPEED, passthroughEmulatedConfig);
1056
1057 // 0 means the config is not being emulated and we assume gen4
1058 bIsGenXorHigher = ((rootPortSpeed == 0 && checkGen <= 4) || rootPortSpeed >= checkGen);
1059
1060 if (rootPortSpeed != 0)
1061 {
1062 NV_PRINTF(LEVEL_INFO, "Root port speed from emulated config space = %d\n", rootPortSpeed);
1063 }
1064 }
1065 else
1066 {
1067 status = clPcieGetRootGenSpeed(pGpu, pCl, &genSpeed);
1068 if (status != NV_OK)
1069 {
1070 NV_PRINTF(LEVEL_ERROR, "Could not get root gen speed - check for GPU gen speed!\n");
1071 // Check for GPU gen speed
1072 regVal = GPU_REG_RD32(pGpu, DEVICE_BASE(NV_PCFG) + NV_XVE_LINK_CONTROL_STATUS);
1073 genSpeed = DRF_VAL(_XVE, _LINK_CONTROL_STATUS, _LINK_SPEED, regVal);
1074 }
1075 NV_PRINTF(LEVEL_INFO, "Gen Speed = %d\n", genSpeed);
1076
1077 if ((genSpeed >= checkGen))
1078 {
1079 bIsGenXorHigher = NV_TRUE;
1080 }
1081 }
1082 return bIsGenXorHigher;
1083 }
1084
1085 /**
1086 * @brief This function applies the gen4+ mapping i.e. switches the LCE passed in
1087 * to use an HSHUB PCE (if available) instead of an FBHUB PCE.
1088 *
1089 * @param[in] pGpu OBJGPU pointer
1090 * @param[in] pKCe KernelCE pointer
1091 * @param[in] pLocalPceLceMap Pointer to PCE-LCE array
1092 * @param[in] pceAvailableMaskPerHshub Pointer to CEs available per HSHUB
1093 * @param[in] lceIndex LCE index for which mapping is being determined
1094 * @param[in] currPceIndex PCE index for which mapping is being determined
1095 */
1096 void
kceApplyGen4orHigherMapping_GA100(OBJGPU * pGpu,KernelCE * pKCe,NvU32 * pLocalPceLceMap,NvU32 * pceAvailableMaskPerHshub,NvU32 lceIndex,NvU32 currPceIndex)1097 kceApplyGen4orHigherMapping_GA100
1098 (
1099 OBJGPU *pGpu,
1100 KernelCE *pKCe,
1101 NvU32 *pLocalPceLceMap,
1102 NvU32 *pceAvailableMaskPerHshub,
1103 NvU32 lceIndex,
1104 NvU32 currPceIndex
1105 )
1106 {
1107 NvBool hsPceAssigned = NV_FALSE;
1108 NvU32 fbPceIndex, hshubId;
1109 NV_STATUS status;
1110
1111 if (NVBIT32(lceIndex) & NV_CE_SYS_LCE_ALLOWED_HSPCE_CONFIG)
1112 {
1113 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
1114 NV2080_CTRL_INTERNAL_HSHUB_GET_NUM_UNITS_PARAMS params;
1115 portMemSet(¶ms, 0, sizeof(params));
1116
1117 NV_ASSERT_OK_OR_ELSE(status,
1118 pRmApi->Control(pRmApi,
1119 pGpu->hInternalClient,
1120 pGpu->hInternalSubdevice,
1121 NV2080_CTRL_CMD_INTERNAL_HSHUB_GET_NUM_UNITS,
1122 ¶ms,
1123 sizeof(params)),
1124 params.numHshubs = 0);
1125
1126 // GA100: If LCE 3, then move this to an HSHUB PCE, if available
1127 fbPceIndex = currPceIndex;
1128 for (hshubId = 0; hshubId < params.numHshubs; hshubId++)
1129 {
1130 if (pceAvailableMaskPerHshub[hshubId] != 0)
1131 {
1132 // We still have HS PCEs available
1133 hsPceAssigned = NV_TRUE;
1134 currPceIndex = CE_GET_LOWEST_AVAILABLE_IDX(pceAvailableMaskPerHshub[hshubId]);
1135 pLocalPceLceMap[currPceIndex] = lceIndex;
1136 // Clear out the assigned PCE
1137 pceAvailableMaskPerHshub[hshubId] &= (~(NVBIT32(currPceIndex)));
1138 break;
1139 }
1140 }
1141 }
1142 if (hsPceAssigned == NV_TRUE)
1143 {
1144 pLocalPceLceMap[fbPceIndex] = NV_CE_PCE2LCE_CONFIG_PCE_ASSIGNED_LCE_NONE;
1145 }
1146 }
1147