1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include "gpu/gpu.h" 25 #include "gpu/ce/kernel_ce.h" 26 #include "gpu/nvlink/kernel_nvlink.h" 27 #include "gpu/ce/kernel_ce_private.h" 28 #include "gpu/bif/kernel_bif.h" 29 #include "platform/chipset/chipset.h" 30 31 #include "published/hopper/gh100/dev_ce.h" 32 #include "published/hopper/gh100/dev_xtl_ep_pcfg_gpu.h" 33 34 // Defines for PCE-LCE mapping algorithm 35 #define NV_CE_MAX_HSHUBS 5 36 #define NV_CE_LCE_MASK_INIT 0xFFFFFFFF 37 #define NV_CE_GRCE_ALLOWED_LCE_MASK 0x03 38 #define NV_CE_MAX_GRCE 2 39 #define NV_CE_EVEN_ASYNC_LCE_MASK 0x55555550 40 #define NV_CE_ODD_ASYNC_LCE_MASK 0xAAAAAAA0 41 #define NV_CE_MAX_LCE_MASK 0x3FF 42 #define NV_CE_PCE_PER_HSHUB 4 43 #define NV_CE_NUM_FBPCE 4 44 #define NV_CE_NUM_PCES_NO_LINK_CASE 12 45 #define NV_CE_MAX_PCE_PER_GRCE 2 46 47 48 49 /*! 50 * @brief Returns the size of the PCE2LCE register array 51 * 52 * 53 * @param[in] pGpu OBJGPU pointer 54 * @param[in] pCe OBJCE pointer 55 * 56 * @return NV_CE_PCE2LCE_CONFIG__SIZE_1 57 * 58 */ 59 NvU32 60 kceGetPce2lceConfigSize1_GH100 61 ( 62 KernelCE *pKCe 63 ) 64 { 65 return NV_CE_PCE2LCE_CONFIG__SIZE_1; 66 } 67 68 /** 69 * @brief This function takes in a link mask and returns the minimum number 70 * of PCE connections required. This is decided based on a round up approach 71 * where each PCE can handle 1.5 links. 72 */ 73 NvU32 74 kceGetNumPceRequired 75 ( 76 NvU32 numLinks 77 ) 78 { 79 switch(numLinks) 80 { 81 case 6: 82 return 4; 83 case 5: 84 case 4: 85 return 3; 86 case 3: 87 return 2; 88 case 2: 89 case 1: 90 default: 91 return 1; 92 } 93 } 94 95 /** 96 * @brief This function returns the pceIndex for a particular link ID 97 * Must always be called with the hshub ID for the calling link ID 98 * 99 * @param[in] pGpu OBJGPU pointer 100 * @param[in] pKCe KernelCE pointer 101 * @param[in] pceAvailableMaskPerHshub Pointer to CEs available per HSHUB 102 * @param[out] pceIndex Pointer to caller pceIndex 103 * @param[out] pHshubId Pointer to caller HSHUB ID 104 */ 105 static void 106 _ceGetAlgorithmPceIndex 107 ( 108 OBJGPU *pGpu, 109 KernelCE *pKCe, 110 NvU32 *pceAvailableMaskPerHshub, 111 NvU32 *pceIndex, 112 NvU8 *pHshubId 113 ) 114 { 115 NvU8 pHshubIdRequested; 116 NvU32 i; 117 118 if ((pceIndex != NULL) && *pceIndex >= kceGetPce2lceConfigSize1_HAL(pKCe)) 119 { 120 NV_PRINTF(LEVEL_ERROR, "Invalid PCE request. pceIndex = %d pceCnt = %d\n", *pceIndex, kceGetPce2lceConfigSize1_HAL(pKCe)); 121 return; 122 } 123 124 if (!(NVBIT32(*pceIndex) & pceAvailableMaskPerHshub[*pHshubId])) 125 { 126 // 127 // 1. We couldn't find an applicable strided PCE in given HSHUB 128 // So, we'll assign the next consecutive PCE on the same HSHUB 129 // 130 *pceIndex = CE_GET_LOWEST_AVAILABLE_IDX(pceAvailableMaskPerHshub[*pHshubId]); 131 if (!(NVBIT32(*pceIndex) & pceAvailableMaskPerHshub[*pHshubId])) 132 { 133 // 2. If this is not a valid PCE on given HSHUB, assign PCE from alternative HSHUB 134 pHshubIdRequested = *pHshubId; 135 for (i = pHshubIdRequested + 1; i != pHshubIdRequested; i++) { 136 if (i > 4) { 137 i = 1; 138 continue; 139 } 140 141 *pceIndex = CE_GET_LOWEST_AVAILABLE_IDX(pceAvailableMaskPerHshub[i]); 142 if (NVBIT32(*pceIndex) & pceAvailableMaskPerHshub[i]) { 143 break; 144 } 145 } 146 147 if (i == pHshubIdRequested) 148 { 149 // If we've reached this point, then we have no more available PCEs to assign 150 NV_PRINTF(LEVEL_ERROR, "No more available PCEs to assign!\n"); 151 NV_ASSERT(0); 152 } 153 } 154 } 155 return; 156 } 157 158 /** 159 * @brief This function assigns LCE 2 and 3 mappings for C2C cases. 160 * 161 * @param[in] pGpu OBJGPU pointer 162 * @param[in] pKCe KernelCE pointer 163 * @param[in] pceAvailableMaskPerHshub Pointer to CEs available per HSHUB 164 * @param[out] pLocalPceLceMap Pointer to PCE-LCE array 165 * @param[out] pLocalExposeCeMask Pointer to LCE Mask 166 */ 167 NV_STATUS 168 kceMapPceLceForC2C_GH100 169 ( 170 OBJGPU *pGpu, 171 KernelCE *pKCe, 172 NvU32 *pceAvailableMaskPerHshub, 173 NvU32 *pLocalPceLceMap, 174 NvU32 *pLocalExposeCeMask 175 ) 176 { 177 NV_STATUS status = NV_OK; 178 KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu); 179 NvU32 pceIndex, i, hshubId, lceMask, lceIndex; 180 NvU32 numNvLinkPeers = 0; 181 NvU32 selectPcePerHshub = 2; 182 NvBool c2cEnabled = pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_IS_C2C_LINK_UP); 183 184 numNvLinkPeers = pKCe->nvlinkNumPeers; 185 if (gpuIsCCFeatureEnabled(pGpu) || (c2cEnabled && numNvLinkPeers == 0 && IS_MIG_IN_USE(pGpu))) 186 { 187 lceMask = NVBIT32(2) | NVBIT32(3); 188 *pLocalExposeCeMask |= lceMask; 189 190 lceIndex = CE_GET_LOWEST_AVAILABLE_IDX(lceMask); 191 pceIndex = NVBIT32(0); 192 pLocalPceLceMap[pceIndex] = lceIndex; 193 lceMask &= (~(NVBIT32(lceIndex))); 194 195 lceIndex = CE_GET_LOWEST_AVAILABLE_IDX(lceMask); 196 pceIndex = NVBIT32(1); 197 pLocalPceLceMap[pceIndex] = lceIndex; 198 } 199 else if (c2cEnabled && numNvLinkPeers == 0) 200 { 201 lceMask = NVBIT32(2); 202 *pLocalExposeCeMask |= lceMask; 203 lceIndex = CE_GET_LOWEST_AVAILABLE_IDX(lceMask); 204 205 for (hshubId = 2; hshubId < NV_CE_MAX_HSHUBS; hshubId++) 206 { 207 for (i = 0; i < selectPcePerHshub; i++) 208 { 209 pceIndex = CE_GET_LOWEST_AVAILABLE_IDX(pceAvailableMaskPerHshub[hshubId]); 210 if (pceIndex < kceGetPce2lceConfigSize1_HAL(pKCe)) 211 { 212 pceAvailableMaskPerHshub[hshubId] &= (~(NVBIT32(pceIndex))); 213 pLocalPceLceMap[pceIndex] = lceIndex; 214 } 215 } 216 } 217 218 lceMask = NVBIT32(4); 219 *pLocalExposeCeMask |= lceMask; 220 lceIndex = CE_GET_LOWEST_AVAILABLE_IDX(lceMask); 221 222 for (hshubId = 2; hshubId < NV_CE_MAX_HSHUBS; hshubId++) 223 { 224 for (i = 0; i < selectPcePerHshub; i++) 225 { 226 pceIndex = CE_GET_LOWEST_AVAILABLE_IDX(pceAvailableMaskPerHshub[hshubId]); 227 if (pceIndex < kceGetPce2lceConfigSize1_HAL(pKCe)) 228 { 229 pceAvailableMaskPerHshub[hshubId] &= (~(NVBIT32(pceIndex))); 230 pLocalPceLceMap[pceIndex] = lceIndex; 231 } 232 } 233 } 234 } 235 else 236 { 237 status = NV_WARN_NOTHING_TO_DO; 238 } 239 240 return status; 241 } 242 243 /** 244 * @brief This function checks for root port gen speed or GPU 245 * gen speed to determine if we should apply genX+ mapping 246 * or genX- mapping 247 * 248 * @param[in] pGpu OBJGPU pointer 249 * @param[in] pKCe KernelCE pointer 250 * @param[in] checkGen gen X for query 251 */ 252 NvBool 253 kceIsGenXorHigherSupported_GH100 254 ( 255 OBJGPU *pGpu, 256 KernelCE *pKCe, 257 NvU32 checkGen 258 ) 259 { 260 OBJSYS *pSys = SYS_GET_INSTANCE(); 261 OBJCL *pCl = SYS_GET_CL(pSys); 262 NvU8 genSpeed = 0; 263 NvU32 busSpeed = 0; 264 NV_STATUS status = NV_OK; 265 NvBool bIsGenXorHigher = NV_FALSE; 266 267 status = clPcieGetRootGenSpeed(pGpu, pCl, &genSpeed); 268 if (status != NV_OK) 269 { 270 NV_PRINTF(LEVEL_ERROR, "Could not get root gen speed - check for GPU gen speed!\n"); 271 // Check for GPU gen speed 272 if (GPU_BUS_CFG_CYCLE_RD32(pGpu, NV_EP_PCFG_GPU_LINK_CONTROL_STATUS, &busSpeed) != NV_OK) 273 { 274 NV_PRINTF(LEVEL_ERROR, "Unable to read NV_EP_PCFG_GPU_LINK_CONTROL_STATUS from config space.\n"); 275 return bIsGenXorHigher; 276 } 277 genSpeed = GPU_DRF_VAL(_EP_PCFG_GPU, _LINK_CONTROL_STATUS, _CURRENT_LINK_SPEED, busSpeed); 278 } 279 NV_PRINTF(LEVEL_INFO, "Gen Speed = %d\n", genSpeed); 280 281 if ((genSpeed >= checkGen)) 282 { 283 bIsGenXorHigher = NV_TRUE; 284 } 285 286 return bIsGenXorHigher; 287 } 288 289 /** 290 * @brief This function assigns PCE-LCE mappings for GRCE LCEs 0 and 1. 291 * This function additionally takes care of mappings for LCE 2 and 3 292 * in the default case. 293 * 294 * @param[in] pGpu OBJGPU pointer 295 * @param[in] pKCe KernelCE pointer 296 * @param[in] pceAvailableMaskPerHshub Pointer to CEs available per HSHUB 297 * @param[out] pLocalPceLceMap Pointer to PCE-LCE array 298 * @param[out] pLocalExposeCeMask Pointer to LCE Mask 299 */ 300 void 301 kceMapPceLceForGRCE_GH100 302 ( 303 OBJGPU *pGpu, 304 KernelCE *pKCe, 305 NvU32 *pceAvailableMaskPerHshub, 306 NvU32 *pLocalPceLceMap, 307 NvU32 *pLocalExposeCeMask, 308 NvU32 *pLocalGrceMap, 309 NvU32 fbPceMask 310 ) 311 { 312 KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu); 313 NvU32 grceIdx, pceIndex, i; 314 NvU32 lceIndex = 0; 315 NvU32 lceMask = 0; 316 NvU32 numNvLinkPeers = 0; 317 NvU32 grceMappings[NV_CE_NUM_FBPCE] = {12, 14, 13, 15}; 318 NvBool gen5OrHigher = kceIsGenXorHigherSupported_HAL(pGpu, pKCe, 5); 319 NvBool c2cEnabled = pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_IS_C2C_LINK_UP); 320 321 numNvLinkPeers = pKCe->nvlinkNumPeers; 322 323 if (gpuIsCCFeatureEnabled(pGpu) || (c2cEnabled && numNvLinkPeers == 0)) 324 { 325 lceMask = NVBIT32(0) | NVBIT32(1); 326 *pLocalExposeCeMask |= lceMask; 327 328 for (grceIdx = 0; grceIdx < NV_CE_MAX_GRCE; grceIdx++) 329 { 330 for (i = 0; i < NV_CE_MAX_PCE_PER_GRCE; i++) 331 { 332 pceIndex = grceMappings[grceIdx * 2 + i]; 333 334 // 335 // floorswept PCE or 336 // PCIe <= Gen4 experience high latency and requires a 337 // different mapping for LCE2 and LCE3 compared to Gen5. 338 // In PCIe <= Gen4 cases, only link 1 PCE to LCE by 339 // skipping every other PCE in the grceMappings array. 340 // 341 if (pceIndex == 0 || (!gen5OrHigher && (i % 2 == 1))) 342 continue; 343 344 lceIndex = CE_GET_LOWEST_AVAILABLE_IDX(lceMask); 345 pLocalPceLceMap[pceIndex] = lceIndex; 346 } 347 348 lceMask &= (~(NVBIT32(lceIndex))); 349 } 350 } 351 else 352 { 353 // Default case which will result in sharing LCE 2 and 3 with LCE 0 and 1 354 lceMask = NVBIT32(2) | NVBIT32(3); 355 *pLocalExposeCeMask |= lceMask; 356 for (grceIdx = 0; grceIdx < NV_CE_MAX_GRCE; grceIdx++) 357 { 358 for (i = 0; i < NV_CE_MAX_PCE_PER_GRCE; i++) 359 { 360 pceIndex = grceMappings[grceIdx * 2 + i]; 361 362 // floorswept PCE or account for PCIe latency in Gen <= 4 363 if (pceIndex == 0 || (!gen5OrHigher && (i % 2 == 1))) 364 continue; 365 366 lceIndex = CE_GET_LOWEST_AVAILABLE_IDX(lceMask); 367 pLocalPceLceMap[pceIndex] = lceIndex; 368 } 369 370 // update lceMask now that all PCEs are assigned to this LCE 371 lceMask &= (~(NVBIT32(lceIndex))); 372 } 373 374 // GRCE Cases 375 lceMask = kceGetGrceSupportedLceMask_HAL(pGpu, pKCe); 376 *pLocalExposeCeMask |= lceMask; 377 378 for (grceIdx = 0; grceIdx < NV_CE_MAX_GRCE; grceIdx++) 379 { 380 for (i = 0; i < NV_CE_MAX_PCE_PER_GRCE; i++) 381 { 382 pceIndex = grceMappings[grceIdx * 2 + i]; 383 fbPceMask &= (~(NVBIT32(pceIndex))); 384 385 // floorswept PCE 386 if (pceIndex == 0 || (!gen5OrHigher && (i % 2 == 1))) 387 continue; 388 389 // Sharing use case 390 if ((NVBIT32(pLocalPceLceMap[pceIndex])) & *pLocalExposeCeMask) 391 { 392 // GRCE is shared - set the status and shared LCE # in register field 393 lceIndex = pLocalPceLceMap[pceIndex]; 394 pLocalGrceMap[grceIdx] = DRF_NUM(_CE, _GRCE_CONFIG, _SHARED, 1) | 395 DRF_NUM(_CE, _GRCE_CONFIG, _SHARED_LCE, lceIndex); 396 } 397 else 398 { 399 // GRCE got its own FBHUB PCE 400 // Store the LCE in the associated PCE for GRCE 401 lceIndex = CE_GET_LOWEST_AVAILABLE_IDX(lceMask); 402 pLocalPceLceMap[pceIndex] = lceIndex; 403 // Reflect non-sharing status in register field 404 pLocalGrceMap[grceIdx] = DRF_NUM(_CE, _GRCE_CONFIG, _SHARED, 0) | 405 DRF_DEF(_CE, _GRCE_CONFIG, _SHARED_LCE, _NONE); 406 } 407 } 408 409 // update lceMask now that all PCEs are assigned to this LCE 410 lceMask &= (~(NVBIT32(lceIndex))); 411 } 412 } 413 } 414 415 /** 416 * @brief This function assigns PCE-LCE mappings for NVLink peers 417 * Based on HSHUBs that the links associated with a peer connect to, 418 * algorithm will attempt to assign a PCE from associated HSHUB taking into 419 * account striding as well. 420 * 421 * @param[in] pGpu OBJGPU pointer 422 * @param[in] pKCe KernelCE pointer 423 * @param[in] pceAvailableMaskPerHshub Pointer to CEs available per HSHUB 424 * @param[out] pLocalPceLceMap Pointer to PCE-LCE array 425 * @param[out] pLocalExposeCeMask Pointer to LCE Mask 426 * 427 * Returns NV_OK if successful in assigning PCEs and LCEs for each of the NVLink peers 428 */ 429 NV_STATUS 430 kceMapPceLceForNvlinkPeers_GH100 431 ( 432 OBJGPU *pGpu, 433 KernelCE *pKCe, 434 NvU32 *pceAvailableMaskPerHshub, 435 NvU32 *pLocalPceLceMap, 436 NvU32 *pLocalExposeCeMask 437 ) 438 { 439 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu); 440 OBJSYS *pSys = SYS_GET_INSTANCE(); 441 NV_STATUS status = NV_OK; 442 NvU32 lceMask = 0; 443 NvU32 pceMask = 0; 444 NvU32 peerLinkMask = 0; 445 KernelCE *pKCeLce = NULL; 446 NvBool bPeerAssigned = NV_FALSE; 447 NvU32 peerAvailableLceMask = NV_CE_LCE_MASK_INIT; 448 OBJGPU *pRemoteGpu; 449 NvU32 numPcePerLink; 450 NvU32 lceIndex, pceIndex; 451 NvU8 hshubId = 0, i; 452 NvU32 linkId, gpuMask, gpuInstance = 0, j; 453 454 NV2080_CTRL_INTERNAL_HSHUB_GET_HSHUB_ID_FOR_LINKS_PARAMS params; 455 456 if (pKernelNvlink == NULL) 457 { 458 return NV_WARN_NOTHING_TO_DO; 459 } 460 461 peerAvailableLceMask = kceGetNvlinkPeerSupportedLceMask_HAL(pGpu, pKCe, peerAvailableLceMask); 462 pKCe->nvlinkNumPeers = 0; 463 464 if (knvlinkIsGpuConnectedToNvswitch(pGpu, pKernelNvlink)) 465 { 466 // 467 // On NVSwitch systems, we only create 1 aperture for all p2p connections. 468 // For PCE2LCE mapping, we should only assign 1 LCE for this connection. 469 // 470 // Since we mark the loopback connections in peerLinkMasks with the appropriate 471 // links (see _nvlinkUpdateSwitchLinkMasks), we can use that to calculate 472 // the PCE2LCE config. 473 // 474 gpuMask = NVBIT32(pGpu->gpuInstance); 475 } 476 else 477 { 478 // On direct connected systems, we'll loop over each GPU in the system 479 // and assign a peer LCE for each connection 480 (void)gpumgrGetGpuAttachInfo(NULL, &gpuMask); 481 } 482 483 while ((pRemoteGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL) 484 { 485 NvU32 numLinksToPeer = knvlinkGetNumLinksToPeer(pGpu, pKernelNvlink, 486 pRemoteGpu); 487 NvU32 maxLceCnt = NV_CE_MAX_LCE_MASK; 488 489 if (numLinksToPeer == 0) 490 { 491 continue; 492 } 493 494 pceMask = 0; 495 lceMask = 0; 496 497 if (peerAvailableLceMask == 0) 498 { 499 // 500 // peerAvailableLceMask is initialized to even async LCEs at the 501 // top of the function. 502 // As a result, if at any point in the loop, this mask == 0, 503 // it implies we have used up all even async LCEs and should move to 504 // using odd async LCEs. 505 // 506 peerAvailableLceMask = kceGetNvlinkPeerSupportedLceMask_HAL(pGpu, pKCe, peerAvailableLceMask); 507 } 508 509 // Each peer gets 1 LCE 510 lceIndex = CE_GET_LOWEST_AVAILABLE_IDX(peerAvailableLceMask); 511 HIGHESTBITIDX_32(maxLceCnt); 512 if (lceIndex < maxLceCnt) 513 { 514 lceMask |= NVBIT32(lceIndex); 515 // Clear out the chosen LCE 516 peerAvailableLceMask &= (~(NVBIT32(lceIndex))); 517 } 518 519 pKCe->nvlinkNumPeers++; 520 521 peerLinkMask = knvlinkGetLinkMaskToPeer(pGpu, pKernelNvlink, pRemoteGpu); 522 if (peerLinkMask == 0) 523 { 524 NV_PRINTF(LEVEL_INFO, "GPU%d has nvlink disabled. Skip programming\n", pRemoteGpu->gpuInstance); 525 continue; 526 } 527 528 portMemSet(¶ms, 0, sizeof(params)); 529 params.linkMask = peerLinkMask; 530 531 status = knvlinkExecGspRmRpc(pGpu, pKernelNvlink, 532 NV2080_CTRL_CMD_INTERNAL_HSHUB_GET_HSHUB_ID_FOR_LINKS, 533 (void *)¶ms, sizeof(params)); 534 NV_ASSERT_OK_OR_RETURN(status); 535 536 // Iterate through links by HSHUB 537 NvU32 linksPerHshub[NV_CE_MAX_HSHUBS] = {0}; 538 539 FOR_EACH_INDEX_IN_MASK(32, linkId, peerLinkMask) 540 { 541 hshubId = params.hshubIds[linkId]; 542 // Update link count for this hshub 543 linksPerHshub[hshubId]++; 544 } 545 FOR_EACH_INDEX_IN_MASK_END; 546 547 for (i = 0; i < NV_CE_MAX_HSHUBS; i++) 548 { 549 if (linksPerHshub[i] == 0) 550 continue; 551 552 pceIndex = CE_GET_LOWEST_AVAILABLE_IDX(pceAvailableMaskPerHshub[i]); 553 numPcePerLink = kceGetNumPceRequired(linksPerHshub[i]); 554 555 for (j = 0; j < numPcePerLink; j++) 556 { 557 _ceGetAlgorithmPceIndex(pGpu, pKCe, pceAvailableMaskPerHshub, &pceIndex, &i); 558 pceMask |= NVBIT32(pceIndex); 559 // Clear out the assigned PCE 560 pceAvailableMaskPerHshub[i] &= (~(NVBIT32(pceIndex))); 561 } 562 563 } 564 565 // Now, assign the PCE-LCE association for the current peer 566 if (pceMask != 0) 567 { 568 // We just need at least one peer to set this to TRUE 569 bPeerAssigned = NV_TRUE; 570 571 FOR_EACH_INDEX_IN_MASK(32, pceIndex, pceMask) 572 { 573 pLocalPceLceMap[pceIndex] = lceIndex; 574 NV_PRINTF(LEVEL_INFO, "GPU%d <-> GPU%d PCE Index: %d LCE Index: %d\n", 575 pGpu->gpuInstance, pRemoteGpu->gpuInstance, pceIndex, lceIndex); 576 } 577 FOR_EACH_INDEX_IN_MASK_END; 578 579 // Store lceMask in the exposeCeMask before moving on 580 *pLocalExposeCeMask |= lceMask; 581 } 582 583 pKCeLce = GPU_GET_KCE(pGpu, lceIndex); 584 pKCeLce->nvlinkPeerMask |= NVBIT(pRemoteGpu->gpuInstance); 585 586 // 587 // Bug 200659256 - Looping over GPUs rather than peers (CL 28776130) 588 // does not handle multi-GPUs/Peer as is the case on switch systems. 589 // We must only take this loop once on switch systems to account for this. 590 // If we need to support multiple peer connections with switch systems 591 // in the future, this code must be revisited 592 // 593 if (pSys->getProperty(pSys, PDB_PROP_SYS_FABRIC_IS_EXTERNALLY_MANAGED)) 594 { 595 break; 596 } 597 598 } 599 600 if (bPeerAssigned == NV_FALSE) 601 { 602 status = NV_WARN_NOTHING_TO_DO; 603 } 604 605 return status; 606 } 607 608 /** 609 * @brief Some clients rely on LCE 4 also being turned on when there 610 * are no NVLink peers. This function sets up the default links. 611 * 612 * @param[in] pGpu OBJGPU pointer 613 * @param[in] pKCe KernelCE pointer 614 * @param[in] pceAvailableMaskPerHshub Pointer to CEs available per HSHUB 615 * @param[out] pLocalPceLceMap Pointer to PCE-LCE array 616 * @param[out] pLocalExposeCeMask Pointer to LCE Mask 617 * 618 * Returns NV_OK if successful in assigning PCEs to a default async LCE (>= 4) 619 */ 620 NV_STATUS 621 kceMapAsyncLceDefault_GH100 622 ( 623 OBJGPU *pGpu, 624 KernelCE *pKCe, 625 NvU32 *pceAvailableMaskPerHshub, 626 NvU32 *pLocalPceLceMap, 627 NvU32 *pLocalExposeCeMask, 628 NvU32 numDefaultPces 629 ) 630 { 631 NvU32 peerAvailableLceMask = NV_CE_LCE_MASK_INIT; 632 NvU32 lceMask = 0; 633 NvU32 pceMask = 0; 634 NvU32 lceIndex, pceIndex, hshubId, i; 635 NvU32 maxLceCnt = NV_CE_MAX_LCE_MASK; 636 637 peerAvailableLceMask = kceGetNvlinkPeerSupportedLceMask_HAL(pGpu, pKCe, peerAvailableLceMask); 638 hshubId = 1; 639 640 // 641 // If no peers were found, then no async LCEs (>= 4) will be turned on. 642 // However, some clients rely on LCE 4 being present even without any 643 // NVLink peers being found. So, turn on the 1st available async LCE (>= 4) 644 // Reference bug 3042556 645 // 646 lceIndex = CE_GET_LOWEST_AVAILABLE_IDX(peerAvailableLceMask); 647 HIGHESTBITIDX_32(maxLceCnt); 648 if (lceIndex < maxLceCnt) 649 { 650 lceMask |= NVBIT32(lceIndex); 651 // Clear out the chosen LCE 652 peerAvailableLceMask &= (~(NVBIT32(lceIndex))); 653 } 654 655 // Assign PCEs to this LCE based on input request 656 for (i = 0; i < numDefaultPces; i++) 657 { 658 if (i % NV_CE_PCE_PER_HSHUB == 0) 659 hshubId++; 660 661 pceIndex = CE_GET_LOWEST_AVAILABLE_IDX(pceAvailableMaskPerHshub[hshubId]); 662 if (pceIndex < kceGetPce2lceConfigSize1_HAL(pKCe)) 663 { 664 pceMask |= NVBIT32(pceIndex); 665 pceAvailableMaskPerHshub[hshubId] &= (~(NVBIT32(pceIndex))); 666 } 667 } 668 669 FOR_EACH_INDEX_IN_MASK(32, pceIndex, pceMask) 670 { 671 pLocalPceLceMap[pceIndex] = lceIndex; 672 NV_PRINTF(LEVEL_INFO, "GPU%d <-> GPU%d PCE Index: %d LCE Index: %d\n", 673 pGpu->gpuInstance, pGpu->gpuInstance, pceIndex, lceIndex); 674 } 675 FOR_EACH_INDEX_IN_MASK_END; 676 677 // Store lceMask in the exposeCeMask before moving on 678 *pLocalExposeCeMask |= lceMask; 679 680 return NV_OK; 681 682 } 683 684 NV_STATUS 685 kceGetMappings_GH100 686 ( 687 OBJGPU *pGpu, 688 KernelCE *pKCe, 689 NVLINK_TOPOLOGY_PARAMS *pTopoParams, 690 NvU32 *pLocalPceLceMap, 691 NvU32 *pLocalGrceMap, 692 NvU32 *pExposeCeMask 693 ) 694 { 695 NV_STATUS status = NV_OK; 696 NV_STATUS statusC2C = NV_OK; 697 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu); 698 699 //Prepare the per-HSHUB/FBHUB available PCE mask 700 kceGetAvailableHubPceMask(pGpu, pTopoParams); 701 702 // Assign PCEs to "PEER"s if nvlink is enabled 703 if (pKernelNvlink && !knvlinkIsForcedConfig(pGpu, pKernelNvlink)) 704 { 705 status = kceMapPceLceForNvlinkPeers_HAL(pGpu, pKCe, 706 pTopoParams->pceAvailableMaskPerHshub, 707 pLocalPceLceMap, 708 pExposeCeMask); 709 } 710 else 711 { 712 status = NV_WARN_NOTHING_TO_DO; 713 } 714 715 // Special C2C cases for LCE 2 and 3 716 statusC2C = kceMapPceLceForC2C_HAL(pGpu, pKCe, 717 pTopoParams->pceAvailableMaskPerHshub, 718 pLocalPceLceMap, pExposeCeMask); 719 720 // Assign PCEs for GRCE case 721 kceMapPceLceForGRCE_HAL(pGpu, pKCe, 722 pTopoParams->pceAvailableMaskPerHshub, 723 pLocalPceLceMap, pExposeCeMask, pLocalGrceMap, pTopoParams->fbhubPceMask); 724 725 if ((status == NV_WARN_NOTHING_TO_DO && statusC2C == NV_WARN_NOTHING_TO_DO) || 726 (status == NV_ERR_NOT_SUPPORTED && statusC2C == NV_ERR_NOT_SUPPORTED)) 727 { 728 // If there's no NVLink peers available, still expose an additional async LCE 729 status = kceMapAsyncLceDefault_HAL(pGpu, pKCe, 730 pTopoParams->pceAvailableMaskPerHshub, 731 pLocalPceLceMap, 732 pExposeCeMask, 733 NV_CE_NUM_PCES_NO_LINK_CASE); 734 } 735 736 NV_PRINTF(LEVEL_INFO, "status = %d, statusC2C = %d\n", status, statusC2C); 737 return NV_OK; 738 } 739