1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #define NVOC_KERNEL_NVLINK_H_PRIVATE_ACCESS_ALLOWED 25 26 // FIXME XXX 27 #define NVOC_KERNEL_IOCTRL_H_PRIVATE_ACCESS_ALLOWED 28 29 #include "os/os.h" 30 #include "core/hal.h" 31 #include "core/locks.h" 32 #include "gpu_mgr/gpu_mgr.h" 33 #include "gpu/gpu.h" 34 #include "kernel/gpu/nvlink/kernel_nvlink.h" 35 #include "kernel/gpu/nvlink/kernel_ioctrl.h" 36 #include "gpu/mem_mgr/mem_mgr.h" 37 #include "gpu/mmu/kern_gmmu.h" 38 #include "gpu/ce/kernel_ce.h" 39 #include "platform/sli/sli.h" 40 #include "gpu/gpu_fabric_probe.h" 41 #include "compute/imex_session_api.h" 42 #include "compute/fabric.h" 43 #include "mem_mgr/mem_multicast_fabric.h" 44 45 /*! 46 * @brief Is NVLINK topology forced? NVLink topology is considered 47 * forced for both legacy forced config and chiplib configs 48 * 49 * @param[in] pGpu OBJGPU 50 * @param[in] pKernelNvlink KernelNvlink pointer 51 * 52 * @return NV_TRUE if topology is forced 53 */ 54 NvBool 55 knvlinkIsForcedConfig_IMPL 56 ( 57 OBJGPU *pGpu, 58 KernelNvlink *pKernelNvlink 59 ) 60 { 61 return (pKernelNvlink->bChiplibConfig); 62 } 63 64 /*! 65 * @brief Determine if NVLink is enabled or disabled by default 66 * 67 * @param[in] pGpu OBJGPU pointer 68 * @param[in] pKernelNvlink KernelNvlink pointer 69 * 70 * @return NV_TRUE if NVLink is enabled on the GPU/platform 71 */ 72 NvBool 73 knvlinkIsNvlinkDefaultEnabled_IMPL 74 ( 75 OBJGPU *pGpu, 76 KernelNvlink *pKernelNvlink 77 ) 78 { 79 // 80 // Currently it is critical that the following lib check be present. 81 // Burying this in the hal below it may get lost as the stub is all 82 // thats required for POR (always true from the hals perspective) 83 // 84 #if !defined(INCLUDE_NVLINK_LIB) 85 86 return NV_FALSE; 87 88 #endif 89 90 // Let the PDB handle the final decision. 91 return pKernelNvlink->getProperty(pKernelNvlink, PDB_PROP_KNVLINK_ENABLED); 92 } 93 94 /*! 95 * @brief Determine if P2P loopback over NVLink is supported for 96 * the given GPU. This function returns true if any link 97 * is connected in loopback mode. 98 * 99 * @param[in] pGpu OBJGPU pointer 100 * @param[in] pKernelNvlink KernelNvlink pointer 101 * 102 * @return NV_TRUE if any link is in loopback mode 103 */ 104 NvBool 105 knvlinkIsP2pLoopbackSupported_IMPL 106 ( 107 OBJGPU *pGpu, 108 KernelNvlink *pKernelNvlink 109 ) 110 { 111 #if defined(INCLUDE_NVLINK_LIB) 112 113 NvU32 i; 114 115 if ((pGpu == NULL) || (pKernelNvlink == NULL)) 116 { 117 return NV_FALSE; 118 } 119 120 // Return false if P2P loopback is disabled through regkey 121 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_NVLINK_P2P_LOOPBACK_DISABLED)) 122 { 123 return NV_FALSE; 124 } 125 126 FOR_EACH_INDEX_IN_MASK(32, i, pKernelNvlink->enabledLinks) 127 { 128 if (knvlinkIsP2pLoopbackSupportedPerLink_IMPL(pGpu, pKernelNvlink, i)) 129 return NV_TRUE; 130 } 131 FOR_EACH_INDEX_IN_MASK_END 132 133 #endif 134 135 return NV_FALSE; 136 } 137 138 /*! 139 * @brief Determine if P2P loopback over NVLink is supported for 140 * the given link. This function returns true if the link 141 * is connected in loopback mode. 142 * 143 * @param[in] pGpu OBJGPU pointer 144 * @param[in] pKernelNvlink KernelNvlink pointer 145 * @param[in] link Link ID 146 * 147 * @return NV_TRUE if the link is in loopback mode 148 */ 149 NvBool 150 knvlinkIsP2pLoopbackSupportedPerLink_IMPL 151 ( 152 OBJGPU *pGpu, 153 KernelNvlink *pKernelNvlink, 154 NvU32 link 155 ) 156 { 157 #if defined(INCLUDE_NVLINK_LIB) 158 159 if ((pGpu == NULL) || (pKernelNvlink == NULL)) 160 { 161 return NV_FALSE; 162 } 163 164 // Return false if P2P loopback is disabled through regkey 165 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_NVLINK_P2P_LOOPBACK_DISABLED)) 166 { 167 return NV_FALSE; 168 } 169 170 // Return false if the given link is disabled 171 if (!(NVBIT(link) & pKernelNvlink->enabledLinks)) 172 { 173 return NV_FALSE; 174 } 175 176 // Check the link connected to the same GPU (loopback) 177 if (pKernelNvlink->nvlinkLinks[link].remoteEndInfo.bConnected) 178 { 179 if (((pKernelNvlink->nvlinkLinks[link].remoteEndInfo.domain == gpuGetDomain(pGpu)) && 180 (pKernelNvlink->nvlinkLinks[link].remoteEndInfo.bus == gpuGetBus(pGpu)) && 181 (pKernelNvlink->nvlinkLinks[link].remoteEndInfo.device == gpuGetDevice(pGpu)) && 182 (pKernelNvlink->nvlinkLinks[link].remoteEndInfo.function == 0)) || 183 pKernelNvlink->PDB_PROP_KNVLINK_FORCED_LOOPBACK_ON_SWITCH_MODE_ENABLED) 184 { 185 return NV_TRUE; 186 } 187 } 188 189 #endif 190 191 return NV_FALSE; 192 } 193 194 /*! 195 * @brief Determine if P2P over NVLINK is supported between 2 GPUs 196 * 197 * @param[in] pGpu OBJGPU pointer for local GPU 198 * @param[in] pKernelNvlink KernelNvlink pointer 199 * @param[in] pPeerGpu OBJGPU pointer for remote GPU 200 * 201 * @return NV_TRUE if P2P is supported between the 2 GPUs 202 */ 203 NvBool 204 knvlinkIsNvlinkP2pSupported_IMPL 205 ( 206 OBJGPU *pGpu, 207 KernelNvlink *pKernelNvlink, 208 OBJGPU *pPeerGpu 209 ) 210 { 211 NV_STATUS status = NV_OK; 212 213 if (pKernelNvlink == NULL) 214 { 215 return NV_FALSE; 216 } 217 218 if (knvlinkIsBandwidthModeOff(pKernelNvlink)) 219 { 220 return NV_FALSE; 221 } 222 223 // Get the Nvlink P2P connections from the core library 224 status = knvlinkGetP2pConnectionStatus(pGpu, pKernelNvlink, pPeerGpu); 225 226 if (status == NV_OK) 227 { 228 return NV_TRUE; 229 } 230 231 return NV_FALSE; 232 } 233 234 static NvBool 235 _knvlinkCheckFabricCliqueId 236 ( 237 OBJGPU *pGpu, 238 OBJGPU *pPeerGpu 239 ) 240 { 241 NvU32 cliqueId, peerCliqueId; 242 NV_STATUS status; 243 244 status = gpuFabricProbeGetFabricCliqueId(pGpu->pGpuFabricProbeInfoKernel, 245 &cliqueId); 246 if (status != NV_OK) 247 { 248 NV_PRINTF(LEVEL_ERROR, "GPU %d failed to get fabric clique Id: 0x%x\n", 249 gpuGetInstance(pGpu), status); 250 return NV_FALSE; 251 } 252 253 status = gpuFabricProbeGetFabricCliqueId(pPeerGpu->pGpuFabricProbeInfoKernel, 254 &peerCliqueId); 255 if (status != NV_OK) 256 { 257 NV_PRINTF(LEVEL_ERROR, "GPU %d failed to get fabric clique Id 0x%x\n", 258 gpuGetInstance(pPeerGpu), status); 259 return NV_FALSE; 260 } 261 262 if (cliqueId != peerCliqueId) 263 { 264 NV_PRINTF(LEVEL_ERROR, "GPU %d and Peer GPU %d cliqueId doesn't match\n", 265 gpuGetInstance(pGpu), gpuGetInstance(pPeerGpu)); 266 return NV_FALSE; 267 } 268 269 return NV_TRUE; 270 } 271 272 /*! 273 * @brief Checks whether necessary the config setup is done to 274 * support P2P over NVSwitch 275 * 276 * @param[in] pGpu OBJGPU pointer for local GPU 277 * @param[in] pKernelNvlink KernelNvlink pointer 278 * @param[in] pPeerGpu OBJGPU pointer for remote GPU 279 * 280 * @return NV_TRUE if P2P over NVSwitch 281 */ 282 NvBool 283 knvlinkCheckNvswitchP2pConfig_IMPL 284 ( 285 OBJGPU *pGpu, 286 KernelNvlink *pKernelNvlink, 287 OBJGPU *pPeerGpu 288 ) 289 { 290 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 291 NvU64 rangeStart = knvlinkGetUniqueFabricBaseAddress(pGpu, pKernelNvlink); 292 NvU64 rangeEnd = rangeStart + (pMemoryManager->Ram.fbTotalMemSizeMb << 20); 293 NvU64 peerRangeStart = knvlinkGetUniqueFabricBaseAddress(pPeerGpu, 294 GPU_GET_KERNEL_NVLINK(pPeerGpu)); 295 296 if (knvlinkIsGpuConnectedToNvswitch(pGpu, pKernelNvlink)) 297 { 298 if (gpuIsSriovEnabled(pGpu)) 299 { 300 // currently vgpu + switch doesn't support GPA addresing. 301 return NV_TRUE; 302 } 303 304 if (gpuFabricProbeIsSupported(pGpu) && gpuFabricProbeIsSupported(pPeerGpu)) 305 { 306 if (!_knvlinkCheckFabricCliqueId(pGpu, pPeerGpu)) 307 { 308 return NV_FALSE; 309 } 310 } 311 312 if (knvlinkGetUniqueFabricBaseAddress(pGpu, pKernelNvlink) == 313 NVLINK_INVALID_FABRIC_ADDR) 314 { 315 NV_PRINTF(LEVEL_ERROR, "GPU %d doesn't have a fabric address\n", 316 gpuGetInstance(pGpu)); 317 318 return NV_FALSE; 319 } 320 321 if ((pGpu != pPeerGpu) && 322 ((peerRangeStart >= rangeStart) && (peerRangeStart < rangeEnd))) 323 { 324 NV_PRINTF(LEVEL_ERROR, 325 "GPU %d doesn't have a unique fabric address\n", 326 gpuGetInstance(pGpu)); 327 328 return NV_FALSE; 329 } 330 } 331 else 332 { 333 if (knvlinkGetUniqueFabricBaseAddress(pGpu, pKernelNvlink) != 334 NVLINK_INVALID_FABRIC_ADDR) 335 { 336 NV_PRINTF(LEVEL_ERROR, 337 "non-NVSwitch GPU %d has a valid fabric address\n", 338 gpuGetInstance(pGpu)); 339 340 return NV_FALSE; 341 } 342 } 343 344 return NV_TRUE; 345 } 346 347 /*! 348 * @brief Get Nvlink P2P connections between 2 GPUs 349 * 350 * @param[in] pGpu OBJGPU pointer for local GPU 351 * @param[in] pKernelNvlink KernelNvlink pointer 352 * @param[in] pPeerGpu OBJGPU pointer for remote GPU 353 * 354 * @return NV_OK if P2P connections are present 355 */ 356 NV_STATUS 357 knvlinkGetP2pConnectionStatus_IMPL 358 ( 359 OBJGPU *pGpu, 360 KernelNvlink *pKernelNvlink, 361 OBJGPU *pPeerGpu 362 ) 363 { 364 NV_STATUS status = NV_OK; 365 OBJGPU *pGpu0 = pGpu; 366 OBJGPU *pGpu1 = pPeerGpu; 367 KernelNvlink *pKernelNvlink0 = pKernelNvlink; 368 KernelNvlink *pKernelNvlink1 = NULL; 369 NvU32 numPeerLinks = 0; 370 371 if (pGpu1 == NULL) 372 { 373 NV_PRINTF(LEVEL_INFO, "Invalid pPeerGpu.\n"); 374 375 return NV_ERR_INVALID_ARGUMENT; 376 } 377 else if ((pGpu0 == pGpu1) && 378 (pGpu0->getProperty(pGpu0, PDB_PROP_GPU_NVLINK_P2P_LOOPBACK_DISABLED))) 379 { 380 // P2P over loopback links are disabled through regkey overrides 381 NV_PRINTF(LEVEL_INFO, "loopback P2P on GPU%u disabled by regkey\n", 382 gpuGetInstance(pGpu0)); 383 384 return NV_ERR_NOT_SUPPORTED; 385 } 386 else 387 { 388 pKernelNvlink1 = GPU_GET_KERNEL_NVLINK(pGpu1); 389 } 390 391 if (pKernelNvlink1 == NULL) 392 { 393 NV_PRINTF(LEVEL_INFO, 394 "Input mask contains a GPU on which NVLink is disabled.\n"); 395 396 return NV_ERR_INVALID_ARGUMENT; 397 } 398 399 if(pKernelNvlink0->bIsGpuDegraded) 400 { 401 NV_PRINTF(LEVEL_INFO, 402 "NVLink P2P is NOT supported between GPU%d and GPU%d\n", 403 gpuGetInstance(pGpu0), gpuGetInstance(pGpu1)); 404 405 return NV_ERR_NOT_SUPPORTED; 406 } 407 408 if(pKernelNvlink1->bIsGpuDegraded) 409 { 410 NV_PRINTF(LEVEL_INFO, 411 "NVLink P2P is NOT supported between GPU%d and GPU%d\n", 412 gpuGetInstance(pGpu0), gpuGetInstance(pGpu1)); 413 414 return NV_ERR_NOT_SUPPORTED; 415 } 416 417 if ((IS_RTLSIM(pGpu0) && !pKernelNvlink0->bForceEnableCoreLibRtlsims) || 418 knvlinkIsForcedConfig(pGpu0, pKernelNvlink0)) 419 { 420 // For non-legacy configs. 421 if (pKernelNvlink0->bChiplibConfig) 422 { 423 NV_PRINTF(LEVEL_INFO, 424 "NVLink P2P is supported between GPU%d and GPU%d\n", 425 gpuGetInstance(pGpu0), gpuGetInstance(pGpu1)); 426 427 return NV_OK; 428 } 429 } 430 431 // Get the remote ends of the links of local GPU from the nvlink core 432 status = knvlinkCoreGetRemoteDeviceInfo(pGpu0, pKernelNvlink0); 433 if (status != NV_OK) 434 { 435 return status; 436 } 437 438 // Post topology link enable on links of local GPU 439 status = knvlinkEnableLinksPostTopology_HAL(pGpu0, pKernelNvlink0, 440 pKernelNvlink0->enabledLinks); 441 if (status != NV_OK) 442 { 443 return status; 444 } 445 446 numPeerLinks = knvlinkGetNumLinksToPeer(pGpu0, pKernelNvlink0, pGpu1); 447 448 // 449 // Maybe knvlinkCoreGetRemoteDeviceInfo was never called on pGpu1. 450 // This can happen on systems where FM doesn't configure GPUs 451 // using RM control calls explicitly. 452 // 453 if ((numPeerLinks == 0) && gpuFabricProbeIsSupported(pGpu1)) 454 { 455 knvlinkCoreGetRemoteDeviceInfo(pGpu1, pKernelNvlink1); 456 457 // Post topology link enable on links of remote GPU 458 status = knvlinkEnableLinksPostTopology_HAL(pGpu1, pKernelNvlink1, 459 pKernelNvlink1->enabledLinks); 460 if (status != NV_OK) 461 { 462 return status; 463 } 464 465 numPeerLinks = knvlinkGetNumLinksToPeer(pGpu0, pKernelNvlink0, pGpu1); 466 } 467 468 if (numPeerLinks > 0) 469 { 470 if (knvlinkGetNumLinksToPeer(pGpu1, pKernelNvlink1, pGpu0) != numPeerLinks) 471 { 472 // Get the remote ends of the links of remote GPU from the nvlink core 473 status = knvlinkCoreGetRemoteDeviceInfo(pGpu1, pKernelNvlink1); 474 if (status != NV_OK) 475 { 476 return status; 477 } 478 479 // Post topology link enable on links of remote GPU 480 status = knvlinkEnableLinksPostTopology_HAL(pGpu1, pKernelNvlink1, 481 pKernelNvlink1->enabledLinks); 482 if (status != NV_OK) 483 { 484 return status; 485 } 486 } 487 488 // Peers should have the same number of links pointing back at us 489 NV_CHECK_OR_RETURN(LEVEL_INFO, 490 (knvlinkGetNumLinksToPeer(pGpu1, pKernelNvlink1, pGpu0) == numPeerLinks), 491 NV_ERR_INVALID_STATE); 492 493 NV_CHECK_OR_RETURN(LEVEL_INFO, 494 knvlinkCheckNvswitchP2pConfig(pGpu0, pKernelNvlink0, pGpu1), 495 NV_ERR_INVALID_STATE); 496 497 NV_CHECK_OR_RETURN(LEVEL_INFO, 498 knvlinkCheckNvswitchP2pConfig(pGpu1, pKernelNvlink1, pGpu0), 499 NV_ERR_INVALID_STATE); 500 501 NV_PRINTF(LEVEL_INFO, 502 "NVLink P2P is supported between GPU%d and GPU%d\n", 503 gpuGetInstance(pGpu0), gpuGetInstance(pGpu1)); 504 505 return NV_OK; 506 } 507 508 NV_PRINTF(LEVEL_INFO, 509 "NVLink P2P is NOT supported between between GPU%d and GPU%d\n", 510 pGpu->gpuInstance, pGpu1->gpuInstance); 511 512 return NV_ERR_NOT_SUPPORTED; 513 } 514 515 /*! 516 * @brief Update the settings for the current established NVLink 517 * topology. This is the top level function that should be 518 * called, instead of applying the settings individually, 519 * since it grabs the required locks 520 * 521 * @param[in] pGpu OBJGPU pointer 522 * @param[in] pKernelNvlink KernelNvlink pointer 523 * 524 * @return NV_OK on success 525 */ 526 NV_STATUS 527 knvlinkUpdateCurrentConfig_IMPL 528 ( 529 OBJGPU *pGpu, 530 KernelNvlink *pKernelNvlink 531 ) 532 { 533 OBJSYS *pSys = SYS_GET_INSTANCE(); 534 KernelCE *pKCe = NULL; 535 NvBool bOwnsLock = NV_FALSE; 536 NV_STATUS status = NV_OK; 537 538 if (osAcquireRmSema(pSys->pSema) == NV_OK) 539 { 540 // 541 // XXX Bug 1795328: Fix P2P path to acquire locks for the GPU 542 // Due to platform differences in the P2P path, the GPU lock is not 543 // consistently held at this point in the call stack. This function 544 // requires exclusive access to RM/PMU data structures to update HSHUB, 545 // and therefore requires the GPU lock to be held at this point. 546 // This check should be removed once the P2P paths have been updated to 547 // acquire the GPU locks consistently for all platforms. 548 // 549 if (!rmDeviceGpuLockIsOwner(pGpu->gpuInstance)) 550 { 551 status = rmDeviceGpuLocksAcquire(pGpu, GPUS_LOCK_FLAGS_NONE, 552 RM_LOCK_MODULES_NVLINK); 553 if (status != NV_OK) 554 { 555 NV_ASSERT(0); 556 goto fail; 557 } 558 559 bOwnsLock = NV_TRUE; 560 } 561 562 // 563 // Links that have remote end detected should have passed RXDET 564 // Update the mask of connected links and bridged links 565 // 566 knvlinkFilterBridgeLinks_HAL(pGpu, pKernelNvlink); 567 568 NV2080_CTRL_NVLINK_UPDATE_CURRENT_CONFIG_PARAMS params; 569 portMemSet(¶ms, 0, sizeof(params)); 570 571 // Reset timeout to clear any accumulated timeouts from link init 572 if (IS_GSP_CLIENT(pGpu)) 573 { 574 threadStateResetTimeout(pGpu); 575 } 576 577 // 578 // RPC into GSP-RM for programming the HSHUB, CONNECTION_CFG and LTCS 579 // registers. 580 // 581 status = knvlinkExecGspRmRpc(pGpu, pKernelNvlink, 582 NV2080_CTRL_CMD_NVLINK_UPDATE_CURRENT_CONFIG, 583 (void *)¶ms, sizeof(params)); 584 if (status != NV_OK) 585 { 586 NV_PRINTF(LEVEL_ERROR, "Updating current NVLink config failed\n"); 587 goto fail; 588 } 589 590 // Sync the GPU property for NVLINK over SYSMEM with GSP-RM 591 pGpu->setProperty(pGpu, PDB_PROP_GPU_NVLINK_SYSMEM, params.bNvlinkSysmemEnabled); 592 593 // Update the PCE-LCE mappings 594 status = kceFindFirstInstance(pGpu, &pKCe); 595 if (status == NV_OK) 596 { 597 status = kceTopLevelPceLceMappingsUpdate(pGpu, pKCe); 598 if (status != NV_OK) 599 { 600 NV_PRINTF(LEVEL_ERROR, "Failed to update PCE-LCE mappings\n"); 601 } 602 } 603 604 fail: 605 if (bOwnsLock) 606 { 607 rmDeviceGpuLocksRelease(pGpu, GPUS_LOCK_FLAGS_NONE, NULL); 608 } 609 610 osReleaseRmSema(pSys->pSema, NULL); 611 } 612 613 return status; 614 } 615 616 const static NVLINK_INBAND_MSG_CALLBACK nvlink_inband_callbacks[] = 617 { 618 { 619 .messageType = NVLINK_INBAND_MSG_TYPE_GPU_PROBE_RSP, 620 .pCallback = gpuFabricProbeReceiveKernelCallback, 621 .wqItemFlags = OS_QUEUE_WORKITEM_FLAGS_LOCK_SEMA | 622 OS_QUEUE_WORKITEM_FLAGS_LOCK_GPU_GROUP_SUBDEVICE_RW 623 }, 624 625 { 626 .messageType = NVLINK_INBAND_MSG_TYPE_MC_TEAM_SETUP_RSP, 627 .pCallback = memorymulticastfabricTeamSetupResponseCallback, 628 .wqItemFlags = OS_QUEUE_WORKITEM_FLAGS_LOCK_SEMA | 629 OS_QUEUE_WORKITEM_FLAGS_LOCK_GPUS_RW 630 }, 631 632 { 633 .messageType = NVLINK_INBAND_MSG_TYPE_GPU_PROBE_UPDATE_REQ, 634 .pCallback = gpuFabricProbeReceiveUpdateKernelCallback, 635 .wqItemFlags = OS_QUEUE_WORKITEM_FLAGS_LOCK_SEMA | 636 OS_QUEUE_WORKITEM_FLAGS_LOCK_GPU_GROUP_SUBDEVICE_RW 637 } 638 }; 639 640 void 641 knvlinkInbandMsgCallbackDispatcher_WORKITEM 642 ( 643 NvU32 gpuInstance, 644 void *pData 645 ) 646 { 647 nvlink_inband_msg_header_t *pHeader; 648 NV2080_CTRL_NVLINK_INBAND_RECEIVED_DATA_PARAMS *pMessage = pData; 649 NvU8 i; 650 const NVLINK_INBAND_MSG_CALLBACK *pCb = NULL; 651 652 // Dispatcher may not be called under GPU lock, so don't access pGpu. 653 654 pHeader = (nvlink_inband_msg_header_t *)pMessage->data; 655 656 for (i = 0; i < NV_ARRAY_ELEMENTS(nvlink_inband_callbacks); i++) 657 { 658 if ((nvlink_inband_callbacks[i].messageType == pHeader->type) && 659 (nvlink_inband_callbacks[i].pCallback != NULL)) 660 { 661 pCb = &nvlink_inband_callbacks[i]; 662 break; 663 } 664 } 665 666 if (pCb == NULL) 667 { 668 NV_PRINTF(LEVEL_ERROR, 669 "No Callback Registered for type %d. Dropping the msg\n", 670 pHeader->type); 671 return; 672 } 673 674 #if defined(DEBUG) || defined(DEVELOP) 675 { 676 NvU8 *pRsvd = NULL; 677 678 // Assert reserved in msgHdr are zero 679 pRsvd = &pHeader->reserved[0]; 680 NV_ASSERT((pRsvd[0] == 0) && portMemCmp(pRsvd, pRsvd + 1, 681 sizeof(pHeader->reserved) - 1) == 0); 682 } 683 #endif 684 685 (void)pCb->pCallback(gpuInstance, NULL, pData); 686 } 687 688 NV_STATUS 689 knvlinkInbandMsgCallbackDispatcher_IMPL 690 ( 691 OBJGPU *pGpu, 692 KernelNvlink *pKernelNvlink, 693 NvU32 dataSize, 694 NvU8 *pMessage 695 ) 696 { 697 NV_STATUS status; 698 nvlink_inband_msg_header_t *pHeader; 699 NV2080_CTRL_NVLINK_INBAND_RECEIVED_DATA_PARAMS *pData = NULL; 700 const NVLINK_INBAND_MSG_CALLBACK *pCb = NULL; 701 NvU8 i; 702 703 pHeader = (nvlink_inband_msg_header_t *)pMessage; 704 705 if (pHeader->type >= NVLINK_INBAND_MSG_TYPE_MAX) 706 { 707 NV_PRINTF(LEVEL_ERROR, "Message type received is Out of Bounds. Dropping the msg\n"); 708 return NV_ERR_INVALID_REQUEST; 709 } 710 711 for (i = 0; i < NV_ARRAY_ELEMENTS(nvlink_inband_callbacks); i++) 712 { 713 if ((nvlink_inband_callbacks[i].messageType == pHeader->type) && 714 (nvlink_inband_callbacks[i].pCallback != NULL)) 715 { 716 pCb = &nvlink_inband_callbacks[i]; 717 break; 718 } 719 } 720 721 if (pCb == NULL) 722 { 723 NV_PRINTF(LEVEL_ERROR, 724 "No Callback Registered for type %d. Dropping the msg\n", 725 pHeader->type); 726 return NV_ERR_INVALID_REQUEST; 727 } 728 729 pData = portMemAllocNonPaged(sizeof(NV2080_CTRL_NVLINK_INBAND_RECEIVED_DATA_PARAMS)); 730 if (pData == NULL) 731 { 732 NV_PRINTF(LEVEL_ERROR, "Out of memory, Dropping message\n"); 733 return NV_ERR_NO_MEMORY; 734 } 735 736 pData->dataSize = dataSize; 737 portMemCopy(pData->data, pData->dataSize, pMessage, dataSize); 738 739 status = osQueueWorkItemWithFlags(pGpu, knvlinkInbandMsgCallbackDispatcher_WORKITEM, pData, 740 pCb->wqItemFlags); 741 if (status != NV_OK) 742 { 743 portMemFree(pData); 744 return status; 745 } 746 747 return NV_OK; 748 } 749 750 NV_STATUS 751 knvlinkSendInbandData_IMPL 752 ( 753 OBJGPU *pGpu, 754 KernelNvlink *pKernelNvlink, 755 NV2080_CTRL_NVLINK_INBAND_SEND_DATA_PARAMS *pParams 756 ) 757 { 758 NV_STATUS status; 759 760 status = knvlinkExecGspRmRpc(pGpu, pKernelNvlink, 761 NV2080_CTRL_CMD_NVLINK_INBAND_SEND_DATA, 762 (void *)pParams, 763 sizeof(*pParams)); 764 765 return status; 766 } 767 /*! 768 * @brief Return the mask of links enabled on the system 769 * 770 * @param[in] pGpu OBJGPU pointer 771 * @param[in] pKernelNvlink KernelNvlink pointer 772 */ 773 NvU32 774 knvlinkGetEnabledLinkMask_IMPL 775 ( 776 OBJGPU *pGpu, 777 KernelNvlink *pKernelNvlink 778 ) 779 { 780 return pKernelNvlink->enabledLinks; 781 } 782 783 /*! 784 * @brief Return the mask of links discovered on the system 785 * 786 * @param[in] pGpu OBJGPU pointer 787 * @param[in] pKernelNvlink KernelNvlink pointer 788 */ 789 NvU32 790 knvlinkGetDiscoveredLinkMask_IMPL 791 ( 792 OBJGPU *pGpu, 793 KernelNvlink *pKernelNvlink 794 ) 795 { 796 return pKernelNvlink->discoveredLinks; 797 } 798 799 /*! 800 * @brief Returns the number of sysmem links 801 * 802 * @param[in] pGpu OBJGPU pointer 803 * @param[in] pKernelNvlink KernelNvlink pointer 804 * 805 * @return The #sysmem NVLinks 806 */ 807 NvU32 808 knvlinkGetNumLinksToSystem_IMPL 809 ( 810 OBJGPU *pGpu, 811 KernelNvlink *pKernelNvlink 812 ) 813 { 814 NvU32 numSysmemLinks = pKernelNvlink->sysmemLinkMask; 815 816 if (numSysmemLinks != 0) 817 { 818 NUMSETBITS_32(numSysmemLinks); 819 } 820 821 return numSysmemLinks; 822 } 823 824 /*! 825 * @brief Returns number of peer links to a remote GPU 826 * 827 * @param[in] pGpu OBJGPU pointer of local GPU 828 * @param[in] pKernelNvlink KernelNvlink pointer 829 * @param[in] pRemoteGpu OBJGPU pointer of remote GPU 830 * 831 * @return The #peer NVLinks to the remote GPU 832 */ 833 NvU32 834 knvlinkGetNumLinksToPeer_IMPL 835 ( 836 OBJGPU *pGpu, 837 KernelNvlink *pKernelNvlink, 838 OBJGPU *pRemoteGpu 839 ) 840 { 841 NvU32 numPeerLinks = 842 knvlinkGetLinkMaskToPeer(pGpu, pKernelNvlink, pRemoteGpu); 843 844 if (numPeerLinks != 0) 845 { 846 NUMSETBITS_32(numPeerLinks); 847 } 848 849 return numPeerLinks; 850 } 851 852 /*! 853 * @brief Gets the mask of peer links between the GPUs 854 * 855 * @param[in] pGpu0 OBJGPU pointer 856 * @param[in] pKernelNvlink0 Nvlink pointer 857 * @param[in] pGpu1 Remote OBJGPU pointer 858 * 859 * @return Returns the mask of peer links between the GPUs 860 */ 861 NvU32 862 knvlinkGetLinkMaskToPeer_IMPL 863 ( 864 OBJGPU *pGpu0, 865 KernelNvlink *pKernelNvlink0, 866 OBJGPU *pGpu1 867 ) 868 { 869 NvU32 peerLinkMask = 0; 870 KernelNvlink *pKernelNvlink1 = NULL; 871 872 pKernelNvlink1 = GPU_GET_KERNEL_NVLINK(pGpu1); 873 874 if (pKernelNvlink1 == NULL) 875 { 876 NV_PRINTF(LEVEL_INFO, 877 "on GPU%d NVLink is disabled.\n", gpuGetInstance(pGpu1)); 878 879 return 0; 880 } 881 882 if(pKernelNvlink0->bIsGpuDegraded) 883 { 884 return peerLinkMask; 885 } 886 887 if(pKernelNvlink1->bIsGpuDegraded) 888 { 889 return peerLinkMask; 890 } 891 892 if (!knvlinkIsForcedConfig(pGpu0, pKernelNvlink0)) 893 { 894 // 895 // If nvlink topology is not forced, then the hshub registers 896 // are updated only when a P2P object is allocated. So, return 897 // the cached value of mask of links connected to a GPU 898 // 899 peerLinkMask = pKernelNvlink0->peerLinkMasks[gpuGetInstance(pGpu1)]; 900 } 901 902 return peerLinkMask; 903 } 904 905 /*! 906 * @brief Sets the mask of peer links between the GPUs 907 * 908 * @param[in] pGpu0 OBJGPU pointer 909 * @param[in] pKernelNvlink0 Nvlink pointer 910 * @param[in] pGpu1 Remote OBJGPU pointer 911 * @param[in] peerLinkMask Mask of links to the peer GPU 912 * 913 * @return NV_OK on success 914 */ 915 NV_STATUS 916 knvlinkSetLinkMaskToPeer_IMPL 917 ( 918 OBJGPU *pGpu0, 919 KernelNvlink *pKernelNvlink0, 920 OBJGPU *pGpu1, 921 NvU32 peerLinkMask 922 ) 923 { 924 NV_STATUS status = NV_OK; 925 926 // Return early if no update needed to the peer link mask 927 if (pKernelNvlink0->peerLinkMasks[gpuGetInstance(pGpu1)] == peerLinkMask) 928 return NV_OK; 929 930 pKernelNvlink0->peerLinkMasks[gpuGetInstance(pGpu1)] = peerLinkMask; 931 932 NV2080_CTRL_NVLINK_UPDATE_PEER_LINK_MASK_PARAMS params; 933 934 portMemSet(¶ms, 0, sizeof(params)); 935 params.gpuInst = gpuGetInstance(pGpu1); 936 params.peerLinkMask = peerLinkMask; 937 938 // Reset timeout to clear any accumulated timeouts from link init 939 if (IS_GSP_CLIENT(pGpu0)) 940 { 941 threadStateResetTimeout(pGpu0); 942 } 943 944 // Sync the peerLinkMask with GSP-RM 945 status = knvlinkExecGspRmRpc(pGpu0, pKernelNvlink0, 946 NV2080_CTRL_CMD_NVLINK_UPDATE_PEER_LINK_MASK, 947 (void *)¶ms, sizeof(params)); 948 if (status != NV_OK) 949 { 950 NV_PRINTF(LEVEL_ERROR, 951 "Failed to sync peerLinksMask from GPU%d to GPU%d\n", 952 gpuGetInstance(pGpu0), gpuGetInstance(pGpu1)); 953 return status; 954 } 955 956 return NV_OK; 957 } 958 959 /*! 960 * @brief Get the mask of links that are peer links 961 * 962 * @param[in] pGpu OBJGPU pointer 963 * @param[in] pKernelNvlink KernelNvlink pointer 964 */ 965 NvU32 966 knvlinkGetPeersNvlinkMaskFromHshub_IMPL 967 ( 968 OBJGPU *pGpu, 969 KernelNvlink *pKernelNvlink 970 ) 971 { 972 NV_STATUS status = NV_OK; 973 NvU32 peerLinkMask = 0; 974 NvU32 i; 975 976 NV2080_CTRL_NVLINK_GET_LINK_AND_CLOCK_INFO_PARAMS params; 977 978 portMemSet(¶ms, 0, sizeof(params)); 979 params.linkMask = pKernelNvlink->enabledLinks; 980 981 status = knvlinkExecGspRmRpc(pGpu, pKernelNvlink, 982 NV2080_CTRL_CMD_NVLINK_GET_LINK_AND_CLOCK_INFO, 983 (void *)¶ms, sizeof(params)); 984 if (status != NV_OK) 985 return 0; 986 987 // Scan enabled links for peer connections 988 FOR_EACH_INDEX_IN_MASK(32, i, pKernelNvlink->enabledLinks) 989 { 990 if (params.linkInfo[i].bLinkConnectedToPeer) 991 peerLinkMask |= NVBIT(i); 992 } 993 FOR_EACH_INDEX_IN_MASK_END; 994 995 return peerLinkMask; 996 } 997 998 /*! 999 * @brief Prepare a GPU's NVLink engine for reset by removing mappings 1000 * to it from other GPUs. 1001 * 1002 * @param[in] pGpu OBJGPU pointer 1003 * @param[in] pKernelNvlink KernelNvlink pointer 1004 * 1005 * return NV_OK on success 1006 */ 1007 NV_STATUS 1008 knvlinkPrepareForXVEReset_IMPL 1009 ( 1010 OBJGPU *pGpu, 1011 KernelNvlink *pKernelNvlink, 1012 NvBool bForceShutdown 1013 ) 1014 { 1015 OBJSYS *pSys = SYS_GET_INSTANCE(); 1016 NV_STATUS retStatus = NV_OK; 1017 OBJGPU *pRemoteGpu; 1018 NV_STATUS status; 1019 NvU32 gpuInstance; 1020 NvU32 gpuMask; 1021 1022 // This is not supported on forced configs 1023 if (knvlinkIsForcedConfig(pGpu, pKernelNvlink)) 1024 { 1025 return NV_OK; 1026 } 1027 1028 // 1029 // Let fabric manager handle link shutdown/reset if the fabric is managed 1030 // externally. 1031 // 1032 if (pKernelNvlink->ipVerNvlink < NVLINK_VERSION_40 && 1033 pSys->getProperty(pSys, PDB_PROP_SYS_FABRIC_IS_EXTERNALLY_MANAGED)) 1034 { 1035 NV_PRINTF(LEVEL_INFO, 1036 "NVLink fabric is externally managed, skipping\n"); 1037 return NV_OK; 1038 } 1039 1040 status = gpumgrGetGpuAttachInfo(NULL, &gpuMask); 1041 NV_ASSERT_OR_RETURN(status == NV_OK, status); 1042 1043 gpuInstance = 0; 1044 while ((pRemoteGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL) 1045 { 1046 KernelNvlink *pRemoteKernelNvlink = GPU_GET_KERNEL_NVLINK(pRemoteGpu); 1047 1048 if ((pRemoteGpu == pGpu) || (pRemoteKernelNvlink == NULL) || 1049 (knvlinkGetNumLinksToPeer(pRemoteGpu, pRemoteKernelNvlink, pGpu) == 0) || 1050 API_GPU_IN_RESET_SANITY_CHECK(pRemoteGpu) || 1051 pRemoteGpu->getProperty(pRemoteGpu, PDB_PROP_GPU_IS_LOST)) 1052 { 1053 continue; 1054 } 1055 1056 // 1057 // Reset the peer masks in HSHUB of the remote GPU. Partial resets 1058 // (only removing the links connected to the GPU being reset) don't 1059 // appear to be sufficient. The reset will work fine, but the next 1060 // time we attempt to initialize this GPU, the copy engines will time 1061 // out while scrubbing FB and a GPU sysmembar (NV_UFLUSH_FB_FLUSH) will 1062 // fail to complete. 1063 // 1064 // The above symptoms haven't been root-caused (yet), but the current 1065 // POR for GPU reset is that once one GPU is reset, the others 1066 // connected to it over NVLink must also be reset before using NVLink 1067 // for peer traffic, so just use the big hammer and squash all HSHUB 1068 // configs on GPU reset. 1069 // 1070 // This allows us to reset the GPUs one by one, with GPU 1071 // initializations in between, without hanging up the GPU trying to 1072 // flush data over links that aren't available anymore. 1073 // 1074 // Starting from Ampere single GPU reset is supported and hence remove 1075 // only the nvlink's of the remote GPU's which are connected to the 1076 // current GPU. 1077 // 1078 1079 if (IsAMPEREorBetter(pGpu)) 1080 { 1081 NvU32 remPeerId = kbusGetPeerId_HAL(pRemoteGpu, GPU_GET_KERNEL_BUS(pRemoteGpu), pGpu); 1082 if (remPeerId != BUS_INVALID_PEER) 1083 status = knvlinkRemoveMapping_HAL(pRemoteGpu, pRemoteKernelNvlink, NV_FALSE, 1084 NVBIT(remPeerId), 1085 NV_FALSE /* bL2Entry */); 1086 } 1087 else 1088 { 1089 status = knvlinkRemoveMapping_HAL(pRemoteGpu, pRemoteKernelNvlink, NV_FALSE, 1090 ((1 << NVLINK_MAX_PEERS_SW) - 1), 1091 NV_FALSE /* bL2Entry */); 1092 } 1093 if (status != NV_OK) 1094 { 1095 NV_PRINTF(LEVEL_ERROR, 1096 "failed to reset HSHUB on GPU%u while preparing for GPU%u XVE reset (0x%x)\n", 1097 gpuGetInstance(pRemoteGpu), gpuGetInstance(pGpu), 1098 status); 1099 1100 retStatus = (retStatus == NV_OK) ? status : retStatus; 1101 } 1102 } 1103 1104 // Remove all NVLink mappings in HSHUB config registers to init values 1105 if (!API_GPU_IN_RESET_SANITY_CHECK(pGpu) && !pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_LOST)) 1106 status = knvlinkRemoveMapping_HAL(pGpu, pKernelNvlink, NV_TRUE, ((1 << NVLINK_MAX_PEERS_SW) - 1), 1107 NV_FALSE /* bL2Entry */); 1108 if (status != NV_OK) 1109 { 1110 NV_PRINTF(LEVEL_ERROR, 1111 "failed to reset HSHUB on GPU%u while preparing XVE reset: %s (0x%x)\n", 1112 gpuGetInstance(pGpu), nvstatusToString(status), status); 1113 1114 retStatus = (retStatus == NV_OK) ? status : retStatus; 1115 } 1116 1117 // 1118 // If GFW is booted and running through link-training, then no need to tear-down the 1119 // links to reset. Exit out early from the function 1120 // 1121 if (!bForceShutdown && pKernelNvlink->getProperty(pKernelNvlink, PDB_PROP_KNVLINK_MINION_GFW_BOOT)) 1122 { 1123 return NV_OK; 1124 } 1125 1126 // Pseudo-clean shutdown the links from this GPU 1127 status = knvlinkCoreShutdownDeviceLinks(pGpu, pKernelNvlink, bForceShutdown); 1128 if (status != NV_OK) 1129 { 1130 NV_PRINTF(LEVEL_ERROR, 1131 "failed to shutdown links on GPU%u while preparing XVE reset: %s (0x%x)\n", 1132 gpuGetInstance(pGpu), nvstatusToString(status), status); 1133 1134 retStatus = (retStatus == NV_OK) ? status : retStatus; 1135 } 1136 1137 // 1138 // Reset links related to this device and its peers (see Bug 2346447) 1139 // The property is disabled on Pascal, since the path hasn't been verified 1140 // and link reset after pseudo-clean shutdown results in DL and TL errors. 1141 // 1142 if (pKernelNvlink->getProperty(pKernelNvlink, PDB_PROP_KNVLINK_LINKRESET_AFTER_SHUTDOWN)) 1143 { 1144 status = knvlinkCoreResetDeviceLinks(pGpu, pKernelNvlink); 1145 if (status != NV_OK) 1146 { 1147 NV_PRINTF(LEVEL_ERROR, 1148 "failed to reset links on GPU%u while preparing XVE reset: %s (0x%x)\n", 1149 gpuGetInstance(pGpu), nvstatusToString(status), status); 1150 1151 retStatus = (retStatus == NV_OK) ? status : retStatus; 1152 } 1153 #if defined(INCLUDE_NVLINK_LIB) 1154 else 1155 { 1156 NvU32 linkId; 1157 1158 // 1159 // The connections have been successfully reset, update connected and disconnected 1160 // links masks on both the devices 1161 // 1162 FOR_EACH_INDEX_IN_MASK(32, linkId, pKernelNvlink->enabledLinks) 1163 { 1164 pKernelNvlink->disconnectedLinkMask |= NVBIT(linkId); 1165 pKernelNvlink->connectedLinksMask &= ~NVBIT(linkId); 1166 1167 if (pKernelNvlink->nvlinkLinks[linkId].remoteEndInfo.deviceType != 1168 NV2080_CTRL_NVLINK_DEVICE_INFO_DEVICE_TYPE_GPU) 1169 { 1170 continue; 1171 } 1172 1173 OBJGPU *pRemoteGpu = gpumgrGetGpuFromBusInfo( 1174 pKernelNvlink->nvlinkLinks[linkId].remoteEndInfo.domain, 1175 pKernelNvlink->nvlinkLinks[linkId].remoteEndInfo.bus, 1176 pKernelNvlink->nvlinkLinks[linkId].remoteEndInfo.device); 1177 1178 if (!API_GPU_IN_RESET_SANITY_CHECK(pRemoteGpu)) 1179 { 1180 KernelNvlink *pRemoteKernelNvlink = GPU_GET_KERNEL_NVLINK(pRemoteGpu); 1181 NvU32 remoteLinkId = pKernelNvlink->nvlinkLinks[linkId].remoteEndInfo.linkNumber; 1182 1183 pRemoteKernelNvlink->disconnectedLinkMask |= NVBIT(remoteLinkId); 1184 pRemoteKernelNvlink->connectedLinksMask &= ~NVBIT(remoteLinkId); 1185 } 1186 } 1187 FOR_EACH_INDEX_IN_MASK_END; 1188 } 1189 #endif 1190 1191 // 1192 // knvlinkCoreResetDeviceLinks() only resets the links which have 1193 // connectivity. 1194 // Pre-Ampere, we may run into a situation where the PLL 1195 // sharing partner links (both) may not be reset due to no connectivity. 1196 // 1197 // Hence, (re-)reset all the links to recover them after shutdown (pre-Ampere) 1198 // 1199 NV2080_CTRL_NVLINK_RESET_LINKS_PARAMS resetLinksparams; 1200 1201 portMemSet(&resetLinksparams, 0, sizeof(resetLinksparams)); 1202 resetLinksparams.linkMask = pKernelNvlink->enabledLinks; 1203 resetLinksparams.flags = NV2080_CTRL_NVLINK_RESET_FLAGS_TOGGLE; 1204 1205 status = knvlinkExecGspRmRpc(pGpu, pKernelNvlink, 1206 NV2080_CTRL_CMD_NVLINK_RESET_LINKS, 1207 (void *)&resetLinksparams, sizeof(resetLinksparams)); 1208 1209 retStatus = (retStatus == NV_OK) ? status : retStatus; 1210 } 1211 1212 return retStatus; 1213 } 1214 1215 /*! 1216 * @brief Set the power features supported on this NVLink IP 1217 * 1218 * @param[in] pGpu OBJGPU pointer 1219 * @param[in] pKernelNvlink KernelNvlink pointer 1220 */ 1221 void 1222 knvlinkSetPowerFeatures_IMPL 1223 ( 1224 OBJGPU *pGpu, 1225 KernelNvlink *pKernelNvlink 1226 ) 1227 { 1228 // Get the Ip Verion from the First available IOCTRL. 1229 switch (pKernelNvlink->ipVerNvlink) 1230 { 1231 case NVLINK_VERSION_22: 1232 { 1233 // NVLink L2 is supported only on MODS and Windows LDDM 1234 if (RMCFG_FEATURE_PLATFORM_WINDOWS || RMCFG_FEATURE_MODS_FEATURES) 1235 { 1236 pKernelNvlink->setProperty(pKernelNvlink, PDB_PROP_KNVLINK_L2_POWER_STATE_ENABLED, 1237 (pKernelNvlink->bDisableL2Mode ? NV_FALSE : NV_TRUE)); 1238 } 1239 1240 break; 1241 } 1242 default: 1243 break; 1244 } 1245 } 1246 1247 /*! 1248 * @brief Checks if NVSWITCH_FABRIC_ADDR field is valid. 1249 * 1250 * @param[in] pGpu OBJGPU pointer 1251 * @param[in] pKernelNvlink KernelNvlink pointer 1252 */ 1253 void 1254 knvlinkDetectNvswitchProxy_IMPL 1255 ( 1256 OBJGPU *pGpu, 1257 KernelNvlink *pKernelNvlink 1258 ) 1259 { 1260 OBJSYS *pSys = SYS_GET_INSTANCE(); 1261 NV_STATUS status = NV_OK; 1262 NvU32 i; 1263 1264 // Initialize fabricBaseAddr to NVLINK_INVALID_FABRIC_ADDR 1265 pKernelNvlink->fabricBaseAddr = NVLINK_INVALID_FABRIC_ADDR; 1266 1267 if (pSys->getProperty(pSys, PDB_PROP_SYS_NVSWITCH_IS_PRESENT) || 1268 pSys->getProperty(pSys, PDB_PROP_SYS_FABRIC_MANAGER_IS_REGISTERED) || 1269 GPU_IS_NVSWITCH_DETECTED(pGpu)) 1270 { 1271 return; 1272 } 1273 1274 if (pKernelNvlink->discoveredLinks == 0) 1275 { 1276 return; 1277 } 1278 1279 // Get the link train status for the enabled link masks 1280 NV2080_CTRL_NVLINK_ARE_LINKS_TRAINED_PARAMS linkTrainedParams; 1281 1282 portMemSet(&linkTrainedParams, 0, sizeof(linkTrainedParams)); 1283 linkTrainedParams.linkMask = pKernelNvlink->enabledLinks; 1284 linkTrainedParams.bActiveOnly = NV_FALSE; 1285 1286 // Reset timeout to clear any accumulated timeouts from link init 1287 if (IS_GSP_CLIENT(pGpu)) 1288 { 1289 threadStateResetTimeout(pGpu); 1290 } 1291 1292 status = knvlinkExecGspRmRpc(pGpu, pKernelNvlink, 1293 NV2080_CTRL_CMD_NVLINK_ARE_LINKS_TRAINED, 1294 (void *)&linkTrainedParams, sizeof(linkTrainedParams)); 1295 if (status != NV_OK) 1296 { 1297 NV_PRINTF(LEVEL_ERROR, "Failed to get the link train status for links\n"); 1298 return; 1299 } 1300 1301 FOR_EACH_INDEX_IN_MASK(32, i, pKernelNvlink->enabledLinks) 1302 { 1303 if (!linkTrainedParams.bIsLinkActive[i]) 1304 { 1305 return; 1306 } 1307 } 1308 FOR_EACH_INDEX_IN_MASK_END; 1309 1310 NV2080_CTRL_INTERNAL_NVLINK_GET_SET_NVSWITCH_FABRIC_ADDR_PARAMS params; 1311 1312 portMemSet(¶ms, 0, sizeof(params)); 1313 params.bGet = NV_TRUE; 1314 params.addr = NVLINK_INVALID_FABRIC_ADDR; 1315 1316 status = knvlinkExecGspRmRpc(pGpu, pKernelNvlink, 1317 NV2080_CTRL_CMD_INTERNAL_NVLINK_GET_SET_NVSWITCH_FABRIC_ADDR, 1318 (void *)¶ms, sizeof(params)); 1319 if (status != NV_OK) 1320 { 1321 NV_PRINTF(LEVEL_ERROR, "Failed to get fabric address for GPU %x\n", 1322 pGpu->gpuInstance); 1323 return; 1324 } 1325 1326 if (params.addr != NVLINK_INVALID_FABRIC_ADDR) 1327 { 1328 pKernelNvlink->fabricBaseAddr = params.addr; 1329 pKernelNvlink->bNvswitchProxy = NV_TRUE; 1330 } 1331 } 1332 1333 /*! 1334 * @brief Sets NVSWITCH_FLA_ADDR field in the scratch register. 1335 * 1336 * @param[in] pGpu OBJGPU pointer 1337 * @param[in] pKernelNvlink KernelNvlink pointer 1338 * @param[in] addr FLA addr 1339 * 1340 * @return Returns NV_OK upon success. 1341 * Otherwise, returns NV_ERR_XXX. 1342 */ 1343 NV_STATUS 1344 knvlinkSetNvswitchFlaAddr_IMPL 1345 ( 1346 OBJGPU *pGpu, 1347 KernelNvlink *pKernelNvlink, 1348 NvU64 addr 1349 ) 1350 { 1351 return NV_OK; 1352 } 1353 1354 /*! 1355 * @brief Gets NVSWITCH_FLA_ADDR field from the scratch register. 1356 * 1357 * @param[in] pGpu OBJGPU pointer 1358 * @param[in] pKernelNvlink KernelNvlink pointer 1359 * 1360 * @return Returns the stashed FLA starting address. 1361 */ 1362 NvU64 1363 knvlinkGetNvswitchFlaAddr_IMPL 1364 ( 1365 OBJGPU *pGpu, 1366 KernelNvlink *pKernelNvlink 1367 ) 1368 { 1369 return 0; 1370 } 1371 1372 /*! 1373 * @brief Checks if fabricBaseAddr is valid. 1374 * 1375 * @param[in] pGpu OBJGPU pointer 1376 * @param[in] pKernelNvlink KernelNvlink pointer 1377 * 1378 * @return Returns true if the fabricBaseAddr is valid. 1379 */ 1380 NvBool 1381 knvlinkIsNvswitchProxyPresent_IMPL 1382 ( 1383 OBJGPU *pGpu, 1384 KernelNvlink *pKernelNvlink 1385 ) 1386 { 1387 return pKernelNvlink->bNvswitchProxy; 1388 } 1389 1390 1391 /*! 1392 * @brief Set unique FLA base address for NVSwitch enabled systems. 1393 * Validates FLA base address and programs the base address 1394 * in switch scratch registers for guest VM to pick it up. 1395 * 1396 * @param[in] pGpu OBJGPU pointer 1397 * @param[in] pKernelNvlink KernelNvlink pointer 1398 * @param[in] flaBaseAddr NvU64 base address 1399 * 1400 * @returns On success, sets unique FLA base address and returns NV_OK. 1401 * On failure, returns NV_ERR_XXX. 1402 */ 1403 NV_STATUS 1404 knvlinkSetUniqueFlaBaseAddress_IMPL 1405 ( 1406 OBJGPU *pGpu, 1407 KernelNvlink *pKernelNvlink, 1408 NvU64 flaBaseAddr 1409 ) 1410 { 1411 NV_STATUS status = NV_OK; 1412 KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu); 1413 1414 NV2080_CTRL_NVLINK_GET_SET_NVSWITCH_FLA_ADDR_PARAMS params; 1415 1416 if (!knvlinkIsForcedConfig(pGpu, pKernelNvlink)) 1417 { 1418 knvlinkCoreGetRemoteDeviceInfo(pGpu, pKernelNvlink); 1419 1420 status = knvlinkEnableLinksPostTopology_HAL(pGpu, pKernelNvlink, 1421 pKernelNvlink->enabledLinks); 1422 if (status != NV_OK) 1423 { 1424 return status; 1425 } 1426 } 1427 1428 status = kbusValidateFlaBaseAddress_HAL(pGpu, pKernelBus, flaBaseAddr); 1429 if (status != NV_OK) 1430 { 1431 NV_PRINTF(LEVEL_ERROR, "FLA base addr validation failed for GPU %x\n", 1432 pGpu->gpuInstance); 1433 return status; 1434 } 1435 1436 if (IsSLIEnabled(pGpu)) 1437 { 1438 NV_PRINTF(LEVEL_ERROR, 1439 "Operation is unsupported on SLI enabled GPU %x\n", 1440 pGpu->gpuInstance); 1441 return NV_ERR_NOT_SUPPORTED; 1442 } 1443 1444 portMemSet(¶ms, 0, sizeof(params)); 1445 params.bGet = NV_FALSE; 1446 params.addr = flaBaseAddr; 1447 1448 status = knvlinkExecGspRmRpc(pGpu, pKernelNvlink, 1449 NV2080_CTRL_CMD_NVLINK_GET_SET_NVSWITCH_FLA_ADDR, 1450 (void *)¶ms, sizeof(params)); 1451 if (status != NV_OK) 1452 { 1453 NV_PRINTF(LEVEL_ERROR, "Failed to stash fla base address for GPU %x\n", 1454 pGpu->gpuInstance); 1455 return status; 1456 } 1457 1458 NV_PRINTF(LEVEL_INFO, "FLA base addr %llx is assigned to GPU %x\n", 1459 flaBaseAddr, pGpu->gpuInstance); 1460 1461 return NV_OK; 1462 } 1463 1464 /*! 1465 * @brief Synchronize the link masks and vbios defined properties 1466 * between CPU and GSP-RMs 1467 * 1468 * @param[in] pGpu OBJGPU pointer 1469 * @param[in] pKernelNvlink KernelNvlink pointer 1470 */ 1471 NV_STATUS 1472 knvlinkSyncLinkMasksAndVbiosInfo_IMPL 1473 ( 1474 OBJGPU *pGpu, 1475 KernelNvlink *pKernelNvlink 1476 ) 1477 { 1478 NV_STATUS status = NV_OK; 1479 1480 NV2080_CTRL_NVLINK_SYNC_LINK_MASKS_AND_VBIOS_INFO_PARAMS params; 1481 1482 portMemSet(¶ms, 0, sizeof(params)); 1483 1484 params.discoveredLinks = pKernelNvlink->discoveredLinks; 1485 params.connectedLinksMask = pKernelNvlink->connectedLinksMask; 1486 params.bridgeSensableLinks = pKernelNvlink->bridgeSensableLinks; 1487 params.bridgedLinks = pKernelNvlink->bridgedLinks; 1488 1489 // Reset timeout to clear any accumulated timeouts from link init 1490 if (IS_GSP_CLIENT(pGpu)) 1491 { 1492 threadStateResetTimeout(pGpu); 1493 } 1494 1495 status = knvlinkExecGspRmRpc(pGpu, pKernelNvlink, 1496 NV2080_CTRL_CMD_NVLINK_SYNC_LINK_MASKS_AND_VBIOS_INFO, 1497 (void *)¶ms, sizeof(params)); 1498 1499 pKernelNvlink->vbiosDisabledLinkMask = params.vbiosDisabledLinkMask; 1500 pKernelNvlink->initializedLinks = params.initializedLinks; 1501 pKernelNvlink->initDisabledLinksMask = params.initDisabledLinksMask; 1502 pKernelNvlink->bEnableSafeModeAtLoad = params.bEnableSafeModeAtLoad; 1503 pKernelNvlink->bEnableTrainingAtLoad = params.bEnableTrainingAtLoad; 1504 1505 return status; 1506 } 1507 1508 /*! 1509 * @brief Update link connection status. 1510 * 1511 * @param[in] pGpu OBJGPU pointer 1512 * @param[in] pKernelNvlink KernelNvlink pointer 1513 * @param[in] linkId Target link Id 1514 */ 1515 NV_STATUS 1516 knvlinkUpdateLinkConnectionStatus_IMPL 1517 ( 1518 OBJGPU *pGpu, 1519 KernelNvlink *pKernelNvlink, 1520 NvU32 linkId 1521 ) 1522 { 1523 NV_STATUS status = NV_OK; 1524 1525 NV2080_CTRL_NVLINK_UPDATE_LINK_CONNECTION_PARAMS params; 1526 1527 portMemSet(¶ms, 0, sizeof(params)); 1528 1529 params.linkId = linkId; 1530 1531 #if defined(INCLUDE_NVLINK_LIB) 1532 1533 params.bConnected = pKernelNvlink->nvlinkLinks[linkId].remoteEndInfo.bConnected; 1534 params.remoteDeviceType = pKernelNvlink->nvlinkLinks[linkId].remoteEndInfo.deviceType; 1535 params.remoteLinkNumber = pKernelNvlink->nvlinkLinks[linkId].remoteEndInfo.linkNumber; 1536 params.remoteChipSid = pKernelNvlink->nvlinkLinks[linkId].remoteEndInfo.chipSid; 1537 params.remoteDomain = pKernelNvlink->nvlinkLinks[linkId].remoteEndInfo.domain; 1538 params.remoteBus = pKernelNvlink->nvlinkLinks[linkId].remoteEndInfo.bus; 1539 params.remoteDevice = pKernelNvlink->nvlinkLinks[linkId].remoteEndInfo.device; 1540 params.remoteFunction = pKernelNvlink->nvlinkLinks[linkId].remoteEndInfo.function; 1541 params.remotePciDeviceId = pKernelNvlink->nvlinkLinks[linkId].remoteEndInfo.pciDeviceId; 1542 params.laneRxdetStatusMask = pKernelNvlink->nvlinkLinks[linkId].laneRxdetStatusMask; 1543 1544 #endif 1545 1546 // Reset timeout to clear any accumulated timeouts from link init 1547 if (IS_GSP_CLIENT(pGpu)) 1548 { 1549 threadStateResetTimeout(pGpu); 1550 } 1551 1552 status = knvlinkExecGspRmRpc(pGpu, pKernelNvlink, 1553 NV2080_CTRL_CMD_NVLINK_UPDATE_LINK_CONNECTION, 1554 (void *)¶ms, sizeof(params)); 1555 if (status != NV_OK) 1556 { 1557 NV_PRINTF(LEVEL_ERROR, "Failed to update Link connection status!\n"); 1558 return status; 1559 } 1560 1561 return NV_OK; 1562 } 1563 1564 /*! 1565 * @brief Execute initial steps to Train links for ALI. 1566 * 1567 * @param[in] pGpu OBJGPU pointer for local GPU 1568 * @param[in] pKernelNvlink KernelNvlink pointer 1569 * @param[in] linkMask Masks of links to enable 1570 * @param[in] bSync Input sync boolean 1571 * 1572 */ 1573 NV_STATUS 1574 knvlinkPreTrainLinksToActiveAli_IMPL 1575 ( 1576 OBJGPU *pGpu, 1577 KernelNvlink *pKernelNvlink, 1578 NvU32 linkMask, 1579 NvBool bSync 1580 ) 1581 { 1582 NV_STATUS status = NV_OK; 1583 1584 NV2080_CTRL_NVLINK_PRE_LINK_TRAIN_ALI_PARAMS params; 1585 1586 portMemSet(¶ms, 0, sizeof(params)); 1587 1588 params.linkMask = linkMask; 1589 params.bSync = bSync; 1590 1591 // Reset timeout to clear any accumulated timeouts from link init 1592 if (IS_GSP_CLIENT(pGpu)) 1593 { 1594 threadStateResetTimeout(pGpu); 1595 } 1596 1597 status = knvlinkExecGspRmRpc(pGpu, pKernelNvlink, 1598 NV2080_CTRL_CMD_NVLINK_PRE_LINK_TRAIN_ALI, 1599 (void *)¶ms, sizeof(params)); 1600 if (status != NV_OK) 1601 { 1602 NV_PRINTF(LEVEL_ERROR, "Failed to execute Pre Link Training ALI steps!\n"); 1603 return status; 1604 } 1605 1606 return NV_OK; 1607 } 1608 1609 /*! 1610 * @brief Train links to active for ALI. 1611 * 1612 * @param[in] pGpu OBJGPU pointer for local GPU 1613 * @param[in] pKernelNvlink KernelNvlink pointer 1614 * @param[in] linkMask Masks of links to enable 1615 * @param[in] bSync Input sync boolean 1616 * 1617 */ 1618 NV_STATUS 1619 knvlinkTrainLinksToActiveAli_IMPL 1620 ( 1621 OBJGPU *pGpu, 1622 KernelNvlink *pKernelNvlink, 1623 NvU32 linkMask, 1624 NvBool bSync 1625 ) 1626 { 1627 NV_STATUS status = NV_OK; 1628 1629 NV2080_CTRL_NVLINK_PRE_LINK_TRAIN_ALI_PARAMS params; 1630 1631 portMemSet(¶ms, 0, sizeof(params)); 1632 1633 params.linkMask = linkMask; 1634 params.bSync = bSync; 1635 1636 // Reset timeout to clear any accumulated timeouts from link init 1637 if (IS_GSP_CLIENT(pGpu)) 1638 { 1639 threadStateResetTimeout(pGpu); 1640 } 1641 1642 status = knvlinkExecGspRmRpc(pGpu, pKernelNvlink, 1643 NV2080_CTRL_CMD_NVLINK_LINK_TRAIN_ALI, 1644 (void *)¶ms, sizeof(params)); 1645 if (status != NV_OK) 1646 { 1647 NV_PRINTF(LEVEL_ERROR, "Failed to change ALI Links to active!\n"); 1648 return status; 1649 } 1650 1651 return NV_OK; 1652 } 1653 1654 /*! 1655 * @brief Update the post Rx Detect link mask. 1656 * 1657 * @param[in] pGpu OBJGPU pointer for local GPU 1658 * @param[in] pKernelNvlink KernelNvlink pointer 1659 * 1660 */ 1661 NV_STATUS 1662 knvlinkUpdatePostRxDetectLinkMask_IMPL 1663 ( 1664 OBJGPU *pGpu, 1665 KernelNvlink *pKernelNvlink 1666 ) 1667 { 1668 NV_STATUS status = NV_OK; 1669 1670 NV2080_CTRL_NVLINK_GET_LINK_MASK_POST_RX_DET_PARAMS params; 1671 1672 portMemSet(¶ms, 0, sizeof(params)); 1673 1674 status = knvlinkExecGspRmRpc(pGpu, pKernelNvlink, 1675 NV2080_CTRL_CMD_NVLINK_GET_LINK_MASK_POST_RX_DET, 1676 (void *)¶ms, sizeof(params)); 1677 if (status != NV_OK) 1678 { 1679 NV_PRINTF(LEVEL_ERROR, "Failed to update Rx Detect Link mask!\n"); 1680 return status; 1681 } 1682 1683 pKernelNvlink->postRxDetLinkMask = params.postRxDetLinkMask; 1684 1685 return NV_OK; 1686 } 1687 1688 /*! 1689 * @brief Copy over the NVLink devices information from GSP-RM. 1690 * 1691 * @param[in] pGpu OBJGPU pointer for local GPU 1692 * @param[in] pKernelNvlink KernelNvlink pointer 1693 */ 1694 NV_STATUS 1695 knvlinkCopyNvlinkDeviceInfo_IMPL 1696 ( 1697 OBJGPU *pGpu, 1698 KernelNvlink *pKernelNvlink 1699 ) 1700 { 1701 NV_STATUS status = NV_OK; 1702 NvU32 i; 1703 1704 NV2080_CTRL_NVLINK_GET_NVLINK_DEVICE_INFO_PARAMS nvlinkInfoParams; 1705 1706 portMemSet(&nvlinkInfoParams, 0, sizeof(nvlinkInfoParams)); 1707 1708 status = knvlinkExecGspRmRpc(pGpu, pKernelNvlink, 1709 NV2080_CTRL_CMD_NVLINK_GET_NVLINK_DEVICE_INFO, 1710 (void *)&nvlinkInfoParams, sizeof(nvlinkInfoParams)); 1711 1712 if (status == NV_ERR_NOT_SUPPORTED) 1713 { 1714 NV_PRINTF(LEVEL_WARNING, "NVLink is unavailable\n"); 1715 return status; 1716 } 1717 else if (status != NV_OK) 1718 { 1719 NV_PRINTF(LEVEL_ERROR, "Failed to retrieve all nvlink device info!\n"); 1720 return status; 1721 } 1722 1723 // Update CPU-RM's NVLink state with the information received from GSP-RM RPC 1724 pKernelNvlink->ioctrlMask = nvlinkInfoParams.ioctrlMask; 1725 pKernelNvlink->ioctrlNumEntries = nvlinkInfoParams.ioctrlNumEntries; 1726 pKernelNvlink->ioctrlSize = nvlinkInfoParams.ioctrlSize; 1727 pKernelNvlink->discoveredLinks = nvlinkInfoParams.discoveredLinks; 1728 pKernelNvlink->ipVerNvlink = nvlinkInfoParams.ipVerNvlink; 1729 1730 for (i = 0; i < NVLINK_MAX_LINKS_SW; i++) 1731 { 1732 pKernelNvlink->nvlinkLinks[i].pGpu = pGpu; 1733 pKernelNvlink->nvlinkLinks[i].bValid = nvlinkInfoParams.linkInfo[i].bValid; 1734 pKernelNvlink->nvlinkLinks[i].linkId = nvlinkInfoParams.linkInfo[i].linkId; 1735 pKernelNvlink->nvlinkLinks[i].ioctrlId = nvlinkInfoParams.linkInfo[i].ioctrlId; 1736 1737 // Copy over the link PLL master and slave relationship for each link 1738 pKernelNvlink->nvlinkLinks[i].pllMasterLinkId = nvlinkInfoParams.linkInfo[i].pllMasterLinkId; 1739 pKernelNvlink->nvlinkLinks[i].pllSlaveLinkId = nvlinkInfoParams.linkInfo[i].pllSlaveLinkId; 1740 1741 // Copy over the ip versions for DLPL devices discovered 1742 pKernelNvlink->nvlinkLinks[i].ipVerDlPl = nvlinkInfoParams.linkInfo[i].ipVerDlPl; 1743 } 1744 1745 return NV_OK; 1746 } 1747 1748 /*! 1749 * @brief Copy over the Ioctrl devices information from GSP-RM. 1750 * 1751 * @param[in] pGpu OBJGPU pointer for local GPU 1752 * @param[in] pKernelNvlink KernelNvlink pointer 1753 */ 1754 NV_STATUS 1755 knvlinkCopyIoctrlDeviceInfo_IMPL 1756 ( 1757 OBJGPU *pGpu, 1758 KernelNvlink *pKernelNvlink 1759 ) 1760 { 1761 KernelIoctrl *pKernelIoctrl = NULL; 1762 NV_STATUS status = NV_OK; 1763 NvU32 ioctrlIdx; 1764 1765 NV2080_CTRL_NVLINK_GET_IOCTRL_DEVICE_INFO_PARAMS ioctrlInfoParams; 1766 1767 // Query the IOCTRL information for each of the IOCTRLs discovered 1768 FOR_EACH_INDEX_IN_MASK(32, ioctrlIdx, pKernelNvlink->ioctrlMask) 1769 { 1770 portMemSet(&ioctrlInfoParams, 0, sizeof(ioctrlInfoParams)); 1771 1772 ioctrlInfoParams.ioctrlIdx = ioctrlIdx; 1773 1774 status = knvlinkExecGspRmRpc(pGpu, pKernelNvlink, 1775 NV2080_CTRL_CMD_NVLINK_GET_IOCTRL_DEVICE_INFO, 1776 (void *)&ioctrlInfoParams, sizeof(ioctrlInfoParams)); 1777 1778 if (status == NV_ERR_NOT_SUPPORTED) 1779 { 1780 NV_PRINTF(LEVEL_WARNING, "NVLink is unavailable\n"); 1781 return status; 1782 } 1783 else if (status != NV_OK) 1784 { 1785 NV_PRINTF(LEVEL_ERROR, "Failed to retrieve device info for IOCTRL %d!\n", ioctrlIdx); 1786 return status; 1787 } 1788 1789 pKernelIoctrl = KNVLINK_GET_IOCTRL(pKernelNvlink, ioctrlIdx); 1790 1791 // Update CPU-RM's NVLink state with the information received from GSP-RM RPC 1792 pKernelIoctrl->PublicId = ioctrlInfoParams.PublicId; 1793 pKernelIoctrl->localDiscoveredLinks = ioctrlInfoParams.localDiscoveredLinks; 1794 pKernelIoctrl->localGlobalLinkOffset = ioctrlInfoParams.localGlobalLinkOffset; 1795 pKernelIoctrl->ioctrlDiscoverySize = ioctrlInfoParams.ioctrlDiscoverySize; 1796 pKernelIoctrl->numDevices = ioctrlInfoParams.numDevices; 1797 1798 // Copy over the ip versions for the ioctrl and minion devices discovered 1799 pKernelIoctrl->ipVerIoctrl = ioctrlInfoParams.ipRevisions.ipVerIoctrl; 1800 pKernelIoctrl->ipVerMinion = ioctrlInfoParams.ipRevisions.ipVerMinion; 1801 1802 if (pKernelIoctrl->ipVerMinion == 0) 1803 { 1804 pKernelIoctrl->setProperty(pKernelIoctrl, PDB_PROP_KIOCTRL_MINION_AVAILABLE, NV_FALSE); 1805 } 1806 } 1807 FOR_EACH_INDEX_IN_MASK_END; 1808 1809 return NV_OK; 1810 } 1811 1812 /** 1813 * @brief Setup topology information for the forced nvlink configurations 1814 * 1815 * @param[in] pGpu OBJGPU pointer for local GPU 1816 * @param[in] pKernelNvlink KernelNvlink pointer 1817 */ 1818 NV_STATUS 1819 knvlinkSetupTopologyForForcedConfig_IMPL 1820 ( 1821 OBJGPU *pGpu, 1822 KernelNvlink *pKernelNvlink 1823 ) 1824 { 1825 NV_STATUS status = NV_OK; 1826 NvU32 i, physLink; 1827 1828 // Start with all links disabled and no forced config in effect 1829 pKernelNvlink->bRegistryLinkOverride = NV_TRUE; 1830 pKernelNvlink->registryLinkMask = 0; 1831 pKernelNvlink->bChiplibConfig = NV_FALSE; 1832 1833 for (i = 0; i < NVLINK_MAX_LINKS_SW; i++) 1834 { 1835 // Filter against the links discovered from IOCTRL 1836 if (!(pKernelNvlink->discoveredLinks & NVBIT(i))) 1837 continue; 1838 1839 // The physical link is guaranteed valid in all cases 1840 physLink = DRF_VAL(_NVLINK, _ARCH_CONNECTION, _PHYSICAL_LINK, pKernelNvlink->pLinkConnection[i]); 1841 1842 // Update link tracking 1843 if (DRF_VAL(_NVLINK, _ARCH_CONNECTION, _ENABLED, pKernelNvlink->pLinkConnection[i])) 1844 { 1845 NV_PRINTF(LEVEL_INFO, 1846 "ARCH_CONNECTION info from chiplib: ENABLED Logical link %d (Physical " 1847 "link %d) = 0x%X\n", i, physLink, 1848 pKernelNvlink->pLinkConnection[i]); 1849 1850 // 1851 // This "link" should be ENABLED. We use the physical link since RM only deals with 1852 // physical links. 1853 // 1854 pKernelNvlink->registryLinkMask |= NVBIT(physLink); 1855 1856 // Config is forced (at least one link requested) 1857 pKernelNvlink->bChiplibConfig = NV_TRUE; 1858 } 1859 else 1860 { 1861 NV_PRINTF(LEVEL_INFO, 1862 "ARCH_CONNECTION info from chiplib: DISABLED Logical link %d (Physical " 1863 "link %d) = 0x%X\n", i, physLink, 1864 pKernelNvlink->pLinkConnection[i]); 1865 } 1866 1867 // Accumulate any PEER links 1868 if (DRF_VAL(_NVLINK, _ARCH_CONNECTION, _PEER_MASK, pKernelNvlink->pLinkConnection[i])) 1869 { 1870 #if defined(INCLUDE_NVLINK_LIB) 1871 // Ensure reginit has the info it needs for the remote side 1872 pKernelNvlink->nvlinkLinks[i].remoteEndInfo.bConnected = NV_TRUE; 1873 pKernelNvlink->nvlinkLinks[i].remoteEndInfo.deviceType = 1874 NV2080_CTRL_NVLINK_DEVICE_INFO_DEVICE_TYPE_GPU; 1875 1876 #endif 1877 } 1878 1879 // Accumulate any CPU links 1880 if (DRF_VAL(_NVLINK, _ARCH_CONNECTION, _CPU, pKernelNvlink->pLinkConnection[i])) 1881 { 1882 #if defined(INCLUDE_NVLINK_LIB) 1883 // Ensure reginit has the info it needs for the remote side 1884 pKernelNvlink->nvlinkLinks[i].remoteEndInfo.bConnected = NV_TRUE; 1885 pKernelNvlink->nvlinkLinks[i].remoteEndInfo.deviceType = pKernelNvlink->forcedSysmemDeviceType; 1886 #endif 1887 } 1888 1889 // RPC into GSP-RM to update the link remote connection status 1890 status = knvlinkUpdateLinkConnectionStatus(pGpu, pKernelNvlink, i); 1891 if (status != NV_OK) 1892 { 1893 return status; 1894 } 1895 } 1896 1897 // Update enabledLinks mask with the mask of forced link configurations 1898 pKernelNvlink->enabledLinks = pKernelNvlink->discoveredLinks & pKernelNvlink->registryLinkMask; 1899 1900 return NV_OK; 1901 } 1902 1903 /*! 1904 * @brief Sync the lane shutdown properties with GSP-RM 1905 * 1906 * @param[in] pGpu OBJGPU pointer 1907 * @param[in] pKernelNvlink KernelNvlink pointer 1908 */ 1909 NV_STATUS 1910 knvlinkSyncLaneShutdownProps_IMPL 1911 ( 1912 OBJGPU *pGpu, 1913 KernelNvlink *pKernelNvlink 1914 ) 1915 { 1916 NV_STATUS status = NV_OK; 1917 1918 NV2080_CTRL_NVLINK_SYNC_NVLINK_SHUTDOWN_PROPS_PARAMS params; 1919 1920 portMemSet(¶ms, 0, sizeof(params)); 1921 1922 params.bLaneShutdownEnabled = 1923 pKernelNvlink->getProperty(pKernelNvlink, PDB_PROP_KNVLINK_LANE_SHUTDOWN_ENABLED); 1924 params.bLaneShutdownOnUnload = 1925 pKernelNvlink->getProperty(pKernelNvlink, PDB_PROP_KNVLINK_LANE_SHUTDOWN_ON_UNLOAD); 1926 1927 status = knvlinkExecGspRmRpc(pGpu, pKernelNvlink, 1928 NV2080_CTRL_CMD_NVLINK_SYNC_NVLINK_SHUTDOWN_PROPS, 1929 (void *)¶ms, sizeof(params)); 1930 if (status != NV_OK) 1931 { 1932 NV_PRINTF(LEVEL_ERROR, "Failed to sync NVLink shutdown properties with GSP!\n"); 1933 return status; 1934 } 1935 1936 return NV_OK; 1937 } 1938 1939 /*! 1940 * @brief Get the number of active links allowed per IOCTRL 1941 * 1942 * @param[in] pGpu OBJGPU pointer 1943 * @param[in] pKernelNvlink KernelNvlink pointer 1944 * 1945 * @returns On success, returns the number of active links per IOCTRL. 1946 * On failure, returns 0. 1947 */ 1948 NvU32 1949 knvlinkGetNumActiveLinksPerIoctrl_IMPL 1950 ( 1951 OBJGPU *pGpu, 1952 KernelNvlink *pKernelNvlink 1953 ) 1954 { 1955 NV_STATUS status; 1956 NV2080_CTRL_INTERNAL_NVLINK_GET_NUM_ACTIVE_LINK_PER_IOCTRL_PARAMS params; 1957 portMemSet(¶ms, 0, sizeof(params)); 1958 status = knvlinkExecGspRmRpc(pGpu, pKernelNvlink, 1959 NV2080_CTRL_INTERNAL_NVLINK_GET_NUM_ACTIVE_LINK_PER_IOCTRL, 1960 (void *)¶ms, sizeof(params)); 1961 if (status != NV_OK) 1962 { 1963 NV_PRINTF(LEVEL_ERROR, "Failed to get the number of active links per IOCTRL\n"); 1964 return 0; 1965 } 1966 return params.numActiveLinksPerIoctrl; 1967 } 1968 1969 /*! 1970 * @brief Get the number of total links per IOCTRL 1971 * 1972 * @param[in] pGpu OBJGPU pointer 1973 * @param[in] pKernelNvlink KernelNvlink pointer 1974 * 1975 * @returns On success, returns the number of total links per IOCTRL. 1976 * On failure, returns 0. 1977 */ 1978 NvU32 1979 knvlinkGetTotalNumLinksPerIoctrl_IMPL 1980 ( 1981 OBJGPU *pGpu, 1982 KernelNvlink *pKernelNvlink 1983 ) 1984 { 1985 NV_STATUS status; 1986 NV2080_CTRL_INTERNAL_NVLINK_GET_TOTAL_NUM_LINK_PER_IOCTRL_PARAMS params; 1987 portMemSet(¶ms, 0, sizeof(params)); 1988 status = knvlinkExecGspRmRpc(pGpu, pKernelNvlink, 1989 NV2080_CTRL_INTERNAL_NVLINK_GET_TOTAL_NUM_LINK_PER_IOCTRL, 1990 (void *)¶ms, sizeof(params)); 1991 if (status != NV_OK) 1992 { 1993 NV_PRINTF(LEVEL_ERROR, "Failed to get the total number of links per IOCTRL\n"); 1994 return 0; 1995 } 1996 return params.numLinksPerIoctrl; 1997 } 1998 1999 /** 2000 * @brief Process the mask of init disabled links 2001 * 2002 * @param[in] pGpu OBJGPU pointer 2003 * @param[in] pKernelNvlink KernelNvlink pointer 2004 */ 2005 NV_STATUS 2006 knvlinkProcessInitDisabledLinks_IMPL 2007 ( 2008 OBJGPU *pGpu, 2009 KernelNvlink *pKernelNvlink 2010 ) 2011 { 2012 NvU32 mask = 0; 2013 NvBool bSkipHwNvlinkDisable = 0; 2014 NV_STATUS status = NV_OK; 2015 2016 NV2080_CTRL_NVLINK_PROCESS_INIT_DISABLED_LINKS_PARAMS params; 2017 2018 status = gpumgrGetGpuInitDisabledNvlinks(pGpu->gpuId, &mask, &bSkipHwNvlinkDisable); 2019 if (status != NV_OK) 2020 { 2021 NV_PRINTF(LEVEL_ERROR, "Failed to get init disabled links from gpumgr\n"); 2022 return status; 2023 } 2024 2025 portMemSet(¶ms, 0, sizeof(params)); 2026 2027 params.initDisabledLinksMask = mask; 2028 params.bSkipHwNvlinkDisable = bSkipHwNvlinkDisable; 2029 2030 status = knvlinkExecGspRmRpc(pGpu, pKernelNvlink, 2031 NV2080_CTRL_CMD_NVLINK_PROCESS_INIT_DISABLED_LINKS, 2032 (void *)¶ms, sizeof(params)); 2033 if (status != NV_OK) 2034 { 2035 NV_PRINTF(LEVEL_ERROR, "Failed to process init disabled links in GSP\n"); 2036 return status; 2037 } 2038 2039 pKernelNvlink->initDisabledLinksMask = params.initDisabledLinksMask; 2040 2041 return NV_OK; 2042 } 2043 2044 void 2045 knvlinkFatalErrorRecovery_WORKITEM 2046 ( 2047 NvU32 gpuInstance, 2048 void *pArgs 2049 ) 2050 { 2051 OBJGPU *pGpu = gpumgrGetGpu(gpuInstance); 2052 rcAndDisableOutstandingClientsWithImportedMemory(pGpu, NV_FABRIC_INVALID_NODE_ID); 2053 } 2054 2055 NV_STATUS 2056 knvlinkFatalErrorRecovery_IMPL 2057 ( 2058 OBJGPU *pGpu, 2059 KernelNvlink *pKernelNvlink 2060 ) 2061 { 2062 NV_STATUS status; 2063 2064 status = osQueueWorkItemWithFlags(pGpu, knvlinkFatalErrorRecovery_WORKITEM, NULL, 2065 (OS_QUEUE_WORKITEM_FLAGS_LOCK_SEMA | 2066 OS_QUEUE_WORKITEM_FLAGS_LOCK_API_RW | 2067 OS_QUEUE_WORKITEM_FLAGS_LOCK_GPU_GROUP_SUBDEVICE_RW)); 2068 2069 return status; 2070 } 2071 2072 // Grab GPU locks before RPCing into GSP-RM for NVLink RPCs 2073 NV_STATUS 2074 knvlinkExecGspRmRpc_IMPL 2075 ( 2076 OBJGPU *pGpu, 2077 KernelNvlink *pKernelNvlink, 2078 NvU32 cmd, 2079 void *paramAddr, 2080 NvU32 paramSize 2081 ) 2082 { 2083 NvU32 gpuMaskRelease = 0; 2084 NvU32 gpuMaskInitial = rmGpuLocksGetOwnedMask(); 2085 NvU32 gpuMask = gpuMaskInitial | NVBIT(pGpu->gpuInstance); 2086 NV_STATUS status = NV_OK; 2087 2088 if (IS_GSP_CLIENT(pGpu)) 2089 { 2090 if (!rmGpuGroupLockIsOwner(pGpu->gpuInstance, GPU_LOCK_GRP_MASK, &gpuMask)) 2091 { 2092 status = rmGpuGroupLockAcquire(pGpu->gpuInstance, 2093 GPU_LOCK_GRP_MASK, 2094 GPU_LOCK_FLAGS_SAFE_LOCK_UPGRADE, 2095 RM_LOCK_MODULES_NVLINK, 2096 &gpuMask); 2097 if (status != NV_OK) 2098 { 2099 NV_PRINTF(LEVEL_ERROR, "Failed to acquire locks for gpumask 0x%x\n", gpuMask); 2100 return status; 2101 } 2102 2103 gpuMaskRelease = (gpuMask & (~gpuMaskInitial)); 2104 } 2105 } 2106 2107 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); 2108 status = pRmApi->Control(pRmApi, 2109 pGpu->hInternalClient, 2110 pGpu->hInternalSubdevice, 2111 cmd, paramAddr, paramSize); 2112 if (gpuMaskRelease) 2113 { 2114 rmGpuGroupLockRelease(gpuMaskRelease, GPUS_LOCK_FLAGS_NONE); 2115 } 2116 2117 return status; 2118 } 2119 2120 void 2121 knvlinkUtoa(NvU8 *str, NvU64 length, NvU64 val) 2122 { 2123 NvU8 temp[NV2080_GPU_MAX_NAME_STRING_LENGTH]; 2124 NvU8 *ptr = temp; 2125 NvU64 i = 0; 2126 2127 NV_ASSERT(str != NULL); 2128 2129 do 2130 { 2131 i = val % 10; 2132 val = val / 10; 2133 *ptr++ = (NvU8)(i + '0'); 2134 } while(val); 2135 2136 NV_ASSERT(length > (NvU64) (ptr - temp)); 2137 2138 while (ptr > temp) 2139 *str++ = *--ptr; 2140 2141 *str = '\0'; 2142 } 2143