1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include "core/core.h" 25 #include "gpu/gpu.h" 26 #include "mem_mgr/vaspace.h" 27 #include "mem_mgr/io_vaspace.h" 28 #include "mem_mgr/gpu_vaspace.h" 29 #include "gpu/mmu/kern_gmmu.h" 30 #include "gpu/bus/kern_bus.h" 31 #include "kernel/gpu/mem_mgr/mem_mgr.h" 32 #include "kernel/gpu/mem_sys/kern_mem_sys.h" 33 #include "kernel/gpu/nvbitmask.h" 34 #include "platform/chipset/chipset.h" 35 #include "rmapi/client.h" 36 #include "nvdevid.h" 37 38 #include "gpu/subdevice/subdevice.h" 39 #include "gpu/gsp/gsp_static_config.h" 40 #include "vgpu/rpc.h" 41 42 #include "nvRmReg.h" 43 44 static NV_STATUS kbusInitRegistryOverrides(OBJGPU *pGpu, KernelBus *pKernelBus); 45 46 NV_STATUS 47 kbusConstructEngine_IMPL(OBJGPU *pGpu, KernelBus *pKernelBus, ENGDESCRIPTOR engDesc) 48 { 49 NV_STATUS status; 50 51 if (IsAMPEREorBetter(pGpu) && pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_ALL_INST_IN_SYSMEM)) 52 { 53 pKernelBus->bBar1PhysicalModeEnabled = NV_TRUE; 54 } 55 56 if (IS_VIRTUAL_WITH_SRIOV(pGpu)) 57 { 58 pKernelBus->bUsePhysicalBar2InitPagetable = NV_TRUE; 59 } 60 61 // allocate HAL private info block 62 status = kbusConstructHal_HAL(pGpu, pKernelBus); 63 if (status != NV_OK) 64 return status; 65 66 kbusInitRegistryOverrides(pGpu, pKernelBus); 67 68 kbusInitPciBars_HAL(pKernelBus); 69 70 // Special handle for VGPU. WAR for bug 3458057, bug 3458029 71 if (IS_VIRTUAL(pGpu)) 72 { 73 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 74 kbusInitBarsSize_HAL(pGpu, pKernelBus)); 75 } 76 77 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 78 kbusInitBarsBaseInfo_HAL(pKernelBus)); 79 80 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 81 kbusSetBarsApertureSize_HAL(pGpu, pKernelBus, GPU_GFID_PF)); 82 83 return NV_OK; 84 } 85 86 /*! 87 * @brief Initialize all registry overrides for this object 88 * 89 * @param[in] pGpu 90 * @param[in,out] pKernelBus 91 */ 92 static NV_STATUS 93 kbusInitRegistryOverrides(OBJGPU *pGpu, KernelBus *pKernelBus) 94 { 95 NvU32 data32; 96 97 switch (DRF_VAL(_REG_STR_RM, _INST_LOC, _BAR_PTE, pGpu->instLocOverrides)) 98 { 99 default: 100 case NV_REG_STR_RM_INST_LOC_BAR_PTE_DEFAULT: 101 // Do not override on default.. 102 break; 103 case NV_REG_STR_RM_INST_LOC_BAR_PTE_VID: 104 pKernelBus->PTEBAR2Aperture = ADDR_FBMEM; 105 pKernelBus->PTEBAR2Attr = NV_MEMORY_WRITECOMBINED; 106 break; 107 108 case NV_REG_STR_RM_INST_LOC_BAR_PTE_COH: 109 if (gpuIsBarPteInSysmemSupported(pGpu) || !gpuIsRegUsesGlobalSurfaceOverridesEnabled(pGpu)) 110 { 111 pKernelBus->PTEBAR2Aperture = ADDR_SYSMEM; 112 pKernelBus->PTEBAR2Attr = NV_MEMORY_CACHED; 113 } 114 else 115 { 116 // 117 // BAR PTEs in sysmem is not supported on all hardware. 118 // HW bug 415430. Once fixed, this property will be set on supported GPUs. 119 // On unsupported GPUs where the GlobalSurfaceOverrides regkey is used, show a warning and don't override. 120 // 121 NV_PRINTF(LEVEL_WARNING, 122 "BAR PTEs not supported in sysmem. Ignoring global override request.\n"); 123 } 124 break; 125 126 case NV_REG_STR_RM_INST_LOC_BAR_PTE_NCOH: 127 if (gpuIsBarPteInSysmemSupported(pGpu) || !gpuIsRegUsesGlobalSurfaceOverridesEnabled(pGpu)) 128 { 129 pKernelBus->PTEBAR2Aperture = ADDR_SYSMEM; 130 pKernelBus->PTEBAR2Attr = NV_MEMORY_UNCACHED; 131 } 132 else 133 { 134 // BAR PTEs in sysmem is not supported on current hardware. See above. 135 NV_PRINTF(LEVEL_WARNING, 136 "BAR PTEs not supported in sysmem. Ignoring global override request.\n"); 137 } 138 break; 139 } 140 141 NV_PRINTF(LEVEL_INFO, "Using aperture %d for BAR2 PTEs\n", 142 pKernelBus->PTEBAR2Aperture); 143 144 switch (DRF_VAL(_REG_STR_RM, _INST_LOC, _BAR_PDE, pGpu->instLocOverrides)) 145 { 146 default: 147 case NV_REG_STR_RM_INST_LOC_BAR_PDE_DEFAULT: 148 // Do not override on default. 149 break; 150 case NV_REG_STR_RM_INST_LOC_BAR_PDE_VID: 151 pKernelBus->PDEBAR2Aperture = ADDR_FBMEM; 152 pKernelBus->PDEBAR2Attr = NV_MEMORY_WRITECOMBINED; 153 break; 154 155 case NV_REG_STR_RM_INST_LOC_BAR_PDE_COH: 156 if (gpuIsBarPteInSysmemSupported(pGpu) || !gpuIsRegUsesGlobalSurfaceOverridesEnabled(pGpu)) 157 { 158 pKernelBus->PDEBAR2Aperture = ADDR_SYSMEM; 159 pKernelBus->PDEBAR2Attr = NV_MEMORY_CACHED; 160 } 161 else 162 { 163 // BAR PDEs in sysmem is not supported on all hardware. See above. 164 NV_PRINTF(LEVEL_WARNING, 165 "BAR PDEs not supported in sysmem. Ignoring global override request.\n"); 166 } 167 break; 168 169 case NV_REG_STR_RM_INST_LOC_BAR_PDE_NCOH: 170 if (gpuIsBarPteInSysmemSupported(pGpu) || !gpuIsRegUsesGlobalSurfaceOverridesEnabled(pGpu)) 171 { 172 pKernelBus->PDEBAR2Aperture = ADDR_SYSMEM; 173 pKernelBus->PDEBAR2Attr = NV_MEMORY_UNCACHED; 174 } 175 else 176 { 177 // BAR PDEs in sysmem is not supported on all hardware. See above. 178 NV_PRINTF(LEVEL_WARNING, 179 "BAR PDEs not supported in sysmem. Ignoring global override request.\n"); 180 } 181 break; 182 } 183 184 if (RMCFG_FEATURE_PLATFORM_WINDOWS_LDDM && !pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_TCC_MODE)) 185 { 186 // 187 // Aligns to unlinked SLI: Volta and up 188 // Next: Plan for all GPUs after validation 189 // 190 pKernelBus->bP2pMailboxClientAllocated = 191 pKernelBus->bP2pMailboxClientAllocatedBug3466714VoltaAndUp; 192 } 193 194 if (osReadRegistryDword(pGpu, NV_REG_STR_P2P_MAILBOX_CLIENT_ALLOCATED, &data32) == NV_OK) 195 { 196 pKernelBus->bP2pMailboxClientAllocated = !!data32; 197 } 198 199 return NV_OK; 200 } 201 202 /** 203 * @brief Gets the BAR1 VA range for a client 204 * 205 * @param[in] pGpu 206 * @param[in] pKernelBus 207 * @param[in] hClient Client handle 208 * @param[out] pBar1VARange BAR1 VA range 209 */ 210 211 NV_STATUS 212 kbusGetBar1VARangeForClient_IMPL(OBJGPU *pGpu, KernelBus *pKernelBus, NvHandle hClient, NV_RANGE *pBar1VARange) 213 { 214 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 215 OBJVASPACE *pBar1VAS = kbusGetBar1VASpace_HAL(pGpu, pKernelBus); 216 217 NV_ASSERT_OR_RETURN(pBar1VAS != NULL, NV_ERR_INVALID_STATE); 218 219 *pBar1VARange = rangeMake(vaspaceGetVaStart(pBar1VAS), vaspaceGetVaLimit(pBar1VAS)); 220 221 if ((pKernelMIGManager != NULL) && kmigmgrIsMIGMemPartitioningEnabled(pGpu, pKernelMIGManager) && 222 !rmclientIsCapableByHandle(hClient, NV_RM_CAP_SYS_SMC_MONITOR) && 223 !kmigmgrIsClientUsingDeviceProfiling(pGpu, pKernelMIGManager, hClient)) 224 { 225 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 226 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 227 MIG_INSTANCE_REF ref; 228 229 *pBar1VARange = memmgrGetMIGPartitionableBAR1Range(pGpu, pMemoryManager); 230 231 NV_ASSERT_OK_OR_RETURN(kmigmgrGetInstanceRefFromClient(pGpu, pKernelMIGManager, 232 hClient, &ref)); 233 NV_ASSERT_OK_OR_RETURN(kmemsysSwizzIdToMIGMemRange(pGpu, pKernelMemorySystem, ref.pKernelMIGGpuInstance->swizzId, 234 *pBar1VARange, pBar1VARange)); 235 } 236 return NV_OK; 237 } 238 239 RmPhysAddr 240 kbusSetupPeerBarAccess_IMPL 241 ( 242 OBJGPU *pLocalGpu, 243 OBJGPU *pRemoteGpu, 244 RmPhysAddr base, 245 NvU64 size, 246 PMEMORY_DESCRIPTOR *ppMemDesc 247 ) 248 { 249 NV_STATUS status; 250 MEMORY_DESCRIPTOR *pMemDesc = *ppMemDesc; 251 IOVAMAPPING *pIovaMapping; 252 253 NV_ASSERT_OR_RETURN(((base & RM_PAGE_MASK) == 0), ~0ULL); 254 255 if (pMemDesc == NULL) 256 { 257 status = memdescCreate(&pMemDesc, pLocalGpu, size, 0, NV_TRUE, 258 ADDR_SYSMEM, NV_MEMORY_UNCACHED, 259 MEMDESC_FLAGS_SKIP_RESOURCE_COMPUTE); 260 NV_ASSERT_OR_RETURN(status == NV_OK, ~0ULL); 261 262 memdescDescribe(pMemDesc, ADDR_SYSMEM, base, size); 263 } 264 else 265 { 266 NV_ASSERT_OR_RETURN( 267 (memdescGetPhysAddr(pMemDesc, AT_GPU, 0) == base) && 268 (memdescGetSize(pMemDesc) == size), ~0ULL); 269 } 270 271 // 272 // Even if IOMMU-remapping fails (which it shouldn't), try to continue 273 // using the CPU physical address. In most cases, this is still sufficient. 274 // 275 status = memdescMapIommu(pMemDesc, pRemoteGpu->busInfo.iovaspaceId); 276 NV_ASSERT(status == NV_OK); 277 278 pIovaMapping = memdescGetIommuMap(pMemDesc, pRemoteGpu->busInfo.iovaspaceId); 279 280 *ppMemDesc = pMemDesc; 281 282 if (pIovaMapping == NULL) 283 { 284 NV_PRINTF(LEVEL_INFO, 285 "no IOVA mapping found for pre-existing P2P domain memdesc\n"); 286 return memdescGetPhysAddr(pMemDesc, AT_GPU, 0); 287 } 288 289 return pIovaMapping->iovaArray[0]; 290 } 291 292 /*! 293 * @brief Get the bus flush aperture flag for the NV_ADDRESS_SPACE 294 * For use with the kbusFlush_HAL() api 295 * 296 * @param[in] addrSpace NV_ADDRESS_SPACE 297 * 298 * @returns bush flush aperture flag 299 */ 300 NvU32 kbusGetFlushAperture_IMPL(KernelBus *pKernelBus, NV_ADDRESS_SPACE addrSpace) 301 { 302 return (addrSpace == ADDR_FBMEM) ? BUS_FLUSH_VIDEO_MEMORY : BUS_FLUSH_SYSTEM_MEMORY; 303 } 304 305 void 306 kbusDestruct_IMPL(KernelBus *pKernelBus) 307 { 308 OBJGPU *pGpu = ENG_GET_GPU(pKernelBus); 309 310 // 311 // We need to clean-up the memory resources for BAR2 as late as possible, 312 // and after all memory descriptors have been reclaimed. 313 // 314 kbusDestructVirtualBar2_HAL(pGpu, pKernelBus, NV_TRUE, GPU_GFID_PF); 315 316 return; 317 } 318 319 /*! Send sysmembar to all sub-devices */ 320 NV_STATUS 321 kbusSendSysmembar_IMPL 322 ( 323 OBJGPU *pGpu, 324 KernelBus *pKernelBus 325 ) 326 { 327 NV_STATUS status = NV_OK; 328 329 // Nothing to be done in guest in the paravirtualization case. 330 if (IS_VIRTUAL_WITHOUT_SRIOV(pGpu)) 331 { 332 return NV_OK; 333 } 334 335 if (kbusIsFbFlushDisabled(pKernelBus)) 336 { 337 // Eliminate FB flushes, but keep mmu invalidates 338 NV_PRINTF(LEVEL_INFO, "disable_fb_flush flag, skipping flush.\n"); 339 return NV_OK; 340 } 341 342 // Wait for the flush to flow through 343 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY); 344 pKernelBus = GPU_GET_KERNEL_BUS(pGpu); 345 if (kbusSendSysmembarSingle_HAL(pGpu, pKernelBus) == NV_ERR_TIMEOUT) 346 { 347 status = NV_ERR_TIMEOUT; 348 } 349 SLI_LOOP_END; 350 pKernelBus = GPU_GET_KERNEL_BUS(pGpu); 351 352 return status; 353 } 354 355 /** 356 * @brief Send sysmembar to a single sub-devices 357 * Trigger RPC to Physical RM. 358 * 359 * @param[in] pGpu 360 * @param[in] pKernelBus 361 */ 362 NV_STATUS 363 kbusSendSysmembarSingle_KERNEL 364 ( 365 OBJGPU *pGpu, 366 KernelBus *pKernelBus 367 ) 368 { 369 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); 370 NV_STATUS status; 371 372 status = pRmApi->Control(pRmApi, pGpu->hInternalClient, pGpu->hInternalSubdevice, 373 NV2080_CTRL_CMD_INTERNAL_BUS_FLUSH_WITH_SYSMEMBAR, 374 NULL, 0); 375 376 return status; 377 } 378 379 /*! 380 * @brief Commit BAR2 381 * 382 * @param[in] pGpu 383 * @param[in] pKernelBus 384 * @param[in] flags GPU state flag (not used by Kernel RM) 385 * 386 * @returns NV_OK on success. 387 */ 388 NV_STATUS 389 kbusCommitBar2_KERNEL 390 ( 391 OBJGPU *pGpu, 392 KernelBus *pKernelBus, 393 NvU32 flags 394 ) 395 { 396 if (!KBUS_BAR0_PRAMIN_DISABLED(pGpu) && 397 !kbusIsBarAccessBlocked(pKernelBus) && 398 !(flags & GPU_STATE_FLAGS_GC6_TRANSITION)) 399 { 400 // we will initialize bar2 to the default big page size of the system 401 NV_ASSERT_OK_OR_RETURN(kbusInitVirtualBar2_HAL(pGpu, pKernelBus)); 402 NV_ASSERT_OK_OR_RETURN(kbusSetupCpuPointerForBusFlush_HAL(pGpu, pKernelBus)); 403 } 404 return NV_OK; 405 } 406 407 /*! Get pci bar size in BYTE */ 408 NvU64 409 kbusGetPciBarSize_IMPL(KernelBus *pKernelBus, NvU32 index) 410 { 411 if (index >= pKernelBus->totalPciBars) 412 { 413 NV_PRINTF(LEVEL_ERROR, "bad index 0x%x\n", index); 414 return 0; 415 } 416 417 return pKernelBus->pciBarSizes[index]; 418 } 419 420 RmPhysAddr 421 kbusGetPciBarOffset_IMPL(KernelBus *pKernelBus, NvU32 index) 422 { 423 RmPhysAddr offset = 0x0; 424 425 if (index < pKernelBus->totalPciBars) 426 { 427 offset = pKernelBus->pciBars[index]; 428 } 429 else 430 { 431 NV_PRINTF(LEVEL_ERROR, "bad index 0x%x\n", index); 432 } 433 434 return offset; 435 } 436 437 /** 438 * @brief Determine bBar1Force64KBMapping base on regkey and bar1 size 439 * Determine if 64KB mappings need to be forced based on total BAR1 size. 440 * Default threshold is 256MB unless overridden by regkey 441 * Force 64KB for SKUs with BAR1 size <= 256MB 442 * 443 * @param[in] pKernelBus 444 */ 445 void 446 kbusDetermineBar1Force64KBMapping_IMPL 447 ( 448 KernelBus *pKernelBus 449 ) 450 { 451 OBJGPU* pGpu = ENG_GET_GPU(pKernelBus); 452 NvU32 data; 453 454 pKernelBus->bBar1Force64KBMapping = NV_TRUE; 455 456 if (osReadRegistryDword(pGpu, NV_REG_STR_RM_64KB_BAR1_MAPPINGS, 457 &data) == NV_OK) 458 { 459 if (data == NV_REG_STR_RM_64KB_BAR1_MAPPINGS_DISABLED) 460 { 461 pKernelBus->bBar1Force64KBMapping = NV_FALSE; 462 } 463 } 464 else 465 { 466 NvU32 bar1SizeMB; 467 bar1SizeMB = (NvU32)(kbusGetPciBarSize(pKernelBus, 1) >> 20); 468 469 if (bar1SizeMB > 256) 470 { 471 pKernelBus->bBar1Force64KBMapping = NV_FALSE; 472 } 473 } 474 } 475 476 477 /** 478 * @brief Determine bar1[gfid].apertureLength base on regkey and bar1 size 479 * 480 * @param[in] pKernelBus 481 * @param[in] gfid 482 */ 483 void 484 kbusDetermineBar1ApertureLength_IMPL 485 ( 486 KernelBus *pKernelBus, 487 NvU32 gfid 488 ) 489 { 490 OBJGPU *pGpu = ENG_GET_GPU(pKernelBus); 491 NvU32 data32; 492 493 if (IS_GFID_VF(gfid)) 494 { 495 pKernelBus->bar1[gfid].apertureLength = pGpu->sriovState.vfBarSize[1]; 496 } 497 else 498 { 499 pKernelBus->bar1[gfid].apertureLength = kbusGetPciBarSize(pKernelBus, 1); 500 } 501 502 // We can shrink BAR1 using this reg key but cannot grow it. 503 if (((NV_OK == osReadRegistryDword(pGpu, 504 NV_REG_STR_RM_BAR1_APERTURE_SIZE_MB, &data32))) && 505 data32 && (((NvU64)data32 << 20) < pKernelBus->bar1[gfid].apertureLength)) 506 { 507 // Set BAR1 aperture length based on the override 508 pKernelBus->bar1[gfid].apertureLength = (NvU64) data32 << 20; 509 } 510 511 } 512 513 /*! 514 * @brief Initialize pciBarSizes[], set pKernelBus->bPciBarSizesValid 515 * Trigger an internal RMAPI to get the data from Physical RM. 516 * 517 * @param[in] pGpu 518 * @param[in] pKernelBus 519 */ 520 NV_STATUS 521 kbusInitBarsSize_KERNEL 522 ( 523 OBJGPU *pGpu, 524 KernelBus *pKernelBus 525 ) 526 { 527 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); 528 NV2080_CTRL_BUS_GET_PCI_BAR_INFO_PARAMS params; 529 NvU32 i; 530 531 NV_ASSERT( ! pKernelBus->bPciBarSizesValid); 532 533 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 534 pRmApi->Control(pRmApi, pGpu->hInternalClient, pGpu->hInternalSubdevice, 535 NV2080_CTRL_CMD_BUS_GET_PCI_BAR_INFO, 536 ¶ms, sizeof(params))); 537 538 for (i = 0; i< params.pciBarCount; i++) 539 { 540 pKernelBus->pciBarSizes[i] = params.pciBarInfo[i].barSizeBytes; 541 } 542 543 pKernelBus->bPciBarSizesValid = NV_TRUE; 544 545 return NV_OK; 546 } 547 548 /*! 549 * @brief Remove P2P mapping to a given peer GPU 550 * 551 * @param[in] pGpu0 (Local) 552 * @param[in] pKernelBus0 (Local) 553 * @param[in] pGpu1 (Remote) 554 * @param[in] peerIdx 555 * 556 * return NV_OK on success 557 */ 558 void 559 kbusDestroyMailbox_IMPL 560 ( 561 OBJGPU *pGpu0, 562 KernelBus *pKernelBus0, 563 OBJGPU *pGpu1, 564 NvU32 peerIdx 565 ) 566 { 567 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu0); 568 NvBool bNeedWarBug999673 = kbusNeedWarForBug999673_HAL(pGpu0, pKernelBus0, pGpu1) || 569 kbusNeedWarForBug999673_HAL(pGpu1, GPU_GET_KERNEL_BUS(pGpu1), pGpu0); 570 NV2080_CTRL_INTERNAL_BUS_DESTROY_P2P_MAILBOX_PARAMS busParams = {0}; 571 NV2080_CTRL_INTERNAL_HSHUB_PEER_CONN_CONFIG_PARAMS hshubParams = {0}; 572 NV_STATUS status; 573 574 kbusDestroyPeerAccess_HAL(pGpu0, pKernelBus0, peerIdx); 575 576 busParams.peerIdx = peerIdx; 577 busParams.bNeedWarBug999673 = bNeedWarBug999673; 578 status = pRmApi->Control(pRmApi, pGpu0->hInternalClient, pGpu0->hInternalSubdevice, 579 NV2080_CTRL_CMD_INTERNAL_BUS_DESTROY_P2P_MAILBOX, 580 &busParams, sizeof(busParams)); 581 NV_ASSERT(status == NV_OK); 582 583 // Create a peer mask for each peer to program their respective peer_connection_cfg registers 584 hshubParams.invalidatePeerMask = NVBIT32(peerIdx); 585 // Program connection_cfg registers 586 status = pRmApi->Control(pRmApi, pGpu0->hInternalClient, pGpu0->hInternalSubdevice, 587 NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG, 588 &hshubParams, sizeof(hshubParams)); 589 NV_ASSERT(status == NV_OK); 590 } 591 592 NvU8 * 593 kbusCpuOffsetInBar2WindowGet_IMPL 594 ( 595 OBJGPU *pGpu, 596 KernelBus *pKernelBus, 597 MEMORY_DESCRIPTOR *pMemDesc 598 ) 599 { 600 NV_ASSERT_OR_RETURN(NULL != pKernelBus->virtualBar2[GPU_GFID_PF].pCpuMapping, NULL); 601 NV_ASSERT_OR_RETURN(ADDR_FBMEM == pMemDesc->_addressSpace, NULL); 602 603 return (NvU8 *)(pKernelBus->virtualBar2[GPU_GFID_PF].pCpuMapping + 604 memdescGetPhysAddr(pMemDesc, AT_GPU, 0)); 605 } 606 607 /*! 608 * Calculates the BAR2 VA limit (in Byte units) which usually means the 609 * cpuVisible area limit in CPU-RM. Can be safely called only after 610 * kbusSetBarsApertureSize_HAL is executed. 611 * 612 * @param pGpu 613 * @param pKernelBus 614 * 615 * @return VA limit of BAR2 616 */ 617 NvU64 618 kbusGetVaLimitForBar2_KERNEL 619 ( 620 OBJGPU *pGpu, 621 KernelBus *pKernelBus 622 ) 623 { 624 NvU64 limit = pKernelBus->bar2[GPU_GFID_PF].cpuVisibleLimit; 625 626 NV_PRINTF(LEVEL_INFO, "va limit: 0x%llx\n", limit); 627 628 // 629 // pKernelBus->bar2.vaLimit is set by this function. 630 // Assert to ensure that this value doesn't get changed. 631 // 632 NV_ASSERT(pKernelBus->bar2[GPU_GFID_PF].vaLimit == 0 || pKernelBus->bar2[GPU_GFID_PF].vaLimit == limit); 633 634 return limit; 635 } 636 637 /*! 638 * Patch CPU-RM's SW cache of BAR1 PDB to GSP-RM's BAR1 PDB so that CPU-RM can 639 * do TLB invalidation to correct VA space. 640 * 641 * @param pGpu 642 * @param pKernelBus 643 * 644 * @return NV_OK if PDB is updated successfully 645 * Or bubble up the error code returned by the callees 646 */ 647 NV_STATUS 648 kbusPatchBar1Pdb_GSPCLIENT 649 ( 650 OBJGPU *pGpu, 651 KernelBus *pKernelBus 652 ) 653 { 654 NV_STATUS status = NV_OK; 655 OBJGVASPACE *pGVAS = dynamicCast(pKernelBus->bar1[GPU_GFID_PF].pVAS, OBJGVASPACE); 656 KernelGmmu *pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu); 657 MEMORY_DESCRIPTOR *pMemDesc = NULL; 658 GVAS_GPU_STATE *pGpuState = gvaspaceGetGpuState(pGVAS, pGpu); 659 const MMU_FMT_LEVEL *pRootFmt = pGpuState->pFmt->pRoot; 660 NvU32 rootSize = pRootFmt->entrySize; 661 MMU_WALK_USER_CTX userCtx = {0}; 662 GspStaticConfigInfo *pGSCI = GPU_GET_GSP_STATIC_INFO(pGpu); 663 664 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 665 memdescCreate(&pMemDesc, pGpu, rootSize, RM_PAGE_SIZE, NV_TRUE, ADDR_FBMEM, 666 kgmmuGetPTEAttr(pKernelGmmu), MEMDESC_FLAGS_NONE)); 667 668 memdescDescribe(pMemDesc, ADDR_FBMEM, pGSCI->bar1PdeBase, rootSize); 669 memdescSetPageSize(pMemDesc, VAS_ADDRESS_TRANSLATION(pKernelBus->bar1[GPU_GFID_PF].pVAS), RM_PAGE_SIZE); 670 671 gvaspaceWalkUserCtxAcquire(pGVAS, pGpu, NULL, &userCtx); 672 673 // 674 // Modify the CPU-RM's walker state with the new backing memory. 675 // This is intended to replace CPU-RM's PDB by GSP-RM's PDB. 676 // 677 status = mmuWalkModifyLevelInstance(pGpuState->pWalk, 678 pRootFmt, 679 vaspaceGetVaStart(pKernelBus->bar1[GPU_GFID_PF].pVAS), 680 (MMU_WALK_MEMDESC*)pMemDesc, 681 mmuFmtLevelSize(pRootFmt), 682 NV_TRUE, 683 NV_TRUE, 684 NV_FALSE); 685 gvaspaceWalkUserCtxRelease(pGVAS, &userCtx); 686 if (NV_OK != status) 687 { 688 NV_PRINTF(LEVEL_ERROR, "Failed to modify CPU-RM's BAR1 PDB to GSP-RM's BAR1 PDB.\n"); 689 return status; 690 } 691 692 gvaspaceInvalidateTlb(pGVAS, pGpu, PTE_DOWNGRADE); 693 694 return status; 695 } 696 697 /*! 698 * Patch CPU-RM's SW cache of BAR2 PDB to GSP-RM's BAR2 PDB so that CPU-RM can 699 * do TLB invalidation to correct VA space. 700 * 701 * For the BAR2 support in RM-offload model, CPU-RM owns the VA range under 702 * PDE3[0] and GSP-RM owns the VA range under PDE3[1]. GSP-RM and CPU-RM 703 * establish their own BAR2 page tables respectively. After CPU-RM establishes 704 * its own table, it passes its PDE3[0] value to GSP-RM, then GSP-RM will fill 705 * this value to PDE3[0] of GSP-RM's table (only GSP-RM's BAR2 table will be 706 * bound to HW) so that HW sees single BAR2 page table for both GSP-RM and 707 * CPU-RM. 708 * 709 * @param pGpu 710 * @param pKernelBus 711 * 712 * @return NV_OK if PDB is updated successfully 713 * Or bubble up the error code returned by the callees 714 */ 715 NV_STATUS 716 kbusPatchBar2Pdb_GSPCLIENT 717 ( 718 OBJGPU *pGpu, 719 KernelBus *pKernelBus 720 ) 721 { 722 NV_STATUS status = NV_OK; 723 PMEMORY_DESCRIPTOR pMemDesc; 724 GspStaticConfigInfo *pGSCI = GPU_GET_GSP_STATIC_INFO(pGpu); 725 const MMU_FMT_LEVEL *pRootFmt = pKernelBus->bar2[GPU_GFID_PF].pFmt->pRoot; 726 NvU64 entryValue; 727 MEMORY_DESCRIPTOR *pOldPdb; 728 729 pOldPdb = pKernelBus->virtualBar2[GPU_GFID_PF].pPDB; 730 731 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 732 memdescCreate(&pMemDesc, pGpu, pKernelBus->bar2[GPU_GFID_PF].pageDirSize, RM_PAGE_SIZE, NV_TRUE, 733 ADDR_FBMEM, pKernelBus->PDEBAR2Attr, MEMDESC_FLAGS_NONE)); 734 735 memdescDescribe(pMemDesc, ADDR_FBMEM, pGSCI->bar2PdeBase, pKernelBus->bar2[GPU_GFID_PF].pageDirSize); 736 737 // Update CPU-RM's SW cache of PDB to GSP-RM's PDB address 738 pKernelBus->virtualBar2[GPU_GFID_PF].pPDB = pMemDesc; 739 740 // 741 // BAR2 page table is not yet working at this point, so retrieving the 742 // PDE3[0] of BAR2 page table via BAR0_WINDOW or GSP-DMA (in case BARs 743 // are blocked) 744 // 745 if (kbusIsBarAccessBlocked(pKernelBus)) 746 { 747 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 748 TRANSFER_SURFACE surf = {0}; 749 750 surf.pMemDesc = pOldPdb; 751 surf.offset = 0; 752 753 NV_ASSERT_OK_OR_RETURN( 754 memmgrMemRead(pMemoryManager, &surf, &entryValue, 755 pRootFmt->entrySize, TRANSFER_FLAGS_NONE)); 756 } 757 else 758 { 759 entryValue = GPU_REG_RD32(pGpu, (NvU32)pKernelBus->bar2[GPU_GFID_PF].bar2OffsetInBar0Window) | 760 ((NvU64)GPU_REG_RD32(pGpu, (NvU32)pKernelBus->bar2[GPU_GFID_PF].bar2OffsetInBar0Window + 4) << 32); 761 } 762 763 // 764 // Provide the PDE3[0] value to GSP-RM so that GSP-RM can merge CPU-RM's 765 // page table to GSP-RM's page table 766 // 767 NV_RM_RPC_UPDATE_BAR_PDE(pGpu, NV_RPC_UPDATE_PDE_BAR_2, entryValue, pRootFmt->virtAddrBitLo, status); 768 769 return NV_OK; 770 } 771 772 773 /*! 774 * @brief Checks whether an engine is available or not. 775 * 776 * The 'engine' is an engine descriptor 777 * This function is different from busProbeRegister in a sense that it doesn't 778 * rely on timeouts after a read of a register in the reg space for engine. 779 * Instead, it 780 * - Return TRUE for all engines which are must present in GPU. 781 * - Get information about CE, MSENC, NVJPG and OFA engines from plugin. 782 * - Rest engines are determined from HAL creation data. 783 * 784 * @param[in] pGpu OBJGPU pointer 785 * @param[in] pKernelBus KernelBus pointer 786 * @param[in] engDesc ENGDESCRIPTOR pointer used to check Engine presence 787 * 788 * @returns NV_TRUE if engine is available. 789 * NV_FALSE if engine is not available or floorswept. 790 * 791 */ 792 NvBool 793 kbusCheckEngine_KERNEL 794 ( 795 OBJGPU *pGpu, 796 KernelBus *pKernelBus, 797 ENGDESCRIPTOR engDesc 798 ) 799 { 800 NvU32 rmEngineCaps[NVGPU_ENGINE_CAPS_MASK_ARRAY_MAX] = {0}; 801 NvU32 nv2080EngineCaps[NVGPU_ENGINE_CAPS_MASK_ARRAY_MAX] = {0}; 802 NvBool bSupported; 803 NV_STATUS status; 804 805 { 806 NvU32 i; 807 GspStaticConfigInfo *pGSCI = GPU_GET_GSP_STATIC_INFO(pGpu); 808 if (pGSCI == NULL) 809 { 810 return NV_FALSE; 811 } 812 813 for (i = 0; i < NVGPU_ENGINE_CAPS_MASK_ARRAY_MAX; i++) 814 { 815 nv2080EngineCaps[i] = pGSCI->engineCaps[i]; 816 } 817 } 818 819 NV_CHECK_OK_OR_ELSE(status, LEVEL_ERROR, 820 gpuGetRmEngineTypeCapMask(nv2080EngineCaps, 821 NVGPU_ENGINE_CAPS_MASK_ARRAY_MAX, 822 rmEngineCaps), 823 return NV_FALSE); 824 825 switch (engDesc) 826 { 827 case ENG_LSFM: 828 case ENG_PMU: 829 case ENG_CLK: 830 case ENG_ACR: 831 case ENG_DISP: 832 return NV_FALSE; 833 // 834 // This function is used in two environments: 835 // (a) vGPU where display is not yet supported. 836 // (b) RM offload (Kernel RM) where display is supported. 837 // 838 case ENG_KERNEL_DISPLAY: 839 return IS_GSP_CLIENT(pGpu); 840 841 case ENG_BIF: 842 case ENG_KERNEL_BIF: 843 case ENG_MC: 844 case ENG_KERNEL_MC: 845 case ENG_PRIV_RING: 846 case ENG_SW_INTR: 847 case ENG_TMR: 848 case ENG_DMA: 849 case ENG_BUS: 850 case ENG_GR(0): 851 case ENG_CIPHER: 852 case ENG_INTR: 853 case ENG_GPULOG: 854 case ENG_GPUMON: 855 case ENG_FIFO: 856 return NV_TRUE; 857 858 case ENG_CE(0): 859 case ENG_CE(1): 860 case ENG_CE(2): 861 case ENG_CE(3): 862 case ENG_CE(4): 863 case ENG_CE(5): 864 case ENG_CE(6): 865 case ENG_CE(7): 866 case ENG_CE(8): 867 case ENG_CE(9): 868 return !!NVGPU_GET_ENGINE_CAPS_MASK(rmEngineCaps, 869 RM_ENGINE_TYPE_COPY(GET_CE_IDX(engDesc))); 870 871 case ENG_MSENC(0): 872 case ENG_MSENC(1): 873 case ENG_MSENC(2): 874 return !!NVGPU_GET_ENGINE_CAPS_MASK(rmEngineCaps, 875 RM_ENGINE_TYPE_NVENC(GET_MSENC_IDX(engDesc))); 876 case ENG_SEC2: 877 return !!NVGPU_GET_ENGINE_CAPS_MASK(rmEngineCaps, 878 RM_ENGINE_TYPE_SEC2); 879 case ENG_NVDEC(0): 880 case ENG_NVDEC(1): 881 case ENG_NVDEC(2): 882 case ENG_NVDEC(3): 883 case ENG_NVDEC(4): 884 case ENG_NVDEC(5): 885 case ENG_NVDEC(6): 886 case ENG_NVDEC(7): 887 return !!NVGPU_GET_ENGINE_CAPS_MASK(rmEngineCaps, 888 RM_ENGINE_TYPE_NVDEC(GET_NVDEC_IDX(engDesc))); 889 890 case ENG_OFA: 891 return !!NVGPU_GET_ENGINE_CAPS_MASK(rmEngineCaps, 892 RM_ENGINE_TYPE_OFA); 893 case ENG_NVJPEG(0): 894 case ENG_NVJPEG(1): 895 case ENG_NVJPEG(2): 896 case ENG_NVJPEG(3): 897 case ENG_NVJPEG(4): 898 case ENG_NVJPEG(5): 899 case ENG_NVJPEG(6): 900 case ENG_NVJPEG(7): 901 return !!NVGPU_GET_ENGINE_CAPS_MASK(rmEngineCaps, 902 RM_ENGINE_TYPE_NVJPEG(GET_NVJPEG_IDX(engDesc))); 903 904 case ENG_GR(1): 905 case ENG_GR(2): 906 case ENG_GR(3): 907 case ENG_GR(4): 908 case ENG_GR(5): 909 case ENG_GR(6): 910 case ENG_GR(7): 911 { 912 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 913 914 NV_ASSERT_OR_RETURN(pKernelFifo != NULL, NV_FALSE); 915 916 return (kfifoCheckEngine_HAL(pGpu, pKernelFifo, 917 engDesc, 918 &bSupported) == NV_OK && 919 bSupported); 920 } 921 922 case ENG_INVALID: 923 NV_PRINTF(LEVEL_ERROR, 924 "Query for ENG_INVALID considered erroneous: %d\n", 925 engDesc); 926 return NV_TRUE; 927 // 928 // Check if engine descriptor is supported by current GPU. 929 // Callee must not send engine descriptor which are not on 930 // HAL lists of GPU. So Add ASSERT there. 931 // 932 default: 933 bSupported = gpuIsEngDescSupported(pGpu, engDesc); 934 935 if (!bSupported) 936 { 937 NV_PRINTF(LEVEL_ERROR, "Unable to check engine ID: %d\n", 938 engDesc); 939 NV_ASSERT(bSupported); 940 } 941 return bSupported; 942 } 943 } 944 945 // 946 // kbusGetDeviceCaps 947 // 948 // This routine gets cap bits in unicast. If bCapsInitialized is passed as 949 // NV_FALSE, the caps will be copied into pHostCaps without OR/ANDing. Otherwise, 950 // the caps bits for the current GPU will be OR/ANDed together with pHostCaps to 951 // create a single set of caps that accurately represents the functionality of 952 // the device. 953 // 954 void 955 kbusGetDeviceCaps_IMPL 956 ( 957 OBJGPU *pGpu, 958 KernelBus *pKernelBus, 959 NvU8 *pHostCaps, 960 NvBool bCapsInitialized 961 ) 962 { 963 OBJSYS *pSys = SYS_GET_INSTANCE(); 964 OBJCL *pCl = SYS_GET_CL(pSys); 965 NvU8 tempCaps[NV0080_CTRL_HOST_CAPS_TBL_SIZE], temp; 966 NvBool bExplicitCacheFlushRequired; 967 968 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 969 970 portMemSet(tempCaps, 0, NV0080_CTRL_HOST_CAPS_TBL_SIZE); 971 972 /*! DMAs to/from cached memory need to have the cache flushed explicitly */ 973 bExplicitCacheFlushRequired = NVCPU_IS_ARM && 974 (RMCFG_FEATURE_PLATFORM_UNIX || RMCFG_FEATURE_PLATFORM_MODS_UNIX); 975 if (bExplicitCacheFlushRequired || 976 (!pCl->getProperty(pCl, PDB_PROP_CL_IS_CHIPSET_IO_COHERENT))) 977 RMCTRL_SET_CAP(tempCaps, NV0080_CTRL_HOST_CAPS, _EXPLICIT_CACHE_FLUSH_REQD); 978 979 if ((pCl->FHBBusInfo.vendorID == PCI_VENDOR_ID_NVIDIA) && 980 ((pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_BR04_PRESENT)) || 981 ((pCl->FHBBusInfo.deviceID >= NVIDIA_C73_CPU_PCI_0_DEVICE_ID_SLI2) && 982 (pCl->FHBBusInfo.deviceID <= NVIDIA_C73_CPU_PCI_0_DEVICE_ID_RESERVED_3)))) 983 { 984 RMCTRL_SET_CAP(tempCaps, NV0080_CTRL_HOST_CAPS, _CPU_WRITE_WAR_BUG_420495); 985 } 986 987 // the RM always supports GPU-coherent mappings 988 RMCTRL_SET_CAP(tempCaps, NV0080_CTRL_HOST_CAPS, _GPU_COHERENT_MAPPING_SUPPORTED); 989 990 // If we don't have existing caps with which to reconcile, then just return 991 if (!bCapsInitialized) 992 { 993 portMemCopy(pHostCaps, NV0080_CTRL_HOST_CAPS_TBL_SIZE, tempCaps, NV0080_CTRL_HOST_CAPS_TBL_SIZE); 994 return; 995 } 996 997 // factor in this GPUs caps: all these are feature caps, so use AND 998 RMCTRL_AND_CAP(pHostCaps, tempCaps, temp, 999 NV0080_CTRL_HOST_CAPS, _P2P_4_WAY); 1000 RMCTRL_AND_CAP(pHostCaps, tempCaps, temp, 1001 NV0080_CTRL_HOST_CAPS, _P2P_8_WAY); 1002 RMCTRL_AND_CAP(pHostCaps, tempCaps, temp, 1003 NV0080_CTRL_HOST_CAPS, _GPU_COHERENT_MAPPING_SUPPORTED); 1004 1005 RMCTRL_OR_CAP(pHostCaps, tempCaps, temp, 1006 NV0080_CTRL_HOST_CAPS, _SEMA_ACQUIRE_BUG_105665); 1007 RMCTRL_OR_CAP(pHostCaps, tempCaps, temp, 1008 NV0080_CTRL_HOST_CAPS, _SYS_SEMA_DEADLOCK_BUG_148216); 1009 RMCTRL_OR_CAP(pHostCaps, tempCaps, temp, 1010 NV0080_CTRL_HOST_CAPS, _SLOWSLI); 1011 RMCTRL_OR_CAP(pHostCaps, tempCaps, temp, 1012 NV0080_CTRL_HOST_CAPS, _SEMA_READ_ONLY_BUG); 1013 RMCTRL_OR_CAP(pHostCaps, tempCaps, temp, 1014 NV0080_CTRL_HOST_CAPS, _MEM2MEM_BUG_365782); 1015 RMCTRL_OR_CAP(pHostCaps, tempCaps, temp, 1016 NV0080_CTRL_HOST_CAPS, _LARGE_NONCOH_UPSTR_WRITE_BUG_114871); 1017 RMCTRL_OR_CAP(pHostCaps, tempCaps, temp, 1018 NV0080_CTRL_HOST_CAPS, _LARGE_UPSTREAM_WRITE_BUG_115115); 1019 RMCTRL_OR_CAP(pHostCaps, tempCaps, temp, 1020 NV0080_CTRL_HOST_CAPS, _SEP_VIDMEM_PB_NOTIFIERS_BUG_83923); 1021 RMCTRL_OR_CAP(pHostCaps, tempCaps, temp, 1022 NV0080_CTRL_HOST_CAPS, _P2P_DEADLOCK_BUG_203825); 1023 RMCTRL_OR_CAP(pHostCaps, tempCaps, temp, 1024 NV0080_CTRL_HOST_CAPS, _COMPRESSED_BL_P2P_BUG_257072); 1025 RMCTRL_OR_CAP(pHostCaps, tempCaps, temp, 1026 NV0080_CTRL_HOST_CAPS, _CROSS_BLITS_BUG_270260); 1027 RMCTRL_OR_CAP(pHostCaps, tempCaps, temp, 1028 NV0080_CTRL_HOST_CAPS, _CPU_WRITE_WAR_BUG_420495); 1029 RMCTRL_OR_CAP(pHostCaps, tempCaps, temp, 1030 NV0080_CTRL_HOST_CAPS, _BAR1_READ_DEADLOCK_BUG_511418); 1031 1032 return; 1033 } 1034 1035 NV_STATUS 1036 kbusMapFbApertureByHandle_IMPL 1037 ( 1038 OBJGPU *pGpu, 1039 KernelBus *pKernelBus, 1040 NvHandle hClient, 1041 NvHandle hMemory, 1042 NvU64 offset, 1043 NvU64 size, 1044 NvU64 *pBar1Va, 1045 Device *pDevice 1046 ) 1047 { 1048 NV_STATUS status; 1049 RsClient *pClient = NULL; 1050 RsResourceRef *pSrcMemoryRef = NULL; 1051 Memory *pSrcMemory = NULL; 1052 MEMORY_DESCRIPTOR *pMemDesc = NULL; 1053 NvU64 fbApertureOffset = 0; 1054 NvU64 fbApertureLength = size; 1055 1056 NV_ASSERT_OK_OR_RETURN(serverGetClientUnderLock(&g_resServ, hClient, &pClient)); 1057 1058 status = clientGetResourceRef(pClient, hMemory, &pSrcMemoryRef); 1059 if (status != NV_OK) 1060 { 1061 return status; 1062 } 1063 1064 pSrcMemory = dynamicCast(pSrcMemoryRef->pResource, Memory); 1065 if (pSrcMemory == NULL) 1066 { 1067 return NV_ERR_INVALID_OBJECT; 1068 } 1069 1070 pMemDesc = pSrcMemory->pMemDesc; 1071 1072 if (memdescGetAddressSpace(pMemDesc) != ADDR_FBMEM) 1073 { 1074 return NV_ERR_INVALID_ARGUMENT; 1075 } 1076 1077 status = kbusMapFbAperture_HAL(pGpu, pKernelBus, pMemDesc, offset, 1078 &fbApertureOffset, &fbApertureLength, 1079 BUS_MAP_FB_FLAGS_MAP_UNICAST, pDevice); 1080 if (status != NV_OK) 1081 { 1082 return status; 1083 } 1084 1085 NV_ASSERT_OR_GOTO(fbApertureLength >= size, failed); 1086 1087 if ((!NV_IS_ALIGNED64(fbApertureOffset, osGetPageSize())) || 1088 (!NV_IS_ALIGNED64(fbApertureLength, osGetPageSize()))) 1089 { 1090 status = NV_ERR_NOT_SUPPORTED; 1091 goto failed; 1092 } 1093 1094 *pBar1Va = gpumgrGetGpuPhysFbAddr(pGpu) + fbApertureOffset; 1095 1096 if (!NV_IS_ALIGNED64(*pBar1Va, osGetPageSize())) 1097 { 1098 status = NV_ERR_INVALID_ADDRESS; 1099 goto failed; 1100 } 1101 1102 return NV_OK; 1103 1104 failed: 1105 // Note: fbApertureLength is not used by kbusUnmapFbAperture_HAL(), so it's passed as 0 1106 kbusUnmapFbAperture_HAL(pGpu, pKernelBus, pMemDesc, 1107 fbApertureOffset, 0, 1108 BUS_MAP_FB_FLAGS_MAP_UNICAST); 1109 1110 return status; 1111 } 1112 1113 NV_STATUS 1114 kbusUnmapFbApertureByHandle_IMPL 1115 ( 1116 OBJGPU *pGpu, 1117 KernelBus *pKernelBus, 1118 NvHandle hClient, 1119 NvHandle hMemory, 1120 NvU64 bar1Va 1121 ) 1122 { 1123 NV_STATUS status; 1124 RsClient *pClient = NULL; 1125 RsResourceRef *pSrcMemoryRef = NULL; 1126 Memory *pSrcMemory = NULL; 1127 MEMORY_DESCRIPTOR *pMemDesc = NULL; 1128 1129 NV_ASSERT_OK_OR_RETURN(serverGetClientUnderLock(&g_resServ, hClient, &pClient)); 1130 1131 status = clientGetResourceRef(pClient, hMemory, &pSrcMemoryRef); 1132 if (status != NV_OK) 1133 { 1134 return status; 1135 } 1136 1137 pSrcMemory = dynamicCast(pSrcMemoryRef->pResource, Memory); 1138 if (pSrcMemory == NULL) 1139 { 1140 return NV_ERR_INVALID_OBJECT; 1141 } 1142 1143 pMemDesc = pSrcMemory->pMemDesc; 1144 1145 // Note: fbApertureLength is not used by kbusUnmapFbAperture_HAL(), so it's passed as 0 1146 status = kbusUnmapFbAperture_HAL(pGpu, pKernelBus, pMemDesc, 1147 bar1Va - gpumgrGetGpuPhysFbAddr(pGpu), 1148 0, BUS_MAP_FB_FLAGS_MAP_UNICAST); 1149 if (status != NV_OK) 1150 { 1151 return status; 1152 } 1153 1154 return NV_OK; 1155 } 1156 1157 /*! 1158 * Helper function to determine if the requested GET_BUS_INFO ctrl call needs to be served 1159 * by GSP/host, then send RPC to GSP/host. Otherwise return directly so that the caller can 1160 * continue the execution on CPU. 1161 * 1162 * @param[in] pGpu OBJGPU pointer 1163 * @param[in/out] pBusInfo Pointer to NV2080_CTRL_BUS_INFO which specifies the index we want to query 1164 * 1165 * @returns RPC status 1166 */ 1167 NV_STATUS 1168 kbusSendBusInfo_IMPL 1169 ( 1170 OBJGPU *pGpu, 1171 KernelBus *pKernelBus, 1172 NV2080_CTRL_BUS_INFO *pBusInfo 1173 ) 1174 { 1175 NV_STATUS status = NV_OK; 1176 NV2080_CTRL_BUS_GET_INFO_V2_PARAMS busGetInfoParams = {0}; 1177 1178 busGetInfoParams.busInfoList[0] = *pBusInfo; 1179 busGetInfoParams.busInfoListSize = 1; 1180 1181 NV_RM_RPC_CONTROL(pGpu, 1182 pGpu->hInternalClient, 1183 pGpu->hInternalSubdevice, 1184 NV2080_CTRL_CMD_BUS_GET_INFO_V2, 1185 &busGetInfoParams, 1186 sizeof(busGetInfoParams), 1187 status); 1188 1189 pBusInfo->data = busGetInfoParams.busInfoList[0].data; 1190 return status; 1191 } 1192 1193 /*! 1194 * @brief Returns the Nvlink peer ID from pGpu0 to pGpu1 1195 * 1196 * @param[in] pGpu0 (local GPU) 1197 * @param[in] pKernelBus0 (local GPU) 1198 * @param[in] pGpu1 (remote GPU) 1199 * @param[in] pKernelBus1 (remote GPU) 1200 * @param[out] nvlinkPeer NvU32 pointer 1201 * 1202 * return NV_OK on success 1203 */ 1204 NV_STATUS 1205 kbusGetNvlinkP2PPeerId_VGPU 1206 ( 1207 OBJGPU *pGpu0, 1208 KernelBus *pKernelBus0, 1209 OBJGPU *pGpu1, 1210 KernelBus *pKernelBus1, 1211 NvU32 *nvlinkPeer, 1212 NvU32 flags 1213 ) 1214 { 1215 *nvlinkPeer = kbusGetPeerId_HAL(pGpu0, pKernelBus0, pGpu1); 1216 if (*nvlinkPeer != BUS_INVALID_PEER) 1217 { 1218 return NV_OK; 1219 } 1220 1221 *nvlinkPeer = kbusGetUnusedPeerId_HAL(pGpu0, pKernelBus0); 1222 1223 // If could not find a free peer ID, return error 1224 if (*nvlinkPeer == BUS_INVALID_PEER) 1225 { 1226 NV_PRINTF(LEVEL_WARNING, 1227 "GPU%d: peerID not available for NVLink P2P\n", 1228 pGpu0->gpuInstance); 1229 return NV_ERR_GENERIC; 1230 } 1231 // Reserve the peer ID for NVLink use 1232 return kbusReserveP2PPeerIds_HAL(pGpu0, pKernelBus0, NVBIT(*nvlinkPeer)); 1233 } 1234 1235 /** 1236 * @brief Check for any P2P references in to remote GPUs 1237 * which are still have a P2P api object alive. 1238 * 1239 * @param[in] pGpu 1240 * @param[in] pKernelBus 1241 */ 1242 NV_STATUS 1243 kbusIsGpuP2pAlive_IMPL 1244 ( 1245 OBJGPU *pGpu, 1246 KernelBus *pKernelBus 1247 ) 1248 { 1249 return (pKernelBus->totalP2pObjectsAliveRefCount > 0); 1250 } 1251 1252 /** 1253 * @brief Setup VF BAR2 during hibernate resume 1254 * 1255 * @param[in] pGpu 1256 * @param[in] pKernelBus 1257 * @param[in] flags 1258 */ 1259 1260