1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2017-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 /*! 25 * Provides TU102+ specific KernelGsp HAL implementations. 26 */ 27 28 #include "gpu/gsp/kernel_gsp.h" 29 30 #include "gpu/bus/kern_bus.h" 31 #include "gpu/disp/kern_disp.h" 32 #include "gpu/mem_mgr/mem_mgr.h" 33 #include "gpu/mem_sys/kern_mem_sys.h" 34 #include "vgpu/rpc.h" 35 #include "rmgspseq.h" 36 #include "core/thread_state.h" 37 #include "os/os.h" 38 #include "nverror.h" 39 #include "gsp/gsp_error.h" 40 41 #include "published/turing/tu102/dev_gsp.h" 42 #include "published/turing/tu102/dev_gsp_addendum.h" 43 #include "published/turing/tu102/dev_riscv_pri.h" 44 #include "published/turing/tu102/dev_fbif_v4.h" 45 #include "published/turing/tu102/dev_falcon_v4.h" 46 #include "published/turing/tu102/dev_fuse.h" 47 #include "published/turing/tu102/dev_ram.h" 48 #include "published/turing/tu102/dev_gc6_island.h" 49 #include "published/turing/tu102/dev_gc6_island_addendum.h" 50 51 #include "gpu/sec2/kernel_sec2.h" 52 53 #define RPC_STRUCTURES 54 #define RPC_GENERIC_UNION 55 #include "g_rpc-structures.h" 56 #undef RPC_STRUCTURES 57 #undef RPC_GENERIC_UNION 58 59 void 60 kgspConfigureFalcon_TU102 61 ( 62 OBJGPU *pGpu, 63 KernelGsp *pKernelGsp 64 ) 65 { 66 KernelFalconEngineConfig falconConfig; 67 68 portMemSet(&falconConfig, 0, sizeof(falconConfig)); 69 70 falconConfig.registerBase = DRF_BASE(NV_PGSP); 71 falconConfig.riscvRegisterBase = NV_FALCON2_GSP_BASE; 72 falconConfig.fbifBase = NV_PGSP_FBIF_BASE; 73 falconConfig.bBootFromHs = NV_FALSE; 74 falconConfig.pmcEnableMask = 0; 75 falconConfig.bIsPmcDeviceEngine = NV_FALSE; 76 falconConfig.physEngDesc = ENG_GSP; 77 78 kflcnConfigureEngine(pGpu, staticCast(pKernelGsp, KernelFalcon), &falconConfig); 79 } 80 81 /*! 82 * Check if the GSP is in debug mode 83 * 84 * @return whether the GSP is in debug mode or not 85 */ 86 NvBool 87 kgspIsDebugModeEnabled_TU102 88 ( 89 OBJGPU *pGpu, 90 KernelGsp *pKernelGsp 91 ) 92 { 93 NvU32 data; 94 95 data = GPU_REG_RD32(pGpu, NV_FUSE_OPT_SECURE_GSP_DEBUG_DIS); 96 97 return FLD_TEST_DRF(_FUSE, _OPT_SECURE_GSP_DEBUG_DIS, _DATA, _NO, data); 98 } 99 100 NV_STATUS 101 kgspAllocBootArgs_TU102 102 ( 103 OBJGPU *pGpu, 104 KernelGsp *pKernelGsp 105 ) 106 { 107 NvP64 pVa = NvP64_NULL; 108 NvP64 pPriv = NvP64_NULL; 109 NV_STATUS nvStatus = NV_OK; 110 NvU64 flags = MEMDESC_FLAGS_NONE; 111 112 // Allocate WPR meta data 113 NV_ASSERT_OK_OR_GOTO(nvStatus, 114 memdescCreate(&pKernelGsp->pWprMetaDescriptor, 115 pGpu, 0x1000, 0x1000, 116 NV_TRUE, ADDR_SYSMEM, NV_MEMORY_CACHED, 117 flags), 118 _kgspAllocBootArgs_exit_cleanup); 119 120 NV_ASSERT_OK_OR_GOTO(nvStatus, 121 memdescAlloc(pKernelGsp->pWprMetaDescriptor), 122 _kgspAllocBootArgs_exit_cleanup); 123 124 NV_ASSERT_OK_OR_GOTO(nvStatus, 125 memdescMap(pKernelGsp->pWprMetaDescriptor, 0, 126 memdescGetSize(pKernelGsp->pWprMetaDescriptor), 127 NV_TRUE, NV_PROTECT_READ_WRITE, 128 &pVa, &pPriv), 129 _kgspAllocBootArgs_exit_cleanup); 130 131 pKernelGsp->pWprMeta = (GspFwWprMeta *)NvP64_VALUE(pVa); 132 pKernelGsp->pWprMetaMappingPriv = pPriv; 133 134 // 135 // Setup libos arguments memory 136 // 137 NV_ASSERT_OK_OR_GOTO(nvStatus, 138 memdescCreate(&pKernelGsp->pLibosInitArgumentsDescriptor, 139 pGpu, 140 LIBOS_INIT_ARGUMENTS_SIZE, 141 LIBOS_INIT_ARGUMENTS_SIZE, 142 NV_TRUE, ADDR_SYSMEM, NV_MEMORY_UNCACHED, 143 flags), 144 _kgspAllocBootArgs_exit_cleanup); 145 146 NV_ASSERT_OK_OR_GOTO(nvStatus, 147 memdescAlloc(pKernelGsp->pLibosInitArgumentsDescriptor), 148 _kgspAllocBootArgs_exit_cleanup); 149 150 NV_ASSERT_OK_OR_GOTO(nvStatus, 151 memdescMap(pKernelGsp->pLibosInitArgumentsDescriptor, 0, 152 memdescGetSize(pKernelGsp->pLibosInitArgumentsDescriptor), 153 NV_TRUE, NV_PROTECT_READ_WRITE, 154 &pVa, &pPriv), 155 _kgspAllocBootArgs_exit_cleanup); 156 157 pKernelGsp->pLibosInitArgumentsCached = (LibosMemoryRegionInitArgument *)NvP64_VALUE(pVa); 158 pKernelGsp->pLibosInitArgumentsMappingPriv = pPriv; 159 160 // Setup bootloader arguments memory. 161 NV_ASSERT(sizeof(GSP_ARGUMENTS_CACHED) <= 0x1000); 162 163 NV_ASSERT_OK_OR_GOTO(nvStatus, 164 memdescCreate(&pKernelGsp->pGspArgumentsDescriptor, 165 pGpu, 0x1000, 0x1000, 166 NV_TRUE, ADDR_SYSMEM, NV_MEMORY_CACHED, 167 flags), 168 _kgspAllocBootArgs_exit_cleanup); 169 170 NV_ASSERT_OK_OR_GOTO(nvStatus, 171 memdescAlloc(pKernelGsp->pGspArgumentsDescriptor), 172 _kgspAllocBootArgs_exit_cleanup); 173 174 NV_ASSERT_OK_OR_GOTO(nvStatus, 175 memdescMap(pKernelGsp->pGspArgumentsDescriptor, 0, 176 memdescGetSize(pKernelGsp->pGspArgumentsDescriptor), 177 NV_TRUE, NV_PROTECT_READ_WRITE, 178 &pVa, &pPriv), 179 _kgspAllocBootArgs_exit_cleanup); 180 181 pKernelGsp->pGspArgumentsCached = (GSP_ARGUMENTS_CACHED *)NvP64_VALUE(pVa); 182 pKernelGsp->pGspArgumentsMappingPriv = pPriv; 183 184 return nvStatus; 185 186 _kgspAllocBootArgs_exit_cleanup: 187 kgspFreeBootArgs_HAL(pGpu, pKernelGsp); 188 return nvStatus; 189 } 190 191 void 192 kgspFreeBootArgs_TU102 193 ( 194 OBJGPU *pGpu, 195 KernelGsp *pKernelGsp 196 ) 197 { 198 // release wpr meta data resources 199 if (pKernelGsp->pWprMeta != NULL) 200 { 201 memdescUnmap(pKernelGsp->pWprMetaDescriptor, 202 NV_TRUE, osGetCurrentProcess(), 203 (void *)pKernelGsp->pWprMeta, 204 pKernelGsp->pWprMetaMappingPriv); 205 pKernelGsp->pWprMeta = NULL; 206 pKernelGsp->pWprMetaMappingPriv = NULL; 207 } 208 if (pKernelGsp->pWprMetaDescriptor != NULL) 209 { 210 memdescFree(pKernelGsp->pWprMetaDescriptor); 211 memdescDestroy(pKernelGsp->pWprMetaDescriptor); 212 pKernelGsp->pWprMetaDescriptor = NULL; 213 } 214 215 // release libos init argument resources 216 if (pKernelGsp->pLibosInitArgumentsCached != NULL) 217 { 218 memdescUnmap(pKernelGsp->pLibosInitArgumentsDescriptor, 219 NV_TRUE, osGetCurrentProcess(), 220 (void *)pKernelGsp->pLibosInitArgumentsCached, 221 pKernelGsp->pLibosInitArgumentsMappingPriv); 222 pKernelGsp->pLibosInitArgumentsCached = NULL; 223 pKernelGsp->pLibosInitArgumentsMappingPriv = NULL; 224 } 225 if (pKernelGsp->pLibosInitArgumentsDescriptor != NULL) 226 { 227 memdescFree(pKernelGsp->pLibosInitArgumentsDescriptor); 228 memdescDestroy(pKernelGsp->pLibosInitArgumentsDescriptor); 229 pKernelGsp->pLibosInitArgumentsDescriptor = NULL; 230 } 231 232 // release init argument page resources 233 if (pKernelGsp->pGspArgumentsCached != NULL) 234 { 235 memdescUnmap(pKernelGsp->pGspArgumentsDescriptor, 236 NV_TRUE, osGetCurrentProcess(), 237 (void *)pKernelGsp->pGspArgumentsCached, 238 pKernelGsp->pGspArgumentsMappingPriv); 239 pKernelGsp->pGspArgumentsCached = NULL; 240 pKernelGsp->pGspArgumentsMappingPriv = NULL; 241 } 242 if (pKernelGsp->pGspArgumentsDescriptor != NULL) 243 { 244 memdescFree(pKernelGsp->pGspArgumentsDescriptor); 245 memdescDestroy(pKernelGsp->pGspArgumentsDescriptor); 246 pKernelGsp->pGspArgumentsDescriptor = NULL; 247 } 248 249 // Release radix3 version of GSP-RM ucode 250 if (pKernelGsp->pGspUCodeRadix3Descriptor != NULL) 251 { 252 memdescFree(pKernelGsp->pGspUCodeRadix3Descriptor); 253 memdescDestroy(pKernelGsp->pGspUCodeRadix3Descriptor); 254 pKernelGsp->pGspUCodeRadix3Descriptor = NULL; 255 } 256 257 // Release signature memory 258 if (pKernelGsp->pSignatureMemdesc != NULL) 259 { 260 memdescFree(pKernelGsp->pSignatureMemdesc); 261 memdescDestroy(pKernelGsp->pSignatureMemdesc); 262 pKernelGsp->pSignatureMemdesc = NULL; 263 } 264 } 265 266 /*! 267 * Determine if GSP reload via SEC2 is completed. 268 */ 269 static NvBool 270 _kgspIsReloadCompleted 271 ( 272 OBJGPU *pGpu, 273 void *pVoid 274 ) 275 { 276 NvU32 reg; 277 278 reg = GPU_REG_RD32(pGpu, NV_PGC6_BSI_SECURE_SCRATCH_14); 279 280 return FLD_TEST_DRF(_PGC6, _BSI_SECURE_SCRATCH_14, _BOOT_STAGE_3_HANDOFF, _VALUE_DONE, reg); 281 } 282 283 /*! 284 * Set command queue head for CPU to GSP message queue 285 * 286 * @param[in] pGpu GPU object pointer 287 * @param[in] pKernelGsp KernelGsp object pointer (not used) 288 * @param[in] queueIdx index 289 * @param[in] value value to set command queue head to. 290 * 291 * @return NV_OK if the operation was successful. 292 */ 293 NV_STATUS 294 kgspSetCmdQueueHead_TU102 295 ( 296 OBJGPU *pGpu, 297 KernelGsp *pKernelGsp, 298 NvU32 queueIdx, 299 NvU32 value 300 ) 301 { 302 NV_ASSERT_OR_RETURN(queueIdx < NV_PGSP_QUEUE_HEAD__SIZE_1, NV_ERR_INVALID_ARGUMENT); 303 304 // Write the value to the correct queue head. 305 GPU_REG_WR32(pGpu, NV_PGSP_QUEUE_HEAD(queueIdx), value); 306 307 return NV_OK; 308 } 309 310 /*! 311 * Load entrypoint address of boot binary into mailbox regs. 312 */ 313 void 314 kgspProgramLibosBootArgsAddr_TU102 315 ( 316 OBJGPU *pGpu, 317 KernelGsp *pKernelGsp 318 ) 319 { 320 NvU64 addr = 321 memdescGetPhysAddr(pKernelGsp->pLibosInitArgumentsDescriptor, AT_GPU, 0); 322 323 GPU_REG_WR32(pGpu, NV_PGSP_FALCON_MAILBOX0, NvU64_LO32(addr)); 324 GPU_REG_WR32(pGpu, NV_PGSP_FALCON_MAILBOX1, NvU64_HI32(addr)); 325 } 326 327 NV_STATUS 328 kgspBootstrapRiscvOSEarly_TU102 329 ( 330 OBJGPU *pGpu, 331 KernelGsp *pKernelGsp, 332 GSP_FIRMWARE *pGspFw 333 ) 334 { 335 NV_STATUS status = NV_OK; 336 KernelFalcon *pKernelFalcon = staticCast(pKernelGsp, KernelFalcon); 337 338 // Only for GSP client builds 339 if (!IS_GSP_CLIENT(pGpu)) 340 { 341 NV_PRINTF(LEVEL_ERROR, "IS_GSP_CLIENT is not set.\n"); 342 return NV_ERR_NOT_SUPPORTED; 343 } 344 345 if (!kflcnIsRiscvCpuEnabled_HAL(pGpu, pKernelFalcon)) 346 { 347 NV_PRINTF(LEVEL_ERROR, "RISC-V core is not enabled.\n"); 348 return NV_ERR_NOT_SUPPORTED; 349 } 350 351 // 352 // Setup for libos bootloader execution including reserving space in the 353 // fb for placement and bootloader args initialization. 354 // 355 kgspPopulateGspRmInitArgs(pGpu, pKernelGsp, NULL); 356 357 { 358 // Execute FWSEC to setup FRTS if we have a FRTS region 359 if (kgspGetFrtsSize_HAL(pGpu, pKernelGsp) > 0) 360 { 361 kflcnReset_HAL(pGpu, pKernelFalcon); 362 363 NV_ASSERT_OK_OR_GOTO(status, 364 kgspExecuteFwsecFrts_HAL(pGpu, 365 pKernelGsp, 366 pKernelGsp->pFwsecUcode, 367 pKernelGsp->pWprMeta->frtsOffset), 368 exit); 369 } 370 } 371 372 kflcnReset_HAL(pGpu, pKernelFalcon); 373 374 // 375 // Stuff the message queue with async init messages that will be run 376 // before OBJGPU is created. 377 // 378 NV_RM_RPC_GSP_SET_SYSTEM_INFO(pGpu, status); 379 if (status != NV_OK) 380 { 381 NV_ASSERT_OK_FAILED("NV_RM_RPC_GSP_SET_SYSTEM_INFO", status); 382 goto exit; 383 } 384 385 NV_RM_RPC_SET_REGISTRY(pGpu, status); 386 if (status != NV_OK) 387 { 388 NV_ASSERT_OK_FAILED("NV_RM_RPC_SET_REGISTRY", status); 389 goto exit; 390 } 391 392 // Initialize libos init args list 393 kgspSetupLibosInitArgs(pGpu, pKernelGsp); 394 395 // Load init args into mailbox regs 396 kgspProgramLibosBootArgsAddr_HAL(pGpu, pKernelGsp); 397 398 // Execute Scrubber if needed 399 if (pKernelGsp->pScrubberUcode != NULL) 400 { 401 NV_ASSERT_OK_OR_GOTO(status, 402 kgspExecuteScrubberIfNeeded_HAL(pGpu, pKernelGsp), 403 exit); 404 } 405 406 { 407 status = kgspExecuteBooterLoad_HAL(pGpu, pKernelGsp, 408 memdescGetPhysAddr(pKernelGsp->pWprMetaDescriptor, AT_GPU, 0)); 409 if (status != NV_OK) 410 { 411 NV_PRINTF(LEVEL_ERROR, "failed to execute Booter Load (ucode for initial boot): 0x%x\n", status); 412 goto exit; 413 } 414 } 415 416 // Ensure the CPU is started 417 if (kflcnIsRiscvActive_HAL(pGpu, pKernelFalcon)) 418 { 419 NV_PRINTF(LEVEL_INFO, "GSP ucode loaded and RISCV started.\n"); 420 } 421 else 422 { 423 NV_PRINTF(LEVEL_ERROR, "Failed to boot GSP.\n"); 424 425 status = NV_ERR_NOT_READY; 426 goto exit; 427 } 428 429 NV_PRINTF(LEVEL_INFO, "Waiting for GSP fw RM to be ready...\n"); 430 431 // Link the status queue. 432 NV_ASSERT_OK_OR_GOTO(status, 433 GspStatusQueueInit(pGpu, &pKernelGsp->pRpc->pMessageQueueInfo), 434 exit); 435 436 NV_ASSERT_OK_OR_GOTO(status, 437 kgspWaitForRmInitDone(pGpu, pKernelGsp), 438 exit); 439 440 NV_PRINTF(LEVEL_INFO, "GSP FW RM ready.\n"); 441 442 exit: 443 return status; 444 } 445 446 void 447 kgspGetGspRmBootUcodeStorage_TU102 448 ( 449 OBJGPU *pGpu, 450 KernelGsp *pKernelGsp, 451 BINDATA_STORAGE **ppBinStorageImage, 452 BINDATA_STORAGE **ppBinStorageDesc 453 ) 454 { 455 const BINDATA_ARCHIVE *pBinArchive = kgspGetBinArchiveGspRmBoot_HAL(pKernelGsp); 456 457 *ppBinStorageImage = (BINDATA_STORAGE *) bindataArchiveGetStorage(pBinArchive, "ucode_image"); 458 *ppBinStorageDesc = (BINDATA_STORAGE *) bindataArchiveGetStorage(pBinArchive, "ucode_desc"); 459 } 460 461 /*! 462 * Calculate the FB layout. Also, copy GSP FW booter image to FB. 463 * 464 * Firmware scrubs the last 256mb of FB, no memory outside of this region 465 * may be used until the FW RM has scrubbed the remainder of memory. 466 * 467 * ---------------------------- <- fbSize (end of FB, 1M aligned) 468 * | VGA WORKSPACE | 469 * ---------------------------- <- vbiosReservedOffset (64K? aligned) 470 * | (potential align. gap) | 471 * ---------------------------- <- gspFwWprEnd (128K aligned) 472 * | FRTS data | (frtsSize is 0 on GA100) 473 * | ------------------------ | <- frtsOffset 474 * | BOOT BIN (e.g. SK + BL) | 475 * ---------------------------- <- bootBinOffset 476 * | GSP FW ELF | 477 * ---------------------------- <- gspFwOffset 478 * | GSP FW (WPR) HEAP | 479 * ---------------------------- <- gspFwHeapOffset** 480 * | Booter-placed metadata | 481 * | (struct GspFwWprMeta) | 482 * ---------------------------- <- gspFwWprStart (128K aligned) 483 * | GSP FW (non-WPR) HEAP | 484 * ---------------------------- <- nonWprHeapOffset, gspFwRsvdStart 485 * 486 * gspFwHeapOffset** contains the RM/Libos Heap. First 16 Mb are for Libos heap 487 * rest is for GSP-RM 488 * @param pGpu GPU object pointer 489 * @param pKernelGsp KernelGsp object pointer 490 * @param pGspFw Pointer to GSP-RM fw image. 491 */ 492 NV_STATUS 493 kgspCalculateFbLayout_TU102 494 ( 495 OBJGPU *pGpu, 496 KernelGsp *pKernelGsp, 497 GSP_FIRMWARE *pGspFw 498 ) 499 { 500 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 501 KernelDisplay *pKernelDisplay = GPU_GET_KERNEL_DISPLAY(pGpu); 502 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 503 GspFwWprMeta *pWprMeta = pKernelGsp->pWprMeta; 504 RM_RISCV_UCODE_DESC *pRiscvDesc = pKernelGsp->pGspRmBootUcodeDesc; 505 NvU64 vbiosReservedOffset; 506 NvU64 mmuLockLo, mmuLockHi; 507 NvBool bIsMmuLockValid; 508 509 ct_assert(sizeof(*pWprMeta) == 256); 510 511 NV_ASSERT_OR_RETURN(IS_GSP_CLIENT(pGpu), NV_ERR_NOT_SUPPORTED); 512 513 NV_ASSERT_OR_RETURN(pKernelGsp->pGspRmBootUcodeImage != NULL, NV_ERR_INVALID_STATE); 514 NV_ASSERT_OR_RETURN(pKernelGsp->gspRmBootUcodeSize != 0, NV_ERR_INVALID_STATE); 515 NV_ASSERT_OR_RETURN(pRiscvDesc != NULL, NV_ERR_INVALID_STATE); 516 517 portMemSet(pWprMeta, 0, sizeof *pWprMeta); 518 519 NV_ASSERT_OK_OR_RETURN(kmemsysGetUsableFbSize_HAL(pGpu, pKernelMemorySystem, &pWprMeta->fbSize)); 520 521 // 522 // Start layout calculations at the top and work down. 523 // Figure out where VGA workspace is located. We do not have to adjust 524 // it ourselves (see vgaRelocateWorkspaceBase_HAL()). 525 // 526 if (gpuFuseSupportsDisplay_HAL(pGpu) && 527 kdispGetVgaWorkspaceBase(pGpu, pKernelDisplay, &pWprMeta->vgaWorkspaceOffset)) 528 { 529 if (pWprMeta->vgaWorkspaceOffset < (pWprMeta->fbSize - DRF_SIZE(NV_PRAMIN))) 530 { 531 const NvU32 VBIOS_WORKSPACE_SIZE = 0x20000; 532 533 // Point NV_PDISP_VGA_WORKSPACE_BASE to end-of-FB 534 pWprMeta->vgaWorkspaceOffset = (pWprMeta->fbSize - VBIOS_WORKSPACE_SIZE); 535 } 536 } 537 else 538 { 539 pWprMeta->vgaWorkspaceOffset = (pWprMeta->fbSize - DRF_SIZE(NV_PRAMIN)); 540 } 541 pWprMeta->vgaWorkspaceSize = pWprMeta->fbSize - pWprMeta->vgaWorkspaceOffset; 542 543 // Check for MMU locked region (locked by VBIOS) 544 NV_ASSERT_OK_OR_RETURN( 545 memmgrReadMmuLock_HAL(pGpu, pMemoryManager, &bIsMmuLockValid, &mmuLockLo, &mmuLockHi)); 546 547 if (bIsMmuLockValid) 548 vbiosReservedOffset = NV_MIN(mmuLockLo, pWprMeta->vgaWorkspaceOffset); 549 else 550 vbiosReservedOffset = pWprMeta->vgaWorkspaceOffset; 551 552 // End of WPR region (128KB aligned) 553 pWprMeta->gspFwWprEnd = NV_ALIGN_DOWN64(vbiosReservedOffset, 0x20000); 554 555 pWprMeta->frtsSize = kgspGetFrtsSize(pGpu, pKernelGsp); 556 pWprMeta->frtsOffset = pWprMeta->gspFwWprEnd - pWprMeta->frtsSize; 557 558 // Offset of boot binary image (4K aligned) 559 pWprMeta->sizeOfBootloader = pKernelGsp->gspRmBootUcodeSize; 560 pWprMeta->bootBinOffset = NV_ALIGN_DOWN64(pWprMeta->frtsOffset - pWprMeta->sizeOfBootloader, 0x1000); 561 562 // Compute GSP firmware image size 563 pWprMeta->sizeOfRadix3Elf = pGspFw->imageSize; 564 565 // 566 // Compute the start of the ELF. Align to 64K to avoid issues with 567 // inherent alignment constraints. 568 // 569 pWprMeta->gspFwOffset = NV_ALIGN_DOWN64(pWprMeta->bootBinOffset - pWprMeta->sizeOfRadix3Elf, 0x10000); 570 571 const NvU64 wprHeapSize = kgspGetWprHeapSize(pGpu, pKernelGsp); 572 573 // GSP-RM heap in WPR, align to 1MB 574 pWprMeta->gspFwHeapOffset = NV_ALIGN_DOWN64(pWprMeta->gspFwOffset - wprHeapSize, 0x100000); 575 pWprMeta->gspFwHeapSize = NV_ALIGN_DOWN64(pWprMeta->gspFwOffset - pWprMeta->gspFwHeapOffset, 0x100000); 576 577 // Number of VF partitions allocating sub-heaps from the WPR heap 578 pWprMeta->gspFwHeapVfPartitionCount = pGpu->bVgpuGspPluginOffloadEnabled ? MAX_PARTITIONS_WITH_GFID : 0; 579 580 // 581 // Start of WPR region (128K alignment requirement, but 1MB aligned so that 582 // the extra padding sits in WPR instead of in between the end of the 583 // non-WPR heap and the start of WPR). 584 // 585 pWprMeta->gspFwWprStart = NV_ALIGN_DOWN64(pWprMeta->gspFwHeapOffset - sizeof *pWprMeta, 0x100000); 586 587 // Non WPR heap (1MB aligned) 588 pWprMeta->nonWprHeapSize = kgspGetNonWprHeapSize(pGpu, pKernelGsp); 589 pWprMeta->nonWprHeapOffset = NV_ALIGN_DOWN64(pWprMeta->gspFwWprStart - pWprMeta->nonWprHeapSize, 0x100000); 590 591 pWprMeta->gspFwRsvdStart = pWprMeta->nonWprHeapOffset; 592 593 // Physical address of GSP-RM firmware in system memory. 594 pWprMeta->sysmemAddrOfRadix3Elf = 595 memdescGetPhysAddr(pKernelGsp->pGspUCodeRadix3Descriptor, AT_GPU, 0); 596 597 // Physical address of boot loader firmware in system memory. 598 pWprMeta->sysmemAddrOfBootloader = 599 memdescGetPhysAddr(pKernelGsp->pGspRmBootUcodeMemdesc, AT_GPU, 0); 600 601 // Set necessary info from bootloader desc 602 pWprMeta->bootloaderCodeOffset = pRiscvDesc->monitorCodeOffset; 603 pWprMeta->bootloaderDataOffset = pRiscvDesc->monitorDataOffset; 604 pWprMeta->bootloaderManifestOffset = pRiscvDesc->manifestOffset; 605 606 if (pKernelGsp->pSignatureMemdesc != NULL) 607 { 608 pWprMeta->sysmemAddrOfSignature = memdescGetPhysAddr(pKernelGsp->pSignatureMemdesc, AT_GPU, 0); 609 pWprMeta->sizeOfSignature = memdescGetSize(pKernelGsp->pSignatureMemdesc); 610 } 611 612 pWprMeta->bootCount = 0; 613 pWprMeta->verified = 0; 614 pWprMeta->revision = GSP_FW_WPR_META_REVISION; 615 pWprMeta->magic = GSP_FW_WPR_META_MAGIC; 616 617 #if 0 618 NV_PRINTF(LEVEL_ERROR, "WPR meta data offset: 0x%016llx\n", pWprMeta->gspFwWprStart); 619 NV_PRINTF(LEVEL_ERROR, " magic: 0x%016llx\n", pWprMeta->magic); 620 NV_PRINTF(LEVEL_ERROR, " revision: 0x%016llx\n", pWprMeta->revision); 621 NV_PRINTF(LEVEL_ERROR, " sysmemAddrOfRadix3Elf: 0x%016llx\n", pWprMeta->sysmemAddrOfRadix3Elf); 622 NV_PRINTF(LEVEL_ERROR, " sizeOfRadix3Elf: 0x%016llx\n", pWprMeta->sizeOfRadix3Elf); 623 NV_PRINTF(LEVEL_ERROR, " sysmemAddrOfBootloader: 0x%016llx\n", pWprMeta->sysmemAddrOfBootloader); 624 NV_PRINTF(LEVEL_ERROR, " sizeOfBootloader: 0x%016llx\n", pWprMeta->sizeOfBootloader); 625 NV_PRINTF(LEVEL_ERROR, " sysmemAddrOfSignature: 0x%016llx\n", pWprMeta->sysmemAddrOfSignature); 626 NV_PRINTF(LEVEL_ERROR, " sizeOfSignature: 0x%016llx\n", pWprMeta->sizeOfSignature); 627 NV_PRINTF(LEVEL_ERROR, " gspFwRsvdStart: 0x%016llx\n", pWprMeta->gspFwRsvdStart); 628 NV_PRINTF(LEVEL_ERROR, " nonWprHeap: 0x%016llx - 0x%016llx (0x%016llx)\n", pWprMeta->nonWprHeapOffset, pWprMeta->nonWprHeapOffset + pWprMeta->nonWprHeapSize - 1, pWprMeta->nonWprHeapSize); 629 NV_PRINTF(LEVEL_ERROR, " gspFwWprStart: 0x%016llx\n", pWprMeta->gspFwWprStart); 630 NV_PRINTF(LEVEL_ERROR, " gspFwHeap: 0x%016llx - 0x%016llx (0x%016llx)\n", pWprMeta->gspFwHeapOffset, pWprMeta->gspFwHeapOffset + pWprMeta->gspFwHeapSize - 1, pWprMeta->gspFwHeapSize); 631 NV_PRINTF(LEVEL_ERROR, " gspFwOffset: 0x%016llx - 0x%016llx (0x%016llx)\n", pWprMeta->gspFwOffset, pWprMeta->gspFwOffset + pWprMeta->sizeOfRadix3Elf - 1, pWprMeta->sizeOfRadix3Elf); 632 NV_PRINTF(LEVEL_ERROR, " bootBinOffset: 0x%016llx - 0x%016llx (0x%016llx)\n", pWprMeta->bootBinOffset, pWprMeta->bootBinOffset + pWprMeta->sizeOfBootloader - 1, pWprMeta->sizeOfBootloader); 633 NV_PRINTF(LEVEL_ERROR, " frtsOffset: 0x%016llx - 0x%016llx (0x%016llx)\n", pWprMeta->frtsOffset, pWprMeta->frtsOffset + pWprMeta->frtsSize - 1, pWprMeta->frtsSize); 634 NV_PRINTF(LEVEL_ERROR, " gspFwWprEnd: 0x%016llx\n", pWprMeta->gspFwWprEnd); 635 NV_PRINTF(LEVEL_ERROR, " fbSize: 0x%016llx\n", pWprMeta->fbSize); 636 NV_PRINTF(LEVEL_ERROR, " vgaWorkspaceOffset: 0x%016llx - 0x%016llx (0x%016llx)\n", pWprMeta->vgaWorkspaceOffset, pWprMeta->vgaWorkspaceOffset + pWprMeta->vgaWorkspaceSize - 1, pWprMeta->vgaWorkspaceSize); 637 NV_PRINTF(LEVEL_ERROR, " bootCount: 0x%016llx\n", pWprMeta->bootCount); 638 NV_PRINTF(LEVEL_ERROR, " verified: 0x%016llx\n", pWprMeta->verified); 639 #endif 640 641 return NV_OK; 642 } 643 644 /*! 645 * Execute GSP sequencer operation 646 * 647 * @param[in] pGpu GPU object pointer 648 * @param[in] pKernelGsp KernelGsp object pointer 649 * @param[in] opCode Sequencer opcode 650 * @param[in] pPayload Pointer to payload 651 * @param[in] payloadSize Size of payload in bytes 652 * 653 * @return NV_OK if the sequencer operation was successful. 654 * Appropriate NV_ERR_xxx value otherwise. 655 */ 656 NV_STATUS 657 kgspExecuteSequencerCommand_TU102 658 ( 659 OBJGPU *pGpu, 660 KernelGsp *pKernelGsp, 661 NvU32 opCode, 662 NvU32 *pPayload, 663 NvU32 payloadSize 664 ) 665 { 666 NV_STATUS status = NV_OK; 667 KernelFalcon *pKernelFalcon = staticCast(pKernelGsp, KernelFalcon); 668 NvU32 secMailbox0 = 0; 669 670 switch (opCode) 671 { 672 case GSP_SEQ_BUF_OPCODE_CORE_RESUME: 673 { 674 { 675 KernelFalcon *pKernelSec2Falcon = staticCast(GPU_GET_KERNEL_SEC2(pGpu), KernelFalcon); 676 677 kflcnSecureReset_HAL(pGpu, pKernelFalcon); 678 kgspProgramLibosBootArgsAddr_HAL(pGpu, pKernelGsp); 679 680 NV_PRINTF(LEVEL_INFO, "---------------Starting SEC2 to resume GSP-RM------------\n"); 681 // Start SEC2 in order to resume GSP-RM 682 kflcnStartCpu_HAL(pGpu, pKernelSec2Falcon); 683 684 // Wait for reload to be completed. 685 status = gpuTimeoutCondWait(pGpu, _kgspIsReloadCompleted, NULL, NULL); 686 687 // Check SEC mailbox. 688 secMailbox0 = kflcnRegRead_HAL(pGpu, pKernelSec2Falcon, NV_PFALCON_FALCON_MAILBOX0); 689 690 if ((status != NV_OK) || (secMailbox0 != NV_OK)) 691 { 692 NV_PRINTF(LEVEL_ERROR, "Timeout waiting for SEC2-RTOS to resume GSP-RM. SEC2 Mailbox0 is : 0x%x\n", secMailbox0); 693 DBG_BREAKPOINT(); 694 return NV_ERR_TIMEOUT; 695 } 696 } 697 698 // Ensure the CPU is started 699 if (kflcnIsRiscvActive_HAL(pGpu, pKernelFalcon)) 700 { 701 NV_PRINTF(LEVEL_INFO, "GSP ucode loaded and RISCV started.\n"); 702 } 703 else 704 { 705 NV_ASSERT_FAILED("Failed to boot GSP"); 706 status = NV_ERR_NOT_READY; 707 } 708 break; 709 } 710 711 default: 712 { 713 status = NV_ERR_INVALID_ARGUMENT; 714 break; 715 } 716 } 717 718 return status; 719 } 720 721 /*! 722 * Reset the GSP HW 723 * 724 * @return NV_OK if the GSP HW was properly reset 725 */ 726 NV_STATUS 727 kgspResetHw_TU102 728 ( 729 OBJGPU *pGpu, 730 KernelGsp *pKernelGsp 731 ) 732 { 733 GPU_FLD_WR_DRF_DEF(pGpu, _PGSP, _FALCON_ENGINE, _RESET, _TRUE); 734 GPU_FLD_WR_DRF_DEF(pGpu, _PGSP, _FALCON_ENGINE, _RESET, _FALSE); 735 736 return NV_OK; 737 } 738 739 void 740 kgspHealthCheck_TU102 741 ( 742 OBJGPU *pGpu, 743 KernelGsp *pKernelGsp 744 ) 745 { 746 NvU32 mb0 = GPU_REG_RD32(pGpu, NV_PGSP_MAILBOX(0)); 747 748 // 749 // Check for an error message in the GSP mailbox. Any error here is severe 750 // enough that it should be reported as an Xid. Clear the error so more can 751 // potentially be reported by GSP, if it was able to recover. In that case, 752 // it's possible that GSP will skip reporting some more errors that happened 753 // before the clear, and it will just update the "skipped" count. 754 // 755 if (FLD_TEST_DRF(_GSP, _ERROR, _TAG, _VAL, mb0)) 756 { 757 NvU32 mb1 = GPU_REG_RD32(pGpu, NV_PGSP_MAILBOX(1)); 758 759 GPU_REG_WR32(pGpu, NV_PGSP_MAILBOX(0), 0); 760 761 NV_PRINTF(LEVEL_NOTICE, 762 "********************************* GSP Failure **********************************\n"); 763 764 nvErrorLog_va((void*)pGpu, GSP_ERROR, 765 "GSP Error: Task %d raised error code 0x%x for reason 0x%x at 0x%x (%d more errors skipped)", 766 DRF_VAL(_GSP, _ERROR, _TASK, mb0), 767 DRF_VAL(_GSP, _ERROR, _CODE, mb0), 768 DRF_VAL(_GSP, _ERROR, _REASON, mb0), 769 mb1, 770 DRF_VAL(_GSP, _ERROR, _SKIPPED, mb0)); 771 772 NV_PRINTF(LEVEL_NOTICE, 773 "********************************************************************************\n"); 774 } 775 } 776 777 /*! 778 * GSP Interrupt Service Routine 779 * 780 * @return 32-bit interrupt status AFTER all known interrupt-sources were 781 * serviced. 782 */ 783 NvU32 784 kgspService_TU102 785 ( 786 OBJGPU *pGpu, 787 KernelGsp *pKernelGsp 788 ) 789 { 790 NvU32 clearBits = 0; 791 NvU32 intrStatus; 792 KernelFalcon *pKernelFalcon = staticCast(pKernelGsp, KernelFalcon); 793 794 // Get the IRQ status and mask the sources not directed to host. 795 intrStatus = kflcnReadIntrStatus_HAL(pGpu, pKernelFalcon); 796 797 // Exit immediately if there is nothing to do 798 if (intrStatus == 0) 799 { 800 return 0; 801 } 802 803 if (!API_GPU_ATTACHED_SANITY_CHECK(pGpu)) 804 { 805 NV_PRINTF(LEVEL_ERROR, "GPU is detached, bailing!\n"); 806 return 0; 807 } 808 809 if (intrStatus & DRF_DEF(_PFALCON, _FALCON_IRQSTAT, _HALT, _TRUE)) 810 { 811 clearBits |= DRF_DEF(_PFALCON, _FALCON_IRQSCLR, _HALT, _SET); 812 813 // 814 // Currently, GSP-RISCV triggers _HALT interrupt to RM when it finds 815 // itself running into a bad state. Triggering _HALT interrupt to RM 816 // provides RM a chance to handle it so we have better debugability 817 // into GSP-RISCV issues. 818 // 819 kgspDumpGspLogs(pGpu, pKernelGsp, NV_FALSE); 820 kgspHealthCheck_HAL(pGpu, pKernelGsp); 821 } 822 if (intrStatus & DRF_DEF(_PFALCON, _FALCON_IRQSTAT, _SWGEN0, _TRUE)) 823 { 824 // 825 // Clear edge triggered interupt BEFORE (and never after) 826 // servicing it to avoid race conditions. 827 // 828 kflcnRegWrite_HAL(pGpu, pKernelFalcon, NV_PFALCON_FALCON_IRQSCLR, 829 DRF_DEF(_PFALCON, _FALCON_IRQSCLR, _SWGEN0, _SET)); 830 831 kgspRpcRecvEvents(pGpu, pKernelGsp); 832 833 // 834 // If lockdown has been engaged (as notified by an RPC event), 835 // we shouldn't access any more GSP registers. 836 // 837 NV_CHECK_OR_RETURN(LEVEL_SILENT, !pKernelGsp->bInLockdown, 0); 838 } 839 840 // Clear any sources that were serviced and get the new status 841 kflcnRegWrite_HAL(pGpu, pKernelFalcon, NV_PFALCON_FALCON_IRQSCLR, clearBits); 842 843 kflcnIntrRetrigger_HAL(pGpu, pKernelFalcon); 844 845 intrStatus = kflcnReadIntrStatus_HAL(pGpu, pKernelFalcon); 846 847 return intrStatus; 848 } 849 850 static NvBool 851 _kgspIsProcessorSuspended 852 ( 853 OBJGPU *pGpu, 854 void *pVoid 855 ) 856 { 857 KernelGsp *pKernelGsp = reinterpretCast(pVoid, KernelGsp *); 858 NvU32 mailbox; 859 860 // Check for LIBOS_INTERRUPT_PROCESSOR_SUSPENDED in mailbox 861 mailbox = kflcnRegRead_HAL(pGpu, staticCast(pKernelGsp, KernelFalcon), 862 NV_PFALCON_FALCON_MAILBOX0); 863 return (mailbox & 0x80000000) == 0x80000000; 864 } 865 866 NV_STATUS 867 kgspWaitForProcessorSuspend_TU102 868 ( 869 OBJGPU *pGpu, 870 KernelGsp *pKernelGsp 871 ) 872 { 873 return gpuTimeoutCondWait(pGpu, _kgspIsProcessorSuspended, pKernelGsp, NULL); 874 } 875 876 #define FWSECLIC_PROG_START_TIMEOUT 50000 // 50ms 877 #define FWSECLIC_PROG_COMPLETE_TIMEOUT 2000000 // 2s 878 879 NV_STATUS 880 kgspWaitForGfwBootOk_TU102 881 ( 882 OBJGPU *pGpu, 883 KernelGsp *pKernelGsp 884 ) 885 { 886 NvU32 timeoutUs = FWSECLIC_PROG_START_TIMEOUT + FWSECLIC_PROG_COMPLETE_TIMEOUT; 887 RMTIMEOUT timeout; 888 NV_STATUS status = NV_OK; 889 890 // Use the OS timer since the GPU timer is not ready yet 891 gpuSetTimeout(pGpu, gpuScaleTimeout(pGpu, timeoutUs), &timeout, 892 GPU_TIMEOUT_FLAGS_OSTIMER); 893 894 while (status == NV_OK) 895 { 896 // 897 // Before reading the actual GFW_BOOT status register, 898 // we want to check that FWSEC has lowered its PLM first. 899 // If not then obviously it has not completed. 900 // 901 if (GPU_FLD_TEST_DRF_DEF(pGpu, 902 _PGC6, 903 _AON_SECURE_SCRATCH_GROUP_05_PRIV_LEVEL_MASK, 904 _READ_PROTECTION_LEVEL0, 905 _ENABLE) 906 ) 907 { 908 if (GPU_FLD_TEST_DRF_DEF(pGpu, 909 _PGC6, 910 _AON_SECURE_SCRATCH_GROUP_05_0_GFW_BOOT, 911 _PROGRESS, 912 _COMPLETED) 913 ) 914 { 915 return NV_OK; 916 } 917 } 918 919 status = gpuCheckTimeout(pGpu, &timeout); 920 if (status == NV_ERR_TIMEOUT) 921 { 922 NV_PRINTF(LEVEL_ERROR, 923 "Timeout waiting for GFW_BOOT to complete\n"); 924 } 925 } 926 927 return status; 928 } 929 930 void 931 kgspFreeSuspendResumeData_TU102 932 ( 933 OBJGPU *pGpu, 934 KernelGsp *pKernelGsp 935 ) 936 { 937 // release sr meta data resources 938 if (pKernelGsp->pSRMetaDescriptor != NULL) 939 { 940 memdescFree(pKernelGsp->pSRMetaDescriptor); 941 memdescDestroy(pKernelGsp->pSRMetaDescriptor); 942 pKernelGsp->pSRMetaDescriptor = NULL; 943 } 944 945 // release sr meta data resources 946 if (pKernelGsp->pSRRadix3Descriptor != NULL) 947 { 948 memdescFree(pKernelGsp->pSRRadix3Descriptor); 949 memdescDestroy(pKernelGsp->pSRRadix3Descriptor); 950 pKernelGsp->pSRRadix3Descriptor = NULL; 951 } 952 } 953 954 NV_STATUS 955 kgspSavePowerMgmtState_TU102 956 ( 957 OBJGPU *pGpu, 958 KernelGsp *pKernelGsp 959 ) 960 { 961 GspFwSRMeta gspfwSRMeta; 962 NvP64 pVa = NvP64_NULL; 963 NvP64 pPriv = NvP64_NULL; 964 NV_STATUS nvStatus = NV_OK; 965 966 // Fill in GspFwSRMeta structure 967 portMemSet(&gspfwSRMeta, 0, sizeof(gspfwSRMeta)); 968 gspfwSRMeta.magic = GSP_FW_SR_META_MAGIC; 969 gspfwSRMeta.revision = GSP_FW_SR_META_REVISION; 970 gspfwSRMeta.sizeOfSuspendResumeData = pKernelGsp->pWprMeta->gspFwWprEnd - pKernelGsp->pWprMeta->gspFwWprStart; 971 972 973 NV_ASSERT_OK_OR_GOTO(nvStatus, 974 kgspCreateRadix3(pGpu, 975 pKernelGsp, 976 &pKernelGsp->pSRRadix3Descriptor, 977 NULL, 978 NULL, 979 gspfwSRMeta.sizeOfSuspendResumeData), 980 exit_fail_cleanup); 981 982 gspfwSRMeta.sysmemAddrOfSuspendResumeData = memdescGetPhysAddr(pKernelGsp->pSRRadix3Descriptor, AT_GPU, 0); 983 984 // Create SR Metadata Area 985 NV_ASSERT_OK_OR_GOTO(nvStatus, 986 memdescCreate(&pKernelGsp->pSRMetaDescriptor, 987 pGpu, 988 sizeof(GspFwSRMeta), 989 256, 990 NV_TRUE, 991 ADDR_SYSMEM, 992 NV_MEMORY_UNCACHED, 993 MEMDESC_FLAGS_NONE), 994 exit_fail_cleanup); 995 996 NV_ASSERT_OK_OR_GOTO(nvStatus, 997 memdescAlloc(pKernelGsp->pSRMetaDescriptor), 998 exit_fail_cleanup); 999 1000 // Copy SR Metadata Structure 1001 NV_ASSERT_OK_OR_GOTO(nvStatus, 1002 memdescMap(pKernelGsp->pSRMetaDescriptor, 1003 0, 1004 memdescGetSize(pKernelGsp->pSRMetaDescriptor), 1005 NV_TRUE, 1006 NV_PROTECT_WRITEABLE, 1007 &pVa, 1008 &pPriv), 1009 exit_fail_cleanup); 1010 1011 portMemCopy(pVa, sizeof(gspfwSRMeta), &gspfwSRMeta, sizeof(gspfwSRMeta)); 1012 1013 memdescUnmap(pKernelGsp->pSRMetaDescriptor, 1014 NV_TRUE, osGetCurrentProcess(), 1015 pVa, pPriv); 1016 1017 NV_ASSERT_OK_OR_GOTO(nvStatus, 1018 kgspExecuteBooterUnloadIfNeeded_HAL(pGpu, 1019 pKernelGsp, 1020 memdescGetPhysAddr(pKernelGsp->pSRMetaDescriptor,AT_GPU, 0)), 1021 exit_fail_cleanup); 1022 1023 return nvStatus; 1024 1025 exit_fail_cleanup: 1026 kgspFreeSuspendResumeData_HAL(pGpu, pKernelGsp); 1027 return nvStatus; 1028 } 1029 1030 NV_STATUS 1031 kgspRestorePowerMgmtState_TU102 1032 ( 1033 OBJGPU *pGpu, 1034 KernelGsp *pKernelGsp 1035 ) 1036 { 1037 NV_STATUS nvStatus = NV_OK; 1038 1039 NV_ASSERT_OK_OR_GOTO(nvStatus, 1040 kgspExecuteBooterLoad_HAL(pGpu, 1041 pKernelGsp, 1042 memdescGetPhysAddr(pKernelGsp->pSRMetaDescriptor, AT_GPU,0)), 1043 exit_cleanup); 1044 1045 exit_cleanup: 1046 kgspFreeSuspendResumeData_HAL(pGpu, pKernelGsp); 1047 return nvStatus; 1048 } 1049