1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2019-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include "resserv/rs_server.h" 25 26 #include "gpu/gsp/kernel_gsp.h" 27 28 #include "kernel/core/thread_state.h" 29 #include "kernel/core/locks.h" 30 #include "kernel/diagnostics/gpu_acct.h" 31 #include "kernel/gpu/fifo/kernel_channel.h" 32 #include "kernel/gpu/intr/engine_idx.h" 33 #include "kernel/gpu/mem_mgr/heap.h" 34 #include "kernel/gpu/mem_mgr/mem_mgr.h" 35 #include "kernel/gpu/mem_sys/kern_mem_sys.h" 36 #include "kernel/gpu/rc/kernel_rc.h" 37 #include "kernel/gpu/nvlink/kernel_nvlink.h" 38 #include "virtualization/hypervisor/hypervisor.h" 39 #include "virtualization/vgpuconfigapi.h" 40 #include "kernel/gpu/disp/kern_disp.h" 41 #include "kernel/gpu/mig_mgr/kernel_mig_manager.h" 42 #include "gpu/external_device/external_device.h" 43 #include "kernel/platform/platform_request_handler.h" 44 #include "class/cl2080.h" // NV20_SUBDEVICE_0 45 #include "liblogdecode.h" 46 #include "libelf.h" 47 #include "nverror.h" 48 #include "nvrm_registry.h" 49 #include "nv-firmware.h" 50 #include "nv-firmware-chip-family-select.h" 51 #include "nvtypes.h" 52 #include "nvVer.h" 53 #include "objrpc.h" 54 #include "objtmr.h" 55 #include "os/os.h" 56 #include "rmgspseq.h" 57 #include "sweng/dispsw.h" 58 #include "kernel/gpu/timed_sema.h" 59 #include "vgpu/rpc.h" 60 #include "kernel/gpu/pmu/kern_pmu.h" 61 #include "gpu/perf/kern_perf.h" 62 #include "core/locks.h" 63 64 #define RPC_STRUCTURES 65 #define RPC_GENERIC_UNION 66 #include "g_rpc-structures.h" 67 #undef RPC_STRUCTURES 68 #undef RPC_GENERIC_UNION 69 70 #define RPC_MESSAGE_STRUCTURES 71 #define RPC_MESSAGE_GENERIC_UNION 72 #include "g_rpc-message-header.h" 73 #undef RPC_MESSAGE_STRUCTURES 74 #undef RPC_MESSAGE_GENERIC_UNION 75 76 #include "gpu/gsp/message_queue_priv.h" 77 78 #include "gpu/conf_compute/conf_compute.h" 79 80 #define RPC_HDR ((rpc_message_header_v*)(pRpc->message_buffer)) 81 82 struct MIG_CI_UPDATE_CALLBACK_PARAMS 83 { 84 NvU32 execPartCount; 85 NvU32 execPartId[NVC637_CTRL_MAX_EXEC_PARTITIONS]; 86 NvU32 gfid; 87 NvBool bDelete; 88 }; 89 90 // 91 // RPC_PARAMS defines the rpc_params pointer and initializes it to the correct 92 // sub-structure. 93 // 94 // RPC_PARAMS intentionally assigns the the latest version structure to the 95 // versioned rpc_params pointer. With the -Werror=incompatible-pointer-types 96 // compiler flag, this checks for mismatched structure versions at compile time. 97 // 98 // For example: 99 // RPC_PARAMS(free, _v03_00); 100 // expands to 101 // rpc_free_v03_00 *rpc_params = &RPC_HDR->rpc_message_data->free_v; 102 // 103 #define RPC_PARAMS(r, v) rpc_##r##v *rpc_params = &RPC_HDR->rpc_message_data->r##_v 104 105 static NV_STATUS _kgspInitRpcInfrastructure(OBJGPU *, KernelGsp *); 106 static void _kgspFreeRpcInfrastructure(OBJGPU *, KernelGsp *); 107 108 static NV_STATUS _kgspConstructRpcObject(OBJGPU *, KernelGsp *, MESSAGE_QUEUE_INFO *, OBJRPC **); 109 110 static NV_STATUS _kgspRpcSendMessage(OBJGPU *, OBJRPC *); 111 static NV_STATUS _kgspRpcRecvPoll(OBJGPU *, OBJRPC *, NvU32); 112 static NV_STATUS _kgspRpcDrainEvents(OBJGPU *, KernelGsp *, NvU32); 113 static void _kgspRpcIncrementTimeoutCountAndRateLimitPrints(OBJGPU *, OBJRPC *); 114 115 static NV_STATUS _kgspAllocSimAccessBuffer(OBJGPU *pGpu, KernelGsp *pKernelGsp); 116 static void _kgspFreeSimAccessBuffer(OBJGPU *pGpu, KernelGsp *pKernelGsp); 117 118 static void _kgspStopLogPolling(OBJGPU *pGpu, KernelGsp *pKernelGsp); 119 120 static void _kgspFreeBootBinaryImage(OBJGPU *pGpu, KernelGsp *pKernelGsp); 121 122 static NV_STATUS _kgspPrepareGspRmBinaryImage(OBJGPU *pGpu, KernelGsp *pKernelGsp, GSP_FIRMWARE *pGspFw); 123 124 static NV_STATUS _kgspCreateSignatureMemdesc(OBJGPU *pGpu, KernelGsp *pKernelGsp, 125 GSP_FIRMWARE *pGspFw); 126 127 static NV_STATUS _kgspFwContainerVerifyVersion(OBJGPU *pGpu, KernelGsp *pKernelGsp, 128 const void *pElfData, NvU64 elfDataSize, 129 const char *pNameInMsg); 130 131 static NV_STATUS _kgspFwContainerGetSection(OBJGPU *pGpu, KernelGsp *pKernelGsp, 132 const void *pElfData, NvU64 elfDataSize, 133 const char *pSectionName, 134 const void **ppSectionData, NvU64 *pSectionSize); 135 136 static NV_STATUS _kgspGetSectionNameForPrefix(OBJGPU *pGpu, KernelGsp *pKernelGsp, 137 char *pSectionNameBuf, NvLength sectionNameBufSize, 138 const char *pSectionPrefix); 139 140 static void 141 _kgspGetActiveRpcDebugData 142 ( 143 OBJRPC *pRpc, 144 NvU32 function, 145 NvU32 *data0, 146 NvU32 *data1 147 ) 148 { 149 switch (function) 150 { 151 case NV_VGPU_MSG_FUNCTION_GSP_RM_CONTROL: 152 { 153 RPC_PARAMS(gsp_rm_control, _v03_00); 154 *data0 = rpc_params->cmd; 155 *data1 = rpc_params->paramsSize; 156 break; 157 } 158 case NV_VGPU_MSG_FUNCTION_GSP_RM_ALLOC: 159 { 160 RPC_PARAMS(gsp_rm_alloc, _v03_00); 161 *data0 = rpc_params->hClass; 162 *data1 = rpc_params->paramsSize; 163 break; 164 } 165 case NV_VGPU_MSG_FUNCTION_FREE: 166 { 167 RPC_PARAMS(free, _v03_00); 168 *data0 = rpc_params->params.hObjectOld; 169 *data1 = rpc_params->params.hObjectParent; 170 break; 171 } 172 case NV_VGPU_MSG_EVENT_GSP_LOCKDOWN_NOTICE: 173 { 174 RPC_PARAMS(gsp_lockdown_notice, _v17_00); 175 *data0 = rpc_params->bLockdownEngaging; 176 *data1 = 0; 177 break; 178 } 179 default: 180 { 181 *data0 = 0; 182 *data1 = 0; 183 break; 184 } 185 } 186 } 187 188 /*! 189 * GSP client RM RPC send routine 190 */ 191 static NV_STATUS 192 _kgspRpcSendMessage 193 ( 194 OBJGPU *pGpu, 195 OBJRPC *pRpc 196 ) 197 { 198 NV_STATUS nvStatus; 199 KernelGsp *pKernelGsp = GPU_GET_KERNEL_GSP(pGpu); 200 201 NV_ASSERT(rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 202 203 NV_ASSERT_OR_RETURN(!osIsGpuShutdown(pGpu), NV_ERR_GPU_IS_LOST); 204 205 // Skip queuing RPC if we are in the GPU reset path. 206 if (API_GPU_IN_RESET_SANITY_CHECK(pGpu)) 207 { 208 NV_PRINTF(LEVEL_INFO, "Skip queuing RPC in the GPU reset path \n"); 209 return NV_ERR_GPU_IS_LOST; 210 } 211 212 nvStatus = GspMsgQueueSendCommand(pRpc->pMessageQueueInfo, pGpu); 213 if (nvStatus != NV_OK) 214 { 215 if (nvStatus == NV_ERR_TIMEOUT || 216 nvStatus == NV_ERR_BUSY_RETRY) 217 { 218 _kgspRpcIncrementTimeoutCountAndRateLimitPrints(pGpu, pRpc); 219 } 220 NV_PRINTF_COND(pRpc->bQuietPrints, LEVEL_INFO, LEVEL_ERROR, 221 "GspMsgQueueSendCommand failed on GPU%d: 0x%x\n", 222 gpuGetInstance(pGpu), nvStatus); 223 return nvStatus; 224 } 225 226 kgspSetCmdQueueHead_HAL(pGpu, pKernelGsp, pRpc->pMessageQueueInfo->queueIdx, 0); 227 228 // Add RPC history entry 229 { 230 NvU32 func = RPC_HDR->function; 231 NvU32 entry; 232 233 entry = pRpc->rpcHistoryCurrent = (pRpc->rpcHistoryCurrent + 1) % RPC_HISTORY_DEPTH; 234 235 portMemSet(&pRpc->rpcHistory[entry], 0, sizeof(pRpc->rpcHistory[0])); 236 pRpc->rpcHistory[entry].function = func; 237 238 _kgspGetActiveRpcDebugData(pRpc, func, 239 &pRpc->rpcHistory[entry].data[0], 240 &pRpc->rpcHistory[entry].data[1]); 241 } 242 243 return NV_OK; 244 } 245 246 static NV_STATUS 247 _kgspRpcRunCpuSequencer 248 ( 249 OBJGPU *pGpu, 250 OBJRPC *pRpc 251 ) 252 { 253 RPC_PARAMS(run_cpu_sequencer, _v17_00); 254 KernelGsp *pKernelGsp = GPU_GET_KERNEL_GSP(pGpu); 255 256 return kgspExecuteSequencerBuffer(pGpu, pKernelGsp, rpc_params); 257 } 258 259 static void 260 _kgspProcessEccNotifier 261 ( 262 OBJGPU *pGpu, 263 void *eventData 264 ) 265 { 266 NV_STATUS nvStatus = NV_OK; 267 MemoryManager *pMemoryMgr = GPU_GET_MEMORY_MANAGER(pGpu); 268 269 if (pMemoryMgr->bEnableDynamicPageOfflining) 270 { 271 Nv2080EccDbeNotification *pParams = (Nv2080EccDbeNotification*)eventData; 272 if ((nvStatus = heapStorePendingBlackList(pGpu, GPU_GET_HEAP(pGpu), pParams->physAddress , 273 pParams->physAddress)) != NV_OK) 274 { 275 if (nvStatus == NV_ERR_RESET_REQUIRED) 276 { 277 NV_PRINTF(LEVEL_INFO, "Since we hit the DED on the reserved region, nothing to handle in this code path... \n"); 278 NV_PRINTF(LEVEL_INFO, "Relying on FBHUB interrupt to kill all the channels and force reset the GPU..\n"); 279 } 280 else 281 { 282 NV_PRINTF(LEVEL_INFO, "Dynamically blacklisting the DED page offset failed with, status: %x\n", nvStatus); 283 DBG_BREAKPOINT(); 284 } 285 } 286 287 } 288 } 289 290 /*! 291 * Receive an event notification from GSP-RM. 292 * 293 * When an event fires in GSP-RM, osNotifyEvent and osEventNotification check 294 * whether the event was originally allocated from client-RM. If so, they post 295 * it to the event queue and take no further action. Client RM picks up the 296 * event here and handles it. 297 */ 298 static NV_STATUS 299 _kgspRpcPostEvent 300 ( 301 OBJGPU *pGpu, 302 OBJRPC *pRpc 303 ) 304 { 305 RPC_PARAMS(post_event, _v17_00); 306 PEVENTNOTIFICATION pNotifyList = NULL; 307 PEVENTNOTIFICATION pNotifyEvent = NULL; 308 Event *pEvent = NULL; 309 NV_STATUS nvStatus = NV_OK; 310 311 // Get the notification list that contains this event. 312 NV_ASSERT_OR_RETURN(CliGetEventInfo(rpc_params->hClient, 313 rpc_params->hEvent, &pEvent), NV_ERR_OBJECT_NOT_FOUND); 314 315 if (pEvent->pNotifierShare != NULL) 316 pNotifyList = pEvent->pNotifierShare->pEventList; 317 318 NV_ASSERT_OR_RETURN(pNotifyList != NULL, NV_ERR_INVALID_POINTER); 319 320 switch (rpc_params->notifyIndex) 321 { 322 case NV2080_NOTIFIERS_ECC_DBE: 323 _kgspProcessEccNotifier(pGpu, rpc_params->eventData); 324 break; 325 } 326 327 // Send the event. 328 if (rpc_params->bNotifyList) 329 { 330 // Send notification to all matching events on the list. 331 nvStatus = osEventNotificationWithInfo(pGpu, pNotifyList, rpc_params->notifyIndex, 332 rpc_params->data, rpc_params->info16, rpc_params->eventData, rpc_params->eventDataSize); 333 } 334 else 335 { 336 // Send event to a specific hEvent. Find hEvent in the notification list. 337 for (pNotifyEvent = pNotifyList; pNotifyEvent; pNotifyEvent = pNotifyEvent->Next) 338 { 339 if (pNotifyEvent->hEvent == rpc_params->hEvent) 340 { 341 nvStatus = osNotifyEvent(pGpu, pNotifyEvent, 0, 342 rpc_params->data, rpc_params->status); 343 break; 344 } 345 } 346 NV_ASSERT_OR_RETURN(pNotifyEvent != NULL, NV_ERR_OBJECT_NOT_FOUND); 347 } 348 349 return nvStatus; 350 } 351 352 /*! 353 * Receive RC notification from GSP-RM. 354 * 355 * RC error handling ("Channel Teardown sequence") is executed in GSP-RM. 356 * Client notifications, OS interaction etc happen in CPU-RM (Kernel RM). 357 */ 358 static NV_STATUS 359 _kgspRpcRCTriggered 360 ( 361 OBJGPU *pGpu, 362 OBJRPC *pRpc 363 ) 364 { 365 RPC_PARAMS(rc_triggered, _v17_02); 366 367 KernelRc *pKernelRc = GPU_GET_KERNEL_RC(pGpu); 368 KernelChannel *pKernelChannel; 369 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 370 CHID_MGR *pChidMgr; 371 NvU32 status = NV_OK; 372 RM_ENGINE_TYPE rmEngineType = gpuGetRmEngineType(rpc_params->nv2080EngineType); 373 374 // check if there's a PCI-E error pending either in device status or in AER 375 krcCheckBusError_HAL(pGpu, pKernelRc); 376 377 status = kfifoGetChidMgrFromType(pGpu, pKernelFifo, 378 ENGINE_INFO_TYPE_RM_ENGINE_TYPE, 379 (NvU32)rmEngineType, 380 &pChidMgr); 381 if (status != NV_OK) 382 return status; 383 384 pKernelChannel = kfifoChidMgrGetKernelChannel(pGpu, pKernelFifo, 385 pChidMgr, 386 rpc_params->chid); 387 NV_CHECK_OR_RETURN(LEVEL_ERROR, 388 pKernelChannel != NULL, 389 NV_ERR_INVALID_CHANNEL); 390 391 return krcErrorSendEventNotifications_HAL(pGpu, pKernelRc, 392 pKernelChannel, 393 rmEngineType, // unused on kernel side 394 rpc_params->exceptType, 395 rpc_params->scope, 396 rpc_params->partitionAttributionId); 397 } 398 399 /*! 400 * Receive Xid notification from GSP-RM 401 * 402 * Passes Xid errors that are triggered on GSP-RM to nvErrorLog for OS interactions 403 * (logging and OS notifications). 404 */ 405 static void 406 _kgspRpcOsErrorLog 407 ( 408 OBJGPU *pGpu, 409 OBJRPC *pRpc 410 ) 411 { 412 RPC_PARAMS(os_error_log, _v17_00); 413 414 KernelRc *pKernelRc = GPU_GET_KERNEL_RC(pGpu); 415 KernelChannel *pKernelChannel = NULL; 416 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 417 CHID_MGR *pChidMgr; 418 419 if (rpc_params->chid != INVALID_CHID) 420 { 421 pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, rpc_params->runlistId); 422 if (pChidMgr != NULL) 423 { 424 pKernelChannel = kfifoChidMgrGetKernelChannel(pGpu, pKernelFifo, 425 pChidMgr, 426 rpc_params->chid); 427 } 428 } 429 430 pKernelRc->pPreviousChannelInError = pKernelChannel; 431 nvErrorLog_va(pGpu, rpc_params->exceptType, "%s", rpc_params->errString); 432 pKernelRc->pPreviousChannelInError = NULL; 433 } 434 435 /*! 436 * Receives RPC events containing periodic perfmon utilization samples, passing them 437 * to GPUACCT for processing. 438 */ 439 static void 440 _kgspRpcGpuacctPerfmonUtilSamples 441 ( 442 OBJGPU *pGpu, 443 OBJRPC *pRpc 444 ) 445 { 446 OBJSYS *pSys = SYS_GET_INSTANCE(); 447 GpuAccounting *pGpuAcct = SYS_GET_GPUACCT(pSys); 448 GPUACCT_GPU_INSTANCE_INFO *pGpuInstanceInfo = &pGpuAcct->gpuInstanceInfo[pGpu->gpuInstance]; 449 RPC_PARAMS(gpuacct_perfmon_util_samples, _v17_00); 450 451 NV2080_CTRL_PERF_GET_GPUMON_PERFMON_UTIL_SAMPLES_V2_PARAMS_v17_00 *src = &rpc_params->params; 452 NV2080_CTRL_PERF_GET_GPUMON_PERFMON_UTIL_SAMPLES_V2_PARAMS *dest; 453 NvU32 i; 454 455 dest = pGpuInstanceInfo->pSamplesParams; 456 if (dest == NULL) 457 { 458 // This RPC event can be received even when the RM hasn't fully started. 459 // For instance, CPU RM can take longer than usual to initialize, 460 // but the GSP RM sampling timer (a 1 sec interval) is about to tick. 461 // In that case, pSamplesParams can not even be allocated by that time. 462 // Ignore this RPC event if pSamplesParams has not been allocated yet. 463 // See GPUSWSEC-1543 for more info. 464 return; 465 } 466 467 portMemSet(dest, 0, sizeof(*dest)); 468 dest->type = src->type; 469 dest->bufSize = src->bufSize; 470 dest->count = src->count; 471 dest->tracker = src->tracker; 472 473 for (i = 0; i < NV2080_CTRL_PERF_GPUMON_SAMPLE_COUNT_PERFMON_UTIL; i++) 474 { 475 dest->samples[i].base.timeStamp = src->samples[i].timeStamp; 476 477 dest->samples[i].fb.util = src->samples[i].fb.util; 478 dest->samples[i].fb.procId = src->samples[i].fb.procId; 479 dest->samples[i].fb.subProcessID = src->samples[i].fb.subProcessID; 480 481 dest->samples[i].gr.util = src->samples[i].gr.util; 482 dest->samples[i].gr.procId = src->samples[i].gr.procId; 483 dest->samples[i].gr.subProcessID = src->samples[i].gr.subProcessID; 484 485 dest->samples[i].nvenc.util = src->samples[i].nvenc.util; 486 dest->samples[i].nvenc.procId = src->samples[i].nvenc.procId; 487 dest->samples[i].nvenc.subProcessID = src->samples[i].nvenc.subProcessID; 488 489 dest->samples[i].nvdec.util = src->samples[i].nvdec.util; 490 dest->samples[i].nvdec.procId = src->samples[i].nvdec.procId; 491 dest->samples[i].nvdec.subProcessID = src->samples[i].nvdec.subProcessID; 492 } 493 494 gpuacctProcessGpuUtil(pGpuInstanceInfo, &dest->samples[0]); 495 } 496 497 /*! 498 * Receives RPC events containing current GPU Boost synchronization limits 499 * that should be cached and considered in the GPU Boost algorithm and runs 500 * the algorithm. 501 */ 502 static void 503 _kgspRpcPerfGpuBoostSyncLimitsCallback 504 ( 505 OBJGPU *pGpu, 506 OBJRPC *pRpc 507 ) 508 { 509 KernelPerf *pKernelPerf = GPU_GET_KERNEL_PERF(pGpu); 510 511 RPC_PARAMS(perf_gpu_boost_sync_limits_callback, _v17_00); 512 513 NV2080_CTRL_INTERNAL_PERF_GPU_BOOST_SYNC_SET_LIMITS_PARAMS_v17_00 *src = &rpc_params->params; 514 NV2080_CTRL_INTERNAL_PERF_GPU_BOOST_SYNC_SET_LIMITS_PARAMS dest; 515 NvU32 i; 516 517 dest.flags = src->flags; 518 dest.bBridgeless = src->bBridgeless; 519 520 for (i = 0; i < NV2080_CTRL_INTERNAL_PERF_SYNC_GPU_BOOST_LIMITS_NUM; i++) 521 { 522 dest.currLimits[i] = src->currLimits[i]; 523 } 524 525 kperfDoSyncGpuBoostLimits(pGpu, pKernelPerf, &dest); 526 527 } 528 529 /*! 530 * Recieves RPC events containing latest change of bridgeless information 531 */ 532 static void 533 _kgspRpcPerfBridgelessInfoUpdate 534 ( 535 OBJGPU *pGpu, 536 OBJRPC *pRpc 537 ) 538 { 539 RPC_PARAMS(perf_bridgeless_info_update, _v17_00); 540 541 kPerfGpuBoostSyncBridgelessUpdateInfo(pGpu, rpc_params->bBridgeless); 542 } 543 544 static void 545 _kgspRpcNvlinkFaultUpCallback 546 ( 547 OBJGPU *pGpu, 548 OBJRPC *pRpc 549 ) 550 { 551 RPC_PARAMS(nvlink_fault_up, _v17_00); 552 553 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu); 554 555 knvlinkHandleFaultUpInterrupt_HAL(pGpu, pKernelNvlink, rpc_params->linkId); 556 } 557 558 static void 559 _kgspRpcNvlinkInbandReceivedData256Callback 560 ( 561 OBJGPU *pGpu, 562 OBJRPC *pRpc 563 ) 564 { 565 RPC_PARAMS(nvlink_inband_received_data_256, _v17_00); 566 567 NV2080_CTRL_NVLINK_INBAND_RECEIVED_DATA_256_PARAMS_v17_00 *dest = &rpc_params->params; 568 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu); 569 570 NV_ASSERT(NV_OK == knvlinkInbandMsgCallbackDispatcher(pGpu, pKernelNvlink, dest->dataSize, dest->data)); 571 } 572 573 static void 574 _kgspRpcNvlinkInbandReceivedData512Callback 575 ( 576 OBJGPU *pGpu, 577 OBJRPC *pRpc 578 ) 579 { 580 RPC_PARAMS(nvlink_inband_received_data_512, _v17_00); 581 582 NV2080_CTRL_NVLINK_INBAND_RECEIVED_DATA_512_PARAMS_v17_00 *dest = &rpc_params->params; 583 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu); 584 585 NV_ASSERT(NV_OK == knvlinkInbandMsgCallbackDispatcher(pGpu, pKernelNvlink, dest->dataSize, dest->data)); 586 } 587 588 static void 589 _kgspRpcNvlinkInbandReceivedData1024Callback 590 ( 591 OBJGPU *pGpu, 592 OBJRPC *pRpc 593 ) 594 { 595 RPC_PARAMS(nvlink_inband_received_data_1024, _v17_00); 596 597 NV2080_CTRL_NVLINK_INBAND_RECEIVED_DATA_1024_PARAMS_v17_00 *dest = &rpc_params->params; 598 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu); 599 600 NV_ASSERT(NV_OK == knvlinkInbandMsgCallbackDispatcher(pGpu, pKernelNvlink, dest->dataSize, dest->data)); 601 } 602 603 static void 604 _kgspRpcNvlinkInbandReceivedData2048Callback 605 ( 606 OBJGPU *pGpu, 607 OBJRPC *pRpc 608 ) 609 { 610 RPC_PARAMS(nvlink_inband_received_data_2048, _v17_00); 611 612 NV2080_CTRL_NVLINK_INBAND_RECEIVED_DATA_2048_PARAMS_v17_00 *dest = &rpc_params->params; 613 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu); 614 615 NV_ASSERT(NV_OK == knvlinkInbandMsgCallbackDispatcher(pGpu, pKernelNvlink, dest->dataSize, dest->data)); 616 } 617 618 static void 619 _kgspRpcNvlinkInbandReceivedData4096Callback 620 ( 621 OBJGPU *pGpu, 622 OBJRPC *pRpc 623 ) 624 { 625 RPC_PARAMS(nvlink_inband_received_data_4096, _v17_00); 626 627 NV2080_CTRL_NVLINK_INBAND_RECEIVED_DATA_4096_PARAMS_v17_00 *dest = &rpc_params->params; 628 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu); 629 630 NV_ASSERT(NV_OK == knvlinkInbandMsgCallbackDispatcher(pGpu, pKernelNvlink, dest->dataSize, dest->data)); 631 } 632 633 /*! 634 * CPU-RM: Receive GPU Degraded status from GSP 635 */ 636 static void 637 _kgspRpcEventIsGpuDegradedCallback 638 ( 639 OBJGPU *pGpu, 640 OBJRPC *pRpc 641 ) 642 { 643 } 644 645 /*! 646 * Receive MMU fault queue notification from GSP-RM. 647 * 648 * Non-replayable fault handling is split between GSP-RM and the UVM driver. 649 * GSP-RM copies designated faults to the UVM driver's shadow buffer, 650 * and sends a notification. CPU-RM, in turn, needs to notify the UVM 651 * driver (schedule the UVM ISR to be run). 652 */ 653 static NV_STATUS 654 _kgspRpcMMUFaultQueued( 655 OBJGPU *pGpu, 656 OBJRPC *pRpc 657 ) 658 { 659 osQueueMMUFaultHandler(pGpu); 660 661 return NV_OK; 662 } 663 664 static NV_STATUS 665 _kgspRpcSimRead 666 ( 667 OBJGPU *pGpu, 668 OBJRPC *pRpc 669 ) 670 { 671 RPC_PARAMS(sim_read, _v1E_01); 672 if (IS_SIMULATION(pGpu)) 673 { 674 const NvU32 count = rpc_params->index + (rpc_params->count / sizeof(NvU32)); 675 NvU32 i; 676 677 KernelGsp *pKernelGsp = GPU_GET_KERNEL_GSP(pGpu); 678 OBJSYS *pSys = SYS_GET_INSTANCE(); 679 OBJOS *pOS = SYS_GET_OS(pSys); 680 681 NV_ASSERT_OR_RETURN(rpc_params->count <= sizeof(pKernelGsp->pSimAccessBuf->data), NV_ERR_BUFFER_TOO_SMALL); 682 683 for (i = rpc_params->index; i < count; i++) 684 { 685 NvU32 data; 686 pOS->osSimEscapeRead(pGpu, rpc_params->path, i, 4, &data); 687 pKernelGsp->pSimAccessBuf->data[i] = data; 688 } 689 690 pKernelGsp->pSimAccessBuf->seq++; 691 return NV_OK; 692 } 693 694 return NV_ERR_NOT_SUPPORTED; 695 } 696 697 static NV_STATUS 698 _kgspRpcSimWrite 699 ( 700 OBJGPU *pGpu, 701 OBJRPC *pRpc 702 ) 703 { 704 RPC_PARAMS(sim_write, _v1E_01); 705 if (IS_SIMULATION(pGpu)) 706 { 707 KernelGsp *pKernelGsp = GPU_GET_KERNEL_GSP(pGpu); 708 OBJSYS *pSys = SYS_GET_INSTANCE(); 709 OBJOS *pOS = SYS_GET_OS(pSys); 710 711 pOS->osSimEscapeWrite(pGpu, rpc_params->path, rpc_params->index, rpc_params->count, rpc_params->data); 712 pKernelGsp->pSimAccessBuf->seq++; 713 return NV_OK; 714 } 715 716 return NV_ERR_NOT_SUPPORTED; 717 } 718 719 static NV_STATUS 720 _kgspRpcSemaphoreScheduleCallback( 721 OBJGPU *pGpu, 722 OBJRPC *pRpc 723 ) 724 { 725 RPC_PARAMS(semaphore_schedule_callback, _v17_00); 726 NV_STATUS status; 727 RsClient *pClient; 728 729 status = serverGetClientUnderLock(&g_resServ, rpc_params->hClient, &pClient); 730 if (status != NV_OK) 731 return status; 732 733 return dispswReleaseSemaphoreAndNotifierFill(pGpu, 734 rpc_params->GPUVA, 735 rpc_params->hVASpace, 736 rpc_params->ReleaseValue, 737 rpc_params->Flags, 738 rpc_params->completionStatus, 739 pClient, rpc_params->hEvent); 740 } 741 742 static NV_STATUS 743 _kgspRpcTimedSemaphoreRelease( 744 OBJGPU *pGpu, 745 OBJRPC *pRpc 746 ) 747 { 748 RPC_PARAMS(timed_semaphore_release, _v01_00); 749 NV_STATUS status; 750 RsClient *pClient; 751 752 status = serverGetClientUnderLock(&g_resServ, rpc_params->hClient, &pClient); 753 if (status != NV_OK) 754 return status; 755 756 return tsemaRelease_HAL(pGpu, 757 rpc_params->semaphoreVA, 758 rpc_params->notifierVA, 759 rpc_params->hVASpace, 760 rpc_params->releaseValue, 761 rpc_params->completionStatus, 762 pClient); 763 } 764 765 766 static NV_STATUS 767 _kgspRpcUcodeLibosPrint 768 ( 769 OBJGPU *pGpu, 770 OBJRPC *pRpc 771 ) 772 { 773 RPC_PARAMS(ucode_libos_print, _v1E_08); 774 775 // Check ucodes registered with the libos print mechanism 776 switch (rpc_params->ucodeEngDesc) 777 { 778 case ENG_PMU: 779 { 780 KernelPmu *pKernelPmu = GPU_GET_KERNEL_PMU(pGpu); 781 NV_CHECK_OR_RETURN(LEVEL_ERROR, pKernelPmu != NULL, NV_ERR_OBJECT_NOT_FOUND); 782 783 kpmuLogBuf(pGpu, pKernelPmu, 784 rpc_params->libosPrintBuf, rpc_params->libosPrintBufSize); 785 786 return NV_OK; 787 } 788 default: 789 NV_ASSERT_FAILED("Attempting to use libos prints with an unsupported ucode!\n"); 790 return NV_ERR_NOT_SUPPORTED; 791 } 792 } 793 794 static NV_STATUS 795 _kgspRpcVgpuGspPluginTriggered 796 ( 797 OBJGPU *pGpu, 798 OBJRPC *pRpc 799 ) 800 { 801 RPC_PARAMS(vgpu_gsp_plugin_triggered, _v17_00); 802 803 if (!IS_VGPU_GSP_PLUGIN_OFFLOAD_ENABLED(pGpu)) 804 return NV_ERR_NOT_SUPPORTED; 805 806 gpuGspPluginTriggeredEvent(pGpu, rpc_params->gfid, rpc_params->notifyIndex); 807 return NV_OK; 808 } 809 810 static NV_STATUS 811 _kgspRpcGspVgpuConfig 812 ( 813 OBJGPU *pGpu, 814 OBJRPC *pRpc 815 ) 816 { 817 RPC_PARAMS(vgpu_config_event, _v17_00); 818 819 NV_ASSERT_OR_RETURN(rpc_params->notifyIndex < NVA081_NOTIFIERS_MAXCOUNT, 820 NV_ERR_INVALID_ARGUMENT); 821 822 CliNotifyVgpuConfigEvent(pGpu, rpc_params->notifyIndex); 823 824 return NV_OK; 825 } 826 827 static NV_STATUS 828 _kgspRpcGspExtdevIntrService 829 ( 830 OBJGPU *pGpu, 831 OBJRPC *pRpc 832 ) 833 { 834 RPC_PARAMS(extdev_intr_service, _v17_00); 835 836 extdevGsyncService(pGpu, rpc_params->lossRegStatus, rpc_params->gainRegStatus, rpc_params->miscRegStatus, rpc_params->rmStatus); 837 838 return NV_OK; 839 } 840 841 static void 842 _kgspRpcGspSendUserSharedData 843 ( 844 OBJGPU *pGpu, 845 OBJRPC *pRpc 846 ) 847 { 848 RPC_PARAMS(gsp_send_user_shared_data, _v17_00); 849 NV00DE_SHARED_DATA *pSharedData = gpushareddataWriteStart(pGpu); 850 GspUserSharedData *pRpcData = (GspUserSharedData*)(&rpc_params->data); 851 852 // Copy over all GSP-owned data 853 pSharedData->gspAssertCount = pRpcData->gspAssertCount; 854 855 gpushareddataWriteFinish(pGpu); 856 } 857 858 static void _kgspRpcMigCiConfigUpdateCallback 859 ( 860 NvU32 gpuInstance, 861 void *pArgs 862 ) 863 { 864 OBJGPU *pGpu = gpumgrGetGpu(gpuInstance); 865 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 866 struct MIG_CI_UPDATE_CALLBACK_PARAMS * pParams = (struct MIG_CI_UPDATE_CALLBACK_PARAMS *)pArgs; 867 868 kmigmgrUpdateCiConfigForVgpu(pGpu, pKernelMIGManager, 869 pParams->execPartCount, pParams->execPartId, 870 pParams->gfid, pParams->bDelete); 871 872 return; 873 } 874 875 static NV_STATUS 876 _kgspRpcMigCiConfigUpdate 877 ( 878 OBJGPU *pGpu, 879 OBJRPC *pRpc 880 ) 881 { 882 NV_STATUS status; 883 OBJOS *pOS = GPU_GET_OS(pGpu); 884 struct MIG_CI_UPDATE_CALLBACK_PARAMS *pParams; 885 886 RPC_PARAMS(vgpu_gsp_mig_ci_config, _v21_03); 887 888 NV_ASSERT_OR_RETURN(rpc_params->execPartCount <= NVC637_CTRL_MAX_EXEC_PARTITIONS, 889 NV_ERR_INVALID_ARGUMENT); 890 891 pParams = portMemAllocNonPaged(sizeof(struct MIG_CI_UPDATE_CALLBACK_PARAMS)); 892 if (pParams == NULL) 893 { 894 return NV_ERR_NO_MEMORY; 895 } 896 897 pParams->execPartCount = rpc_params->execPartCount; 898 portMemCopy(pParams->execPartId, (sizeof(NvU32) * rpc_params->execPartCount), 899 rpc_params->execPartId, (sizeof(NvU32) * rpc_params->execPartCount)); 900 pParams->gfid = rpc_params->gfid; 901 pParams->bDelete = rpc_params->bDelete; 902 status = pOS->osQueueWorkItemWithFlags(pGpu, 903 _kgspRpcMigCiConfigUpdateCallback, 904 (void *)pParams, 905 OS_QUEUE_WORKITEM_FLAGS_LOCK_API_RW | OS_QUEUE_WORKITEM_FLAGS_LOCK_GPUS_RW); 906 if (status != NV_OK) 907 { 908 portMemFree(pParams); 909 } 910 911 return status; 912 } 913 914 static NV_STATUS 915 _kgspRpcRgLineIntr 916 ( 917 OBJGPU *pGpu, 918 OBJRPC *pRpc 919 ) 920 { 921 RPC_PARAMS(rg_line_intr, _v17_00); 922 923 KernelDisplay *pKernelDisplay = GPU_GET_KERNEL_DISPLAY(pGpu); 924 NV_CHECK_OR_RETURN(LEVEL_ERROR, pKernelDisplay != NULL, NV_ERR_OBJECT_NOT_FOUND); 925 926 kdispInvokeRgLineCallback(pKernelDisplay, rpc_params->head, rpc_params->rgIntr, NV_FALSE); 927 928 return NV_OK; 929 } 930 931 static NV_STATUS 932 _kgspRpcEventPlatformRequestHandlerStateSyncCallback 933 ( 934 OBJGPU* pGpu, 935 OBJRPC* pRpc 936 ) 937 { 938 OBJSYS *pSys = SYS_GET_INSTANCE(); 939 PlatformRequestHandler* pPlatformRequestHandler 940 = SYS_GET_PFM_REQ_HNDLR(pSys); 941 942 RPC_PARAMS(pfm_req_hndlr_state_sync_callback, _v21_04); 943 944 NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_PARAMS_v21_04 *src = &rpc_params->params; 945 NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_PARAMS dst = { 0 }; 946 947 dst.flags = src->flags; 948 dst.syncData.type = src->syncData.type; 949 950 // Copy in the rpc data 951 switch (src->syncData.type) 952 { 953 case NV2080_CTRL_INTERNAL_PFM_REQ_HNDLR_STATE_SYNC_DATA_TYPE_SMBPBI: 954 { 955 dst.syncData.data.smbpbi.sensorId = 956 src->syncData.data.smbpbi.sensorId; 957 dst.syncData.data.smbpbi.limit = 958 src->syncData.data.smbpbi.limit; 959 break; 960 } 961 default: 962 { 963 // Nothing for now 964 break; 965 } 966 } 967 968 pfmreqhndlrStateSync(pPlatformRequestHandler, pGpu, &dst); 969 return NV_OK; 970 } 971 972 static void 973 _kgspRpcGspLockdownNotice 974 ( 975 OBJGPU *pGpu, 976 OBJRPC *pRpc 977 ) 978 { 979 KernelGsp *pKernelGsp = GPU_GET_KERNEL_GSP(pGpu); 980 RPC_PARAMS(gsp_lockdown_notice, _v17_00); 981 982 // 983 // While the GSP is in lockdown, we cannot access some of its registers, 984 // including interrupt status and control. We shouldn't receive any more 985 // SWGEN0 interrupts while the core is in lockdown. 986 // 987 pKernelGsp->bInLockdown = rpc_params->bLockdownEngaging; 988 989 NV_PRINTF(LEVEL_INFO, "GSP lockdown %s\n", 990 pKernelGsp->bInLockdown ? "engaged" : "disengaged"); 991 } 992 993 static 994 const char *_getRpcName 995 ( 996 NvU32 id 997 ) 998 { 999 static const char *rpcName[] = 1000 { 1001 #define X(UNIT, a) #a, 1002 #define E(a) #a, 1003 #undef _RPC_GLOBAL_ENUMS_H_ 1004 #include "vgpu/rpc_global_enums.h" 1005 #undef X 1006 #undef E 1007 }; 1008 1009 if (id < NV_VGPU_MSG_FUNCTION_NUM_FUNCTIONS) 1010 { 1011 return rpcName[id]; 1012 } 1013 else if ((id > NV_VGPU_MSG_EVENT_FIRST_EVENT) && (id < NV_VGPU_MSG_EVENT_NUM_EVENTS)) 1014 { 1015 NvU32 index = id - (NV_VGPU_MSG_EVENT_FIRST_EVENT - NV_VGPU_MSG_FUNCTION_NUM_FUNCTIONS) + 1; 1016 return rpcName[index]; 1017 } 1018 1019 return "Unknown"; 1020 } 1021 1022 /*! 1023 * GSP client process RPC events 1024 */ 1025 static NV_STATUS 1026 _kgspProcessRpcEvent 1027 ( 1028 OBJGPU *pGpu, 1029 OBJRPC *pRpc 1030 ) 1031 { 1032 rpc_message_header_v *pMsgHdr = RPC_HDR; 1033 NV_STATUS nvStatus = NV_OK; 1034 1035 NV_PRINTF(LEVEL_INFO, "received event from GPU%d: 0x%x (%s) status: 0x%x size: %d\n", 1036 gpuGetInstance(pGpu), pMsgHdr->function, _getRpcName(pMsgHdr->function), pMsgHdr->rpc_result, pMsgHdr->length); 1037 1038 switch(pMsgHdr->function) 1039 { 1040 case NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER: 1041 nvStatus = _kgspRpcRunCpuSequencer(pGpu, pRpc); 1042 break; 1043 1044 case NV_VGPU_MSG_EVENT_POST_EVENT: 1045 nvStatus = _kgspRpcPostEvent(pGpu, pRpc); 1046 break; 1047 1048 case NV_VGPU_MSG_EVENT_RC_TRIGGERED: 1049 nvStatus = _kgspRpcRCTriggered(pGpu, pRpc); 1050 break; 1051 1052 case NV_VGPU_MSG_EVENT_MMU_FAULT_QUEUED: 1053 nvStatus = _kgspRpcMMUFaultQueued(pGpu, pRpc); 1054 break; 1055 1056 case NV_VGPU_MSG_EVENT_SIM_READ: 1057 nvStatus = _kgspRpcSimRead(pGpu, pRpc); 1058 break; 1059 1060 case NV_VGPU_MSG_EVENT_SIM_WRITE: 1061 nvStatus = _kgspRpcSimWrite(pGpu, pRpc); 1062 break; 1063 1064 case NV_VGPU_MSG_EVENT_OS_ERROR_LOG: 1065 _kgspRpcOsErrorLog(pGpu, pRpc); 1066 break; 1067 1068 case NV_VGPU_MSG_EVENT_GPUACCT_PERFMON_UTIL_SAMPLES: 1069 _kgspRpcGpuacctPerfmonUtilSamples(pGpu, pRpc); 1070 break; 1071 1072 case NV_VGPU_MSG_EVENT_PERF_GPU_BOOST_SYNC_LIMITS_CALLBACK: 1073 _kgspRpcPerfGpuBoostSyncLimitsCallback(pGpu, pRpc); 1074 break; 1075 1076 case NV_VGPU_MSG_EVENT_PERF_BRIDGELESS_INFO_UPDATE: 1077 _kgspRpcPerfBridgelessInfoUpdate(pGpu, pRpc); 1078 break; 1079 1080 case NV_VGPU_MSG_EVENT_SEMAPHORE_SCHEDULE_CALLBACK: 1081 _kgspRpcSemaphoreScheduleCallback(pGpu, pRpc); 1082 break; 1083 1084 case NV_VGPU_MSG_EVENT_TIMED_SEMAPHORE_RELEASE: 1085 _kgspRpcTimedSemaphoreRelease(pGpu, pRpc); 1086 break; 1087 1088 case NV_VGPU_MSG_EVENT_NVLINK_FAULT_UP: 1089 _kgspRpcNvlinkFaultUpCallback(pGpu, pRpc); 1090 break; 1091 1092 case NV_VGPU_MSG_EVENT_NVLINK_INBAND_RECEIVED_DATA_256: 1093 _kgspRpcNvlinkInbandReceivedData256Callback(pGpu, pRpc); 1094 break; 1095 1096 case NV_VGPU_MSG_EVENT_NVLINK_INBAND_RECEIVED_DATA_512: 1097 _kgspRpcNvlinkInbandReceivedData512Callback(pGpu, pRpc); 1098 break; 1099 1100 case NV_VGPU_MSG_EVENT_NVLINK_INBAND_RECEIVED_DATA_1024: 1101 _kgspRpcNvlinkInbandReceivedData1024Callback(pGpu, pRpc); 1102 break; 1103 1104 case NV_VGPU_MSG_EVENT_NVLINK_INBAND_RECEIVED_DATA_2048: 1105 _kgspRpcNvlinkInbandReceivedData2048Callback(pGpu, pRpc); 1106 break; 1107 1108 case NV_VGPU_MSG_EVENT_NVLINK_INBAND_RECEIVED_DATA_4096: 1109 _kgspRpcNvlinkInbandReceivedData4096Callback(pGpu, pRpc); 1110 break; 1111 1112 case NV_VGPU_MSG_EVENT_NVLINK_IS_GPU_DEGRADED : 1113 _kgspRpcEventIsGpuDegradedCallback(pGpu, pRpc); 1114 break; 1115 1116 case NV_VGPU_MSG_EVENT_RG_LINE_INTR: 1117 _kgspRpcRgLineIntr(pGpu, pRpc); 1118 break; 1119 1120 case NV_VGPU_MSG_EVENT_UCODE_LIBOS_PRINT: 1121 nvStatus = _kgspRpcUcodeLibosPrint(pGpu, pRpc); 1122 break; 1123 1124 case NV_VGPU_MSG_EVENT_VGPU_GSP_PLUGIN_TRIGGERED: 1125 nvStatus = _kgspRpcVgpuGspPluginTriggered(pGpu, pRpc); 1126 break; 1127 1128 case NV_VGPU_MSG_EVENT_VGPU_CONFIG: 1129 nvStatus = _kgspRpcGspVgpuConfig(pGpu, pRpc); 1130 break; 1131 1132 case NV_VGPU_MSG_EVENT_EXTDEV_INTR_SERVICE: 1133 nvStatus = _kgspRpcGspExtdevIntrService(pGpu, pRpc); 1134 break; 1135 1136 case NV_VGPU_MSG_EVENT_PFM_REQ_HNDLR_STATE_SYNC_CALLBACK: 1137 nvStatus = _kgspRpcEventPlatformRequestHandlerStateSyncCallback(pGpu, pRpc); 1138 break; 1139 1140 case NV_VGPU_MSG_EVENT_MIG_CI_CONFIG_UPDATE: 1141 nvStatus = _kgspRpcMigCiConfigUpdate(pGpu, pRpc); 1142 break; 1143 1144 case NV_VGPU_MSG_EVENT_GSP_SEND_USER_SHARED_DATA: 1145 _kgspRpcGspSendUserSharedData(pGpu, pRpc); 1146 break; 1147 1148 case NV_VGPU_MSG_EVENT_GSP_LOCKDOWN_NOTICE: 1149 _kgspRpcGspLockdownNotice(pGpu, pRpc); 1150 break; 1151 1152 case NV_VGPU_MSG_EVENT_GSP_INIT_DONE: // Handled by _kgspRpcRecvPoll. 1153 default: 1154 // 1155 // We will get here if the previous RPC timed out. The response 1156 // eventually comes in as an unexpected event. The error handling 1157 // for the timeout has already happened, and returning an error here 1158 // causes subsequent messages to fail. So return NV_OK. 1159 // 1160 NV_PRINTF(LEVEL_ERROR, "Unexpected RPC event from GPU%d: 0x%x (%s)\n", 1161 gpuGetInstance(pGpu), pMsgHdr->function, _getRpcName(pMsgHdr->function)); 1162 break; 1163 } 1164 1165 return nvStatus; 1166 } 1167 1168 /*! 1169 * Handle a single RPC event from GSP unless the event is [an RPC return for] expectedFunc, 1170 * or there are no events available in the buffer. 1171 * 1172 * @return 1173 * NV_OK if the event is successfully handled. 1174 * NV_WARN_NOTHING_TO_DO if there are no events available. 1175 * NV_WARN_MORE_PROCESSING_REQUIRED if the event is expectedFunc: it is unhandled and in the staging area. 1176 * (Another status) if event reading or processing fails. 1177 */ 1178 static NV_STATUS 1179 _kgspRpcDrainOneEvent 1180 ( 1181 OBJGPU *pGpu, 1182 OBJRPC *pRpc, 1183 NvU32 expectedFunc 1184 ) 1185 { 1186 NV_STATUS nvStatus; 1187 1188 // Issue a memory barrier to ensure we see any queue updates. 1189 // Note: Without the fence, the CPU may get stuck in an infinite loop 1190 // waiting for a message that has already arrived. 1191 portAtomicMemoryFenceFull(); 1192 1193 nvStatus = GspMsgQueueReceiveStatus(pRpc->pMessageQueueInfo); 1194 1195 if (nvStatus == NV_OK) 1196 { 1197 rpc_message_header_v *pMsgHdr = RPC_HDR; 1198 if (pMsgHdr->function == expectedFunc) 1199 return NV_WARN_MORE_PROCESSING_REQUIRED; 1200 1201 nvStatus = _kgspProcessRpcEvent(pGpu, pRpc); 1202 if (nvStatus != NV_OK) 1203 { 1204 NV_PRINTF(LEVEL_ERROR, 1205 "Failed to process received event 0x%x (%s) from GPU%d: status=0x%x\n", 1206 pMsgHdr->function, _getRpcName(pMsgHdr->function), gpuGetInstance(pGpu), nvStatus); 1207 } 1208 } 1209 1210 // 1211 // We don't expect the NV_WARN_MORE_PROCESSING_REQUIRED from either called function. 1212 // If we get it we need to suppress it to avoid confusing our caller, for whom it has special meaning. 1213 // 1214 NV_ASSERT_OR_ELSE(nvStatus != NV_WARN_MORE_PROCESSING_REQUIRED, 1215 nvStatus = NV_ERR_GENERIC); 1216 1217 return nvStatus; 1218 } 1219 1220 /*! 1221 * Handle RPC events from GSP until the event is [an RPC return for] expectedFunc, 1222 * or there are no events available in the buffer. 1223 * 1224 * Also dump GSP logs, and check for severe errors coming from GSP. 1225 * 1226 * @return 1227 * NV_OK if one or more events are handled and there are none left. 1228 * NV_WARN_MORE_PROCESSING_REQUIRED if an expectedFunc event is found: it is unhandled and in the staging area. 1229 * (Zero or more preceding events were successfully handled.) 1230 * (Another status) if event reading or processing fails. 1231 */ 1232 static NV_STATUS 1233 _kgspRpcDrainEvents 1234 ( 1235 OBJGPU *pGpu, 1236 KernelGsp *pKernelGsp, 1237 NvU32 expectedFunc 1238 ) 1239 { 1240 NV_STATUS nvStatus = NV_OK; 1241 OBJRPC *pRpc = GPU_GET_RPC(pGpu); 1242 1243 while (nvStatus == NV_OK) 1244 { 1245 nvStatus = _kgspRpcDrainOneEvent(pGpu, pRpc, expectedFunc); 1246 kgspDumpGspLogs(pGpu, pKernelGsp, NV_FALSE); 1247 } 1248 1249 kgspHealthCheck_HAL(pGpu, pKernelGsp); 1250 1251 if (nvStatus == NV_WARN_NOTHING_TO_DO) 1252 nvStatus = NV_OK; 1253 1254 return nvStatus; 1255 } 1256 1257 /*! 1258 * Log Xid 119 - GSP RPC Timeout 1259 */ 1260 static void 1261 _kgspLogXid119 1262 ( 1263 OBJGPU *pGpu, 1264 OBJRPC *pRpc, 1265 NvU32 expectedFunc 1266 ) 1267 { 1268 NvU32 historyEntry = pRpc->rpcHistoryCurrent; 1269 1270 if (pRpc->timeoutCount == 1) 1271 { 1272 NV_PRINTF(LEVEL_ERROR, 1273 "********************************* GSP Failure **********************************\n"); 1274 } 1275 1276 NV_ASSERT(expectedFunc == pRpc->rpcHistory[historyEntry].function); 1277 1278 nvErrorLog_va((void*)pGpu, GSP_RPC_TIMEOUT, 1279 "Timeout waiting for RPC from GSP%d! Expected function %d (%s) (0x%x 0x%x).", 1280 gpuGetInstance(pGpu), 1281 expectedFunc, 1282 _getRpcName(expectedFunc), 1283 pRpc->rpcHistory[historyEntry].data[0], 1284 pRpc->rpcHistory[historyEntry].data[1]); 1285 1286 if (pRpc->timeoutCount == 1) 1287 { 1288 NvU32 activeData[2]; 1289 NvU32 historyIndex; 1290 rpc_message_header_v *pMsgHdr = RPC_HDR; 1291 1292 _kgspGetActiveRpcDebugData(pRpc, pMsgHdr->function, 1293 &activeData[0], &activeData[1]); 1294 1295 if ((expectedFunc != pMsgHdr->function) || 1296 (pRpc->rpcHistory[historyEntry].data[0] != activeData[0]) || 1297 (pRpc->rpcHistory[historyEntry].data[1] != activeData[1])) 1298 { 1299 NV_PRINTF(LEVEL_ERROR, 1300 "Current RPC function %d (%s) or data (0x%x 0x%x) does not match expected function %d (%s) or data (0x%x 0x%x).\n", 1301 pMsgHdr->function, _getRpcName(pMsgHdr->function), 1302 activeData[0], activeData[1], 1303 expectedFunc, _getRpcName(expectedFunc), 1304 pRpc->rpcHistory[historyEntry].data[0], 1305 pRpc->rpcHistory[historyEntry].data[1]); 1306 } 1307 1308 NV_PRINTF(LEVEL_ERROR, "RPC history (CPU -> GSP%d):\n", gpuGetInstance(pGpu)); 1309 NV_PRINTF(LEVEL_ERROR, "\tentry\tfunc\t\t\t\tdata\n"); 1310 for (historyIndex = 0; historyIndex < RPC_HISTORY_DEPTH; historyIndex++) 1311 { 1312 historyEntry = (pRpc->rpcHistoryCurrent + RPC_HISTORY_DEPTH - historyIndex) % RPC_HISTORY_DEPTH; 1313 NV_PRINTF(LEVEL_ERROR, "\t%c%-2d\t%2d %-22s\t0x%08x 0x%08x\n", 1314 ((historyIndex == 0) ? ' ' : '-'), 1315 historyIndex, 1316 pRpc->rpcHistory[historyEntry].function, 1317 _getRpcName(pRpc->rpcHistory[historyEntry].function), 1318 pRpc->rpcHistory[historyEntry].data[0], 1319 pRpc->rpcHistory[historyEntry].data[1]); 1320 } 1321 1322 osAssertFailed(); 1323 1324 NV_PRINTF(LEVEL_ERROR, 1325 "********************************************************************************\n"); 1326 } 1327 } 1328 1329 static void 1330 _kgspRpcIncrementTimeoutCountAndRateLimitPrints 1331 ( 1332 OBJGPU *pGpu, 1333 OBJRPC *pRpc 1334 ) 1335 { 1336 pRpc->timeoutCount++; 1337 1338 if ((pRpc->timeoutCount == (RPC_TIMEOUT_LIMIT_PRINT_RATE_THRESH + 1)) && 1339 (RPC_TIMEOUT_LIMIT_PRINT_RATE_SKIP > 0)) 1340 { 1341 // make sure we warn Xid and NV_PRINTF/NVLOG consumers that we are rate limiting prints 1342 if (GPU_GET_KERNEL_RC(pGpu)->bLogEvents) 1343 { 1344 portDbgPrintf( 1345 "NVRM: Rate limiting GSP RPC error prints for GPU at PCI:%04x:%02x:%02x (printing 1 of every %d). The GPU likely needs to be reset.\n", 1346 gpuGetDomain(pGpu), 1347 gpuGetBus(pGpu), 1348 gpuGetDevice(pGpu), 1349 RPC_TIMEOUT_LIMIT_PRINT_RATE_SKIP + 1); 1350 } 1351 NV_PRINTF(LEVEL_WARNING, 1352 "Rate limiting GSP RPC error prints (printing 1 of every %d)\n", 1353 RPC_TIMEOUT_LIMIT_PRINT_RATE_SKIP + 1); 1354 } 1355 1356 pRpc->bQuietPrints = ((pRpc->timeoutCount > RPC_TIMEOUT_LIMIT_PRINT_RATE_THRESH) && 1357 ((pRpc->timeoutCount % (RPC_TIMEOUT_LIMIT_PRINT_RATE_SKIP + 1)) != 0)); 1358 } 1359 1360 /*! 1361 * GSP client RM RPC poll routine 1362 */ 1363 static NV_STATUS 1364 _kgspRpcRecvPoll 1365 ( 1366 OBJGPU *pGpu, 1367 OBJRPC *pRpc, 1368 NvU32 expectedFunc 1369 ) 1370 { 1371 KernelGsp *pKernelGsp = GPU_GET_KERNEL_GSP(pGpu); 1372 NV_STATUS nvStatus; 1373 RMTIMEOUT timeout; 1374 NvU32 timeoutUs; 1375 NvU32 timeoutFlags; 1376 NvBool bSlowGspRpc = IS_EMULATION(pGpu) || IS_SIMULATION(pGpu); 1377 1378 // 1379 // We do not allow recursive polling. This can happen if e.g. 1380 // 1. CPU-RM issues RPC-A to GSP and polls waiting for it to finish 1381 // 2. While servicing RPC-A, GSP emits an async event back to CPU-RM 1382 // 3. CPU-RM services the async event and sends another synchronous RPC-B 1383 // 4. RPC-A response will come first, but CPU-RM is now waiting on RPC-B 1384 // 1385 // We don't have a good way to handle this and should just be deferring the 1386 // second RPC until the first one is done, via e.g. osQueueWorkItem(). 1387 // This assert is meant to catch and loudly fail such cases. 1388 // 1389 NV_ASSERT_OR_RETURN(!pKernelGsp->bPollingForRpcResponse, NV_ERR_INVALID_STATE); 1390 pKernelGsp->bPollingForRpcResponse = NV_TRUE; 1391 1392 // GSP-RM init in emulation/simulation environment is extremely slow, 1393 // so need to increment timeout. 1394 // Apply the timeout extension to other RPCs as well, mostly so that 1395 // we'll reset the thread state after each RPC, not just while waiting 1396 // for the INIT_DONE event. 1397 // 1398 if (bSlowGspRpc) 1399 { 1400 NvU32 timeoutResult; 1401 1402 // On slow Apollo emulators, GSP-RM init could take more than an hour 1403 NV_ASSERT(portSafeMulU32(GSP_SCALE_TIMEOUT_EMU_SIM, 1500000, &timeoutResult)); 1404 timeoutUs = timeoutResult; 1405 } 1406 else 1407 { 1408 NvU32 defaultus = pGpu->timeoutData.defaultus; 1409 1410 if (IS_VGPU_GSP_PLUGIN_OFFLOAD_ENABLED(pGpu)) 1411 { 1412 // Ensure at least 3.1s for vGPU-GSP before adding leeway (Bug 3928607) 1413 timeoutUs = NV_MAX(3100 * 1000, defaultus) + (defaultus / 2); 1414 } 1415 else 1416 { 1417 // We should only ever timeout this when GSP is in really bad state, so if it just 1418 // happens to timeout on default timeout it should be OK for us to give it a little 1419 // more time - make this timeout 1.5 of the default to allow some leeway. 1420 timeoutUs = defaultus + defaultus / 2; 1421 } 1422 } 1423 1424 NV_ASSERT(rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 1425 1426 timeoutFlags = GPU_TIMEOUT_FLAGS_BYPASS_THREAD_STATE; 1427 if (pRpc->bQuietPrints) 1428 timeoutFlags |= GPU_TIMEOUT_FLAGS_BYPASS_JOURNAL_LOG; 1429 1430 gpuSetTimeout(pGpu, timeoutUs, &timeout, timeoutFlags); 1431 1432 for (;;) 1433 { 1434 nvStatus = _kgspRpcDrainEvents(pGpu, pKernelGsp, expectedFunc); 1435 1436 switch (nvStatus) { 1437 case NV_WARN_MORE_PROCESSING_REQUIRED: 1438 nvStatus = NV_OK; 1439 goto done; 1440 case NV_OK: 1441 // Check timeout and continue outer loop. 1442 break; 1443 default: 1444 goto done; 1445 } 1446 1447 osSpinLoop(); 1448 1449 nvStatus = gpuCheckTimeout(pGpu, &timeout); 1450 if (nvStatus == NV_ERR_TIMEOUT) 1451 { 1452 _kgspRpcIncrementTimeoutCountAndRateLimitPrints(pGpu, pRpc); 1453 1454 if (!pRpc->bQuietPrints) 1455 { 1456 _kgspLogXid119(pGpu, pRpc, expectedFunc); 1457 } 1458 1459 goto done; 1460 } 1461 1462 if (osIsGpuShutdown(pGpu)) 1463 { 1464 nvStatus = NV_ERR_GPU_IS_LOST; 1465 goto done; 1466 } 1467 } 1468 1469 pRpc->timeoutCount = 0; 1470 1471 done: 1472 pKernelGsp->bPollingForRpcResponse = NV_FALSE; 1473 1474 if (bSlowGspRpc) 1475 { 1476 // Avoid cumulative timeout due to slow RPC 1477 threadStateResetTimeout(pGpu); 1478 } 1479 1480 return nvStatus; 1481 } 1482 1483 /*! 1484 * Initialize RPC objects required for interfacing with GSP. 1485 */ 1486 static NV_STATUS 1487 _kgspInitRpcInfrastructure 1488 ( 1489 OBJGPU *pGpu, 1490 KernelGsp *pKernelGsp 1491 ) 1492 { 1493 NV_STATUS nvStatus = NV_OK; 1494 MESSAGE_QUEUE_COLLECTION *pMQCollection = NULL; 1495 1496 nvStatus = GspMsgQueuesInit(pGpu, &pMQCollection); 1497 if (nvStatus != NV_OK) 1498 { 1499 NV_PRINTF(LEVEL_ERROR, "GspMsgQueueInit failed\n"); 1500 goto done; 1501 } 1502 1503 pKernelGsp->pMQCollection = pMQCollection; 1504 1505 // Init RM RPC object 1506 nvStatus = _kgspConstructRpcObject(pGpu, pKernelGsp, 1507 &pMQCollection->rpcQueues[RPC_TASK_RM_QUEUE_IDX], 1508 &pKernelGsp->pRpc); 1509 if (nvStatus != NV_OK) 1510 { 1511 NV_PRINTF(LEVEL_ERROR, "init task RM RPC infrastructure failed\n"); 1512 goto done; 1513 } 1514 1515 // Init task_isr RPC object 1516 if (pKernelGsp->bIsTaskIsrQueueRequired) 1517 { 1518 nvStatus = _kgspConstructRpcObject(pGpu, pKernelGsp, 1519 &pMQCollection->rpcQueues[RPC_TASK_ISR_QUEUE_IDX], 1520 &pKernelGsp->pLocklessRpc); 1521 if (nvStatus != NV_OK) 1522 { 1523 NV_PRINTF(LEVEL_ERROR, "init task ISR RPC infrastructure failed\n"); 1524 goto done; 1525 } 1526 } 1527 1528 done: 1529 if (nvStatus != NV_OK) 1530 { 1531 _kgspFreeRpcInfrastructure(pGpu, pKernelGsp); 1532 } 1533 1534 return nvStatus; 1535 } 1536 1537 1538 /*! 1539 * Initialize stripped down version of RPC infra init for GSP clients. 1540 */ 1541 static NV_STATUS 1542 _kgspConstructRpcObject 1543 ( 1544 OBJGPU *pGpu, 1545 KernelGsp *pKernelGsp, 1546 MESSAGE_QUEUE_INFO *pMQI, 1547 OBJRPC **ppRpc 1548 ) 1549 { 1550 OBJRPC *pRpc; 1551 1552 NV_ASSERT_OR_RETURN(pMQI != NULL, NV_ERR_INVALID_ARGUMENT); 1553 1554 pRpc = initRpcObject(pGpu); 1555 if (pRpc == NULL) 1556 { 1557 NV_PRINTF(LEVEL_ERROR, "initRpcObject failed\n"); 1558 return NV_ERR_INSUFFICIENT_RESOURCES; 1559 } 1560 1561 pRpc->pMessageQueueInfo = pMQI; 1562 1563 portMemSet(&pRpc->rpcHistory, 0, sizeof(pRpc->rpcHistory)); 1564 pRpc->rpcHistoryCurrent = RPC_HISTORY_DEPTH - 1; 1565 pRpc->message_buffer = (NvU32 *)pRpc->pMessageQueueInfo->pRpcMsgBuf; 1566 pRpc->maxRpcSize = GSP_MSG_QUEUE_RPC_SIZE_MAX; 1567 1568 rpcSendMessage_FNPTR(pRpc) = _kgspRpcSendMessage; 1569 rpcRecvPoll_FNPTR(pRpc) = _kgspRpcRecvPoll; 1570 1571 *ppRpc = pRpc; 1572 1573 return NV_OK; 1574 } 1575 1576 static void 1577 _kgspFreeRpcInfrastructure 1578 ( 1579 OBJGPU *pGpu, 1580 KernelGsp *pKernelGsp 1581 ) 1582 { 1583 if (pKernelGsp->pRpc != NULL) 1584 { 1585 rpcDestroy(pGpu, pKernelGsp->pRpc); 1586 portMemFree(pKernelGsp->pRpc); 1587 pKernelGsp->pRpc = NULL; 1588 } 1589 if (pKernelGsp->pLocklessRpc != NULL) 1590 { 1591 rpcDestroy(pGpu, pKernelGsp->pLocklessRpc); 1592 portMemFree(pKernelGsp->pLocklessRpc); 1593 pKernelGsp->pLocklessRpc = NULL; 1594 } 1595 GspMsgQueuesCleanup(&pKernelGsp->pMQCollection); 1596 } 1597 1598 /*! 1599 * Convert init arg name to 64bit id value. 1600 * 1601 * @param[in] name String representing name of init arg 1602 */ 1603 static NvU64 1604 _kgspGenerateInitArgId(const char *name) 1605 { 1606 NvU64 id = 0; 1607 NvU8 c; 1608 NvU32 i; 1609 1610 // Convert at most 8 characters from name into id. 1611 for (i = 0; i < (sizeof(NvU64) / sizeof(NvU8)); ++i) 1612 { 1613 c = (NvU8)*name++; 1614 if (c == '\0') 1615 { 1616 break; 1617 } 1618 id = (id << 8) | c; 1619 } 1620 1621 return id; 1622 } 1623 1624 /*! 1625 * Free vgpu partition LIBOS task logging structures 1626 */ 1627 static void 1628 _kgspFreeLibosVgpuPartitionLoggingStructures 1629 ( 1630 OBJGPU *pGpu, 1631 KernelGsp *pKernelGsp, 1632 NvU32 gfid 1633 ) 1634 { 1635 libosLogDestroy(&pKernelGsp->logDecodeVgpuPartition[gfid - 1]); 1636 1637 // release the vgpu task log buffer memory 1638 RM_LIBOS_LOG_MEM *pGspPluginVgpuTaskLog = &pKernelGsp->gspPluginVgpuTaskLogMem[gfid - 1]; 1639 1640 if (pGspPluginVgpuTaskLog->pTaskLogBuffer != NULL) 1641 { 1642 memdescUnmapInternal(pGpu, pGspPluginVgpuTaskLog->pTaskLogDescriptor, TRANSFER_FLAGS_NONE); 1643 1644 pGspPluginVgpuTaskLog->pTaskLogBuffer = NULL; 1645 } 1646 1647 if (pGspPluginVgpuTaskLog->pTaskLogDescriptor != NULL) 1648 { 1649 memdescFree(pGspPluginVgpuTaskLog->pTaskLogDescriptor); 1650 memdescDestroy(pGspPluginVgpuTaskLog->pTaskLogDescriptor); 1651 pGspPluginVgpuTaskLog->pTaskLogDescriptor = NULL; 1652 } 1653 1654 // release the init task log buffer memory 1655 RM_LIBOS_LOG_MEM *pGspPluginInitTaskLog = &pKernelGsp->gspPluginInitTaskLogMem[gfid - 1]; 1656 1657 if (pGspPluginInitTaskLog->pTaskLogBuffer != NULL) 1658 { 1659 memdescUnmapInternal(pGpu, pGspPluginInitTaskLog->pTaskLogDescriptor, TRANSFER_FLAGS_NONE); 1660 1661 pGspPluginInitTaskLog->pTaskLogBuffer = NULL; 1662 } 1663 1664 if (pGspPluginInitTaskLog->pTaskLogDescriptor != NULL) 1665 { 1666 memdescFree(pGspPluginInitTaskLog->pTaskLogDescriptor); 1667 memdescDestroy(pGspPluginInitTaskLog->pTaskLogDescriptor); 1668 pGspPluginInitTaskLog->pTaskLogDescriptor = NULL; 1669 } 1670 } 1671 1672 /*! 1673 * Free vgpu partition LIBOS task logging structures 1674 */ 1675 NV_STATUS 1676 kgspFreeVgpuPartitionLogging_IMPL 1677 ( 1678 OBJGPU *pGpu, 1679 KernelGsp *pKernelGsp, 1680 NvU32 gfid 1681 ) 1682 { 1683 if (gfid > MAX_PARTITIONS_WITH_GFID) 1684 { 1685 return NV_ERR_INVALID_ARGUMENT; 1686 } 1687 else 1688 { 1689 // Make sure there is no lingering debug output. 1690 kgspDumpGspLogs(pGpu, pKernelGsp, NV_FALSE); 1691 1692 _kgspFreeLibosVgpuPartitionLoggingStructures(pGpu, pKernelGsp, gfid); 1693 return NV_OK; 1694 } 1695 } 1696 1697 /*! 1698 * Initialize vgpu partition LIBOS task logging structures 1699 */ 1700 NV_STATUS 1701 kgspInitVgpuPartitionLogging_IMPL 1702 ( 1703 OBJGPU *pGpu, 1704 KernelGsp *pKernelGsp, 1705 NvU32 gfid, 1706 NvU64 initTaskLogBUffOffset, 1707 NvU64 initTaskLogBUffSize, 1708 NvU64 vgpuTaskLogBUffOffset, 1709 NvU64 vgpuTaskLogBuffSize 1710 ) 1711 { 1712 NV_STATUS nvStatus = NV_OK; 1713 RM_LIBOS_LOG_MEM *pGspPluginVgpuTaskLog = NULL; 1714 RM_LIBOS_LOG_MEM *pGspPluginInitTaskLog = NULL; 1715 char vm_string[8], sourceName[SOURCE_NAME_MAX_LENGTH]; 1716 1717 if (gfid > MAX_PARTITIONS_WITH_GFID) 1718 { 1719 return NV_ERR_INVALID_ARGUMENT; 1720 } 1721 1722 // Source name is used to generate a tag that is a unique identifier for nvlog buffers. 1723 // As the source name 'GSP' is already in use, we will need a custom source name. 1724 nvDbgSnprintf(sourceName, SOURCE_NAME_MAX_LENGTH, "V%02d", gfid); 1725 libosLogCreateEx(&pKernelGsp->logDecodeVgpuPartition[gfid - 1], sourceName); 1726 1727 // Setup logging for vgpu task in vgpu partition 1728 { 1729 pGspPluginVgpuTaskLog = &pKernelGsp->gspPluginVgpuTaskLogMem[gfid - 1]; 1730 NvP64 pVa = NvP64_NULL; 1731 1732 1733 NV_ASSERT_OK_OR_GOTO(nvStatus, 1734 memdescCreate(&pGspPluginVgpuTaskLog->pTaskLogDescriptor, 1735 pGpu, 1736 vgpuTaskLogBuffSize, 1737 RM_PAGE_SIZE, 1738 NV_TRUE, ADDR_FBMEM, NV_MEMORY_CACHED, 1739 MEMDESC_FLAGS_NONE), 1740 error_cleanup); 1741 1742 1743 memdescDescribe(pGspPluginVgpuTaskLog->pTaskLogDescriptor, ADDR_FBMEM, vgpuTaskLogBUffOffset, vgpuTaskLogBuffSize); 1744 1745 pVa = memdescMapInternal(pGpu, pGspPluginVgpuTaskLog->pTaskLogDescriptor, TRANSFER_FLAGS_NONE); 1746 1747 if (pVa != NvP64_NULL) 1748 { 1749 pGspPluginVgpuTaskLog->pTaskLogBuffer = pVa; 1750 portMemSet(pGspPluginVgpuTaskLog->pTaskLogBuffer, 0, vgpuTaskLogBuffSize); 1751 1752 pGspPluginVgpuTaskLog->id8 = _kgspGenerateInitArgId("LOGVGPU"); 1753 1754 nvDbgSnprintf(vm_string, sizeof(vm_string), "VGPU%d", gfid); 1755 1756 libosLogAddLogEx(&pKernelGsp->logDecodeVgpuPartition[gfid - 1], 1757 pGspPluginVgpuTaskLog->pTaskLogBuffer, 1758 memdescGetSize(pGspPluginVgpuTaskLog->pTaskLogDescriptor), 1759 pGpu->gpuInstance, 1760 (gpuGetChipArch(pGpu) >> GPU_ARCH_SHIFT), 1761 gpuGetChipImpl(pGpu), 1762 vm_string, 1763 ".fwlogging_vgpu"); 1764 } 1765 else 1766 { 1767 NV_PRINTF(LEVEL_ERROR, "Failed to map memory for vgpu task log buffer for vGPU partition \n"); 1768 nvStatus = NV_ERR_INSUFFICIENT_RESOURCES; 1769 goto error_cleanup; 1770 } 1771 } 1772 1773 // Setup logging for init task in vgpu partition 1774 { 1775 pGspPluginInitTaskLog = &pKernelGsp->gspPluginInitTaskLogMem[gfid - 1]; 1776 NvP64 pVa = NvP64_NULL; 1777 1778 NV_ASSERT_OK_OR_GOTO(nvStatus, 1779 memdescCreate(&pGspPluginInitTaskLog->pTaskLogDescriptor, 1780 pGpu, 1781 initTaskLogBUffSize, 1782 RM_PAGE_SIZE, 1783 NV_TRUE, ADDR_FBMEM, NV_MEMORY_CACHED, 1784 MEMDESC_FLAGS_NONE), 1785 error_cleanup); 1786 1787 memdescDescribe(pGspPluginInitTaskLog->pTaskLogDescriptor, ADDR_FBMEM, initTaskLogBUffOffset, initTaskLogBUffSize); 1788 1789 pVa = memdescMapInternal(pGpu, pGspPluginInitTaskLog->pTaskLogDescriptor, TRANSFER_FLAGS_NONE); 1790 1791 if (pVa != NvP64_NULL) 1792 { 1793 pGspPluginInitTaskLog->pTaskLogBuffer = pVa; 1794 portMemSet(pGspPluginInitTaskLog->pTaskLogBuffer, 0, initTaskLogBUffSize); 1795 1796 pGspPluginInitTaskLog->id8 = _kgspGenerateInitArgId("LOGINIT"); 1797 1798 nvDbgSnprintf(vm_string, sizeof(vm_string), "INIT%d", gfid); 1799 1800 libosLogAddLogEx(&pKernelGsp->logDecodeVgpuPartition[gfid - 1], 1801 pGspPluginInitTaskLog->pTaskLogBuffer, 1802 memdescGetSize(pGspPluginInitTaskLog->pTaskLogDescriptor), 1803 pGpu->gpuInstance, 1804 (gpuGetChipArch(pGpu) >> GPU_ARCH_SHIFT), 1805 gpuGetChipImpl(pGpu), 1806 vm_string, 1807 ".fwlogging_init"); 1808 } 1809 else 1810 { 1811 NV_PRINTF(LEVEL_ERROR, "Failed to map memory for init task log buffer for vGPU partition \n"); 1812 nvStatus = NV_ERR_INSUFFICIENT_RESOURCES; 1813 goto error_cleanup; 1814 } 1815 } 1816 1817 { 1818 libosLogInit(&pKernelGsp->logDecodeVgpuPartition[gfid - 1], pKernelGsp->pLogElf, pKernelGsp->logElfDataSize); 1819 // nvlog buffers are now setup using the appropriate sourceName to avoid tag-value clash. 1820 // Now sourceName can be modified to preserve the 'GSP-VGPUx' logging convention. 1821 portStringCopy(pKernelGsp->logDecodeVgpuPartition[gfid - 1].sourceName, 1822 SOURCE_NAME_MAX_LENGTH, 1823 "GSP", SOURCE_NAME_MAX_LENGTH); 1824 } 1825 1826 error_cleanup: 1827 if (nvStatus != NV_OK) 1828 _kgspFreeLibosVgpuPartitionLoggingStructures(pGpu, pKernelGsp, gfid); 1829 1830 return nvStatus; 1831 } 1832 1833 /*! 1834 * Free LIBOS task logging structures 1835 */ 1836 static void 1837 _kgspFreeLibosLoggingStructures 1838 ( 1839 OBJGPU *pGpu, 1840 KernelGsp *pKernelGsp 1841 ) 1842 { 1843 NvU8 idx; 1844 1845 _kgspStopLogPolling(pGpu, pKernelGsp); 1846 1847 // Make sure there is no lingering debug output. 1848 kgspDumpGspLogs(pGpu, pKernelGsp, NV_FALSE); 1849 1850 libosLogDestroy(&pKernelGsp->logDecode); 1851 1852 for (idx = 0; idx < LOGIDX_SIZE; idx++) 1853 { 1854 RM_LIBOS_LOG_MEM *pLog = &pKernelGsp->rmLibosLogMem[idx]; 1855 1856 // release log memory for each task. 1857 if (pLog->pTaskLogBuffer != NULL) 1858 { 1859 memdescUnmap(pLog->pTaskLogDescriptor, 1860 NV_TRUE, osGetCurrentProcess(), 1861 (void *)pLog->pTaskLogBuffer, 1862 pLog->pTaskLogMappingPriv); 1863 pLog->pTaskLogBuffer = NULL; 1864 pLog->pTaskLogMappingPriv = NULL; 1865 } 1866 1867 if (pLog->pTaskLogDescriptor != NULL) 1868 { 1869 memdescFree(pLog->pTaskLogDescriptor); 1870 memdescDestroy(pLog->pTaskLogDescriptor); 1871 pLog->pTaskLogDescriptor = NULL; 1872 } 1873 } 1874 1875 portMemFree(pKernelGsp->pLogElf); 1876 pKernelGsp->pLogElf = NULL; 1877 } 1878 1879 /*! 1880 * Initialize LIBOS task logging structures 1881 */ 1882 static NV_STATUS 1883 _kgspInitLibosLoggingStructures 1884 ( 1885 OBJGPU *pGpu, 1886 KernelGsp *pKernelGsp 1887 ) 1888 { 1889 static const struct 1890 { 1891 const char *szMemoryId; 1892 const char *szPrefix; 1893 NvU32 size; 1894 const char *elfSectionName; 1895 } logInitValues[] = 1896 { 1897 {"LOGINIT", "INIT", 0x10000, ".fwlogging_init"}, // 64KB for stack traces 1898 #if defined(DEVELOP) || defined(DEBUG) 1899 // The interrupt task is in the rm elf, so they share the same logging elf too 1900 {"LOGINTR", "INTR", 0x40000, ".fwlogging_rm"}, // 256KB ISR debug log on develop/debug builds 1901 {"LOGRM", "RM", 0x40000, ".fwlogging_rm"} // 256KB RM debug log on develop/debug builds 1902 #else 1903 // The interrupt task is in the rm elf, so they share the same logging elf too 1904 {"LOGINTR", "INTR", 0x10000, ".fwlogging_rm"}, // 64KB ISR debug log on develop/debug builds 1905 {"LOGRM", "RM", 0x10000, ".fwlogging_rm"} // 64KB RM debug log on release builds 1906 #endif 1907 }; 1908 ct_assert(NV_ARRAY_ELEMENTS(logInitValues) <= LIBOS_LOG_MAX_LOGS); 1909 ct_assert(NV_ARRAY_ELEMENTS(logInitValues) == LOGIDX_SIZE); 1910 1911 NV_STATUS nvStatus = NV_OK; 1912 NvU8 idx; 1913 NvU64 flags = MEMDESC_FLAGS_NONE; 1914 1915 libosLogCreate(&pKernelGsp->logDecode); 1916 1917 flags |= MEMDESC_FLAGS_ALLOC_IN_UNPROTECTED_MEMORY; 1918 1919 for (idx = 0; idx < LOGIDX_SIZE; idx++) 1920 { 1921 RM_LIBOS_LOG_MEM *pLog = &pKernelGsp->rmLibosLogMem[idx]; 1922 NvP64 pVa = NvP64_NULL; 1923 NvP64 pPriv = NvP64_NULL; 1924 1925 // Setup logging memory for each task. 1926 NV_ASSERT_OK_OR_GOTO(nvStatus, 1927 memdescCreate(&pLog->pTaskLogDescriptor, 1928 pGpu, 1929 logInitValues[idx].size, 1930 RM_PAGE_SIZE, 1931 NV_TRUE, ADDR_SYSMEM, NV_MEMORY_CACHED, 1932 flags), 1933 error_cleanup); 1934 1935 NV_ASSERT_OK_OR_GOTO(nvStatus, 1936 memdescAlloc(pLog->pTaskLogDescriptor), 1937 error_cleanup); 1938 1939 NV_ASSERT_OK_OR_GOTO(nvStatus, 1940 memdescMap(pLog->pTaskLogDescriptor, 0, 1941 memdescGetSize(pLog->pTaskLogDescriptor), 1942 NV_TRUE, NV_PROTECT_READ_WRITE, 1943 &pVa, &pPriv), 1944 error_cleanup); 1945 1946 pLog->pTaskLogBuffer = pVa; 1947 pLog->pTaskLogMappingPriv = pPriv; 1948 portMemSet(pLog->pTaskLogBuffer, 0, memdescGetSize(pLog->pTaskLogDescriptor)); 1949 1950 // Pass the PTE table for the log buffer in the log buffer, after the put pointer. 1951 memdescGetPhysAddrs(pLog->pTaskLogDescriptor, 1952 AT_GPU, 1953 0, 1954 RM_PAGE_SIZE, 1955 NV_CEIL(memdescGetSize(pLog->pTaskLogDescriptor), RM_PAGE_SIZE), 1956 &pLog->pTaskLogBuffer[1]); 1957 1958 pLog->id8 = _kgspGenerateInitArgId(logInitValues[idx].szMemoryId); 1959 1960 libosLogAddLogEx(&pKernelGsp->logDecode, 1961 pLog->pTaskLogBuffer, 1962 memdescGetSize(pLog->pTaskLogDescriptor), 1963 pGpu->gpuInstance, 1964 (gpuGetChipArch(pGpu) >> GPU_ARCH_SHIFT), 1965 gpuGetChipImpl(pGpu), 1966 logInitValues[idx].szPrefix, 1967 logInitValues[idx].elfSectionName); 1968 } 1969 1970 error_cleanup: 1971 if (nvStatus != NV_OK) 1972 _kgspFreeLibosLoggingStructures(pGpu, pKernelGsp); 1973 1974 return nvStatus; 1975 } 1976 1977 static NV_STATUS 1978 _kgspInitLibosLogDecoder 1979 ( 1980 OBJGPU *pGpu, 1981 KernelGsp *pKernelGsp, 1982 GSP_FIRMWARE *pGspFw 1983 ) 1984 { 1985 // If there's no log ELF or it's already been wired, skip wiring it now 1986 if ((pGspFw->pLogElf == NULL) || (pKernelGsp->pLogElf != NULL)) 1987 return NV_OK; 1988 1989 // Setup symbol decoder 1990 const void *pLogData = NULL; 1991 NvU64 logSize = 0; 1992 1993 NV_ASSERT_OK_OR_RETURN( 1994 _kgspFwContainerVerifyVersion(pGpu, pKernelGsp, 1995 pGspFw->pLogElf, 1996 pGspFw->logElfSize, 1997 "GSP firmware log")); 1998 1999 NV_ASSERT_OK_OR_RETURN( 2000 _kgspFwContainerGetSection(pGpu, pKernelGsp, 2001 pGspFw->pLogElf, 2002 pGspFw->logElfSize, 2003 GSP_LOGGING_SECTION_NAME, 2004 &pLogData, 2005 &logSize)); 2006 2007 pKernelGsp->pLogElf = portMemAllocNonPaged(logSize); 2008 pKernelGsp->logElfDataSize = logSize; 2009 2010 NV_ASSERT_OR_RETURN(pKernelGsp->pLogElf != NULL, NV_ERR_NO_MEMORY); 2011 2012 portMemCopy(pKernelGsp->pLogElf, logSize, pLogData, logSize); 2013 libosLogInit(&pKernelGsp->logDecode, pKernelGsp->pLogElf, logSize); 2014 2015 return NV_OK; 2016 } 2017 2018 static NV_STATUS 2019 _kgspAllocSimAccessBuffer(OBJGPU *pGpu, KernelGsp *pKernelGsp) 2020 { 2021 NvP64 pVa = NvP64_NULL; 2022 NvP64 pPriv = NvP64_NULL; 2023 NV_STATUS nvStatus; 2024 2025 if (!IS_SIMULATION(pGpu)) 2026 { 2027 pKernelGsp->pMemDesc_simAccessBuf = NULL; 2028 pKernelGsp->pSimAccessBuf = NULL; 2029 pKernelGsp->pSimAccessBufPriv = NULL; 2030 return NV_ERR_NOT_SUPPORTED; 2031 } 2032 2033 NV_ASSERT_OK_OR_GOTO(nvStatus, 2034 memdescCreate(&pKernelGsp->pMemDesc_simAccessBuf, 2035 pGpu, 2036 sizeof(SimAccessBuffer), 2037 RM_PAGE_SIZE, 2038 NV_TRUE, ADDR_SYSMEM, NV_MEMORY_UNCACHED, 2039 MEMDESC_FLAGS_NONE), 2040 error_cleanup); 2041 2042 NV_ASSERT_OK_OR_GOTO(nvStatus, 2043 memdescAlloc(pKernelGsp->pMemDesc_simAccessBuf), 2044 error_cleanup); 2045 2046 NV_ASSERT_OK_OR_GOTO(nvStatus, 2047 memdescMap(pKernelGsp->pMemDesc_simAccessBuf, 0, 2048 memdescGetSize(pKernelGsp->pMemDesc_simAccessBuf), 2049 NV_TRUE, NV_PROTECT_READ_WRITE, 2050 &pVa, &pPriv), 2051 error_cleanup); 2052 2053 pKernelGsp->pSimAccessBuf = (SimAccessBuffer*)pVa; 2054 pKernelGsp->pSimAccessBufPriv = pPriv; 2055 2056 portMemSet(pKernelGsp->pSimAccessBuf, 0, memdescGetSize(pKernelGsp->pMemDesc_simAccessBuf)); 2057 2058 error_cleanup: 2059 if (nvStatus != NV_OK) 2060 _kgspFreeSimAccessBuffer(pGpu, pKernelGsp); 2061 2062 return nvStatus; 2063 } 2064 2065 static void 2066 _kgspFreeSimAccessBuffer(OBJGPU *pGpu, KernelGsp *pKernelGsp) 2067 { 2068 if (!IS_SIMULATION(pGpu)) 2069 { 2070 return; 2071 } 2072 2073 if (pKernelGsp->pMemDesc_simAccessBuf != NULL) 2074 { 2075 memdescFree(pKernelGsp->pMemDesc_simAccessBuf); 2076 memdescDestroy(pKernelGsp->pMemDesc_simAccessBuf); 2077 } 2078 2079 pKernelGsp->pMemDesc_simAccessBuf = NULL; 2080 pKernelGsp->pSimAccessBuf = NULL; 2081 pKernelGsp->pSimAccessBufPriv = NULL; 2082 } 2083 2084 /*! 2085 * Create KernelGsp object and initialize RPC infrastructure 2086 */ 2087 NV_STATUS 2088 kgspConstructEngine_IMPL 2089 ( 2090 OBJGPU *pGpu, 2091 KernelGsp *pKernelGsp, 2092 ENGDESCRIPTOR engDesc 2093 ) 2094 { 2095 NV_STATUS nvStatus = NV_OK; 2096 2097 if (!IS_GSP_CLIENT(pGpu)) 2098 return NV_ERR_NOT_SUPPORTED; 2099 2100 kgspConfigureFalcon_HAL(pGpu, pKernelGsp); 2101 2102 // Init RPC objects used to communicate with GSP. 2103 nvStatus = _kgspInitRpcInfrastructure(pGpu, pKernelGsp); 2104 if (nvStatus != NV_OK) 2105 { 2106 NV_PRINTF(LEVEL_ERROR, "init RPC infrastructure failed\n"); 2107 goto done; 2108 } 2109 2110 // Init logging memory used by GSP 2111 nvStatus = _kgspInitLibosLoggingStructures(pGpu, pKernelGsp); 2112 if (nvStatus != NV_OK) 2113 { 2114 NV_PRINTF(LEVEL_ERROR, "init libos logging structures failed: 0x%x\n", nvStatus); 2115 goto done; 2116 } 2117 2118 // Clear out the gspStaticInfo. We will populate this once GSP-RM is up. 2119 portMemSet(&pKernelGsp->gspStaticInfo, 0, 2120 sizeof(pKernelGsp->gspStaticInfo)); 2121 2122 nvStatus = kgspAllocBootArgs_HAL(pGpu, pKernelGsp); 2123 if (nvStatus != NV_OK) 2124 { 2125 NV_PRINTF(LEVEL_ERROR, "boot arg alloc failed: 0x%x\n", nvStatus); 2126 goto done; 2127 } 2128 2129 if (IS_SIMULATION(pGpu)) 2130 { 2131 nvStatus = _kgspAllocSimAccessBuffer(pGpu, pKernelGsp); 2132 if (nvStatus != NV_OK) 2133 { 2134 NV_PRINTF(LEVEL_ERROR, "sim access buffer alloc failed: 0x%x\n", nvStatus); 2135 goto done; 2136 } 2137 } 2138 2139 done: 2140 if (nvStatus != NV_OK) 2141 { 2142 kgspFreeBootArgs_HAL(pGpu, pKernelGsp); 2143 _kgspFreeLibosLoggingStructures(pGpu, pKernelGsp); 2144 _kgspFreeRpcInfrastructure(pGpu, pKernelGsp); 2145 } 2146 2147 return nvStatus; 2148 } 2149 2150 /*! 2151 * Initialize GSP-RM 2152 * 2153 * @param[in] pGpu GPU object pointer 2154 * @param[in] pKernelGsp KernelGsp object pointer 2155 * @param[in] pGspFw GSP firmware structure pointer 2156 * 2157 * @return NV_OK if GSP fw RM offload successfully initialized. 2158 * Appropriate NV_ERR_xxx value otherwise. 2159 */ 2160 NV_STATUS 2161 kgspInitRm_IMPL 2162 ( 2163 OBJGPU *pGpu, 2164 KernelGsp *pKernelGsp, 2165 GSP_FIRMWARE *pGspFw 2166 ) 2167 { 2168 NV_STATUS status = NV_OK; 2169 OBJTMR *pTmr = GPU_GET_TIMER(pGpu); 2170 GPU_MASK gpusLockedMask = 0; 2171 2172 if (!IS_GSP_CLIENT(pGpu)) 2173 return NV_OK; 2174 2175 if ((pGspFw == NULL) || (pGspFw->pBuf == NULL) || (pGspFw->size == 0)) 2176 { 2177 NV_PRINTF(LEVEL_ERROR, "need firmware to initialize GSP\n"); 2178 return NV_ERR_INVALID_ARGUMENT; 2179 } 2180 2181 pKernelGsp->bInInit = NV_TRUE; 2182 2183 // Need to hold the GPU instance lock in order to write to the RPC queue 2184 NV_ASSERT_OK_OR_GOTO(status, 2185 rmGpuGroupLockAcquire(pGpu->gpuInstance, GPU_LOCK_GRP_SUBDEVICE, 2186 GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_INIT, &gpusLockedMask), 2187 done); 2188 2189 // Set the GPU time to the wall-clock time before loading GSP ucode. 2190 tmrSetCurrentTime_HAL(pGpu, pTmr); 2191 2192 /* 2193 * For GSP-RM boot, we must trigger FRTS (if it exists for the chip) 2194 * before loading GSP-RM so that FRTS data and GSP-RM code/data/heap can coexist 2195 * in WPR2. FRTS is triggered by running a VBIOS-provided ucode called FWSEC. 2196 * 2197 * Here, we extract a VBIOS image from ROM, and parse it for FWSEC. 2198 */ 2199 if (pKernelGsp->pFwsecUcode == NULL) 2200 { 2201 KernelGspVbiosImg *pVbiosImg = NULL; 2202 2203 // Try and extract a VBIOS image. 2204 status = kgspExtractVbiosFromRom_HAL(pGpu, pKernelGsp, &pVbiosImg); 2205 2206 if (status == NV_OK) 2207 { 2208 // Got a VBIOS image, now parse it for FWSEC. 2209 status = kgspParseFwsecUcodeFromVbiosImg(pGpu, pKernelGsp, pVbiosImg, 2210 &pKernelGsp->pFwsecUcode); 2211 kgspFreeVbiosImg(pVbiosImg); 2212 if (status != NV_OK) 2213 { 2214 NV_PRINTF(LEVEL_ERROR, "failed to parse FWSEC ucode from VBIOS image: 0x%x\n", 2215 status); 2216 goto done; 2217 } 2218 } 2219 else if (status == NV_ERR_NOT_SUPPORTED) 2220 { 2221 // 2222 // Extracting VBIOS image from ROM is not supported. 2223 // Sanity check we don't depend on it for FRTS, and proceed without FWSEC. 2224 // 2225 NV_ASSERT_OR_GOTO(kgspGetFrtsSize(pGpu, pKernelGsp) == 0, done); 2226 status = NV_OK; 2227 } 2228 else 2229 { 2230 NV_PRINTF(LEVEL_ERROR, "failed to extract VBIOS image from ROM: 0x%x\n", 2231 status); 2232 goto done; 2233 } 2234 2235 } 2236 2237 /* 2238 * We use a set of Booter ucodes to boot GSP-RM as well as manage its lifecycle. 2239 * 2240 * Booter Load loads, verifies, and boots GSP-RM in WPR2. 2241 * Booter Unload tears down WPR2 for driver unload. 2242 * 2243 * Here we prepare the Booter ucode images in SYSMEM so they may be loaded onto 2244 * SEC2 (Load / Unload) and NVDEC0 (Unload). 2245 */ 2246 if (pKernelGsp->bPartitionedFmc) 2247 { 2248 // 2249 // The secure boot ucode is included in the partitioned FMC, no need for 2250 // separate Booter ucodes. 2251 // 2252 } 2253 else 2254 { 2255 if (pKernelGsp->pBooterLoadUcode == NULL) 2256 { 2257 status = kgspAllocateBooterLoadUcodeImage(pGpu, pKernelGsp, 2258 &pKernelGsp->pBooterLoadUcode); 2259 if (status != NV_OK) 2260 { 2261 NV_PRINTF(LEVEL_ERROR, "failed to allocate Booter Load ucode: 0x%x\n", status); 2262 goto done; 2263 } 2264 } 2265 2266 if (pKernelGsp->pBooterUnloadUcode == NULL) 2267 { 2268 status = kgspAllocateBooterUnloadUcodeImage(pGpu, pKernelGsp, 2269 &pKernelGsp->pBooterUnloadUcode); 2270 if (status != NV_OK) 2271 { 2272 NV_PRINTF(LEVEL_ERROR, "failed to allocate Booter Unload ucode: 0x%x\n", status); 2273 goto done; 2274 } 2275 } 2276 } 2277 2278 // Prepare boot binary image. 2279 status = kgspPrepareBootBinaryImage(pGpu, pKernelGsp); 2280 if (status != NV_OK) 2281 { 2282 NV_PRINTF(LEVEL_ERROR, "Error preparing boot binary image\n"); 2283 goto done; 2284 } 2285 2286 // Prepare GSP-RM image. 2287 status = _kgspPrepareGspRmBinaryImage(pGpu, pKernelGsp, pGspFw); 2288 if (status != NV_OK) 2289 { 2290 NV_PRINTF(LEVEL_ERROR, "Error preparing GSP-RM image\n"); 2291 goto done; 2292 } 2293 2294 status = kgspCalculateFbLayout(pGpu, pKernelGsp, pGspFw); 2295 if (status != NV_OK) 2296 { 2297 NV_PRINTF(LEVEL_ERROR, "Error calculating FB layout\n"); 2298 goto done; 2299 } 2300 2301 // Prepare Scrubber ucode image if pre-scrubbed memory is insufficient 2302 if (pKernelGsp->pScrubberUcode == NULL) 2303 { 2304 NvU64 neededSize = pKernelGsp->pWprMeta->fbSize - pKernelGsp->pWprMeta->gspFwRsvdStart; 2305 NvU64 prescrubbedSize = kgspGetPrescrubbedTopFbSize(pGpu, pKernelGsp); 2306 2307 if (neededSize > prescrubbedSize) 2308 { 2309 NV_PRINTF(LEVEL_INFO, 2310 "allocating Scrubber ucode as pre-scrubbed memory (0x%llx bytes) is insufficient (0x%llx bytes needed)\n", 2311 prescrubbedSize, neededSize); 2312 2313 status = kgspAllocateScrubberUcodeImage(pGpu, pKernelGsp, 2314 &pKernelGsp->pScrubberUcode); 2315 if (status != NV_OK) 2316 { 2317 NV_PRINTF(LEVEL_ERROR, "failed to allocate Scrubber ucode: 0x%x\n", status); 2318 goto done; 2319 } 2320 } 2321 else 2322 { 2323 NV_PRINTF(LEVEL_INFO, 2324 "skipping allocating Scrubber ucode as pre-scrubbed memory (0x%llx bytes) is sufficient (0x%llx bytes needed)\n", 2325 prescrubbedSize, neededSize); 2326 } 2327 } 2328 2329 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, _kgspInitLibosLogDecoder(pGpu, pKernelGsp, pGspFw), done); 2330 2331 // Wait for GFW_BOOT OK status 2332 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, kgspWaitForGfwBootOk_HAL(pGpu, pKernelGsp), done); 2333 2334 // Fail early if WPR2 is up 2335 if (kgspIsWpr2Up_HAL(pGpu, pKernelGsp)) 2336 { 2337 NV_PRINTF(LEVEL_ERROR, "unexpected WPR2 already up, cannot proceed with booting gsp\n"); 2338 NV_PRINTF(LEVEL_ERROR, "(the GPU is likely in a bad state and may need to be reset)\n"); 2339 status = NV_ERR_INVALID_STATE; 2340 goto done; 2341 } 2342 2343 // bring up ucode with RM offload task 2344 status = kgspBootstrapRiscvOSEarly_HAL(pGpu, pKernelGsp, pGspFw); 2345 if (status != NV_OK) 2346 { 2347 NV_PRINTF(LEVEL_ERROR, "cannot bootstrap riscv/gsp: 0x%x\n", status); 2348 kgspHealthCheck_HAL(pGpu, pKernelGsp); 2349 goto done; 2350 } 2351 2352 // at this point we should be able to exchange RPCs with RM offload task 2353 NV_RM_RPC_SET_GUEST_SYSTEM_INFO(pGpu, status); 2354 if (status != NV_OK) 2355 { 2356 NV_PRINTF(LEVEL_ERROR, "SET_GUEST_SYSTEM_INFO failed: 0x%x\n", status); 2357 goto done; 2358 } 2359 2360 NV_RM_RPC_GET_GSP_STATIC_INFO(pGpu, status); 2361 if (status != NV_OK) 2362 { 2363 NV_PRINTF(LEVEL_ERROR, "GET_GSP_STATIC_INFO failed: 0x%x\n", status); 2364 goto done; 2365 } 2366 2367 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, kgspStartLogPolling(pGpu, pKernelGsp), done); 2368 2369 done: 2370 pKernelGsp->bInInit = NV_FALSE; 2371 2372 if (status != NV_OK) 2373 { 2374 KernelPmu *pKernelPmu = GPU_GET_KERNEL_PMU(pGpu); 2375 2376 // Preserve any captured GSP-RM logs 2377 libosPreserveLogs(&pKernelGsp->logDecode); 2378 2379 if (pKernelPmu != NULL) 2380 { 2381 // If PMU init fails, kgsp init will also fail 2382 libosPreserveLogs(&pKernelPmu->logDecode); 2383 } 2384 } 2385 2386 if (gpusLockedMask != 0) 2387 { 2388 rmGpuGroupLockRelease(gpusLockedMask, GPUS_LOCK_FLAGS_NONE); 2389 } 2390 2391 return status; 2392 } 2393 2394 /*! 2395 * Unload GSP-RM 2396 */ 2397 NV_STATUS 2398 kgspUnloadRm_IMPL 2399 ( 2400 OBJGPU *pGpu, 2401 KernelGsp *pKernelGsp 2402 ) 2403 { 2404 NV_STATUS rpcStatus = NV_OK; 2405 NV_STATUS status; 2406 2407 NV_PRINTF(LEVEL_INFO, "unloading GSP-RM\n"); 2408 NV_RM_RPC_UNLOADING_GUEST_DRIVER(pGpu, rpcStatus, NV_FALSE, NV_FALSE, 0); 2409 2410 // Wait for GSP-RM processor to suspend 2411 kgspWaitForProcessorSuspend_HAL(pGpu, pKernelGsp); 2412 2413 // Dump GSP-RM logs and reset before invoking FWSEC-SB 2414 kgspDumpGspLogs(pGpu, pKernelGsp, NV_FALSE); 2415 2416 // 2417 // Avoid cascading timeouts when attempting to invoke the below ucodes if 2418 // we are unloading due to a GSP-RM timeout. 2419 // 2420 threadStateResetTimeout(pGpu); 2421 2422 // Because of COT, RM cannot reset GSP-RISCV and FSP has exclusive access to reset and reboot GSP for next run. 2423 if(!(pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_COT_ENABLED))) 2424 { 2425 kflcnReset_HAL(pGpu, staticCast(pKernelGsp, KernelFalcon)); 2426 } 2427 2428 // Invoke FWSEC-SB to put back PreOsApps during driver unload 2429 status = kgspExecuteFwsecSb_HAL(pGpu, pKernelGsp, pKernelGsp->pFwsecUcode); 2430 if (status != NV_OK) 2431 { 2432 NV_PRINTF(LEVEL_ERROR, "failed to execute FWSEC-SB for PreOsApps during driver unload: 0x%x\n", status); 2433 NV_ASSERT(0); 2434 } 2435 2436 if (pKernelGsp->bPartitionedFmc) 2437 { 2438 // 2439 // GSP-RM invokes the partitioned FMC to unload directly as part of the 2440 // NV_RM_RPC_UNLOADING_GUEST_DRIVER call above. 2441 // 2442 status = rpcStatus; 2443 } 2444 else 2445 { 2446 // After instructing GSP-RM to unload itself, run Booter Unload to teardown WPR2 2447 status = kgspExecuteBooterUnloadIfNeeded_HAL(pGpu, pKernelGsp, 0); 2448 } 2449 2450 if (rpcStatus != NV_OK) 2451 { 2452 return rpcStatus; 2453 } 2454 2455 return status; 2456 } 2457 2458 /*! 2459 * Free RPC infrastructure and KernelGsp object 2460 */ 2461 void 2462 kgspDestruct_IMPL 2463 ( 2464 KernelGsp *pKernelGsp 2465 ) 2466 { 2467 OBJGPU *pGpu = ENG_GET_GPU(pKernelGsp); 2468 2469 if (!IS_GSP_CLIENT(pGpu)) 2470 return; 2471 2472 kgspFreeFlcnUcode(pKernelGsp->pFwsecUcode); 2473 pKernelGsp->pFwsecUcode = NULL; 2474 2475 kgspFreeFlcnUcode(pKernelGsp->pBooterLoadUcode); 2476 pKernelGsp->pBooterLoadUcode = NULL; 2477 2478 kgspFreeFlcnUcode(pKernelGsp->pBooterUnloadUcode); 2479 pKernelGsp->pBooterUnloadUcode = NULL; 2480 2481 kgspFreeFlcnUcode(pKernelGsp->pScrubberUcode); 2482 pKernelGsp->pScrubberUcode = NULL; 2483 2484 kgspFreeBootArgs_HAL(pGpu, pKernelGsp); 2485 2486 _kgspFreeLibosLoggingStructures(pGpu, pKernelGsp); 2487 _kgspFreeRpcInfrastructure(pGpu, pKernelGsp); 2488 _kgspFreeBootBinaryImage(pGpu, pKernelGsp); 2489 _kgspFreeSimAccessBuffer(pGpu, pKernelGsp); 2490 2491 kgspFreeSuspendResumeData_HAL(pGpu, pKernelGsp); 2492 } 2493 2494 /*! 2495 * Dump logs coming from GSP-RM 2496 * 2497 * @param[in] pGpu OBJGPU pointer 2498 * @param[in] pKernelGsp KernelGsp pointer 2499 * @param[in] bSyncNvLog NV_TRUE: Copy a snapshot of the libos logs 2500 * into the nvLog wrap buffers. 2501 */ 2502 void 2503 kgspDumpGspLogs_IMPL 2504 ( 2505 OBJGPU *pGpu, 2506 KernelGsp *pKernelGsp, 2507 NvBool bSyncNvLog 2508 ) 2509 { 2510 if (!IS_GSP_CLIENT(pGpu)) 2511 { 2512 return; 2513 } 2514 2515 if (pKernelGsp->bInInit || pKernelGsp->pLogElf || bSyncNvLog) 2516 { 2517 libosExtractLogs(&pKernelGsp->logDecode, bSyncNvLog); 2518 } 2519 2520 if (IS_VGPU_GSP_PLUGIN_OFFLOAD_ENABLED(pGpu)) 2521 { 2522 // Dump logs from vGPU partition 2523 for (NvU32 i = 0; i < MAX_PARTITIONS_WITH_GFID; i++) 2524 { 2525 libosExtractLogs(&pKernelGsp->logDecodeVgpuPartition[i], bSyncNvLog); 2526 } 2527 } 2528 2529 } 2530 2531 /*! 2532 * Populate GSP-RM init arguments. 2533 */ 2534 void 2535 kgspPopulateGspRmInitArgs_IMPL 2536 ( 2537 OBJGPU *pGpu, 2538 KernelGsp *pKernelGsp, 2539 GSP_SR_INIT_ARGUMENTS *pGspInitArgs 2540 ) 2541 { 2542 GSP_ARGUMENTS_CACHED *pGspArgs = pKernelGsp->pGspArgumentsCached; 2543 MESSAGE_QUEUE_INIT_ARGUMENTS *pMQInitArgs = &pGspArgs->messageQueueInitArguments; 2544 MESSAGE_QUEUE_COLLECTION *pMQCollection = pKernelGsp->pMQCollection; 2545 GSP_SR_INIT_ARGUMENTS *pSrInitArgs = &pGspArgs->srInitArguments; 2546 2547 // Setup the message queue arguments 2548 pMQInitArgs->sharedMemPhysAddr = pMQCollection->sharedMemPA; 2549 pMQInitArgs->pageTableEntryCount = pMQCollection->pageTableEntryCount; 2550 pMQInitArgs->cmdQueueOffset = pMQCollection->pageTableSize; 2551 pMQInitArgs->statQueueOffset = pMQInitArgs->cmdQueueOffset + pMQCollection->rpcQueues[RPC_TASK_RM_QUEUE_IDX].commandQueueSize; 2552 if (pKernelGsp->bIsTaskIsrQueueRequired) 2553 { 2554 pMQInitArgs->locklessCmdQueueOffset = pMQInitArgs->statQueueOffset + pMQCollection->rpcQueues[RPC_TASK_RM_QUEUE_IDX].statusQueueSize; 2555 pMQInitArgs->locklessStatQueueOffset = pMQInitArgs->locklessCmdQueueOffset + pMQCollection->rpcQueues[RPC_TASK_ISR_QUEUE_IDX].commandQueueSize; 2556 } 2557 else 2558 { 2559 pMQInitArgs->locklessCmdQueueOffset = 0; 2560 pMQInitArgs->locklessStatQueueOffset = 0; 2561 } 2562 2563 if (pGspInitArgs == NULL) 2564 { 2565 pSrInitArgs->bInPMTransition = NV_FALSE; 2566 pSrInitArgs->oldLevel = 0; 2567 pSrInitArgs->flags = 0; 2568 } 2569 else 2570 { 2571 pSrInitArgs->bInPMTransition = NV_TRUE; 2572 pSrInitArgs->oldLevel = pGspInitArgs->oldLevel; 2573 pSrInitArgs->flags = pGspInitArgs->flags; 2574 } 2575 2576 pGspArgs->gpuInstance = pGpu->gpuInstance; 2577 2578 portMemSet(&pGspArgs->profilerArgs, 0, sizeof(pGspArgs->profilerArgs)); 2579 2580 if (pKernelGsp->pProfilerSamples != NULL && 2581 pKernelGsp->pProfilerSamplesMD != NULL) 2582 { 2583 pGspArgs->profilerArgs.pa = memdescGetPhysAddr(pKernelGsp->pProfilerSamplesMD, AT_GPU, 0); 2584 pGspArgs->profilerArgs.size = memdescGetSize(pKernelGsp->pProfilerSamplesMD); 2585 } 2586 } 2587 2588 /*! 2589 * Prepare boot binary image for GSP-RM boot. 2590 * 2591 * @return NV_OK if boot binary image prepared successfully. 2592 * Appropriate NV_ERR_xxx value otherwise. 2593 */ 2594 NV_STATUS 2595 kgspPrepareBootBinaryImage_IMPL 2596 ( 2597 OBJGPU *pGpu, 2598 KernelGsp *pKernelGsp 2599 ) 2600 { 2601 NV_STATUS status; 2602 BINDATA_STORAGE *pBinStorageImage; 2603 BINDATA_STORAGE *pBinStorageDesc; 2604 NvU32 bufSize; 2605 NvU32 bufSizeAligned; 2606 NvU8 *pDesc = NULL; 2607 NvP64 pVa = NvP64_NULL; 2608 NvP64 pPriv = NvP64_NULL; 2609 NvU64 flags = MEMDESC_FLAGS_NONE; 2610 2611 NV_ASSERT_OR_RETURN(pKernelGsp->pGspRmBootUcodeImage == NULL, NV_ERR_INVALID_STATE); 2612 NV_ASSERT_OR_RETURN(pKernelGsp->pGspRmBootUcodeDesc == NULL, NV_ERR_INVALID_STATE); 2613 2614 // get the bindata storage for the image/descriptor 2615 kgspGetGspRmBootUcodeStorage_HAL(pGpu, pKernelGsp, &pBinStorageImage, &pBinStorageDesc); 2616 2617 // copy the image to sysmem 2618 bufSize = bindataGetBufferSize(pBinStorageImage); 2619 bufSizeAligned = NV_ALIGN_UP(bufSize, 0x1000); 2620 2621 flags |= MEMDESC_FLAGS_ALLOC_IN_UNPROTECTED_MEMORY; 2622 2623 NV_ASSERT_OK_OR_GOTO(status, 2624 memdescCreate(&pKernelGsp->pGspRmBootUcodeMemdesc, 2625 pGpu, 2626 bufSizeAligned, 2627 RM_PAGE_SIZE, 2628 NV_TRUE, ADDR_SYSMEM, NV_MEMORY_CACHED, 2629 flags), 2630 fail); 2631 2632 NV_ASSERT_OK_OR_GOTO(status, 2633 memdescAlloc(pKernelGsp->pGspRmBootUcodeMemdesc), 2634 fail); 2635 2636 NV_ASSERT_OK_OR_GOTO(status, 2637 memdescMap(pKernelGsp->pGspRmBootUcodeMemdesc, 0, 2638 memdescGetSize(pKernelGsp->pGspRmBootUcodeMemdesc), 2639 NV_TRUE, NV_PROTECT_READ_WRITE, 2640 &pVa, &pPriv), 2641 fail); 2642 2643 pKernelGsp->gspRmBootUcodeSize = bufSize; 2644 pKernelGsp->pGspRmBootUcodeImage = (NvU8 *)NvP64_VALUE(pVa);; 2645 pKernelGsp->pGspRmBootUcodeMemdescPriv = pPriv; 2646 2647 NV_ASSERT_OK_OR_GOTO(status, 2648 bindataWriteToBuffer(pBinStorageImage, 2649 pKernelGsp->pGspRmBootUcodeImage, 2650 bufSize), 2651 fail); 2652 2653 // copy the image descriptor 2654 bufSize = bindataGetBufferSize(pBinStorageDesc); 2655 pDesc = portMemAllocNonPaged(bufSize); 2656 if (pDesc == NULL) 2657 { 2658 NV_PRINTF(LEVEL_ERROR, "Failed to allocate ucode desc buffer\n"); 2659 status = NV_ERR_NO_MEMORY; 2660 goto fail; 2661 } 2662 2663 pKernelGsp->pGspRmBootUcodeDesc = (RM_RISCV_UCODE_DESC*)pDesc; 2664 2665 NV_ASSERT_OK_OR_GOTO(status, 2666 bindataWriteToBuffer(pBinStorageDesc, pDesc, bufSize), 2667 fail); 2668 2669 return status; 2670 2671 fail: 2672 _kgspFreeBootBinaryImage(pGpu, pKernelGsp); 2673 return status; 2674 } 2675 2676 static void 2677 _kgspFreeBootBinaryImage 2678 ( 2679 OBJGPU *pGpu, 2680 KernelGsp *pKernelGsp 2681 ) 2682 { 2683 portMemFree(pKernelGsp->pGspRmBootUcodeDesc); 2684 pKernelGsp->pGspRmBootUcodeDesc = NULL; 2685 2686 if (pKernelGsp->pGspRmBootUcodeImage != NULL) 2687 { 2688 memdescUnmap(pKernelGsp->pGspRmBootUcodeMemdesc, 2689 NV_TRUE, osGetCurrentProcess(), 2690 (void *)pKernelGsp->pGspRmBootUcodeImage, 2691 pKernelGsp->pGspRmBootUcodeMemdescPriv); 2692 pKernelGsp->pGspRmBootUcodeImage = NULL; 2693 pKernelGsp->pGspRmBootUcodeMemdescPriv = NULL; 2694 } 2695 if (pKernelGsp->pGspRmBootUcodeMemdesc != NULL) 2696 { 2697 memdescFree(pKernelGsp->pGspRmBootUcodeMemdesc); 2698 memdescDestroy(pKernelGsp->pGspRmBootUcodeMemdesc); 2699 pKernelGsp->pGspRmBootUcodeMemdesc = NULL; 2700 } 2701 2702 pKernelGsp->gspRmBootUcodeSize = 0; 2703 } 2704 2705 static NV_STATUS 2706 _kgspCreateSignatureMemdesc 2707 ( 2708 OBJGPU *pGpu, 2709 KernelGsp *pKernelGsp, 2710 GSP_FIRMWARE *pGspFw 2711 ) 2712 { 2713 NV_STATUS status = NV_OK; 2714 NvU8 *pSignatureVa = NULL; 2715 NvU64 flags = MEMDESC_FLAGS_NONE; 2716 2717 flags |= MEMDESC_FLAGS_ALLOC_IN_UNPROTECTED_MEMORY; 2718 2719 // NOTE: align to 256 because that's the alignment needed for Booter DMA 2720 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2721 memdescCreate(&pKernelGsp->pSignatureMemdesc, pGpu, 2722 NV_ALIGN_UP(pGspFw->signatureSize, 256), 256, 2723 NV_TRUE, ADDR_SYSMEM, NV_MEMORY_CACHED, flags)); 2724 2725 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, 2726 memdescAlloc(pKernelGsp->pSignatureMemdesc), fail_create); 2727 2728 pSignatureVa = memdescMapInternal(pGpu, pKernelGsp->pSignatureMemdesc, TRANSFER_FLAGS_NONE); 2729 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, 2730 (pSignatureVa != NULL) ? NV_OK : NV_ERR_INSUFFICIENT_RESOURCES, 2731 fail_alloc); 2732 2733 portMemCopy(pSignatureVa, memdescGetSize(pKernelGsp->pSignatureMemdesc), 2734 pGspFw->pSignatureData, pGspFw->signatureSize); 2735 2736 memdescUnmapInternal(pGpu, pKernelGsp->pSignatureMemdesc, 0); 2737 pSignatureVa = NULL; 2738 2739 return status; 2740 2741 fail_alloc: 2742 memdescFree(pKernelGsp->pSignatureMemdesc); 2743 2744 fail_create: 2745 memdescDestroy(pKernelGsp->pSignatureMemdesc); 2746 pKernelGsp->pSignatureMemdesc = NULL; 2747 2748 return status; 2749 } 2750 2751 /*! 2752 * Verify that the version embedded in the .fwversion section of the ELF given 2753 * by pElfData and elfDataSize matches our NV_VERSION_STRING. 2754 */ 2755 static NV_STATUS 2756 _kgspFwContainerVerifyVersion 2757 ( 2758 OBJGPU *pGpu, 2759 KernelGsp *pKernelGsp, 2760 const void *pElfData, 2761 NvU64 elfDataSize, 2762 const char *pNameInMsg 2763 ) 2764 { 2765 const char *pFwversion; 2766 NvU64 fwversionSize; 2767 NvU64 expectedVersionLength = portStringLength(NV_VERSION_STRING); 2768 2769 { 2770 const void *pFwversionRaw; 2771 2772 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2773 _kgspFwContainerGetSection(pGpu, pKernelGsp, 2774 pElfData, 2775 elfDataSize, 2776 GSP_VERSION_SECTION_NAME, 2777 &pFwversionRaw, 2778 &fwversionSize)); 2779 2780 pFwversion = (const char *) pFwversionRaw; 2781 } 2782 2783 // Check that text in .fwversion section of ELF matches our NV_VERSION_STRING 2784 if ((fwversionSize != expectedVersionLength + 1) || 2785 (portStringCompare(pFwversion, NV_VERSION_STRING, expectedVersionLength) != 0)) 2786 { 2787 // Sanity check .fwversion before attempting to print it in the error message 2788 if ((fwversionSize > 0) && 2789 (fwversionSize < 64) && 2790 (pFwversion[fwversionSize - 1] == '\0')) 2791 { 2792 NV_PRINTF(LEVEL_ERROR, "%s version mismatch: got version %s, expected version %s\n", 2793 pNameInMsg, pFwversion, NV_VERSION_STRING); 2794 } 2795 else 2796 { 2797 NV_PRINTF(LEVEL_ERROR, "%s version unknown or malformed, expected version %s\n", 2798 pNameInMsg, NV_VERSION_STRING); 2799 } 2800 return NV_ERR_INVALID_DATA; 2801 } 2802 2803 return NV_OK; 2804 } 2805 2806 /*! 2807 * Get the name of the section corresponding to the given section name 2808 * prefix and the current chip. 2809 */ 2810 static NV_STATUS 2811 _kgspGetSectionNameForPrefix 2812 ( 2813 OBJGPU *pGpu, 2814 KernelGsp *pKernelGsp, 2815 char *pSectionNameBuf, // out 2816 NvLength sectionNameBufSize, 2817 const char *pSectionPrefix 2818 ) 2819 { 2820 NvLength sectionPrefixLength; 2821 2822 nv_firmware_chip_family_t chipFamily; 2823 const char *pChipFamilyName; 2824 NvLength chipFamilyNameLength; 2825 2826 NvLength totalSize; 2827 2828 NV_ASSERT_OR_RETURN(pSectionNameBuf != NULL, NV_ERR_INVALID_ARGUMENT); 2829 NV_ASSERT_OR_RETURN(sectionNameBufSize > 0, NV_ERR_INVALID_ARGUMENT); 2830 NV_ASSERT_OR_RETURN(pSectionPrefix != NULL, NV_ERR_INVALID_ARGUMENT); 2831 2832 chipFamily = nv_firmware_get_chip_family(gpuGetChipArch(pGpu), 2833 gpuGetChipImpl(pGpu)); 2834 NV_ASSERT_OR_RETURN(chipFamily != NV_FIRMWARE_CHIP_FAMILY_NULL, 2835 NV_ERR_INVALID_STATE); 2836 2837 pChipFamilyName = nv_firmware_chip_family_to_string(chipFamily); 2838 NV_ASSERT_OR_RETURN(pChipFamilyName != NULL, NV_ERR_INVALID_STATE); 2839 2840 sectionPrefixLength = portStringLength(pSectionPrefix); 2841 chipFamilyNameLength = portStringLength(pChipFamilyName); 2842 2843 totalSize = sectionPrefixLength + chipFamilyNameLength + 1; 2844 NV_ASSERT_OR_RETURN(sectionNameBufSize >= sectionPrefixLength + 1, 2845 NV_ERR_BUFFER_TOO_SMALL); 2846 NV_ASSERT_OR_RETURN(sectionNameBufSize >= totalSize, 2847 NV_ERR_BUFFER_TOO_SMALL); 2848 2849 portStringCopy(pSectionNameBuf, sectionNameBufSize, 2850 pSectionPrefix, sectionPrefixLength + 1); 2851 portStringCat(pSectionNameBuf, sectionNameBufSize, 2852 pChipFamilyName, chipFamilyNameLength + 1); 2853 2854 return NV_OK; 2855 } 2856 2857 static NV_STATUS 2858 _kgspPrepareGspRmBinaryImage 2859 ( 2860 OBJGPU *pGpu, 2861 KernelGsp *pKernelGsp, 2862 GSP_FIRMWARE *pGspFw 2863 ) 2864 { 2865 char signatureSectionName[32]; 2866 2867 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2868 _kgspFwContainerVerifyVersion(pGpu, pKernelGsp, 2869 pGspFw->pBuf, 2870 pGspFw->size, 2871 "GSP firmware image")); 2872 2873 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2874 _kgspFwContainerGetSection(pGpu, pKernelGsp, 2875 pGspFw->pBuf, 2876 pGspFw->size, 2877 GSP_IMAGE_SECTION_NAME, 2878 &pGspFw->pImageData, 2879 &pGspFw->imageSize)); 2880 2881 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2882 _kgspGetSectionNameForPrefix(pGpu, pKernelGsp, 2883 signatureSectionName, sizeof(signatureSectionName), 2884 kgspGetSignatureSectionNamePrefix_HAL(pGpu, pKernelGsp))); 2885 2886 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2887 _kgspFwContainerGetSection(pGpu, pKernelGsp, 2888 pGspFw->pBuf, 2889 pGspFw->size, 2890 signatureSectionName, 2891 &pGspFw->pSignatureData, 2892 &pGspFw->signatureSize)); 2893 2894 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2895 _kgspCreateSignatureMemdesc(pGpu, pKernelGsp, 2896 pGspFw)); 2897 2898 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2899 kgspCreateRadix3(pGpu, pKernelGsp, &pKernelGsp->pGspUCodeRadix3Descriptor, 2900 NULL, pGspFw->pImageData, pGspFw->imageSize)); 2901 2902 return NV_OK; 2903 } 2904 2905 NV_STATUS 2906 kgspCreateRadix3_IMPL 2907 ( 2908 OBJGPU *pGpu, 2909 KernelGsp *pKernelGsp, 2910 MEMORY_DESCRIPTOR **ppMemdescRadix3, 2911 MEMORY_DESCRIPTOR *pMemdescData, 2912 const void *pData, 2913 NvU64 size 2914 ) 2915 { 2916 const NvU64 entriesLog2 = LIBOS_MEMORY_REGION_RADIX_PAGE_LOG2 - 3; 2917 NvU8 *pRadix3Buf; 2918 NvP64 pVaKernel; 2919 NvP64 pPrivKernel; 2920 NvU64 ptSize; 2921 NvU64 allocSize; 2922 NvU64 nPages = 0; 2923 NvU64 dataOffset = 0; 2924 NvU32 i; 2925 NV_STATUS status = NV_OK; 2926 NvU64 flags = MEMDESC_FLAGS_KERNEL_MODE; 2927 2928 // radix3 working array. 2929 struct 2930 { 2931 NvU64 nPages; 2932 NvU64 offset; 2933 } radix3[4]; 2934 2935 NV_ASSERT_OR_RETURN(ppMemdescRadix3 != NULL, NV_ERR_INVALID_PARAMETER); 2936 NV_ASSERT_OR_ELSE_STR(!((pMemdescData != NULL) && (pData != NULL)), 2937 "Specify pMemdescData or pData, or none, but not both", 2938 return NV_ERR_INVALID_PARAMETER); 2939 2940 // If the size is not specified, get it from the memory descriptor. 2941 if ((size == 0) && (pMemdescData != NULL)) 2942 size = memdescGetSize(pMemdescData); 2943 NV_ASSERT_OR_RETURN(size > 0, NV_ERR_OUT_OF_RANGE); 2944 2945 // Clear working structure. 2946 portMemSet(radix3, 0, sizeof radix3); 2947 2948 // Populate npages, high to low. 2949 i = NV_ARRAY_ELEMENTS(radix3) - 1; 2950 radix3[i].nPages = (size + LIBOS_MEMORY_REGION_RADIX_PAGE_SIZE - 1) >> 2951 LIBOS_MEMORY_REGION_RADIX_PAGE_LOG2; 2952 for (; i > 0; i--) 2953 radix3[i - 1].nPages = ((radix3[i].nPages - 1) >> entriesLog2) + 1; 2954 2955 // Populate offset, low to high. 2956 for (i = 1; i < NV_ARRAY_ELEMENTS(radix3); i++) 2957 { 2958 nPages += radix3[i - 1].nPages; 2959 radix3[i].offset = nPages << LIBOS_MEMORY_REGION_RADIX_PAGE_LOG2; 2960 } 2961 2962 NV_ASSERT_OR_RETURN(radix3[0].nPages == 1, NV_ERR_OUT_OF_RANGE); 2963 2964 // Allocate space for PTEs and PDEs. 2965 ptSize = nPages << LIBOS_MEMORY_REGION_RADIX_PAGE_LOG2; 2966 allocSize = ptSize; 2967 2968 if (pMemdescData == NULL) 2969 { 2970 // We don't have a separate descriptor for the data. We need PTEs, 2971 // so include space for data in the new descriptor. 2972 allocSize += radix3[3].nPages << LIBOS_MEMORY_REGION_RADIX_PAGE_LOG2; 2973 } 2974 2975 flags |= MEMDESC_FLAGS_ALLOC_IN_UNPROTECTED_MEMORY; 2976 2977 NV_ASSERT_OK_OR_GOTO(status, 2978 memdescCreate(ppMemdescRadix3, pGpu, allocSize, 2979 LIBOS_MEMORY_REGION_RADIX_PAGE_SIZE, 2980 NV_MEMORY_NONCONTIGUOUS, 2981 ADDR_SYSMEM, 2982 NV_MEMORY_CACHED, 2983 flags), 2984 done); 2985 2986 NV_ASSERT_OK_OR_GOTO(status, memdescAlloc(*ppMemdescRadix3), error_ret); 2987 2988 // Create kernel mapping. 2989 NV_ASSERT_OK_OR_GOTO(status, 2990 memdescMap(*ppMemdescRadix3, 0, allocSize, NV_TRUE, NV_PROTECT_WRITEABLE, 2991 &pVaKernel, &pPrivKernel), 2992 error_ret); 2993 2994 if (pVaKernel == NvP64_NULL) 2995 { 2996 NV_PRINTF(LEVEL_ERROR, "VA error for radix3 shared buffer\n"); 2997 status = NV_ERR_NO_MEMORY; 2998 goto error_ret; 2999 } 3000 3001 pRadix3Buf = KERNEL_POINTER_FROM_NvP64(NvU8 *, pVaKernel); 3002 3003 // Zap out page table. 3004 portMemSet(pRadix3Buf, 0, ptSize); 3005 3006 // Fill in PDEs. 3007 for (i = 0; i < NV_ARRAY_ELEMENTS(radix3) - 2; i++) 3008 { 3009 memdescGetPhysAddrs(*ppMemdescRadix3, 3010 AT_GPU, // addressTranslation 3011 radix3[i + 1].offset, // offset 3012 RM_PAGE_SIZE, // stride 3013 radix3[i + 1].nPages, // count 3014 (RmPhysAddr *)(pRadix3Buf + radix3[i].offset)); // physical address table 3015 } 3016 3017 dataOffset = radix3[3].offset; 3018 3019 if (pData != NULL) 3020 { 3021 // Optionally copy data into the radix3 buffer. 3022 portMemCopy(pRadix3Buf + dataOffset, size, pData, size); 3023 3024 // If we only have part of the last page, clear the rest. 3025 NvU32 clearSize = allocSize - dataOffset - size; 3026 if (clearSize != 0) 3027 portMemSet(pRadix3Buf + dataOffset + size, 0, clearSize); 3028 3029 pMemdescData = *ppMemdescRadix3; 3030 } 3031 3032 memdescGetPhysAddrs(*ppMemdescRadix3, 3033 AT_GPU, // addressTranslation 3034 dataOffset, // offset 3035 RM_PAGE_SIZE, // stride 3036 radix3[3].nPages, // count 3037 (RmPhysAddr *)(pRadix3Buf + radix3[2].offset)); // physical address table 3038 3039 // 3040 // No reason to keep this memory mapped on the CPU side. Only GSP will 3041 // access it after this point. 3042 // 3043 memdescUnmap(*ppMemdescRadix3, NV_TRUE, osGetCurrentProcess(), 3044 pVaKernel, pPrivKernel); 3045 done: 3046 return status; 3047 3048 error_ret: 3049 if (*ppMemdescRadix3 != NULL) 3050 { 3051 memdescFree(*ppMemdescRadix3); 3052 memdescDestroy(*ppMemdescRadix3); 3053 *ppMemdescRadix3 = NULL; 3054 } 3055 3056 return status; 3057 } 3058 3059 static NV_STATUS 3060 _kgspFwContainerGetSection 3061 ( 3062 OBJGPU *pGpu, 3063 KernelGsp *pKernelGsp, 3064 const void *pElfData, 3065 NvU64 elfDataSize, 3066 const char *pSectionName, 3067 const void **ppSectionData, 3068 NvU64 *pSectionSize 3069 ) 3070 { 3071 const NvU8 *pGspBuf = pElfData; 3072 const LibosElf64Header *pElfHeader; 3073 const LibosElf64SectionHeader *pElfSectionHeader; 3074 NvU64 elfSectionHeaderTableLength; 3075 NvU64 elfSectionHeaderMaxIdx; 3076 NvU64 elfSectionNamesTableOffset; 3077 NvU64 elfSectionNamesTableSize; 3078 NvU64 elfSectionNamesTableMaxIdx; 3079 static const NvU32 elfMagicNumber = 0x464C457F; 3080 static const NvU8 elfClass64 = 0x2; 3081 static const NvU8 elfLittleEndian = 0x1; 3082 const char *pCurrentSectionName; 3083 NvLength sectionNameLength; 3084 NvS16 idx; 3085 3086 NV_CHECK_OR_RETURN(LEVEL_ERROR, pElfData != NULL, NV_ERR_INVALID_ARGUMENT); 3087 NV_CHECK_OR_RETURN(LEVEL_ERROR, elfDataSize > 0, NV_ERR_INVALID_ARGUMENT); 3088 NV_CHECK_OR_RETURN(LEVEL_ERROR, pSectionName != NULL, NV_ERR_INVALID_ARGUMENT); 3089 NV_CHECK_OR_RETURN(LEVEL_ERROR, ppSectionData != NULL, NV_ERR_INVALID_ARGUMENT); 3090 NV_CHECK_OR_RETURN(LEVEL_ERROR, pSectionSize != NULL, NV_ERR_INVALID_ARGUMENT); 3091 NV_CHECK_OR_RETURN(LEVEL_ERROR, elfDataSize >= sizeof(LibosElf64Header), NV_ERR_INVALID_DATA); 3092 3093 sectionNameLength = portStringLength(pSectionName); 3094 3095 pElfHeader = (const LibosElf64Header*) pGspBuf; 3096 3097 // Check for the elf identifier at the beginning of the file 3098 NV_CHECK_OR_RETURN(LEVEL_ERROR, *(NvU32*)&pElfHeader->ident == elfMagicNumber, NV_ERR_INVALID_DATA); 3099 // Make sure the data is formatted as little endian 3100 NV_CHECK_OR_RETURN(LEVEL_ERROR, pElfHeader->ident[5] == elfLittleEndian, NV_ERR_INVALID_DATA); 3101 // Check the class type, only ELFCLASS64 is supported 3102 NV_CHECK_OR_RETURN(LEVEL_ERROR, pElfHeader->ident[4] == elfClass64, NV_ERR_INVALID_DATA); 3103 3104 // Make sure that the elf section header table is valid 3105 NV_CHECK_OR_RETURN(LEVEL_ERROR, pElfHeader->shentsize == sizeof(LibosElf64SectionHeader), NV_ERR_INVALID_DATA); 3106 NV_CHECK_OR_RETURN(LEVEL_ERROR, portSafeMulU64(pElfHeader->shentsize, pElfHeader->shnum, &elfSectionHeaderTableLength), NV_ERR_INVALID_DATA); 3107 NV_CHECK_OR_RETURN(LEVEL_ERROR, portSafeAddU64(pElfHeader->shoff, elfSectionHeaderTableLength - 1, &elfSectionHeaderMaxIdx), NV_ERR_INVALID_DATA); 3108 NV_CHECK_OR_RETURN(LEVEL_ERROR, elfDataSize >= elfSectionHeaderMaxIdx, NV_ERR_INVALID_DATA); 3109 NV_CHECK_OR_RETURN(LEVEL_ERROR, pElfHeader->shstrndx <= pElfHeader->shnum, NV_ERR_INVALID_DATA); 3110 3111 // Get the offset and size of the table that holds the section names and make sure they are valid 3112 pElfSectionHeader = (const LibosElf64SectionHeader*) &pGspBuf[pElfHeader->shoff + (pElfHeader->shstrndx * pElfHeader->shentsize)]; 3113 elfSectionNamesTableOffset = pElfSectionHeader->offset; 3114 elfSectionNamesTableSize = pElfSectionHeader->size; 3115 NV_CHECK_OR_RETURN(LEVEL_ERROR, portSafeAddU64(elfSectionNamesTableOffset, elfSectionNamesTableSize - 1, &elfSectionNamesTableMaxIdx), NV_ERR_INVALID_DATA); 3116 NV_CHECK_OR_RETURN(LEVEL_ERROR, elfDataSize >= elfSectionNamesTableMaxIdx, NV_ERR_INVALID_DATA); 3117 3118 // Iterate through all of the section headers to find the signatures 3119 pElfSectionHeader = (const LibosElf64SectionHeader*) &pGspBuf[elfSectionHeaderMaxIdx + 1 - sizeof(*pElfSectionHeader)]; 3120 3121 for (idx = pElfHeader->shnum - 1; idx >= 0; idx--, pElfSectionHeader--) 3122 { 3123 NvU64 currentSectionNameMaxLength; 3124 NvU64 elfSectionMaxIdx; 3125 3126 // Make sure the header name index fits within the section names table 3127 NV_CHECK_OR_RETURN(LEVEL_ERROR, elfSectionNamesTableSize - 1 >= pElfSectionHeader->name, NV_ERR_INVALID_DATA); 3128 currentSectionNameMaxLength = elfSectionNamesTableSize - pElfSectionHeader->name - 1; 3129 pCurrentSectionName = (const char *) &pGspBuf[elfSectionNamesTableOffset + pElfSectionHeader->name]; 3130 3131 // Make sure the elf section size and offset are valid 3132 if (pElfSectionHeader->size > 0) 3133 { 3134 NV_CHECK_OR_RETURN(LEVEL_ERROR, portSafeAddU64(pElfSectionHeader->offset, pElfSectionHeader->size - 1, &elfSectionMaxIdx), NV_ERR_INVALID_DATA); 3135 } 3136 else 3137 { 3138 elfSectionMaxIdx = pElfSectionHeader->offset; 3139 } 3140 NV_CHECK_OR_RETURN(LEVEL_ERROR, elfDataSize >= elfSectionMaxIdx, NV_ERR_INVALID_DATA); 3141 3142 // Check whether the section name matches the expected section name 3143 if ((sectionNameLength <= currentSectionNameMaxLength) && 3144 (portStringCompare(pCurrentSectionName, pSectionName, sectionNameLength) == 0) && 3145 (pCurrentSectionName[sectionNameLength] == '\0')) 3146 { 3147 *ppSectionData = &pGspBuf[pElfSectionHeader->offset]; 3148 *pSectionSize = pElfSectionHeader->size; 3149 3150 return NV_OK; 3151 } 3152 } 3153 3154 return NV_ERR_OBJECT_NOT_FOUND; 3155 } 3156 3157 /*! 3158 * Setup libos init arguments. 3159 */ 3160 void 3161 kgspSetupLibosInitArgs_IMPL 3162 ( 3163 OBJGPU *pGpu, 3164 KernelGsp *pKernelGsp 3165 ) 3166 { 3167 LibosMemoryRegionInitArgument *pLibosInitArgs = pKernelGsp->pLibosInitArgumentsCached; 3168 NvU8 idx; 3169 3170 portMemSet(pLibosInitArgs, 0, LIBOS_INIT_ARGUMENTS_SIZE); 3171 3172 // Add memory areas for logging each LIBOS task. 3173 // @note LOGINIT must be first for early init logging to work. 3174 // @note: These should be switched to radix regions to remove the need 3175 // for large apertures in the RM task for logging. 3176 for (idx = 0; idx < LOGIDX_SIZE; idx++) 3177 { 3178 pLibosInitArgs[idx].kind = LIBOS_MEMORY_REGION_CONTIGUOUS; 3179 pLibosInitArgs[idx].loc = LIBOS_MEMORY_REGION_LOC_SYSMEM; 3180 pLibosInitArgs[idx].id8 = pKernelGsp->rmLibosLogMem[idx].id8; 3181 pLibosInitArgs[idx].pa = pKernelGsp->rmLibosLogMem[idx].pTaskLogBuffer[1]; 3182 pLibosInitArgs[idx].size = memdescGetSize(pKernelGsp->rmLibosLogMem[idx].pTaskLogDescriptor); 3183 } 3184 3185 // insert GSP-RM ELF args address; id must match libos-config.py entry 3186 pLibosInitArgs[idx].kind = LIBOS_MEMORY_REGION_CONTIGUOUS; 3187 pLibosInitArgs[idx].loc = LIBOS_MEMORY_REGION_LOC_SYSMEM; 3188 pLibosInitArgs[idx].id8 = _kgspGenerateInitArgId("RMARGS"); 3189 pLibosInitArgs[idx].pa = memdescGetPhysAddr(pKernelGsp->pGspArgumentsDescriptor, AT_GPU, 0); 3190 pLibosInitArgs[idx].size = memdescGetSize(pKernelGsp->pGspArgumentsDescriptor); 3191 3192 portAtomicMemoryFenceFull(); 3193 } 3194 3195 /*! 3196 * Receive and process RPC event from GSP-RM. 3197 * 3198 * This function is called from interrupt bottom-half handler (DPC) and 3199 * would race with normal RPC flow, _kgspRpcRecvPoll(). 3200 * This race is currently avoided only because DPC is executed under 3201 * gpus lock, so RPC and Bottom-half handler are mutually exclusive 3202 * control flows. 3203 */ 3204 void 3205 kgspRpcRecvEvents_IMPL 3206 ( 3207 OBJGPU *pGpu, 3208 KernelGsp *pKernelGsp 3209 ) 3210 { 3211 NV_ASSERT(rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 3212 // 3213 // We should never have an event with code NV_VGPU_MSG_FUNCTION_NUM_FUNCTIONS. 3214 // If we do the assert will fail on NV_WARN_MORE_PROCESSING_REQUIRED, 3215 // in addition to general error codes. 3216 // 3217 NV_ASSERT_OK(_kgspRpcDrainEvents(pGpu, pKernelGsp, NV_VGPU_MSG_FUNCTION_NUM_FUNCTIONS)); 3218 } 3219 3220 /*! 3221 * Wait for GSP-RM initialization to complete. 3222 */ 3223 NV_STATUS 3224 kgspWaitForRmInitDone_IMPL 3225 ( 3226 OBJGPU *pGpu, 3227 KernelGsp *pKernelGsp 3228 ) 3229 { 3230 OBJRPC *pRpc = pKernelGsp->pRpc; 3231 3232 // 3233 // Kernel RM can timeout when GSP-RM has an error condition. Give GSP-RM 3234 // a chance to report the error before we pull the rug out from under it. 3235 // 3236 threadStateResetTimeout(pGpu); 3237 3238 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3239 rpcRecvPoll(pGpu, pRpc, NV_VGPU_MSG_EVENT_GSP_INIT_DONE)); 3240 3241 // 3242 // Now check if RPC really succeeded (NV_VGPU_MSG_RESULT_* are defined to 3243 // equivalent NV_STATUS codes in RM). 3244 // 3245 NV_ASSERT_OK_OR_RETURN(RPC_HDR->rpc_result); 3246 3247 pGpu->gspRmInitialized = NV_TRUE; 3248 if (hypervisorIsVgxHyper() && pGpu->getProperty(pGpu, PDB_PROP_GPU_EXTENDED_GSP_RM_INITIALIZATION_TIMEOUT_FOR_VGX)) 3249 { 3250 // Decrease timeout values for VGX driver 3251 timeoutInitializeGpuDefault(&pGpu->timeoutData, pGpu); 3252 } 3253 3254 return NV_OK; 3255 } 3256 3257 /*! 3258 * Execute a sequencer buffer coming from GSP 3259 * 3260 * @param[in] pGpu GPU object pointer 3261 * @param[in] pKernelGsp KernelGsp object pointer 3262 * @param[in] pRunCpuSeqParams Sequence buffer RPC parameters 3263 * 3264 * @return NV_OK if the GSP sequencer buffer has been executed successfully 3265 * NV_ERR_INVALID_STATE if the sequencer buffer is not allocated 3266 * NV_ERR_INVALID_DATA is the sequencer buffer is malformed 3267 */ 3268 NV_STATUS 3269 kgspExecuteSequencerBuffer_IMPL 3270 ( 3271 OBJGPU *pGpu, 3272 KernelGsp *pKernelGsp, 3273 void *pRunCpuSeqParams 3274 ) 3275 { 3276 rpc_run_cpu_sequencer_v17_00 *pParams = (rpc_run_cpu_sequencer_v17_00 *)pRunCpuSeqParams; 3277 NvU32 *pCmd = pParams->commandBuffer; 3278 NvU32 buffer_end = pParams->cmdIndex; 3279 NvU32 current_cmd_index = 0; 3280 NV_STATUS nvStatus = NV_OK; 3281 NvU32 payloadSize; 3282 3283 NV_ASSERT_OR_RETURN(IS_GSP_CLIENT(pGpu), NV_ERR_NOT_SUPPORTED); 3284 NV_ASSERT_OR_RETURN((pParams->bufferSizeDWord != 0), NV_ERR_INVALID_STATE); 3285 NV_ASSERT_OR_RETURN(buffer_end < pParams->bufferSizeDWord, NV_ERR_INVALID_DATA); 3286 3287 while (current_cmd_index < buffer_end) 3288 { 3289 NvU32 opCode = pCmd[current_cmd_index++]; 3290 payloadSize = GSP_SEQUENCER_PAYLOAD_SIZE_DWORDS(opCode); 3291 3292 NV_ASSERT_OR_RETURN(current_cmd_index + payloadSize <= buffer_end, NV_ERR_INVALID_DATA); 3293 3294 // 3295 // Handling of sequencer commands is split between those commands 3296 // that are common to all architectures (handled directly here) and 3297 // those commands that are arch-specific and are handled via the 3298 // kgspExecuteSequencerCommand_HAL() call below. 3299 // 3300 switch (opCode) 3301 { 3302 // 2 arguments 3303 case GSP_SEQ_BUF_OPCODE_REG_WRITE: 3304 { 3305 GSP_SEQ_BUF_PAYLOAD_REG_WRITE regWrite; 3306 portMemCopy(®Write, sizeof(GSP_SEQ_BUF_PAYLOAD_REG_WRITE), &pCmd[current_cmd_index], sizeof(GSP_SEQ_BUF_PAYLOAD_REG_WRITE)); 3307 3308 GPU_REG_WR32(pGpu, regWrite.addr, regWrite.val); 3309 break; 3310 } 3311 3312 // 3 arguments 3313 case GSP_SEQ_BUF_OPCODE_REG_MODIFY: 3314 { 3315 GSP_SEQ_BUF_PAYLOAD_REG_MODIFY regModify; 3316 NvU32 regVal; 3317 3318 portMemCopy(®Modify, sizeof(GSP_SEQ_BUF_PAYLOAD_REG_MODIFY), &pCmd[current_cmd_index], sizeof(GSP_SEQ_BUF_PAYLOAD_REG_MODIFY)); 3319 3320 regVal = GPU_REG_RD32(pGpu, regModify.addr); 3321 regVal = regVal & ~regModify.mask; 3322 regVal = regVal | regModify.val; 3323 GPU_REG_WR32(pGpu, regModify.addr, regVal); 3324 break; 3325 } 3326 3327 // 5 arguments 3328 case GSP_SEQ_BUF_OPCODE_REG_POLL: 3329 { 3330 GSP_SEQ_BUF_PAYLOAD_REG_POLL regPoll; 3331 NvU32 regval; 3332 RMTIMEOUT timeout; 3333 3334 portMemCopy(®Poll, sizeof(GSP_SEQ_BUF_PAYLOAD_REG_POLL), &pCmd[current_cmd_index], sizeof(GSP_SEQ_BUF_PAYLOAD_REG_POLL)); 3335 3336 regval = GPU_REG_RD32(pGpu, regPoll.addr); 3337 3338 gpuSetTimeout(pGpu, regPoll.timeout, &timeout, 0); 3339 while ((regval & regPoll.mask) != regPoll.val) 3340 { 3341 nvStatus = gpuCheckTimeout(pGpu, &timeout); 3342 if (nvStatus == NV_ERR_TIMEOUT) 3343 { 3344 NV_PRINTF(LEVEL_ERROR, "Timeout waiting for register to settle, value = 0x%x, err_code = 0x%x\n", 3345 regval, regPoll.error); 3346 DBG_BREAKPOINT(); 3347 return nvStatus; 3348 } 3349 osSpinLoop(); 3350 regval = GPU_REG_RD32(pGpu, regPoll.addr); 3351 } 3352 break; 3353 } 3354 3355 case GSP_SEQ_BUF_OPCODE_DELAY_US: 3356 { 3357 GSP_SEQ_BUF_PAYLOAD_DELAY_US delayUs; 3358 portMemCopy(&delayUs, sizeof(GSP_SEQ_BUF_PAYLOAD_DELAY_US), &pCmd[current_cmd_index], sizeof(GSP_SEQ_BUF_PAYLOAD_DELAY_US)); 3359 3360 osDelayUs(delayUs.val); 3361 break; 3362 } 3363 3364 case GSP_SEQ_BUF_OPCODE_REG_STORE: 3365 { 3366 GSP_SEQ_BUF_PAYLOAD_REG_STORE regStore; 3367 portMemCopy(®Store, sizeof(GSP_SEQ_BUF_PAYLOAD_REG_STORE), &pCmd[current_cmd_index], sizeof(GSP_SEQ_BUF_PAYLOAD_REG_STORE)); 3368 3369 NV_ASSERT_OR_RETURN(regStore.index < GSP_SEQ_BUF_REG_SAVE_SIZE, NV_ERR_INVALID_ARGUMENT); 3370 3371 pParams->regSaveArea[regStore.index] = GPU_REG_RD32(pGpu, regStore.addr); 3372 break; 3373 } 3374 3375 default: 3376 // 3377 // Route this command to the arch-specific handler. 3378 // 3379 NV_ASSERT_OK_OR_RETURN(kgspExecuteSequencerCommand_HAL(pGpu, pKernelGsp, opCode, &pCmd[current_cmd_index], payloadSize * sizeof (*pCmd))); 3380 break; 3381 } 3382 current_cmd_index += payloadSize; 3383 } 3384 3385 return NV_OK; 3386 } 3387 3388 #if LIBOS_LOG_DECODE_ENABLE 3389 static void 3390 _kgspLogPollingCallback 3391 ( 3392 OBJGPU *pGpu, 3393 void *data 3394 ) 3395 { 3396 KernelGsp *pKernelGsp = GPU_GET_KERNEL_GSP(pGpu); 3397 kgspDumpGspLogs(pGpu, pKernelGsp, NV_FALSE); 3398 } 3399 3400 NV_STATUS 3401 kgspStartLogPolling_IMPL 3402 ( 3403 OBJGPU *pGpu, 3404 KernelGsp *pKernelGsp 3405 ) 3406 { 3407 NV_STATUS status; 3408 status = osSchedule1SecondCallback(pGpu, _kgspLogPollingCallback, NULL, NV_OS_1HZ_REPEAT); 3409 return status; 3410 } 3411 3412 static void 3413 _kgspStopLogPolling 3414 ( 3415 OBJGPU *pGpu, 3416 KernelGsp *pKernelGsp 3417 ) 3418 { 3419 osRemove1SecondRepeatingCallback(pGpu, _kgspLogPollingCallback, NULL); 3420 } 3421 3422 #else // LIBOS_LOG_DECODE_ENABLE 3423 3424 NV_STATUS 3425 kgspStartLogPolling_IMPL 3426 ( 3427 OBJGPU *pGpu, 3428 KernelGsp *pKernelGsp 3429 ) 3430 { 3431 return NV_OK; 3432 } 3433 3434 static void 3435 _kgspStopLogPolling 3436 ( 3437 OBJGPU *pGpu, 3438 KernelGsp *pKernelGsp 3439 ) 3440 { 3441 return; 3442 } 3443 #endif // LIBOS_LOG_DECODE_ENABLE 3444 3445 /*! 3446 * Provides an opportunity to register some IntrService during intrStateInit. 3447 */ 3448 void 3449 kgspRegisterIntrService_IMPL 3450 ( 3451 OBJGPU *pGpu, 3452 KernelGsp *pKernelGsp, 3453 IntrServiceRecord pRecords[MC_ENGINE_IDX_MAX] 3454 ) 3455 { 3456 NvU32 engineIdx = MC_ENGINE_IDX_GSP; 3457 3458 if (!IS_GSP_CLIENT(pGpu)) 3459 return; 3460 3461 NV_ASSERT(pRecords[engineIdx].pInterruptService == NULL); 3462 pRecords[engineIdx].pInterruptService = staticCast(pKernelGsp, IntrService); 3463 } 3464 3465 /*! 3466 * Service GSP interrupts. 3467 * 3468 * @returns Zero, or any implementation-chosen nonzero value. If the same nonzero value is returned enough 3469 * times the interrupt is considered stuck. 3470 */ 3471 NvU32 3472 kgspServiceInterrupt_IMPL 3473 ( 3474 OBJGPU *pGpu, 3475 KernelGsp *pKernelGsp, 3476 IntrServiceServiceInterruptArguments *pParams 3477 ) 3478 { 3479 NV_ASSERT_OR_RETURN(pParams != NULL, 0); 3480 NV_ASSERT_OR_RETURN(pParams->engineIdx == MC_ENGINE_IDX_GSP, 0); 3481 3482 return kgspService_HAL(pGpu, pKernelGsp); 3483 } 3484 3485 /*! 3486 * Calculates the GSP FW heap size based on the GPU's resources. 3487 */ 3488 static NvU64 3489 _kgspCalculateFwHeapSize 3490 ( 3491 OBJGPU *pGpu, 3492 KernelGsp *pKernelGsp, 3493 NvU32 maxGspFwHeapSizeMB 3494 ) 3495 { 3496 // For VGPU, use the static pre-calculated size 3497 if (pGpu->bVgpuGspPluginOffloadEnabled) 3498 return GSP_FW_HEAP_SIZE_VGPU_DEFAULT; 3499 3500 // 3501 // The baremetal heap calculation is a function of the architecture, FB 3502 // size, and a chunk for backing client allocations (pre-calibrated for the 3503 // architecture through rough profiling). 3504 // 3505 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 3506 NvU64 fbSize = 0; 3507 3508 NV_ASSERT_OK(kmemsysGetUsableFbSize_HAL(pGpu, pKernelMemorySystem, &fbSize)); 3509 const NvU32 fbSizeGB = (NvU32)(NV_ALIGN_UP64(fbSize, 1 << 30) >> 30); 3510 3511 // 3512 // Reclaimable binary data will end up padding the heap (in some cases, 3513 // significantly), but due to memory fragmentation we can't rely on it to 3514 // linearly reduce the amount needed in the primary heap, so it is not a 3515 // factor here. Instead, it's just extra margin to keep us from exhausting 3516 // the heap at runtime. 3517 // 3518 NvU64 heapSize = pKernelGsp->fwHeapParamOsCarveoutSize + 3519 pKernelGsp->fwHeapParamBaseSize + 3520 NV_ALIGN_UP(GSP_FW_HEAP_PARAM_SIZE_PER_GB_FB * fbSizeGB, 1 << 20) + 3521 NV_ALIGN_UP(GSP_FW_HEAP_PARAM_CLIENT_ALLOC_SIZE, 1 << 20); 3522 3523 // Clamp to the minimum, even if the calculations say we can do with less 3524 const NvU32 minGspFwHeapSizeMB = kgspGetMinWprHeapSizeMB_HAL(pGpu, pKernelGsp); 3525 heapSize = NV_MAX(heapSize, (NvU64)minGspFwHeapSizeMB << 20); 3526 3527 // Clamp to the maximum heap size, if necessary 3528 heapSize = NV_MIN(heapSize, (NvU64)maxGspFwHeapSizeMB << 20); 3529 3530 NV_PRINTF(LEVEL_INFO, "GSP FW heap %lluMB of %uGB\n", 3531 heapSize >> 20, fbSizeGB); 3532 3533 return heapSize; 3534 } 3535 3536 /*! 3537 * Returns the size in bytes of the GSP FW heap: 3538 * - the registry override, if present 3539 * - otherwise, calculate the FW heap size for this GPU, limiting it to stay 3540 * within the pre-scrubbed area at the end of FB, if needed 3541 * 3542 * @param[in] posteriorFbSize - size in bytes of the memory reserved between the 3543 * end of the GSP FW heap and the end of FB, or 0 3544 * to disable limiting of the heap range to within 3545 * the pre-scrubbed area at the end of FB 3546 */ 3547 NvU64 3548 kgspGetFwHeapSize_IMPL 3549 ( 3550 OBJGPU *pGpu, 3551 KernelGsp *pKernelGsp, 3552 NvU64 posteriorFbSize 3553 ) 3554 { 3555 NvU32 maxScrubbedHeapSizeMB = NV_U32_MAX; 3556 NvU32 heapSizeMB = 0; 3557 3558 // 3559 // The pre-scrubbed region at the end of FB may limit the heap size, if no 3560 // scrubber ucode is supported to unlock the rest of memory prior to booting 3561 // GSP-RM. 3562 // 3563 if (!pKernelGsp->bScrubberUcodeSupported && (posteriorFbSize != 0)) 3564 { 3565 const NvU64 prescrubbedSize = kgspGetPrescrubbedTopFbSize(pGpu, pKernelGsp); 3566 if (prescrubbedSize < NV_U64_MAX) 3567 maxScrubbedHeapSizeMB = (NvU32)((prescrubbedSize - posteriorFbSize) >> 20); 3568 } 3569 3570 // Get the heap size override from the registry, if any 3571 if ((osReadRegistryDword(pGpu, NV_REG_STR_GSP_FIRMWARE_HEAP_SIZE_MB, &heapSizeMB) == NV_OK) && 3572 (heapSizeMB != NV_REG_STR_GSP_FIRMWARE_HEAP_SIZE_MB_DEFAULT)) 3573 { 3574 const NvU32 minGspFwHeapSizeMB = kgspGetMinWprHeapSizeMB_HAL(pGpu, pKernelGsp); 3575 const NvU32 maxGspFwHeapSizeMB = NV_MIN(kgspGetMaxWprHeapSizeMB_HAL(pGpu, pKernelGsp), 3576 maxScrubbedHeapSizeMB); 3577 3578 NV_ASSERT(minGspFwHeapSizeMB < maxGspFwHeapSizeMB); 3579 3580 if (heapSizeMB > maxGspFwHeapSizeMB) 3581 { 3582 NV_PRINTF(LEVEL_WARNING, "Firmware heap size clamped to maximum (%uMB)\n", 3583 maxGspFwHeapSizeMB); 3584 heapSizeMB = maxGspFwHeapSizeMB; 3585 } 3586 else if (heapSizeMB < minGspFwHeapSizeMB) 3587 { 3588 NV_PRINTF(LEVEL_WARNING, "Firmware heap size clamped to minimum (%uMB)\n", 3589 minGspFwHeapSizeMB); 3590 heapSizeMB = minGspFwHeapSizeMB; 3591 } 3592 else 3593 { 3594 NV_PRINTF(LEVEL_WARNING, "Firmware heap size overridden (%uMB)\n", 3595 heapSizeMB); 3596 } 3597 3598 return ((NvU64)heapSizeMB) << 20; 3599 } 3600 3601 return _kgspCalculateFwHeapSize(pGpu, pKernelGsp, maxScrubbedHeapSizeMB); 3602 } 3603