1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2019-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include "gpu/gsp/kernel_gsp.h" 25 26 #include "kernel/core/thread_state.h" 27 #include "kernel/core/locks.h" 28 #include "kernel/diagnostics/gpu_acct.h" 29 #include "kernel/gpu/fifo/kernel_channel.h" 30 #include "kernel/gpu/intr/engine_idx.h" 31 #include "kernel/gpu/mem_mgr/heap.h" 32 #include "kernel/gpu/mem_mgr/mem_mgr.h" 33 #include "kernel/gpu/rc/kernel_rc.h" 34 #include "kernel/gpu/nvlink/kernel_nvlink.h" 35 #include "virtualization/vgpuconfigapi.h" 36 #include "kernel/gpu/disp/kern_disp.h" 37 #include "gpu/external_device/external_device.h" 38 #include "class/cl2080.h" // NV20_SUBDEVICE_0 39 #include "liblogdecode.h" 40 #include "libelf.h" 41 #include "nverror.h" 42 #include "nv-firmware.h" 43 #include "nv-firmware-chip-family-select.h" 44 #include "nvtypes.h" 45 #include "nvVer.h" 46 #include "objrpc.h" 47 #include "objtmr.h" 48 #include "os/os.h" 49 #include "rmgspseq.h" 50 #include "sweng/dispsw.h" 51 #include "kernel/gpu/timed_sema.h" 52 #include "vgpu/rpc.h" 53 #include "kernel/gpu/pmu/kern_pmu.h" 54 #include "gpu/perf/kern_perf.h" 55 #include "core/locks.h" 56 57 #define RPC_STRUCTURES 58 #define RPC_GENERIC_UNION 59 #include "g_rpc-structures.h" 60 #undef RPC_STRUCTURES 61 #undef RPC_GENERIC_UNION 62 63 #define RPC_MESSAGE_STRUCTURES 64 #define RPC_MESSAGE_GENERIC_UNION 65 #include "g_rpc-message-header.h" 66 #undef RPC_MESSAGE_STRUCTURES 67 #undef RPC_MESSAGE_GENERIC_UNION 68 69 #include "gpu/gsp/message_queue_priv.h" 70 71 #define RPC_HDR ((rpc_message_header_v*)(pRpc->message_buffer)) 72 73 // 74 // RPC_PARAMS defines the rpc_params pointer and initializes it to the correct 75 // sub-structure. 76 // 77 // RPC_PARAMS intentionally assigns the the latest version structure to the 78 // versioned rpc_params pointer. With the -Werror=incompatible-pointer-types 79 // compiler flag, this checks for mismatched structure versions at compile time. 80 // 81 // For example: 82 // RPC_PARAMS(free, _v03_00); 83 // expands to 84 // rpc_free_v03_00 *rpc_params = &RPC_HDR->rpc_message_data->free_v; 85 // 86 #define RPC_PARAMS(r, v) rpc_##r##v *rpc_params = &RPC_HDR->rpc_message_data->r##_v 87 88 static NV_STATUS _kgspInitRpcInfrastructure(OBJGPU *, KernelGsp *); 89 static void _kgspFreeRpcInfrastructure(OBJGPU *, KernelGsp *); 90 91 static NV_STATUS _kgspRpcSendMessage(OBJGPU *, OBJRPC *); 92 static NV_STATUS _kgspRpcRecvPoll(OBJGPU *, OBJRPC *, NvU32); 93 static NV_STATUS _kgspRpcDrainEvents(OBJGPU *, KernelGsp *, NvU32); 94 95 static NV_STATUS _kgspAllocSimAccessBuffer(OBJGPU *pGpu, KernelGsp *pKernelGsp); 96 static void _kgspFreeSimAccessBuffer(OBJGPU *pGpu, KernelGsp *pKernelGsp); 97 98 static void _kgspStopLogPolling(OBJGPU *pGpu, KernelGsp *pKernelGsp); 99 100 static void _kgspFreeBootBinaryImage(OBJGPU *pGpu, KernelGsp *pKernelGsp); 101 102 static NV_STATUS _kgspPrepareGspRmBinaryImage(OBJGPU *pGpu, KernelGsp *pKernelGsp, GSP_FIRMWARE *pGspFw); 103 104 static NV_STATUS _kgspCreateRadix3(OBJGPU *pGpu, MEMORY_DESCRIPTOR **ppMemdescRadix3, 105 MEMORY_DESCRIPTOR *pMemdescData, const void *pData, NvU64 size); 106 107 static NV_STATUS _kgspCreateSignatureMemdesc(OBJGPU *pGpu, KernelGsp *pKernelGsp, 108 GSP_FIRMWARE *pGspFw); 109 110 static NV_STATUS _kgspFwContainerVerifyVersion(OBJGPU *pGpu, KernelGsp *pKernelGsp, 111 const void *pElfData, NvU64 elfDataSize, 112 const char *pNameInMsg); 113 114 static NV_STATUS _kgspFwContainerGetSection(OBJGPU *pGpu, KernelGsp *pKernelGsp, 115 const void *pElfData, NvU64 elfDataSize, 116 const char *pSectionName, 117 const void **ppSectionData, NvU64 *pSectionSize); 118 119 static NV_STATUS _kgspGetSectionNameForPrefix(OBJGPU *pGpu, KernelGsp *pKernelGsp, 120 char *pSectionNameBuf, NvLength sectionNameBufSize, 121 const char *pSectionPrefix); 122 123 static void 124 _kgspGetActiveRpcDebugData 125 ( 126 OBJRPC *pRpc, 127 NvU32 function, 128 NvU32 *data0, 129 NvU32 *data1 130 ) 131 { 132 switch (function) 133 { 134 case NV_VGPU_MSG_FUNCTION_GSP_RM_CONTROL: 135 { 136 rpc_gsp_rm_control_v03_00 *rpc_params = &rpc_message->gsp_rm_control_v03_00; 137 *data0 = rpc_params->cmd; 138 *data1 = rpc_params->paramsSize; 139 break; 140 } 141 case NV_VGPU_MSG_FUNCTION_GSP_RM_ALLOC: 142 { 143 rpc_gsp_rm_alloc_v03_00 *rpc_params = &rpc_message->gsp_rm_alloc_v03_00; 144 *data0 = rpc_params->hClass; 145 *data1 = rpc_params->paramsSize; 146 break; 147 } 148 case NV_VGPU_MSG_FUNCTION_FREE: 149 { 150 rpc_free_v03_00 *rpc_params = &rpc_message->free_v03_00; 151 *data0 = rpc_params->params.hObjectOld; 152 *data1 = rpc_params->params.hObjectParent; 153 break; 154 } 155 default: 156 { 157 *data0 = 0; 158 *data1 = 0; 159 break; 160 } 161 } 162 } 163 164 /*! 165 * GSP client RM RPC send routine 166 */ 167 static NV_STATUS 168 _kgspRpcSendMessage 169 ( 170 OBJGPU *pGpu, 171 OBJRPC *pRpc 172 ) 173 { 174 NV_STATUS nvStatus; 175 KernelGsp *pKernelGsp = GPU_GET_KERNEL_GSP(pGpu); 176 177 NV_ASSERT(rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 178 179 NV_ASSERT_OR_RETURN(!osIsGpuShutdown(pGpu), NV_ERR_GPU_IS_LOST); 180 181 nvStatus = GspMsgQueueSendCommand(pRpc->pMessageQueueInfo, pGpu); 182 if (nvStatus != NV_OK) 183 { 184 NV_PRINTF(LEVEL_ERROR, "GspMsgQueueSendCommand failed: 0x%x\n", nvStatus); 185 return nvStatus; 186 } 187 188 // GSPRM TODO: Use this call to pass the actual index. 189 kgspSetCmdQueueHead_HAL(pGpu, pKernelGsp, 0, 0); 190 191 // Add RPC history entry 192 { 193 NvU32 func = vgpu_rpc_message_header_v->function; 194 NvU32 entry; 195 196 entry = pRpc->rpcHistoryCurrent = (pRpc->rpcHistoryCurrent + 1) % RPC_HISTORY_DEPTH; 197 198 portMemSet(&pRpc->rpcHistory[entry], 0, sizeof(pRpc->rpcHistory[0])); 199 pRpc->rpcHistory[entry].function = func; 200 201 _kgspGetActiveRpcDebugData(pRpc, func, 202 &pRpc->rpcHistory[entry].data[0], 203 &pRpc->rpcHistory[entry].data[1]); 204 } 205 206 return NV_OK; 207 } 208 209 static NV_STATUS 210 _kgspRpcRunCpuSequencer 211 ( 212 OBJGPU *pGpu, 213 OBJRPC *pRpc 214 ) 215 { 216 RPC_PARAMS(run_cpu_sequencer, _v17_00); 217 KernelGsp *pKernelGsp = GPU_GET_KERNEL_GSP(pGpu); 218 219 return kgspExecuteSequencerBuffer(pGpu, pKernelGsp, rpc_params); 220 } 221 222 static void 223 _kgspProcessEccNotifier 224 ( 225 OBJGPU *pGpu, 226 void *eventData 227 ) 228 { 229 NV_STATUS nvStatus = NV_OK; 230 MemoryManager *pMemoryMgr = GPU_GET_MEMORY_MANAGER(pGpu); 231 232 if (pMemoryMgr->bEnableDynamicPageOfflining) 233 { 234 Nv2080EccDbeNotification *pParams = (Nv2080EccDbeNotification*)eventData; 235 if ((nvStatus = heapStorePendingBlackList(pGpu, GPU_GET_HEAP(pGpu), pParams->physAddress , 236 pParams->physAddress)) != NV_OK) 237 { 238 if (nvStatus == NV_ERR_RESET_REQUIRED) 239 { 240 NV_PRINTF(LEVEL_INFO, "Since we hit the DED on the reserved region, nothing to handle in this code path... \n"); 241 NV_PRINTF(LEVEL_INFO, "Relying on FBHUB interrupt to kill all the channels and force reset the GPU..\n"); 242 } 243 else 244 { 245 NV_PRINTF(LEVEL_INFO, "Dynamically blacklisting the DED page offset failed with, status: %x\n", nvStatus); 246 DBG_BREAKPOINT(); 247 } 248 } 249 250 } 251 } 252 253 /*! 254 * Receive an event notification from GSP-RM. 255 * 256 * When an event fires in GSP-RM, osNotifyEvent and osEventNotification check 257 * whether the event was originally allocated from client-RM. If so, they post 258 * it to the event queue and take no further action. Client RM picks up the 259 * event here and handles it. 260 */ 261 static NV_STATUS 262 _kgspRpcPostEvent 263 ( 264 OBJGPU *pGpu, 265 OBJRPC *pRpc 266 ) 267 { 268 RPC_PARAMS(post_event, _v17_00); 269 PEVENTNOTIFICATION pNotifyList = NULL; 270 PEVENTNOTIFICATION pNotifyEvent = NULL; 271 Event *pEvent = NULL; 272 NV_STATUS nvStatus = NV_OK; 273 274 // Get the notification list that contains this event. 275 NV_ASSERT_OR_RETURN(CliGetEventInfo(rpc_params->hClient, 276 rpc_params->hEvent, &pEvent), NV_ERR_OBJECT_NOT_FOUND); 277 278 if (pEvent->pNotifierShare != NULL) 279 pNotifyList = pEvent->pNotifierShare->pEventList; 280 281 NV_ASSERT_OR_RETURN(pNotifyList != NULL, NV_ERR_INVALID_POINTER); 282 283 switch (rpc_params->notifyIndex) 284 { 285 case NV2080_NOTIFIERS_ECC_DBE: 286 _kgspProcessEccNotifier(pGpu, rpc_params->eventData); 287 break; 288 } 289 290 // Send the event. 291 if (rpc_params->bNotifyList) 292 { 293 // Send notification to all matching events on the list. 294 nvStatus = osEventNotification(pGpu, pNotifyList, rpc_params->notifyIndex, 295 rpc_params->eventData, rpc_params->eventDataSize); 296 } 297 else 298 { 299 // Send event to a specific hEvent. Find hEvent in the notification list. 300 for (pNotifyEvent = pNotifyList; pNotifyEvent; pNotifyEvent = pNotifyEvent->Next) 301 { 302 if (pNotifyEvent->hEvent == rpc_params->hEvent) 303 { 304 nvStatus = osNotifyEvent(pGpu, pNotifyEvent, 0, 305 rpc_params->data, rpc_params->status); 306 break; 307 } 308 } 309 NV_ASSERT_OR_RETURN(pNotifyEvent != NULL, NV_ERR_OBJECT_NOT_FOUND); 310 } 311 312 return nvStatus; 313 } 314 315 /*! 316 * Receive RC notification from GSP-RM. 317 * 318 * RC error handling ("Channel Teardown sequence") is executed in GSP-RM. 319 * Client notifications, OS interaction etc happen in CPU-RM (Kernel RM). 320 */ 321 static NV_STATUS 322 _kgspRpcRCTriggered 323 ( 324 OBJGPU *pGpu, 325 OBJRPC *pRpc 326 ) 327 { 328 RPC_PARAMS(rc_triggered, _v17_02); 329 330 KernelRc *pKernelRc = GPU_GET_KERNEL_RC(pGpu); 331 KernelChannel *pKernelChannel; 332 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 333 CHID_MGR *pChidMgr; 334 NvU32 status = NV_OK; 335 RM_ENGINE_TYPE rmEngineType = gpuGetRmEngineType(rpc_params->nv2080EngineType); 336 337 // check if there's a PCI-E error pending either in device status or in AER 338 krcCheckBusError_HAL(pGpu, pKernelRc); 339 340 status = kfifoGetChidMgrFromType(pGpu, pKernelFifo, 341 ENGINE_INFO_TYPE_RM_ENGINE_TYPE, 342 (NvU32)rmEngineType, 343 &pChidMgr); 344 if (status != NV_OK) 345 return status; 346 347 pKernelChannel = kfifoChidMgrGetKernelChannel(pGpu, pKernelFifo, 348 pChidMgr, 349 rpc_params->chid); 350 NV_CHECK_OR_RETURN(LEVEL_ERROR, 351 pKernelChannel != NULL, 352 NV_ERR_INVALID_CHANNEL); 353 354 return krcErrorSendEventNotifications_HAL(pGpu, pKernelRc, 355 pKernelChannel, 356 rmEngineType, // unused on kernel side 357 rpc_params->exceptType, 358 rpc_params->scope, 359 rpc_params->partitionAttributionId); 360 } 361 362 /*! 363 * Receive Xid notification from GSP-RM 364 * 365 * Passes Xid errors that are triggered on GSP-RM to nvErrorLog for OS interactions 366 * (logging and OS notifications). 367 */ 368 static void 369 _kgspRpcOsErrorLog 370 ( 371 OBJGPU *pGpu, 372 OBJRPC *pRpc 373 ) 374 { 375 RPC_PARAMS(os_error_log, _v17_00); 376 377 KernelRc *pKernelRc = GPU_GET_KERNEL_RC(pGpu); 378 KernelChannel *pKernelChannel = NULL; 379 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 380 CHID_MGR *pChidMgr; 381 382 if (rpc_params->chid != INVALID_CHID) 383 { 384 pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, rpc_params->runlistId); 385 if (pChidMgr != NULL) 386 { 387 pKernelChannel = kfifoChidMgrGetKernelChannel(pGpu, pKernelFifo, 388 pChidMgr, 389 rpc_params->chid); 390 } 391 } 392 393 pKernelRc->pPreviousChannelInError = pKernelChannel; 394 nvErrorLog_va(pGpu, rpc_params->exceptType, "%s", rpc_params->errString); 395 pKernelRc->pPreviousChannelInError = NULL; 396 } 397 398 /*! 399 * Receives RPC events containing periodic perfmon utilization samples, passing them 400 * to GPUACCT for processing. 401 */ 402 static void 403 _kgspRpcGpuacctPerfmonUtilSamples 404 ( 405 OBJGPU *pGpu, 406 OBJRPC *pRpc 407 ) 408 { 409 OBJSYS *pSys = SYS_GET_INSTANCE(); 410 GpuAccounting *pGpuAcct = SYS_GET_GPUACCT(pSys); 411 GPUACCT_GPU_INSTANCE_INFO *pGpuInstanceInfo = &pGpuAcct->gpuInstanceInfo[pGpu->gpuInstance]; 412 RPC_PARAMS(gpuacct_perfmon_util_samples, _v17_00); 413 414 NV2080_CTRL_PERF_GET_GPUMON_PERFMON_UTIL_SAMPLES_V2_PARAMS_v17_00 *src = &rpc_params->params; 415 NV2080_CTRL_PERF_GET_GPUMON_PERFMON_UTIL_SAMPLES_V2_PARAMS *dest; 416 NvU32 i; 417 418 dest = pGpuInstanceInfo->pSamplesParams; 419 if (dest == NULL) 420 { 421 // This RPC event can be received even when the RM hasn't fully started. 422 // For instance, CPU RM can take longer than usual to initialize, 423 // but the GSP RM sampling timer (a 1 sec interval) is about to tick. 424 // In that case, pSamplesParams can not even be allocated by that time. 425 // Ignore this RPC event if pSamplesParams has not been allocated yet. 426 // See GPUSWSEC-1543 for more info. 427 return; 428 } 429 430 portMemSet(dest, 0, sizeof(*dest)); 431 dest->type = src->type; 432 dest->bufSize = src->bufSize; 433 dest->count = src->count; 434 dest->tracker = src->tracker; 435 436 for (i = 0; i < NV2080_CTRL_PERF_GPUMON_SAMPLE_COUNT_PERFMON_UTIL; i++) 437 { 438 dest->samples[i].base.timeStamp = src->samples[i].timeStamp; 439 440 dest->samples[i].fb.util = src->samples[i].fb.util; 441 dest->samples[i].fb.procId = src->samples[i].fb.procId; 442 dest->samples[i].fb.subProcessID = src->samples[i].fb.subProcessID; 443 444 dest->samples[i].gr.util = src->samples[i].gr.util; 445 dest->samples[i].gr.procId = src->samples[i].gr.procId; 446 dest->samples[i].gr.subProcessID = src->samples[i].gr.subProcessID; 447 448 dest->samples[i].nvenc.util = src->samples[i].nvenc.util; 449 dest->samples[i].nvenc.procId = src->samples[i].nvenc.procId; 450 dest->samples[i].nvenc.subProcessID = src->samples[i].nvenc.subProcessID; 451 452 dest->samples[i].nvdec.util = src->samples[i].nvdec.util; 453 dest->samples[i].nvdec.procId = src->samples[i].nvdec.procId; 454 dest->samples[i].nvdec.subProcessID = src->samples[i].nvdec.subProcessID; 455 } 456 457 gpuacctProcessGpuUtil(pGpuInstanceInfo, &dest->samples[0]); 458 } 459 460 /*! 461 * Receives RPC events containing current GPU Boost synchronization limits 462 * that should be cached and considered in the GPU Boost algorithm and runs 463 * the algorithm. 464 */ 465 static void 466 _kgspRpcPerfGpuBoostSyncLimitsCallback 467 ( 468 OBJGPU *pGpu, 469 OBJRPC *pRpc 470 ) 471 { 472 KernelPerf *pKernelPerf = GPU_GET_KERNEL_PERF(pGpu); 473 474 RPC_PARAMS(perf_gpu_boost_sync_limits_callback, _v17_00); 475 476 NV2080_CTRL_INTERNAL_PERF_GPU_BOOST_SYNC_SET_LIMITS_PARAMS_v17_00 *src = &rpc_params->params; 477 NV2080_CTRL_INTERNAL_PERF_GPU_BOOST_SYNC_SET_LIMITS_PARAMS dest; 478 NvU32 i; 479 480 dest.flags = src->flags; 481 dest.bBridgeless = src->bBridgeless; 482 483 for (i = 0; i < NV2080_CTRL_INTERNAL_PERF_SYNC_GPU_BOOST_LIMITS_NUM; i++) 484 { 485 dest.currLimits[i] = src->currLimits[i]; 486 } 487 488 kperfDoSyncGpuBoostLimits(pGpu, pKernelPerf, &dest); 489 490 } 491 492 /*! 493 * Recieves RPC events containing latest change of bridgeless information 494 */ 495 static void 496 _kgspRpcPerfBridgelessInfoUpdate 497 ( 498 OBJGPU *pGpu, 499 OBJRPC *pRpc 500 ) 501 { 502 RPC_PARAMS(perf_bridgeless_info_update, _v17_00); 503 504 kPerfGpuBoostSyncBridgelessUpdateInfo(pGpu, rpc_params->bBridgeless); 505 } 506 507 static void 508 _kgspRpcNvlinkInbandReceivedData256Callback 509 ( 510 OBJGPU *pGpu, 511 OBJRPC *pRpc 512 ) 513 { 514 RPC_PARAMS(nvlink_inband_received_data_256, _v17_00); 515 516 NV2080_CTRL_NVLINK_INBAND_RECEIVED_DATA_256_PARAMS_v17_00 *dest = &rpc_params->params; 517 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu); 518 519 NV_ASSERT(NV_OK == knvlinkInbandMsgCallbackDispatcher(pGpu, pKernelNvlink, dest->dataSize, dest->data)); 520 } 521 522 static void 523 _kgspRpcNvlinkInbandReceivedData512Callback 524 ( 525 OBJGPU *pGpu, 526 OBJRPC *pRpc 527 ) 528 { 529 RPC_PARAMS(nvlink_inband_received_data_512, _v17_00); 530 531 NV2080_CTRL_NVLINK_INBAND_RECEIVED_DATA_512_PARAMS_v17_00 *dest = &rpc_params->params; 532 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu); 533 534 NV_ASSERT(NV_OK == knvlinkInbandMsgCallbackDispatcher(pGpu, pKernelNvlink, dest->dataSize, dest->data)); 535 } 536 537 static void 538 _kgspRpcNvlinkInbandReceivedData1024Callback 539 ( 540 OBJGPU *pGpu, 541 OBJRPC *pRpc 542 ) 543 { 544 RPC_PARAMS(nvlink_inband_received_data_1024, _v17_00); 545 546 NV2080_CTRL_NVLINK_INBAND_RECEIVED_DATA_1024_PARAMS_v17_00 *dest = &rpc_params->params; 547 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu); 548 549 NV_ASSERT(NV_OK == knvlinkInbandMsgCallbackDispatcher(pGpu, pKernelNvlink, dest->dataSize, dest->data)); 550 } 551 552 static void 553 _kgspRpcNvlinkInbandReceivedData2048Callback 554 ( 555 OBJGPU *pGpu, 556 OBJRPC *pRpc 557 ) 558 { 559 RPC_PARAMS(nvlink_inband_received_data_2048, _v17_00); 560 561 NV2080_CTRL_NVLINK_INBAND_RECEIVED_DATA_2048_PARAMS_v17_00 *dest = &rpc_params->params; 562 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu); 563 564 NV_ASSERT(NV_OK == knvlinkInbandMsgCallbackDispatcher(pGpu, pKernelNvlink, dest->dataSize, dest->data)); 565 } 566 567 static void 568 _kgspRpcNvlinkInbandReceivedData4096Callback 569 ( 570 OBJGPU *pGpu, 571 OBJRPC *pRpc 572 ) 573 { 574 RPC_PARAMS(nvlink_inband_received_data_4096, _v17_00); 575 576 NV2080_CTRL_NVLINK_INBAND_RECEIVED_DATA_4096_PARAMS_v17_00 *dest = &rpc_params->params; 577 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu); 578 579 NV_ASSERT(NV_OK == knvlinkInbandMsgCallbackDispatcher(pGpu, pKernelNvlink, dest->dataSize, dest->data)); 580 } 581 582 /*! 583 * CPU-RM: Receive GPU Degraded status from GSP 584 */ 585 static void 586 _kgspRpcEventIsGpuDegradedCallback 587 ( 588 OBJGPU *pGpu, 589 OBJRPC *pRpc 590 ) 591 { 592 } 593 594 /*! 595 * Receive MMU fault queue notification from GSP-RM. 596 * 597 * Non-replayable fault handling is split between GSP-RM and the UVM driver. 598 * GSP-RM copies designated faults to the UVM driver's shadow buffer, 599 * and sends a notification. CPU-RM, in turn, needs to notify the UVM 600 * driver (schedule the UVM ISR to be run). 601 */ 602 static NV_STATUS 603 _kgspRpcMMUFaultQueued( 604 OBJGPU *pGpu, 605 OBJRPC *pRpc 606 ) 607 { 608 osQueueMMUFaultHandler(pGpu); 609 610 return NV_OK; 611 } 612 613 static NV_STATUS 614 _kgspRpcSimRead 615 ( 616 OBJGPU *pGpu, 617 OBJRPC *pRpc 618 ) 619 { 620 RPC_PARAMS(sim_read, _v1E_01); 621 if (IS_SIMULATION(pGpu)) 622 { 623 const NvU32 count = rpc_params->index + (rpc_params->count / sizeof(NvU32)); 624 NvU32 i; 625 626 KernelGsp *pKernelGsp = GPU_GET_KERNEL_GSP(pGpu); 627 OBJSYS *pSys = SYS_GET_INSTANCE(); 628 OBJOS *pOS = SYS_GET_OS(pSys); 629 630 NV_ASSERT_OR_RETURN(rpc_params->count <= sizeof(pKernelGsp->pSimAccessBuf->data), NV_ERR_BUFFER_TOO_SMALL); 631 632 for (i = rpc_params->index; i < count; i++) 633 { 634 NvU32 data; 635 pOS->osSimEscapeRead(pGpu, rpc_params->path, i, 4, &data); 636 pKernelGsp->pSimAccessBuf->data[i] = data; 637 } 638 639 pKernelGsp->pSimAccessBuf->seq++; 640 return NV_OK; 641 } 642 643 return NV_ERR_NOT_SUPPORTED; 644 } 645 646 static NV_STATUS 647 _kgspRpcSimWrite 648 ( 649 OBJGPU *pGpu, 650 OBJRPC *pRpc 651 ) 652 { 653 RPC_PARAMS(sim_write, _v1E_01); 654 if (IS_SIMULATION(pGpu)) 655 { 656 KernelGsp *pKernelGsp = GPU_GET_KERNEL_GSP(pGpu); 657 OBJSYS *pSys = SYS_GET_INSTANCE(); 658 OBJOS *pOS = SYS_GET_OS(pSys); 659 660 pOS->osSimEscapeWrite(pGpu, rpc_params->path, rpc_params->index, rpc_params->count, rpc_params->data); 661 pKernelGsp->pSimAccessBuf->seq++; 662 return NV_OK; 663 } 664 665 return NV_ERR_NOT_SUPPORTED; 666 } 667 668 static NV_STATUS 669 _kgspRpcSemaphoreScheduleCallback( 670 OBJGPU *pGpu, 671 OBJRPC *pRpc 672 ) 673 { 674 RPC_PARAMS(semaphore_schedule_callback, _v17_00); 675 676 return dispswReleaseSemaphoreAndNotifierFill(pGpu, 677 rpc_params->GPUVA, 678 rpc_params->hVASpace, 679 rpc_params->ReleaseValue, 680 rpc_params->Flags, 681 rpc_params->completionStatus, 682 rpc_params->hClient, 683 rpc_params->hEvent); 684 } 685 686 static NV_STATUS 687 _kgspRpcTimedSemaphoreRelease( 688 OBJGPU *pGpu, 689 OBJRPC *pRpc 690 ) 691 { 692 RPC_PARAMS(timed_semaphore_release, _v01_00); 693 694 return tsemaRelease_HAL(pGpu, 695 rpc_params->semaphoreVA, 696 rpc_params->notifierVA, 697 rpc_params->hVASpace, 698 rpc_params->releaseValue, 699 rpc_params->completionStatus, 700 rpc_params->hClient); 701 } 702 703 704 static NV_STATUS 705 _kgspRpcUcodeLibosPrint 706 ( 707 OBJGPU *pGpu, 708 OBJRPC *pRpc 709 ) 710 { 711 RPC_PARAMS(ucode_libos_print, _v1E_08); 712 713 // Check ucodes registered with the libos print mechanism 714 switch (rpc_params->ucodeEngDesc) 715 { 716 case ENG_PMU: 717 { 718 KernelPmu *pKernelPmu = GPU_GET_KERNEL_PMU(pGpu); 719 NV_CHECK_OR_RETURN(LEVEL_ERROR, pKernelPmu != NULL, NV_ERR_OBJECT_NOT_FOUND); 720 721 kpmuLogBuf(pGpu, pKernelPmu, 722 rpc_params->libosPrintBuf, rpc_params->libosPrintBufSize); 723 724 return NV_OK; 725 } 726 default: 727 NV_ASSERT_FAILED("Attempting to use libos prints with an unsupported ucode!\n"); 728 return NV_ERR_NOT_SUPPORTED; 729 } 730 } 731 732 static NV_STATUS 733 _kgspRpcVgpuGspPluginTriggered 734 ( 735 OBJGPU *pGpu, 736 OBJRPC *pRpc 737 ) 738 { 739 RPC_PARAMS(vgpu_gsp_plugin_triggered, _v17_00); 740 741 if (!IS_VGPU_GSP_PLUGIN_OFFLOAD_ENABLED(pGpu)) 742 return NV_ERR_NOT_SUPPORTED; 743 744 gpuGspPluginTriggeredEvent(pGpu, rpc_params->gfid, rpc_params->notifyIndex); 745 return NV_OK; 746 } 747 748 static NV_STATUS 749 _kgspRpcGspVgpuConfig 750 ( 751 OBJGPU *pGpu, 752 OBJRPC *pRpc 753 ) 754 { 755 RPC_PARAMS(vgpu_config_event, _v17_00); 756 757 NV_ASSERT_OR_RETURN(rpc_params->notifyIndex < NVA081_NOTIFIERS_MAXCOUNT, 758 NV_ERR_INVALID_ARGUMENT); 759 760 CliNotifyVgpuConfigEvent(pGpu, rpc_params->notifyIndex); 761 762 return NV_OK; 763 } 764 765 static NV_STATUS 766 _kgspRpcGspExtdevIntrService 767 ( 768 OBJGPU *pGpu, 769 OBJRPC *pRpc 770 ) 771 { 772 RPC_PARAMS(extdev_intr_service, _v17_00); 773 774 extdevGsyncService(pGpu, rpc_params->lossRegStatus, rpc_params->gainRegStatus, rpc_params->miscRegStatus, rpc_params->rmStatus); 775 776 return NV_OK; 777 } 778 779 static NV_STATUS 780 _kgspRpcRgLineIntr 781 ( 782 OBJGPU *pGpu, 783 OBJRPC *pRpc 784 ) 785 { 786 RPC_PARAMS(rg_line_intr, _v17_00); 787 788 KernelDisplay *pKernelDisplay = GPU_GET_KERNEL_DISPLAY(pGpu); 789 NV_CHECK_OR_RETURN(LEVEL_ERROR, pKernelDisplay != NULL, NV_ERR_OBJECT_NOT_FOUND); 790 791 kdispInvokeRgLineCallback(pKernelDisplay, rpc_params->head, rpc_params->rgIntr, NV_FALSE); 792 793 return NV_OK; 794 } 795 796 static NV_STATUS 797 _kgspRpcEventPlatformRequestHandlerStateSyncCallback 798 ( 799 OBJGPU* pGpu, 800 OBJRPC* pRpc 801 ) 802 { 803 return NV_OK; 804 } 805 806 static 807 const char *_getRpcName 808 ( 809 NvU32 id 810 ) 811 { 812 static const char *rpcName[] = 813 { 814 #define X(UNIT, a) #a, 815 #define E(a) #a, 816 #undef _RPC_GLOBAL_ENUMS_H_ 817 #include "vgpu/rpc_global_enums.h" 818 #undef X 819 #undef E 820 }; 821 822 if (id < NV_VGPU_MSG_FUNCTION_NUM_FUNCTIONS) 823 { 824 return rpcName[id]; 825 } 826 else if ((id > NV_VGPU_MSG_EVENT_FIRST_EVENT) && (id < NV_VGPU_MSG_EVENT_NUM_EVENTS)) 827 { 828 NvU32 index = id - (NV_VGPU_MSG_EVENT_FIRST_EVENT - NV_VGPU_MSG_FUNCTION_NUM_FUNCTIONS) + 1; 829 return rpcName[index]; 830 } 831 832 return "Unknown"; 833 } 834 835 /*! 836 * GSP client process RPC events 837 */ 838 static NV_STATUS 839 _kgspProcessRpcEvent 840 ( 841 OBJGPU *pGpu, 842 OBJRPC *pRpc 843 ) 844 { 845 rpc_message_header_v *pMsgHdr = RPC_HDR; 846 NV_STATUS nvStatus = NV_OK; 847 848 NV_PRINTF(LEVEL_INFO, "received event: 0x%x (%s) status: 0x%x size: %d\n", 849 pMsgHdr->function, _getRpcName(pMsgHdr->function), pMsgHdr->rpc_result, pMsgHdr->length); 850 851 switch(pMsgHdr->function) 852 { 853 case NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER: 854 nvStatus = _kgspRpcRunCpuSequencer(pGpu, pRpc); 855 break; 856 857 case NV_VGPU_MSG_EVENT_POST_EVENT: 858 nvStatus = _kgspRpcPostEvent(pGpu, pRpc); 859 break; 860 861 case NV_VGPU_MSG_EVENT_RC_TRIGGERED: 862 nvStatus = _kgspRpcRCTriggered(pGpu, pRpc); 863 break; 864 865 case NV_VGPU_MSG_EVENT_MMU_FAULT_QUEUED: 866 nvStatus = _kgspRpcMMUFaultQueued(pGpu, pRpc); 867 break; 868 869 case NV_VGPU_MSG_EVENT_SIM_READ: 870 nvStatus = _kgspRpcSimRead(pGpu, pRpc); 871 break; 872 873 case NV_VGPU_MSG_EVENT_SIM_WRITE: 874 nvStatus = _kgspRpcSimWrite(pGpu, pRpc); 875 break; 876 877 case NV_VGPU_MSG_EVENT_OS_ERROR_LOG: 878 _kgspRpcOsErrorLog(pGpu, pRpc); 879 break; 880 881 case NV_VGPU_MSG_EVENT_GPUACCT_PERFMON_UTIL_SAMPLES: 882 _kgspRpcGpuacctPerfmonUtilSamples(pGpu, pRpc); 883 break; 884 885 case NV_VGPU_MSG_EVENT_PERF_GPU_BOOST_SYNC_LIMITS_CALLBACK: 886 _kgspRpcPerfGpuBoostSyncLimitsCallback(pGpu, pRpc); 887 break; 888 889 case NV_VGPU_MSG_EVENT_PERF_BRIDGELESS_INFO_UPDATE: 890 _kgspRpcPerfBridgelessInfoUpdate(pGpu, pRpc); 891 break; 892 893 case NV_VGPU_MSG_EVENT_SEMAPHORE_SCHEDULE_CALLBACK: 894 _kgspRpcSemaphoreScheduleCallback(pGpu, pRpc); 895 break; 896 897 case NV_VGPU_MSG_EVENT_TIMED_SEMAPHORE_RELEASE: 898 _kgspRpcTimedSemaphoreRelease(pGpu, pRpc); 899 break; 900 901 case NV_VGPU_MSG_EVENT_NVLINK_INBAND_RECEIVED_DATA_256: 902 _kgspRpcNvlinkInbandReceivedData256Callback(pGpu, pRpc); 903 break; 904 905 case NV_VGPU_MSG_EVENT_NVLINK_INBAND_RECEIVED_DATA_512: 906 _kgspRpcNvlinkInbandReceivedData512Callback(pGpu, pRpc); 907 break; 908 909 case NV_VGPU_MSG_EVENT_NVLINK_INBAND_RECEIVED_DATA_1024: 910 _kgspRpcNvlinkInbandReceivedData1024Callback(pGpu, pRpc); 911 break; 912 913 case NV_VGPU_MSG_EVENT_NVLINK_INBAND_RECEIVED_DATA_2048: 914 _kgspRpcNvlinkInbandReceivedData2048Callback(pGpu, pRpc); 915 break; 916 917 case NV_VGPU_MSG_EVENT_NVLINK_INBAND_RECEIVED_DATA_4096: 918 _kgspRpcNvlinkInbandReceivedData4096Callback(pGpu, pRpc); 919 break; 920 921 case NV_VGPU_MSG_EVENT_NVLINK_IS_GPU_DEGRADED : 922 _kgspRpcEventIsGpuDegradedCallback(pGpu, pRpc); 923 break; 924 925 case NV_VGPU_MSG_EVENT_RG_LINE_INTR: 926 _kgspRpcRgLineIntr(pGpu, pRpc); 927 break; 928 929 case NV_VGPU_MSG_EVENT_UCODE_LIBOS_PRINT: 930 nvStatus = _kgspRpcUcodeLibosPrint(pGpu, pRpc); 931 break; 932 933 case NV_VGPU_MSG_EVENT_VGPU_GSP_PLUGIN_TRIGGERED: 934 nvStatus = _kgspRpcVgpuGspPluginTriggered(pGpu, pRpc); 935 break; 936 937 case NV_VGPU_MSG_EVENT_VGPU_CONFIG: 938 nvStatus = _kgspRpcGspVgpuConfig(pGpu, pRpc); 939 break; 940 941 case NV_VGPU_MSG_EVENT_EXTDEV_INTR_SERVICE: 942 nvStatus = _kgspRpcGspExtdevIntrService(pGpu, pRpc); 943 break; 944 945 case NV_VGPU_MSG_EVENT_PFM_REQ_HNDLR_STATE_SYNC_CALLBACK: 946 nvStatus = _kgspRpcEventPlatformRequestHandlerStateSyncCallback(pGpu, pRpc); 947 break; 948 949 case NV_VGPU_MSG_EVENT_GSP_INIT_DONE: // Handled by _kgspRpcRecvPoll. 950 default: 951 // 952 // We will get here if the previous RPC timed out. The response 953 // eventually comes in as an unexpected event. The error handling 954 // for the timeout has already happened, and returning an error here 955 // causes subsequent messages to fail. So return NV_OK. 956 // 957 NV_PRINTF(LEVEL_ERROR, "Unexpected RPC event 0x%x (%s)\n", 958 pMsgHdr->function, _getRpcName(pMsgHdr->function)); 959 break; 960 } 961 962 return nvStatus; 963 } 964 965 /*! 966 * Handle a single RPC event from GSP unless the event is [an RPC return for] expectedFunc, 967 * or there are no events available in the buffer. 968 * 969 * @return 970 * NV_OK if the event is successfully handled. 971 * NV_WARN_NOTHING_TO_DO if there are no events available. 972 * NV_WARN_MORE_PROCESSING_REQUIRED if the event is expectedFunc: it is unhandled and in the staging area. 973 * (Another status) if event reading or processing fails. 974 */ 975 static NV_STATUS 976 _kgspRpcDrainOneEvent 977 ( 978 OBJGPU *pGpu, 979 OBJRPC *pRpc, 980 NvU32 expectedFunc 981 ) 982 { 983 NV_STATUS nvStatus; 984 985 // Issue a memory barrier to ensure we see any queue updates. 986 // Note: Without the fence, the CPU may get stuck in an infinite loop 987 // waiting for a message that has already arrived. 988 portAtomicMemoryFenceFull(); 989 990 nvStatus = GspMsgQueueReceiveStatus(pRpc->pMessageQueueInfo); 991 992 if (nvStatus == NV_OK) 993 { 994 rpc_message_header_v *pMsgHdr = RPC_HDR; 995 if (pMsgHdr->function == expectedFunc) 996 return NV_WARN_MORE_PROCESSING_REQUIRED; 997 998 nvStatus = _kgspProcessRpcEvent(pGpu, pRpc); 999 if (nvStatus != NV_OK) 1000 { 1001 NV_PRINTF(LEVEL_ERROR, 1002 "Failed to process received event 0x%x (%s): status=0x%x\n", 1003 pMsgHdr->function, _getRpcName(pMsgHdr->function), nvStatus); 1004 } 1005 } 1006 1007 // 1008 // We don't expect the NV_WARN_MORE_PROCESSING_REQUIRED from either called function. 1009 // If we get it we need to suppress it to avoid confusing our caller, for whom it has special meaning. 1010 // 1011 NV_ASSERT_OR_ELSE(nvStatus != NV_WARN_MORE_PROCESSING_REQUIRED, 1012 nvStatus = NV_ERR_GENERIC); 1013 1014 return nvStatus; 1015 } 1016 1017 /*! 1018 * Handle RPC events from GSP until the event is [an RPC return for] expectedFunc, 1019 * or there are no events available in the buffer. 1020 * 1021 * Also dump GSP logs, and check for severe errors coming from GSP. 1022 * 1023 * @return 1024 * NV_OK if one or more events are handled and there are none left. 1025 * NV_WARN_MORE_PROCESSING_REQUIRED if an expectedFunc event is found: it is unhandled and in the staging area. 1026 * (Zero or more preceding events were successfully handled.) 1027 * (Another status) if event reading or processing fails. 1028 */ 1029 static NV_STATUS 1030 _kgspRpcDrainEvents 1031 ( 1032 OBJGPU *pGpu, 1033 KernelGsp *pKernelGsp, 1034 NvU32 expectedFunc 1035 ) 1036 { 1037 NV_STATUS nvStatus = NV_OK; 1038 OBJRPC *pRpc = GPU_GET_RPC(pGpu); 1039 1040 while (nvStatus == NV_OK) 1041 { 1042 nvStatus = _kgspRpcDrainOneEvent(pGpu, pRpc, expectedFunc); 1043 kgspDumpGspLogs(pGpu, pKernelGsp, NV_FALSE); 1044 } 1045 1046 kgspHealthCheck_HAL(pGpu, pKernelGsp); 1047 1048 if (nvStatus == NV_WARN_NOTHING_TO_DO) 1049 nvStatus = NV_OK; 1050 1051 return nvStatus; 1052 } 1053 1054 /*! 1055 * Log Xid 119 - GSP RPC Timeout 1056 */ 1057 static void 1058 _kgspLogXid119 1059 ( 1060 OBJGPU *pGpu, 1061 OBJRPC *pRpc, 1062 NvU32 expectedFunc 1063 ) 1064 { 1065 KernelGsp *pKernelGsp = GPU_GET_KERNEL_GSP(pGpu); 1066 NvU32 historyEntry = pRpc->rpcHistoryCurrent; 1067 NvU32 activeData[2]; 1068 1069 if (!pKernelGsp->bXid119Printed) 1070 { 1071 NV_PRINTF(LEVEL_NOTICE, 1072 "********************************* GSP Failure **********************************\n"); 1073 } 1074 1075 NV_ASSERT(expectedFunc == pRpc->rpcHistory[historyEntry].function); 1076 1077 _kgspGetActiveRpcDebugData(pRpc, expectedFunc, 1078 &activeData[0], &activeData[1]); 1079 1080 nvErrorLog_va((void*)pGpu, GSP_RPC_TIMEOUT, 1081 "Timeout waiting for RPC from GSP! Expected function %d (%s) (0x%x 0x%x).", 1082 expectedFunc, 1083 _getRpcName(expectedFunc), 1084 pRpc->rpcHistory[historyEntry].data[0], 1085 pRpc->rpcHistory[historyEntry].data[1]); 1086 1087 if (!pKernelGsp->bXid119Printed) 1088 { 1089 NvU32 historyIndex; 1090 1091 if ((expectedFunc != vgpu_rpc_message_header_v->function) || 1092 (pRpc->rpcHistory[historyEntry].data[0] != activeData[0]) || 1093 (pRpc->rpcHistory[historyEntry].data[1] != activeData[1])) 1094 { 1095 NV_PRINTF(LEVEL_ERROR, 1096 "Current RPC function %d (%s) or data (0x%x 0x%x) does not match expected function %d (%s) or data (0x%x 0x%x).\n", 1097 vgpu_rpc_message_header_v->function, _getRpcName(vgpu_rpc_message_header_v->function), 1098 activeData[0], activeData[1], 1099 expectedFunc, _getRpcName(expectedFunc), 1100 pRpc->rpcHistory[historyEntry].data[0], 1101 pRpc->rpcHistory[historyEntry].data[1]); 1102 } 1103 1104 NV_PRINTF(LEVEL_ERROR, "RPC history (CPU -> GSP):\n"); 1105 NV_PRINTF(LEVEL_ERROR, "\tentry\tfunc\t\t\t\tdata\n"); 1106 for (historyIndex = 0; historyIndex < RPC_HISTORY_DEPTH; historyIndex++) 1107 { 1108 historyEntry = (pRpc->rpcHistoryCurrent + RPC_HISTORY_DEPTH - historyIndex) % RPC_HISTORY_DEPTH; 1109 NV_PRINTF(LEVEL_ERROR, "\t%c%-2d\t%2d %-22s\t0x%08x 0x%08x\n", 1110 ((historyIndex == 0) ? ' ' : '-'), 1111 historyIndex, 1112 pRpc->rpcHistory[historyEntry].function, 1113 _getRpcName(pRpc->rpcHistory[historyEntry].function), 1114 pRpc->rpcHistory[historyEntry].data[0], 1115 pRpc->rpcHistory[historyEntry].data[1]); 1116 } 1117 1118 NV_PRINTF(LEVEL_ERROR, "Dumping stack:\n"); 1119 osAssertFailed(); 1120 1121 NV_PRINTF(LEVEL_NOTICE, 1122 "********************************************************************************\n"); 1123 } 1124 1125 pKernelGsp->bXid119Printed = NV_TRUE; 1126 } 1127 1128 /*! 1129 * GSP client RM RPC poll routine 1130 */ 1131 static NV_STATUS 1132 _kgspRpcRecvPoll 1133 ( 1134 OBJGPU *pGpu, 1135 OBJRPC *pRpc, 1136 NvU32 expectedFunc 1137 ) 1138 { 1139 KernelGsp *pKernelGsp = GPU_GET_KERNEL_GSP(pGpu); 1140 NV_STATUS nvStatus; 1141 RMTIMEOUT timeout; 1142 NvU32 timeoutUs = GPU_TIMEOUT_DEFAULT; 1143 NvBool bSlowGspRpc = IS_EMULATION(pGpu) || IS_SIMULATION(pGpu); 1144 1145 // 1146 // We do not allow recursive polling. This can happen if e.g. 1147 // 1. CPU-RM issues RPC-A to GSP and polls waiting for it to finish 1148 // 2. While servicing RPC-A, GSP emits an async event back to CPU-RM 1149 // 3. CPU-RM services the async event and sends another synchronous RPC-B 1150 // 4. RPC-A response will come first, but CPU-RM is now waiting on RPC-B 1151 // 1152 // We don't have a good way to handle this and should just be deferring the 1153 // second RPC until the first one is done, via e.g. osQueueWorkItem(). 1154 // This assert is meant to catch and loudly fail such cases. 1155 // 1156 NV_ASSERT_OR_RETURN(!pKernelGsp->bPollingForRpcResponse, NV_ERR_INVALID_STATE); 1157 pKernelGsp->bPollingForRpcResponse = NV_TRUE; 1158 1159 // GSP-RM init in emulation/simulation environment is extremely slow, 1160 // so need to increment timeout. 1161 // Apply the timeout extension to other RPCs as well, mostly so that 1162 // we'll reset the thread state after each RPC, not just while waiting 1163 // for the INIT_DONE event. 1164 // 1165 if (bSlowGspRpc) 1166 { 1167 NvU32 timeoutResult; 1168 1169 // On slow Apollo emulators, GSP-RM init could take more than an hour 1170 NV_ASSERT(portSafeMulU32(GSP_SCALE_TIMEOUT_EMU_SIM, 1500000, &timeoutResult)); 1171 timeoutUs = timeoutResult; 1172 } 1173 else 1174 { 1175 // We should only ever timeout this when GSP is in really bad state, so if it just 1176 // happens to timeout on default timeout it should be OK for us to give it a little 1177 // more time - make this timeout 1.5 of the default to allow some leeway. 1178 NvU32 defaultus = pGpu->timeoutData.defaultus; 1179 1180 timeoutUs = defaultus + defaultus / 2; 1181 } 1182 1183 NV_ASSERT(rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 1184 gpuSetTimeout(pGpu, timeoutUs, &timeout, GPU_TIMEOUT_FLAGS_BYPASS_THREAD_STATE); 1185 1186 for (;;) 1187 { 1188 nvStatus = _kgspRpcDrainEvents(pGpu, pKernelGsp, expectedFunc); 1189 1190 switch (nvStatus) { 1191 case NV_WARN_MORE_PROCESSING_REQUIRED: 1192 nvStatus = NV_OK; 1193 goto done; 1194 case NV_OK: 1195 // Check timeout and continue outer loop. 1196 break; 1197 default: 1198 goto done; 1199 } 1200 1201 osSpinLoop(); 1202 1203 nvStatus = gpuCheckTimeout(pGpu, &timeout); 1204 if (nvStatus == NV_ERR_TIMEOUT) 1205 { 1206 _kgspLogXid119(pGpu, pRpc, expectedFunc); 1207 goto done; 1208 } 1209 1210 if (osIsGpuShutdown(pGpu)) 1211 { 1212 nvStatus = NV_ERR_GPU_IS_LOST; 1213 goto done; 1214 } 1215 } 1216 1217 done: 1218 pKernelGsp->bPollingForRpcResponse = NV_FALSE; 1219 1220 if (bSlowGspRpc) 1221 { 1222 // Avoid cumulative timeout due to slow RPC 1223 threadStateResetTimeout(pGpu); 1224 } 1225 1226 return nvStatus; 1227 } 1228 1229 /*! 1230 * Initialize stripped down version of RPC infra init for GSP clients. 1231 */ 1232 static NV_STATUS 1233 _kgspInitRpcInfrastructure 1234 ( 1235 OBJGPU *pGpu, 1236 KernelGsp *pKernelGsp 1237 ) 1238 { 1239 NV_STATUS nvStatus = NV_OK; 1240 1241 pKernelGsp->pRpc = initRpcObject(pGpu); 1242 if (pKernelGsp->pRpc == NULL) 1243 { 1244 NV_PRINTF(LEVEL_ERROR, "initRpcObject failed\n"); 1245 return NV_ERR_INSUFFICIENT_RESOURCES; 1246 } 1247 1248 OBJRPC *pRpc = pKernelGsp->pRpc; 1249 1250 portMemSet(&pRpc->rpcHistory, 0, sizeof(pRpc->rpcHistory)); 1251 pRpc->rpcHistoryCurrent = RPC_HISTORY_DEPTH - 1; 1252 1253 pRpc->pMessageQueueInfo = NULL; 1254 1255 nvStatus = GspMsgQueueInit(pGpu, &pRpc->pMessageQueueInfo); 1256 if (nvStatus != NV_OK) 1257 { 1258 NV_PRINTF(LEVEL_ERROR, "GspMsgQueueInit failed\n"); 1259 _kgspFreeRpcInfrastructure(pGpu, pKernelGsp); 1260 return nvStatus; 1261 } 1262 1263 pRpc->messageQueuePhysMem = pRpc->pMessageQueueInfo->sharedMemPA; 1264 pRpc->message_buffer = (NvU32 *)pRpc->pMessageQueueInfo->pRpcMsgBuf; 1265 pRpc->maxRpcSize = GSP_MSG_QUEUE_RPC_SIZE_MAX; 1266 pRpc->init_msg_buf = (NvU32 *)pRpc->pMessageQueueInfo->pInitMsgBuf; 1267 pRpc->init_msg_buf_pa = pRpc->pMessageQueueInfo->initMsgBufPA; 1268 1269 portMemSet(&pKernelGsp->gspStaticInfo, 0, 1270 sizeof(pKernelGsp->gspStaticInfo)); 1271 1272 rpcSendMessage_FNPTR(pKernelGsp->pRpc) = _kgspRpcSendMessage; 1273 rpcRecvPoll_FNPTR(pKernelGsp->pRpc) = _kgspRpcRecvPoll; 1274 1275 return NV_OK; 1276 } 1277 1278 static void 1279 _kgspFreeRpcInfrastructure 1280 ( 1281 OBJGPU *pGpu, 1282 KernelGsp *pKernelGsp 1283 ) 1284 { 1285 if (pKernelGsp->pRpc != NULL) 1286 { 1287 GspMsgQueueCleanup(&pKernelGsp->pRpc->pMessageQueueInfo); 1288 rpcDestroy(pGpu, pKernelGsp->pRpc); 1289 portMemFree(pKernelGsp->pRpc); 1290 pKernelGsp->pRpc = NULL; 1291 } 1292 } 1293 1294 /*! 1295 * Convert init arg name to 64bit id value. 1296 * 1297 * @param[in] name String representing name of init arg 1298 */ 1299 static NvU64 1300 _kgspGenerateInitArgId(const char *name) 1301 { 1302 NvU64 id = 0; 1303 NvU8 c; 1304 NvU32 i; 1305 1306 // Convert at most 8 characters from name into id. 1307 for (i = 0; i < (sizeof(NvU64) / sizeof(NvU8)); ++i) 1308 { 1309 c = (NvU8)*name++; 1310 if (c == '\0') 1311 { 1312 break; 1313 } 1314 id = (id << 8) | c; 1315 } 1316 1317 return id; 1318 } 1319 1320 /*! 1321 * Free vgpu partition LIBOS task logging structures 1322 */ 1323 static void 1324 _kgspFreeLibosVgpuPartitionLoggingStructures 1325 ( 1326 OBJGPU *pGpu, 1327 KernelGsp *pKernelGsp, 1328 NvU32 gfid 1329 ) 1330 { 1331 libosLogDestroy(&pKernelGsp->logDecodeVgpuPartition[gfid - 1]); 1332 1333 // release the vgpu task log buffer memory 1334 RM_LIBOS_LOG_MEM *pGspPluginVgpuTaskLog = &pKernelGsp->gspPluginVgpuTaskLogMem[gfid - 1]; 1335 1336 if (pGspPluginVgpuTaskLog->pTaskLogBuffer != NULL) 1337 { 1338 memdescUnmapInternal(pGpu, pGspPluginVgpuTaskLog->pTaskLogDescriptor, TRANSFER_FLAGS_NONE); 1339 1340 pGspPluginVgpuTaskLog->pTaskLogBuffer = NULL; 1341 } 1342 1343 if (pGspPluginVgpuTaskLog->pTaskLogDescriptor != NULL) 1344 { 1345 memdescFree(pGspPluginVgpuTaskLog->pTaskLogDescriptor); 1346 memdescDestroy(pGspPluginVgpuTaskLog->pTaskLogDescriptor); 1347 pGspPluginVgpuTaskLog->pTaskLogDescriptor = NULL; 1348 } 1349 1350 // release the init task log buffer memory 1351 RM_LIBOS_LOG_MEM *pGspPluginInitTaskLog = &pKernelGsp->gspPluginInitTaskLogMem[gfid - 1]; 1352 1353 if (pGspPluginInitTaskLog->pTaskLogBuffer != NULL) 1354 { 1355 memdescUnmapInternal(pGpu, pGspPluginInitTaskLog->pTaskLogDescriptor, TRANSFER_FLAGS_NONE); 1356 1357 pGspPluginInitTaskLog->pTaskLogBuffer = NULL; 1358 } 1359 1360 if (pGspPluginInitTaskLog->pTaskLogDescriptor != NULL) 1361 { 1362 memdescFree(pGspPluginInitTaskLog->pTaskLogDescriptor); 1363 memdescDestroy(pGspPluginInitTaskLog->pTaskLogDescriptor); 1364 pGspPluginInitTaskLog->pTaskLogDescriptor = NULL; 1365 } 1366 } 1367 1368 /*! 1369 * Free vgpu partition LIBOS task logging structures 1370 */ 1371 NV_STATUS 1372 kgspFreeVgpuPartitionLogging_IMPL 1373 ( 1374 OBJGPU *pGpu, 1375 KernelGsp *pKernelGsp, 1376 NvU32 gfid 1377 ) 1378 { 1379 if (gfid > MAX_PARTITIONS_WITH_GFID) 1380 { 1381 return NV_ERR_INVALID_ARGUMENT; 1382 } 1383 else 1384 { 1385 // Make sure there is no lingering debug output. 1386 kgspDumpGspLogs(pGpu, pKernelGsp, NV_FALSE); 1387 1388 _kgspFreeLibosVgpuPartitionLoggingStructures(pGpu, pKernelGsp, gfid); 1389 return NV_OK; 1390 } 1391 } 1392 1393 /*! 1394 * Initialize vgpu partition LIBOS task logging structures 1395 */ 1396 NV_STATUS 1397 kgspInitVgpuPartitionLogging_IMPL 1398 ( 1399 OBJGPU *pGpu, 1400 KernelGsp *pKernelGsp, 1401 NvU32 gfid, 1402 NvU64 initTaskLogBUffOffset, 1403 NvU64 initTaskLogBUffSize, 1404 NvU64 vgpuTaskLogBUffOffset, 1405 NvU64 vgpuTaskLogBuffSize 1406 ) 1407 { 1408 NV_STATUS nvStatus = NV_OK; 1409 RM_LIBOS_LOG_MEM *pGspPluginVgpuTaskLog = NULL; 1410 RM_LIBOS_LOG_MEM *pGspPluginInitTaskLog = NULL; 1411 char vm_string[8], sourceName[SOURCE_NAME_MAX_LENGTH]; 1412 1413 if (gfid > MAX_PARTITIONS_WITH_GFID) 1414 { 1415 return NV_ERR_INVALID_ARGUMENT; 1416 } 1417 1418 // Source name is used to generate a tag that is a unique identifier for nvlog buffers. 1419 // As the source name 'GSP' is already in use, we will need a custom source name. 1420 nvDbgSnprintf(sourceName, SOURCE_NAME_MAX_LENGTH, "V%02d", gfid); 1421 libosLogCreateEx(&pKernelGsp->logDecodeVgpuPartition[gfid - 1], sourceName); 1422 1423 // Setup logging for vgpu task in vgpu partition 1424 { 1425 pGspPluginVgpuTaskLog = &pKernelGsp->gspPluginVgpuTaskLogMem[gfid - 1]; 1426 NvP64 pVa = NvP64_NULL; 1427 1428 1429 NV_ASSERT_OK_OR_GOTO(nvStatus, 1430 memdescCreate(&pGspPluginVgpuTaskLog->pTaskLogDescriptor, 1431 pGpu, 1432 vgpuTaskLogBuffSize, 1433 RM_PAGE_SIZE, 1434 NV_TRUE, ADDR_FBMEM, NV_MEMORY_CACHED, 1435 MEMDESC_FLAGS_NONE), 1436 error_cleanup); 1437 1438 1439 memdescDescribe(pGspPluginVgpuTaskLog->pTaskLogDescriptor, ADDR_FBMEM, vgpuTaskLogBUffOffset, vgpuTaskLogBuffSize); 1440 1441 pVa = memdescMapInternal(pGpu, pGspPluginVgpuTaskLog->pTaskLogDescriptor, TRANSFER_FLAGS_NONE); 1442 1443 if (pVa != NvP64_NULL) 1444 { 1445 pGspPluginVgpuTaskLog->pTaskLogBuffer = pVa; 1446 portMemSet(pGspPluginVgpuTaskLog->pTaskLogBuffer, 0, vgpuTaskLogBuffSize); 1447 1448 pGspPluginVgpuTaskLog->id8 = _kgspGenerateInitArgId("LOGVGPU"); 1449 1450 nvDbgSnprintf(vm_string, sizeof(vm_string), "VGPU%d", gfid); 1451 1452 libosLogAddLogEx(&pKernelGsp->logDecodeVgpuPartition[gfid - 1], 1453 pGspPluginVgpuTaskLog->pTaskLogBuffer, 1454 memdescGetSize(pGspPluginVgpuTaskLog->pTaskLogDescriptor), 1455 pGpu->gpuInstance, 1456 (gpuGetChipArch(pGpu) >> GPU_ARCH_SHIFT), 1457 gpuGetChipImpl(pGpu), 1458 vm_string, 1459 ".fwlogging_vgpu"); 1460 } 1461 else 1462 { 1463 NV_PRINTF(LEVEL_ERROR, "Failed to map memory for vgpu task log buffer for vGPU partition \n"); 1464 nvStatus = NV_ERR_INSUFFICIENT_RESOURCES; 1465 goto error_cleanup; 1466 } 1467 } 1468 1469 // Setup logging for init task in vgpu partition 1470 { 1471 pGspPluginInitTaskLog = &pKernelGsp->gspPluginInitTaskLogMem[gfid - 1]; 1472 NvP64 pVa = NvP64_NULL; 1473 1474 NV_ASSERT_OK_OR_GOTO(nvStatus, 1475 memdescCreate(&pGspPluginInitTaskLog->pTaskLogDescriptor, 1476 pGpu, 1477 initTaskLogBUffSize, 1478 RM_PAGE_SIZE, 1479 NV_TRUE, ADDR_FBMEM, NV_MEMORY_CACHED, 1480 MEMDESC_FLAGS_NONE), 1481 error_cleanup); 1482 1483 memdescDescribe(pGspPluginInitTaskLog->pTaskLogDescriptor, ADDR_FBMEM, initTaskLogBUffOffset, initTaskLogBUffSize); 1484 1485 pVa = memdescMapInternal(pGpu, pGspPluginInitTaskLog->pTaskLogDescriptor, TRANSFER_FLAGS_NONE); 1486 1487 if (pVa != NvP64_NULL) 1488 { 1489 pGspPluginInitTaskLog->pTaskLogBuffer = pVa; 1490 portMemSet(pGspPluginInitTaskLog->pTaskLogBuffer, 0, initTaskLogBUffSize); 1491 1492 pGspPluginInitTaskLog->id8 = _kgspGenerateInitArgId("LOGINIT"); 1493 1494 nvDbgSnprintf(vm_string, sizeof(vm_string), "INIT%d", gfid); 1495 1496 libosLogAddLogEx(&pKernelGsp->logDecodeVgpuPartition[gfid - 1], 1497 pGspPluginInitTaskLog->pTaskLogBuffer, 1498 memdescGetSize(pGspPluginInitTaskLog->pTaskLogDescriptor), 1499 pGpu->gpuInstance, 1500 (gpuGetChipArch(pGpu) >> GPU_ARCH_SHIFT), 1501 gpuGetChipImpl(pGpu), 1502 vm_string, 1503 ".fwlogging_init"); 1504 } 1505 else 1506 { 1507 NV_PRINTF(LEVEL_ERROR, "Failed to map memory for init task log buffer for vGPU partition \n"); 1508 nvStatus = NV_ERR_INSUFFICIENT_RESOURCES; 1509 goto error_cleanup; 1510 } 1511 } 1512 1513 { 1514 libosLogInit(&pKernelGsp->logDecodeVgpuPartition[gfid - 1], pKernelGsp->pLogElf, pKernelGsp->logElfDataSize); 1515 // nvlog buffers are now setup using the appropriate sourceName to avoid tag-value clash. 1516 // Now sourceName can be modified to preserve the 'GSP-VGPUx' logging convention. 1517 portStringCopy(pKernelGsp->logDecodeVgpuPartition[gfid - 1].sourceName, 1518 SOURCE_NAME_MAX_LENGTH, 1519 "GSP", SOURCE_NAME_MAX_LENGTH); 1520 } 1521 1522 error_cleanup: 1523 if (nvStatus != NV_OK) 1524 _kgspFreeLibosVgpuPartitionLoggingStructures(pGpu, pKernelGsp, gfid); 1525 1526 return nvStatus; 1527 } 1528 1529 /*! 1530 * Free LIBOS task logging structures 1531 */ 1532 static void 1533 _kgspFreeLibosLoggingStructures 1534 ( 1535 OBJGPU *pGpu, 1536 KernelGsp *pKernelGsp 1537 ) 1538 { 1539 NvU8 idx; 1540 1541 _kgspStopLogPolling(pGpu, pKernelGsp); 1542 1543 // Make sure there is no lingering debug output. 1544 kgspDumpGspLogs(pGpu, pKernelGsp, NV_FALSE); 1545 1546 libosLogDestroy(&pKernelGsp->logDecode); 1547 1548 for (idx = 0; idx < LOGIDX_SIZE; idx++) 1549 { 1550 RM_LIBOS_LOG_MEM *pLog = &pKernelGsp->rmLibosLogMem[idx]; 1551 1552 // release log memory for each task. 1553 if (pLog->pTaskLogBuffer != NULL) 1554 { 1555 memdescUnmap(pLog->pTaskLogDescriptor, 1556 NV_TRUE, osGetCurrentProcess(), 1557 (void *)pLog->pTaskLogBuffer, 1558 pLog->pTaskLogMappingPriv); 1559 pLog->pTaskLogBuffer = NULL; 1560 pLog->pTaskLogMappingPriv = NULL; 1561 } 1562 1563 if (pLog->pTaskLogDescriptor != NULL) 1564 { 1565 memdescFree(pLog->pTaskLogDescriptor); 1566 memdescDestroy(pLog->pTaskLogDescriptor); 1567 pLog->pTaskLogDescriptor = NULL; 1568 } 1569 } 1570 1571 portMemFree(pKernelGsp->pLogElf); 1572 pKernelGsp->pLogElf = NULL; 1573 } 1574 1575 /*! 1576 * Initialize LIBOS task logging structures 1577 */ 1578 static NV_STATUS 1579 _kgspInitLibosLoggingStructures 1580 ( 1581 OBJGPU *pGpu, 1582 KernelGsp *pKernelGsp, 1583 GSP_FIRMWARE *pGspFw 1584 ) 1585 { 1586 static const struct 1587 { 1588 const char *szMemoryId; 1589 const char *szPrefix; 1590 NvU32 size; 1591 const char *elfSectionName; 1592 } logInitValues[] = 1593 { 1594 {"LOGINIT", "INIT", 0x10000, ".fwlogging_init"}, // 64KB for stack traces 1595 #if defined(DEVELOP) || defined(DEBUG) 1596 // The interrupt task is in the rm elf, so they share the same logging elf too 1597 {"LOGINTR", "INTR", 0x40000, ".fwlogging_rm"}, // 256KB ISR debug log on develop/debug builds 1598 {"LOGRM", "RM", 0x40000, ".fwlogging_rm"} // 256KB RM debug log on develop/debug builds 1599 #else 1600 // The interrupt task is in the rm elf, so they share the same logging elf too 1601 {"LOGINTR", "INTR", 0x10000, ".fwlogging_rm"}, // 64KB ISR debug log on develop/debug builds 1602 {"LOGRM", "RM", 0x10000, ".fwlogging_rm"} // 64KB RM debug log on release builds 1603 #endif 1604 }; 1605 ct_assert(NV_ARRAY_ELEMENTS(logInitValues) <= LIBOS_LOG_MAX_LOGS); 1606 ct_assert(NV_ARRAY_ELEMENTS(logInitValues) == LOGIDX_SIZE); 1607 1608 NV_STATUS nvStatus = NV_OK; 1609 NvU8 idx; 1610 1611 libosLogCreate(&pKernelGsp->logDecode); 1612 1613 for (idx = 0; idx < LOGIDX_SIZE; idx++) 1614 { 1615 RM_LIBOS_LOG_MEM *pLog = &pKernelGsp->rmLibosLogMem[idx]; 1616 NvP64 pVa = NvP64_NULL; 1617 NvP64 pPriv = NvP64_NULL; 1618 1619 // 1620 // Setup logging memory for each task. 1621 // Use MEMDESC_FLAGS_CPU_ONLY -- too early to call memdescMapIommu. 1622 // 1623 NV_ASSERT_OK_OR_GOTO(nvStatus, 1624 memdescCreate(&pLog->pTaskLogDescriptor, 1625 pGpu, 1626 logInitValues[idx].size, 1627 RM_PAGE_SIZE, 1628 NV_TRUE, ADDR_SYSMEM, NV_MEMORY_CACHED, 1629 MEMDESC_FLAGS_NONE), 1630 error_cleanup); 1631 1632 NV_ASSERT_OK_OR_GOTO(nvStatus, 1633 memdescAlloc(pLog->pTaskLogDescriptor), 1634 error_cleanup); 1635 1636 NV_ASSERT_OK_OR_GOTO(nvStatus, 1637 memdescMap(pLog->pTaskLogDescriptor, 0, 1638 memdescGetSize(pLog->pTaskLogDescriptor), 1639 NV_TRUE, NV_PROTECT_READ_WRITE, 1640 &pVa, &pPriv), 1641 error_cleanup); 1642 1643 pLog->pTaskLogBuffer = pVa; 1644 pLog->pTaskLogMappingPriv = pPriv; 1645 portMemSet(pLog->pTaskLogBuffer, 0, memdescGetSize(pLog->pTaskLogDescriptor)); 1646 1647 // Pass the PTE table for the log buffer in the log buffer, after the put pointer. 1648 memdescGetPhysAddrs(pLog->pTaskLogDescriptor, 1649 AT_GPU, 1650 0, 1651 RM_PAGE_SIZE, 1652 NV_CEIL(memdescGetSize(pLog->pTaskLogDescriptor), RM_PAGE_SIZE), 1653 &pLog->pTaskLogBuffer[1]); 1654 1655 pLog->id8 = _kgspGenerateInitArgId(logInitValues[idx].szMemoryId); 1656 1657 libosLogAddLogEx(&pKernelGsp->logDecode, 1658 pLog->pTaskLogBuffer, 1659 memdescGetSize(pLog->pTaskLogDescriptor), 1660 pGpu->gpuInstance, 1661 (gpuGetChipArch(pGpu) >> GPU_ARCH_SHIFT), 1662 gpuGetChipImpl(pGpu), 1663 logInitValues[idx].szPrefix, 1664 logInitValues[idx].elfSectionName); 1665 } 1666 1667 // Setup symbol decoder 1668 if (pGspFw->pLogElf) 1669 { 1670 const void *pLogData = NULL; 1671 NvU64 logSize = 0; 1672 1673 NV_ASSERT_OK_OR_GOTO( 1674 nvStatus, 1675 _kgspFwContainerVerifyVersion(pGpu, pKernelGsp, 1676 pGspFw->pLogElf, 1677 pGspFw->logElfSize, 1678 "GSP firmware log"), 1679 error_cleanup); 1680 1681 NV_ASSERT_OK_OR_GOTO( 1682 nvStatus, 1683 _kgspFwContainerGetSection(pGpu, pKernelGsp, 1684 pGspFw->pLogElf, 1685 pGspFw->logElfSize, 1686 GSP_LOGGING_SECTION_NAME, 1687 &pLogData, 1688 &logSize), 1689 error_cleanup); 1690 1691 pKernelGsp->pLogElf = portMemAllocNonPaged(logSize); 1692 pKernelGsp->logElfDataSize = logSize; 1693 1694 if (pKernelGsp->pLogElf == NULL) 1695 { 1696 NV_PRINTF(LEVEL_ERROR, "Failed to allocate memory for log elf"); 1697 nvStatus = NV_ERR_NO_MEMORY; 1698 goto error_cleanup; 1699 } 1700 portMemCopy(pKernelGsp->pLogElf, logSize, pLogData, logSize); 1701 1702 if (pKernelGsp->pLogElf) 1703 { 1704 libosLogInit(&pKernelGsp->logDecode, pKernelGsp->pLogElf, logSize); 1705 } 1706 } 1707 1708 error_cleanup: 1709 if (nvStatus != NV_OK) 1710 _kgspFreeLibosLoggingStructures(pGpu, pKernelGsp); 1711 1712 return nvStatus; 1713 } 1714 1715 static NV_STATUS 1716 _kgspAllocSimAccessBuffer(OBJGPU *pGpu, KernelGsp *pKernelGsp) 1717 { 1718 NvP64 pVa = NvP64_NULL; 1719 NvP64 pPriv = NvP64_NULL; 1720 NV_STATUS nvStatus; 1721 1722 if (!IS_SIMULATION(pGpu)) 1723 { 1724 pKernelGsp->pMemDesc_simAccessBuf = NULL; 1725 pKernelGsp->pSimAccessBuf = NULL; 1726 pKernelGsp->pSimAccessBufPriv = NULL; 1727 return NV_ERR_NOT_SUPPORTED; 1728 } 1729 1730 NV_ASSERT_OK_OR_GOTO(nvStatus, 1731 memdescCreate(&pKernelGsp->pMemDesc_simAccessBuf, 1732 pGpu, 1733 sizeof(SimAccessBuffer), 1734 RM_PAGE_SIZE, 1735 NV_TRUE, ADDR_SYSMEM, NV_MEMORY_UNCACHED, 1736 MEMDESC_FLAGS_NONE), 1737 error_cleanup); 1738 1739 NV_ASSERT_OK_OR_GOTO(nvStatus, 1740 memdescAlloc(pKernelGsp->pMemDesc_simAccessBuf), 1741 error_cleanup); 1742 1743 NV_ASSERT_OK_OR_GOTO(nvStatus, 1744 memdescMap(pKernelGsp->pMemDesc_simAccessBuf, 0, 1745 memdescGetSize(pKernelGsp->pMemDesc_simAccessBuf), 1746 NV_TRUE, NV_PROTECT_READ_WRITE, 1747 &pVa, &pPriv), 1748 error_cleanup); 1749 1750 pKernelGsp->pSimAccessBuf = (SimAccessBuffer*)pVa; 1751 pKernelGsp->pSimAccessBufPriv = pPriv; 1752 1753 portMemSet(pKernelGsp->pSimAccessBuf, 0, memdescGetSize(pKernelGsp->pMemDesc_simAccessBuf)); 1754 1755 error_cleanup: 1756 if (nvStatus != NV_OK) 1757 _kgspFreeSimAccessBuffer(pGpu, pKernelGsp); 1758 1759 return nvStatus; 1760 } 1761 1762 static void 1763 _kgspFreeSimAccessBuffer(OBJGPU *pGpu, KernelGsp *pKernelGsp) 1764 { 1765 if (!IS_SIMULATION(pGpu)) 1766 { 1767 return; 1768 } 1769 1770 if (pKernelGsp->pMemDesc_simAccessBuf != NULL) 1771 { 1772 memdescFree(pKernelGsp->pMemDesc_simAccessBuf); 1773 memdescDestroy(pKernelGsp->pMemDesc_simAccessBuf); 1774 } 1775 1776 pKernelGsp->pMemDesc_simAccessBuf = NULL; 1777 pKernelGsp->pSimAccessBuf = NULL; 1778 pKernelGsp->pSimAccessBufPriv = NULL; 1779 } 1780 1781 /*! 1782 * Create KernelGsp object and initialize RPC infrastructure 1783 */ 1784 NV_STATUS 1785 kgspConstructEngine_IMPL 1786 ( 1787 OBJGPU *pGpu, 1788 KernelGsp *pKernelGsp, 1789 ENGDESCRIPTOR engDesc 1790 ) 1791 { 1792 NV_STATUS nvStatus = NV_OK; 1793 1794 if (!IS_GSP_CLIENT(pGpu)) 1795 return NV_ERR_NOT_SUPPORTED; 1796 1797 kgspConfigureFalcon_HAL(pGpu, pKernelGsp); 1798 1799 nvStatus = _kgspInitRpcInfrastructure(pGpu, pKernelGsp); 1800 if (nvStatus != NV_OK) 1801 { 1802 NV_PRINTF(LEVEL_ERROR, "init RPC infrastructure failed\n"); 1803 return nvStatus; 1804 } 1805 1806 nvStatus = kgspAllocBootArgs_HAL(pGpu, pKernelGsp); 1807 if (nvStatus != NV_OK) 1808 { 1809 NV_PRINTF(LEVEL_ERROR, "boot arg alloc failed: 0x%x\n", nvStatus); 1810 _kgspFreeRpcInfrastructure(pGpu, pKernelGsp); 1811 return nvStatus; 1812 } 1813 1814 if (IS_SIMULATION(pGpu)) 1815 { 1816 nvStatus = _kgspAllocSimAccessBuffer(pGpu, pKernelGsp); 1817 if (nvStatus != NV_OK) 1818 { 1819 NV_PRINTF(LEVEL_ERROR, "sim access buffer alloc failed: 0x%x\n", nvStatus); 1820 kgspFreeBootArgs_HAL(pGpu, pKernelGsp); 1821 _kgspFreeRpcInfrastructure(pGpu, pKernelGsp); 1822 return nvStatus; 1823 } 1824 } 1825 1826 return NV_OK; 1827 } 1828 1829 /*! 1830 * Initialize GSP-RM 1831 * 1832 * @param[in] pGpu GPU object pointer 1833 * @param[in] pKernelGsp KernelGsp object pointer 1834 * @param[in] pGspFw GSP firmware structure pointer 1835 * 1836 * @return NV_OK if GSP fw RM offload successfully initialized. 1837 * Appropriate NV_ERR_xxx value otherwise. 1838 */ 1839 NV_STATUS 1840 kgspInitRm_IMPL 1841 ( 1842 OBJGPU *pGpu, 1843 KernelGsp *pKernelGsp, 1844 GSP_FIRMWARE *pGspFw 1845 ) 1846 { 1847 NV_STATUS status = NV_OK; 1848 OBJTMR *pTmr = GPU_GET_TIMER(pGpu); 1849 GPU_MASK gpusLockedMask = 0; 1850 1851 if (!IS_GSP_CLIENT(pGpu)) 1852 return NV_OK; 1853 1854 if ((pGspFw == NULL) || (pGspFw->pBuf == NULL) || (pGspFw->size == 0)) 1855 { 1856 NV_PRINTF(LEVEL_ERROR, "need firmware to initialize GSP\n"); 1857 return NV_ERR_INVALID_ARGUMENT; 1858 } 1859 1860 pKernelGsp->bInInit = NV_TRUE; 1861 1862 // Need to hold the GPU instance lock in order to write to the RPC queue 1863 NV_ASSERT_OK_OR_GOTO(status, 1864 rmGpuGroupLockAcquire(pGpu->gpuInstance, GPU_LOCK_GRP_SUBDEVICE, 1865 GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_INIT, &gpusLockedMask), 1866 done); 1867 1868 // Set the GPU time to the wall-clock time before loading GSP ucode. 1869 tmrSetCurrentTime_HAL(pGpu, pTmr); 1870 1871 /* 1872 * For GSP-RM boot, we must trigger FRTS (if it exists for the chip) 1873 * before loading GSP-RM so that FRTS data and GSP-RM code/data/heap can coexist 1874 * in WPR2. FRTS is triggered by running a VBIOS-provided ucode called FWSEC. 1875 * 1876 * Here, we extract a VBIOS image from ROM, and parse it for FWSEC. 1877 */ 1878 if (pKernelGsp->pFwsecUcode == NULL) 1879 { 1880 KernelGspVbiosImg *pVbiosImg = NULL; 1881 1882 // Try and extract a VBIOS image. 1883 status = kgspExtractVbiosFromRom_HAL(pGpu, pKernelGsp, &pVbiosImg); 1884 1885 if (status == NV_OK) 1886 { 1887 // Got a VBIOS image, now parse it for FWSEC. 1888 status = kgspParseFwsecUcodeFromVbiosImg(pGpu, pKernelGsp, pVbiosImg, 1889 &pKernelGsp->pFwsecUcode); 1890 kgspFreeVbiosImg(pVbiosImg); 1891 if (status != NV_OK) 1892 { 1893 NV_PRINTF(LEVEL_ERROR, "failed to parse FWSEC ucode from VBIOS image: 0x%x\n", 1894 status); 1895 goto done; 1896 } 1897 } 1898 else if (status == NV_ERR_NOT_SUPPORTED) 1899 { 1900 // 1901 // Extracting VBIOS image from ROM is not supported. 1902 // Sanity check we don't depend on it for FRTS, and proceed without FWSEC. 1903 // 1904 NV_ASSERT_OR_GOTO(kgspGetFrtsSize(pGpu, pKernelGsp) == 0, done); 1905 status = NV_OK; 1906 } 1907 else 1908 { 1909 NV_PRINTF(LEVEL_ERROR, "failed to extract VBIOS image from ROM: 0x%x\n", 1910 status); 1911 goto done; 1912 } 1913 1914 } 1915 1916 /* 1917 * We use a set of Booter ucodes to boot GSP-RM as well as manage its lifecycle. 1918 * 1919 * Booter Load loads, verifies, and boots GSP-RM in WPR2. 1920 * Booter Unload tears down WPR2 for driver unload. 1921 * 1922 * Here we prepare the Booter ucode images in SYSMEM so they may be loaded onto 1923 * SEC2 (Load / Unload) and NVDEC0 (Unload). 1924 */ 1925 if (pKernelGsp->bPartitionedFmc) 1926 { 1927 // 1928 // The secure boot ucode is included in the partitioned FMC, no need for 1929 // separate Booter ucodes. 1930 // 1931 } 1932 else 1933 { 1934 if (pKernelGsp->pBooterLoadUcode == NULL) 1935 { 1936 status = kgspAllocateBooterLoadUcodeImage(pGpu, pKernelGsp, 1937 &pKernelGsp->pBooterLoadUcode); 1938 if (status != NV_OK) 1939 { 1940 NV_PRINTF(LEVEL_ERROR, "failed to allocate Booter Load ucode: 0x%x\n", status); 1941 goto done; 1942 } 1943 } 1944 1945 if (pKernelGsp->pBooterUnloadUcode == NULL) 1946 { 1947 status = kgspAllocateBooterUnloadUcodeImage(pGpu, pKernelGsp, 1948 &pKernelGsp->pBooterUnloadUcode); 1949 if (status != NV_OK) 1950 { 1951 NV_PRINTF(LEVEL_ERROR, "failed to allocate Booter Unload ucode: 0x%x\n", status); 1952 goto done; 1953 } 1954 } 1955 1956 // execute Booter Unload if needed to reset from unclean shutdown 1957 kgspExecuteBooterUnloadIfNeeded_HAL(pGpu, pKernelGsp); 1958 } 1959 1960 // Prepare boot binary image. 1961 status = kgspPrepareBootBinaryImage(pGpu, pKernelGsp); 1962 if (status != NV_OK) 1963 { 1964 NV_PRINTF(LEVEL_ERROR, "Error preparing boot binary image\n"); 1965 goto done; 1966 } 1967 1968 // Prepare GSP-RM image. 1969 status = _kgspPrepareGspRmBinaryImage(pGpu, pKernelGsp, pGspFw); 1970 if (status != NV_OK) 1971 { 1972 NV_PRINTF(LEVEL_ERROR, "Error preparing GSP-RM image\n"); 1973 goto done; 1974 } 1975 1976 status = kgspCalculateFbLayout(pGpu, pKernelGsp, pGspFw); 1977 if (status != NV_OK) 1978 { 1979 NV_PRINTF(LEVEL_ERROR, "Error calculating FB layout\n"); 1980 goto done; 1981 } 1982 1983 // Prepare Scrubber ucode image if pre-scrubbed memory is insufficient 1984 if (pKernelGsp->pScrubberUcode == NULL) 1985 { 1986 NvU64 neededSize = pKernelGsp->pWprMeta->fbSize - pKernelGsp->pWprMeta->gspFwRsvdStart; 1987 NvU64 prescrubbedSize = kgspGetPrescrubbedTopFbSize(pGpu, pKernelGsp); 1988 1989 if (neededSize > prescrubbedSize) 1990 { 1991 NV_PRINTF(LEVEL_INFO, 1992 "allocating Scrubber ucode as pre-scrubbed memory (0x%llx bytes) is insufficient (0x%llx bytes needed)\n", 1993 prescrubbedSize, neededSize); 1994 1995 status = kgspAllocateScrubberUcodeImage(pGpu, pKernelGsp, 1996 &pKernelGsp->pScrubberUcode); 1997 if (status != NV_OK) 1998 { 1999 NV_PRINTF(LEVEL_ERROR, "failed to allocate Scrubber ucode: 0x%x\n", status); 2000 goto done; 2001 } 2002 } 2003 else 2004 { 2005 NV_PRINTF(LEVEL_INFO, 2006 "skipping allocating Scrubber ucode as pre-scrubbed memory (0x%llx bytes) is sufficient (0x%llx bytes needed)\n", 2007 prescrubbedSize, neededSize); 2008 } 2009 } 2010 2011 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, kgspInitLogging(pGpu, pKernelGsp, pGspFw), done); 2012 2013 // Wait for GFW_BOOT OK status 2014 kgspWaitForGfwBootOk_HAL(pGpu, pKernelGsp); 2015 2016 // bring up ucode with RM offload task 2017 status = kgspBootstrapRiscvOSEarly_HAL(pGpu, pKernelGsp, pGspFw); 2018 if (status != NV_OK) 2019 { 2020 NV_PRINTF(LEVEL_ERROR, "cannot bootstrap riscv/gsp: 0x%x\n", status); 2021 kgspHealthCheck_HAL(pGpu, pKernelGsp); 2022 goto done; 2023 } 2024 2025 // at this point we should be able to exchange RPCs with RM offload task 2026 NV_RM_RPC_SET_GUEST_SYSTEM_INFO(pGpu, status); 2027 if (status != NV_OK) 2028 { 2029 NV_PRINTF(LEVEL_ERROR, "SET_GUEST_SYSTEM_INFO failed: 0x%x\n", status); 2030 goto done; 2031 } 2032 2033 NV_RM_RPC_GET_GSP_STATIC_INFO(pGpu, status); 2034 if (status != NV_OK) 2035 { 2036 NV_PRINTF(LEVEL_ERROR, "GET_GSP_STATIC_INFO failed: 0x%x\n", status); 2037 goto done; 2038 } 2039 2040 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, kgspStartLogPolling(pGpu, pKernelGsp), done); 2041 2042 done: 2043 pKernelGsp->bInInit = NV_FALSE; 2044 2045 if (status != NV_OK) 2046 { 2047 KernelPmu *pKernelPmu = GPU_GET_KERNEL_PMU(pGpu); 2048 2049 // Preserve any captured GSP-RM logs 2050 libosPreserveLogs(&pKernelGsp->logDecode); 2051 2052 if (pKernelPmu != NULL) 2053 { 2054 // If PMU init fails, kgsp init will also fail 2055 libosPreserveLogs(&pKernelPmu->logDecode); 2056 } 2057 } 2058 2059 if (gpusLockedMask != 0) 2060 { 2061 rmGpuGroupLockRelease(gpusLockedMask, GPUS_LOCK_FLAGS_NONE); 2062 } 2063 2064 return status; 2065 } 2066 2067 NV_STATUS 2068 kgspInitLogging_IMPL 2069 ( 2070 OBJGPU *pGpu, 2071 KernelGsp *pKernelGsp, 2072 GSP_FIRMWARE *pGspFw 2073 ) 2074 { 2075 NV_STATUS nvStatus; 2076 2077 nvStatus = _kgspInitLibosLoggingStructures(pGpu, pKernelGsp, pGspFw); 2078 if (nvStatus != NV_OK) 2079 { 2080 NV_PRINTF(LEVEL_ERROR, "init LIBOS logging structures failed: 0x%x\n", nvStatus); 2081 return nvStatus; 2082 } 2083 2084 return nvStatus; 2085 } 2086 2087 /*! 2088 * Unload GSP-RM 2089 */ 2090 NV_STATUS 2091 kgspUnloadRm_IMPL 2092 ( 2093 OBJGPU *pGpu, 2094 KernelGsp *pKernelGsp 2095 ) 2096 { 2097 NV_STATUS rpcStatus = NV_OK; 2098 NV_STATUS status; 2099 2100 NV_PRINTF(LEVEL_INFO, "unloading GSP-RM\n"); 2101 NV_RM_RPC_UNLOADING_GUEST_DRIVER(pGpu, rpcStatus, NV_FALSE, NV_FALSE, 0); 2102 2103 // Wait for GSP-RM processor to suspend 2104 kgspWaitForProcessorSuspend_HAL(pGpu, pKernelGsp); 2105 2106 // Dump GSP-RM logs and reset before invoking FWSEC-SB 2107 kgspDumpGspLogs(pGpu, pKernelGsp, NV_FALSE); 2108 2109 // 2110 // Avoid cascading timeouts when attempting to invoke the below ucodes if 2111 // we are unloading due to a GSP-RM timeout. 2112 // 2113 threadStateResetTimeout(pGpu); 2114 2115 // Because of COT, RM cannot reset GSP-RISCV and FSP has exclusive access to reset and reboot GSP for next run. 2116 if(!(pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_COT_ENABLED))) 2117 { 2118 kflcnReset_HAL(pGpu, staticCast(pKernelGsp, KernelFalcon)); 2119 } 2120 2121 // Invoke FWSEC-SB to put back PreOsApps during driver unload 2122 status = kgspExecuteFwsecSb_HAL(pGpu, pKernelGsp, pKernelGsp->pFwsecUcode); 2123 if (status != NV_OK) 2124 { 2125 NV_PRINTF(LEVEL_ERROR, "failed to execute FWSEC-SB for PreOsApps during driver unload: 0x%x\n", status); 2126 NV_ASSERT(0); 2127 } 2128 2129 if (pKernelGsp->bPartitionedFmc) 2130 { 2131 // 2132 // GSP-RM invokes the partitioned FMC to unload directly as part of the 2133 // NV_RM_RPC_UNLOADING_GUEST_DRIVER call above. 2134 // 2135 status = rpcStatus; 2136 } 2137 else 2138 { 2139 // After instructing GSP-RM to unload itself, run Booter Unload to teardown WPR2 2140 status = kgspExecuteBooterUnloadIfNeeded_HAL(pGpu, pKernelGsp); 2141 } 2142 2143 if (rpcStatus != NV_OK) 2144 { 2145 return rpcStatus; 2146 } 2147 2148 return status; 2149 } 2150 2151 /*! 2152 * Free RPC infrastructure and KernelGsp object 2153 */ 2154 void 2155 kgspDestruct_IMPL 2156 ( 2157 KernelGsp *pKernelGsp 2158 ) 2159 { 2160 OBJGPU *pGpu = ENG_GET_GPU(pKernelGsp); 2161 2162 if (!IS_GSP_CLIENT(pGpu)) 2163 return; 2164 2165 kgspFreeFlcnUcode(pKernelGsp->pFwsecUcode); 2166 pKernelGsp->pFwsecUcode = NULL; 2167 2168 kgspFreeFlcnUcode(pKernelGsp->pBooterLoadUcode); 2169 pKernelGsp->pBooterLoadUcode = NULL; 2170 2171 kgspFreeFlcnUcode(pKernelGsp->pBooterUnloadUcode); 2172 pKernelGsp->pBooterUnloadUcode = NULL; 2173 2174 kgspFreeBootArgs_HAL(pGpu, pKernelGsp); 2175 2176 _kgspFreeLibosLoggingStructures(pGpu, pKernelGsp); 2177 _kgspFreeRpcInfrastructure(pGpu, pKernelGsp); 2178 _kgspFreeBootBinaryImage(pGpu, pKernelGsp); 2179 _kgspFreeSimAccessBuffer(pGpu, pKernelGsp); 2180 } 2181 2182 /*! 2183 * Dump logs coming from GSP-RM 2184 * 2185 * @param[in] pGpu OBJGPU pointer 2186 * @param[in] pKernelGsp KernelGsp pointer 2187 * @param[in] bSyncNvLog NV_TRUE: Copy a snapshot of the libos logs 2188 * into the nvLog wrap buffers. 2189 */ 2190 void 2191 kgspDumpGspLogs_IMPL 2192 ( 2193 OBJGPU *pGpu, 2194 KernelGsp *pKernelGsp, 2195 NvBool bSyncNvLog 2196 ) 2197 { 2198 if (!IS_GSP_CLIENT(pGpu)) 2199 { 2200 return; 2201 } 2202 2203 if (pKernelGsp->bInInit || pKernelGsp->pLogElf || bSyncNvLog) 2204 { 2205 libosExtractLogs(&pKernelGsp->logDecode, bSyncNvLog); 2206 } 2207 2208 if (IS_VGPU_GSP_PLUGIN_OFFLOAD_ENABLED(pGpu)) 2209 { 2210 // Dump logs from vGPU partition 2211 for (NvU32 i = 0; i < MAX_PARTITIONS_WITH_GFID; i++) 2212 { 2213 libosExtractLogs(&pKernelGsp->logDecodeVgpuPartition[i], bSyncNvLog); 2214 } 2215 } 2216 2217 } 2218 2219 /*! 2220 * Populate GSP-RM init arguments. 2221 */ 2222 void 2223 kgspPopulateGspRmInitArgs_IMPL 2224 ( 2225 OBJGPU *pGpu, 2226 KernelGsp *pKernelGsp, 2227 GSP_SR_INIT_ARGUMENTS *pGspInitArgs 2228 ) 2229 { 2230 GSP_ARGUMENTS_CACHED *pGspArgs = pKernelGsp->pGspArgumentsCached; 2231 MESSAGE_QUEUE_INIT_ARGUMENTS *pMQInitArgs = &pGspArgs->messageQueueInitArguments; 2232 MESSAGE_QUEUE_INFO *pMQI = pKernelGsp->pRpc->pMessageQueueInfo; 2233 GSP_SR_INIT_ARGUMENTS *pSrInitArgs = &pGspArgs->srInitArguments; 2234 2235 // Setup the message queue arguments 2236 pMQInitArgs->sharedMemPhysAddr = pKernelGsp->pRpc->messageQueuePhysMem; 2237 pMQInitArgs->pageTableEntryCount = pMQI->pageTableEntryCount; 2238 pMQInitArgs->cmdQueueOffset = pMQI->pageTableSize; 2239 pMQInitArgs->statQueueOffset = pMQInitArgs->cmdQueueOffset + pMQI->commandQueueSize; 2240 2241 if (pGspInitArgs == NULL) 2242 { 2243 pSrInitArgs->bInPMTransition = NV_FALSE; 2244 pSrInitArgs->oldLevel = 0; 2245 pSrInitArgs->flags = 0; 2246 } 2247 else 2248 { 2249 pSrInitArgs->bInPMTransition = NV_TRUE; 2250 pSrInitArgs->oldLevel = pGspInitArgs->oldLevel; 2251 pSrInitArgs->flags = pGspInitArgs->flags; 2252 } 2253 2254 pGspArgs->gpuInstance = pGpu->gpuInstance; 2255 } 2256 2257 /*! 2258 * Prepare boot binary image for GSP-RM boot. 2259 * 2260 * @return NV_OK if boot binary image prepared successfully. 2261 * Appropriate NV_ERR_xxx value otherwise. 2262 */ 2263 NV_STATUS 2264 kgspPrepareBootBinaryImage_IMPL 2265 ( 2266 OBJGPU *pGpu, 2267 KernelGsp *pKernelGsp 2268 ) 2269 { 2270 NV_STATUS status; 2271 BINDATA_STORAGE *pBinStorageImage; 2272 BINDATA_STORAGE *pBinStorageDesc; 2273 NvU32 bufSize; 2274 NvU32 bufSizeAligned; 2275 NvU8 *pDesc = NULL; 2276 NvP64 pVa = NvP64_NULL; 2277 NvP64 pPriv = NvP64_NULL; 2278 2279 NV_ASSERT_OR_RETURN(pKernelGsp->pGspRmBootUcodeImage == NULL, NV_ERR_INVALID_STATE); 2280 NV_ASSERT_OR_RETURN(pKernelGsp->pGspRmBootUcodeDesc == NULL, NV_ERR_INVALID_STATE); 2281 2282 // get the bindata storage for the image/descriptor 2283 kgspGetGspRmBootUcodeStorage_HAL(pGpu, pKernelGsp, &pBinStorageImage, &pBinStorageDesc); 2284 2285 // copy the image to sysmem 2286 bufSize = bindataGetBufferSize(pBinStorageImage); 2287 bufSizeAligned = NV_ALIGN_UP(bufSize, 0x1000); 2288 2289 NV_ASSERT_OK_OR_GOTO(status, 2290 memdescCreate(&pKernelGsp->pGspRmBootUcodeMemdesc, 2291 pGpu, 2292 bufSizeAligned, 2293 RM_PAGE_SIZE, 2294 NV_TRUE, ADDR_SYSMEM, NV_MEMORY_CACHED, 2295 MEMDESC_FLAGS_NONE), 2296 fail); 2297 2298 NV_ASSERT_OK_OR_GOTO(status, 2299 memdescAlloc(pKernelGsp->pGspRmBootUcodeMemdesc), 2300 fail); 2301 2302 NV_ASSERT_OK_OR_GOTO(status, 2303 memdescMap(pKernelGsp->pGspRmBootUcodeMemdesc, 0, 2304 memdescGetSize(pKernelGsp->pGspRmBootUcodeMemdesc), 2305 NV_TRUE, NV_PROTECT_READ_WRITE, 2306 &pVa, &pPriv), 2307 fail); 2308 2309 pKernelGsp->gspRmBootUcodeSize = bufSize; 2310 pKernelGsp->pGspRmBootUcodeImage = (NvU8 *)NvP64_VALUE(pVa);; 2311 pKernelGsp->pGspRmBootUcodeMemdescPriv = pPriv; 2312 2313 NV_ASSERT_OK_OR_GOTO(status, 2314 bindataWriteToBuffer(pBinStorageImage, 2315 pKernelGsp->pGspRmBootUcodeImage, 2316 bufSize), 2317 fail); 2318 2319 // copy the image descriptor 2320 bufSize = bindataGetBufferSize(pBinStorageDesc); 2321 pDesc = portMemAllocNonPaged(bufSize); 2322 if (pDesc == NULL) 2323 { 2324 NV_PRINTF(LEVEL_ERROR, "Failed to allocate ucode desc buffer\n"); 2325 status = NV_ERR_NO_MEMORY; 2326 goto fail; 2327 } 2328 2329 pKernelGsp->pGspRmBootUcodeDesc = (RM_RISCV_UCODE_DESC*)pDesc; 2330 2331 NV_ASSERT_OK_OR_GOTO(status, 2332 bindataWriteToBuffer(pBinStorageDesc, pDesc, bufSize), 2333 fail); 2334 2335 return status; 2336 2337 fail: 2338 _kgspFreeBootBinaryImage(pGpu, pKernelGsp); 2339 return status; 2340 } 2341 2342 static void 2343 _kgspFreeBootBinaryImage 2344 ( 2345 OBJGPU *pGpu, 2346 KernelGsp *pKernelGsp 2347 ) 2348 { 2349 portMemFree(pKernelGsp->pGspRmBootUcodeDesc); 2350 pKernelGsp->pGspRmBootUcodeDesc = NULL; 2351 2352 if (pKernelGsp->pGspRmBootUcodeImage != NULL) 2353 { 2354 memdescUnmap(pKernelGsp->pGspRmBootUcodeMemdesc, 2355 NV_TRUE, osGetCurrentProcess(), 2356 (void *)pKernelGsp->pGspRmBootUcodeImage, 2357 pKernelGsp->pGspRmBootUcodeMemdescPriv); 2358 pKernelGsp->pGspRmBootUcodeImage = NULL; 2359 pKernelGsp->pGspRmBootUcodeMemdescPriv = NULL; 2360 } 2361 if (pKernelGsp->pGspRmBootUcodeMemdesc != NULL) 2362 { 2363 memdescFree(pKernelGsp->pGspRmBootUcodeMemdesc); 2364 memdescDestroy(pKernelGsp->pGspRmBootUcodeMemdesc); 2365 pKernelGsp->pGspRmBootUcodeMemdesc = NULL; 2366 } 2367 2368 pKernelGsp->gspRmBootUcodeSize = 0; 2369 } 2370 2371 static NV_STATUS 2372 _kgspCreateSignatureMemdesc 2373 ( 2374 OBJGPU *pGpu, 2375 KernelGsp *pKernelGsp, 2376 GSP_FIRMWARE *pGspFw 2377 ) 2378 { 2379 NV_STATUS status = NV_OK; 2380 NvU8 *pSignatureVa = NULL; 2381 2382 // NOTE: align to 256 because that's the alignment needed for Booter DMA 2383 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2384 memdescCreate(&pKernelGsp->pSignatureMemdesc, pGpu, 2385 NV_ALIGN_UP(pGspFw->signatureSize, 256), 256, 2386 NV_TRUE, ADDR_SYSMEM, NV_MEMORY_CACHED, MEMDESC_FLAGS_NONE)); 2387 2388 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, 2389 memdescAlloc(pKernelGsp->pSignatureMemdesc), fail_create); 2390 2391 pSignatureVa = memdescMapInternal(pGpu, pKernelGsp->pSignatureMemdesc, TRANSFER_FLAGS_NONE); 2392 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, 2393 (pSignatureVa != NULL) ? NV_OK : NV_ERR_INSUFFICIENT_RESOURCES, 2394 fail_alloc); 2395 2396 portMemCopy(pSignatureVa, memdescGetSize(pKernelGsp->pSignatureMemdesc), 2397 pGspFw->pSignatureData, pGspFw->signatureSize); 2398 2399 memdescUnmapInternal(pGpu, pKernelGsp->pSignatureMemdesc, 0); 2400 pSignatureVa = NULL; 2401 2402 return status; 2403 2404 fail_alloc: 2405 memdescFree(pKernelGsp->pSignatureMemdesc); 2406 2407 fail_create: 2408 memdescDestroy(pKernelGsp->pSignatureMemdesc); 2409 pKernelGsp->pSignatureMemdesc = NULL; 2410 2411 return status; 2412 } 2413 2414 /*! 2415 * Verify that the version embedded in the .fwversion section of the ELF given 2416 * by pElfData and elfDataSize matches our NV_VERSION_STRING. 2417 */ 2418 static NV_STATUS 2419 _kgspFwContainerVerifyVersion 2420 ( 2421 OBJGPU *pGpu, 2422 KernelGsp *pKernelGsp, 2423 const void *pElfData, 2424 NvU64 elfDataSize, 2425 const char *pNameInMsg 2426 ) 2427 { 2428 const char *pFwversion; 2429 NvU64 fwversionSize; 2430 NvU64 expectedVersionLength = portStringLength(NV_VERSION_STRING); 2431 2432 { 2433 const void *pFwversionRaw; 2434 2435 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2436 _kgspFwContainerGetSection(pGpu, pKernelGsp, 2437 pElfData, 2438 elfDataSize, 2439 GSP_VERSION_SECTION_NAME, 2440 &pFwversionRaw, 2441 &fwversionSize)); 2442 2443 pFwversion = (const char *) pFwversionRaw; 2444 } 2445 2446 // Check that text in .fwversion section of ELF matches our NV_VERSION_STRING 2447 if ((fwversionSize != expectedVersionLength + 1) || 2448 (portStringCompare(pFwversion, NV_VERSION_STRING, expectedVersionLength) != 0)) 2449 { 2450 // Sanity check .fwversion before attempting to print it in the error message 2451 if ((fwversionSize > 0) && 2452 (fwversionSize < 64) && 2453 (pFwversion[fwversionSize - 1] == '\0')) 2454 { 2455 NV_PRINTF(LEVEL_ERROR, "%s version mismatch: got version %s, expected version %s\n", 2456 pNameInMsg, pFwversion, NV_VERSION_STRING); 2457 } 2458 else 2459 { 2460 NV_PRINTF(LEVEL_ERROR, "%s version unknown or malformed, expected version %s\n", 2461 pNameInMsg, NV_VERSION_STRING); 2462 } 2463 return NV_ERR_INVALID_DATA; 2464 } 2465 2466 return NV_OK; 2467 } 2468 2469 /*! 2470 * Get the name of the section corresponding to the given section name 2471 * prefix and the current chip. 2472 */ 2473 static NV_STATUS 2474 _kgspGetSectionNameForPrefix 2475 ( 2476 OBJGPU *pGpu, 2477 KernelGsp *pKernelGsp, 2478 char *pSectionNameBuf, // out 2479 NvLength sectionNameBufSize, 2480 const char *pSectionPrefix 2481 ) 2482 { 2483 NvLength sectionPrefixLength; 2484 2485 nv_firmware_chip_family_t chipFamily; 2486 const char *pChipFamilyName; 2487 NvLength chipFamilyNameLength; 2488 2489 NvLength totalSize; 2490 2491 NV_ASSERT_OR_RETURN(pSectionNameBuf != NULL, NV_ERR_INVALID_ARGUMENT); 2492 NV_ASSERT_OR_RETURN(sectionNameBufSize > 0, NV_ERR_INVALID_ARGUMENT); 2493 NV_ASSERT_OR_RETURN(pSectionPrefix != NULL, NV_ERR_INVALID_ARGUMENT); 2494 2495 chipFamily = nv_firmware_get_chip_family(gpuGetChipArch(pGpu), 2496 gpuGetChipImpl(pGpu)); 2497 NV_ASSERT_OR_RETURN(chipFamily != NV_FIRMWARE_CHIP_FAMILY_NULL, 2498 NV_ERR_INVALID_STATE); 2499 2500 pChipFamilyName = nv_firmware_chip_family_to_string(chipFamily); 2501 NV_ASSERT_OR_RETURN(pChipFamilyName != NULL, NV_ERR_INVALID_STATE); 2502 2503 sectionPrefixLength = portStringLength(pSectionPrefix); 2504 chipFamilyNameLength = portStringLength(pChipFamilyName); 2505 2506 totalSize = sectionPrefixLength + chipFamilyNameLength + 1; 2507 NV_ASSERT_OR_RETURN(sectionNameBufSize >= sectionPrefixLength + 1, 2508 NV_ERR_BUFFER_TOO_SMALL); 2509 NV_ASSERT_OR_RETURN(sectionNameBufSize >= totalSize, 2510 NV_ERR_BUFFER_TOO_SMALL); 2511 2512 portStringCopy(pSectionNameBuf, sectionNameBufSize, 2513 pSectionPrefix, sectionPrefixLength + 1); 2514 portStringCat(pSectionNameBuf, sectionNameBufSize, 2515 pChipFamilyName, chipFamilyNameLength + 1); 2516 2517 return NV_OK; 2518 } 2519 2520 static NV_STATUS 2521 _kgspPrepareGspRmBinaryImage 2522 ( 2523 OBJGPU *pGpu, 2524 KernelGsp *pKernelGsp, 2525 GSP_FIRMWARE *pGspFw 2526 ) 2527 { 2528 char signatureSectionName[32]; 2529 2530 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2531 _kgspFwContainerVerifyVersion(pGpu, pKernelGsp, 2532 pGspFw->pBuf, 2533 pGspFw->size, 2534 "GSP firmware image")); 2535 2536 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2537 _kgspFwContainerGetSection(pGpu, pKernelGsp, 2538 pGspFw->pBuf, 2539 pGspFw->size, 2540 GSP_IMAGE_SECTION_NAME, 2541 &pGspFw->pImageData, 2542 &pGspFw->imageSize)); 2543 2544 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2545 _kgspGetSectionNameForPrefix(pGpu, pKernelGsp, 2546 signatureSectionName, sizeof(signatureSectionName), 2547 kgspGetSignatureSectionNamePrefix_HAL(pGpu, pKernelGsp))); 2548 2549 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2550 _kgspFwContainerGetSection(pGpu, pKernelGsp, 2551 pGspFw->pBuf, 2552 pGspFw->size, 2553 signatureSectionName, 2554 &pGspFw->pSignatureData, 2555 &pGspFw->signatureSize)); 2556 2557 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2558 _kgspCreateSignatureMemdesc(pGpu, pKernelGsp, 2559 pGspFw)); 2560 2561 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2562 _kgspCreateRadix3(pGpu, &pKernelGsp->pGspUCodeRadix3Descriptor, 2563 NULL, pGspFw->pImageData, pGspFw->imageSize)); 2564 2565 return NV_OK; 2566 } 2567 2568 static NV_STATUS 2569 _kgspCreateRadix3 2570 ( 2571 OBJGPU *pGpu, 2572 MEMORY_DESCRIPTOR **ppMemdescRadix3, 2573 MEMORY_DESCRIPTOR *pMemdescData, 2574 const void *pData, 2575 NvU64 size 2576 ) 2577 { 2578 const NvU64 entriesLog2 = LIBOS_MEMORY_REGION_RADIX_PAGE_LOG2 - 3; 2579 NvU8 *pRadix3Buf; 2580 NvP64 pVaKernel; 2581 NvP64 pPrivKernel; 2582 NvU64 ptSize; 2583 NvU64 allocSize; 2584 NvU64 nPages = 0; 2585 NvU64 dataOffset = 0; 2586 NvU32 i; 2587 NV_STATUS status = NV_OK; 2588 2589 // radix3 working array. 2590 struct 2591 { 2592 NvU64 nPages; 2593 NvU64 offset; 2594 } radix3[4]; 2595 2596 NV_ASSERT_OR_RETURN(ppMemdescRadix3 != NULL, NV_ERR_INVALID_PARAMETER); 2597 NV_ASSERT_OR_ELSE_STR((pMemdescData != NULL) != (pData != NULL), 2598 "Specify pMemdescData or pData, but not both", 2599 return NV_ERR_INVALID_PARAMETER); 2600 2601 // If the size is not specified, get it from the memory descriptor. 2602 if ((size == 0) && (pMemdescData != NULL)) 2603 size = memdescGetSize(pMemdescData); 2604 NV_ASSERT_OR_RETURN(size > 0, NV_ERR_OUT_OF_RANGE); 2605 2606 // Clear working structure. 2607 portMemSet(radix3, 0, sizeof radix3); 2608 2609 // Populate npages, high to low. 2610 i = NV_ARRAY_ELEMENTS(radix3) - 1; 2611 radix3[i].nPages = (size + LIBOS_MEMORY_REGION_RADIX_PAGE_SIZE - 1) >> 2612 LIBOS_MEMORY_REGION_RADIX_PAGE_LOG2; 2613 for (; i > 0; i--) 2614 radix3[i - 1].nPages = ((radix3[i].nPages - 1) >> entriesLog2) + 1; 2615 2616 // Populate offset, low to high. 2617 for (i = 1; i < NV_ARRAY_ELEMENTS(radix3); i++) 2618 { 2619 nPages += radix3[i - 1].nPages; 2620 radix3[i].offset = nPages << LIBOS_MEMORY_REGION_RADIX_PAGE_LOG2; 2621 } 2622 2623 NV_ASSERT_OR_RETURN(radix3[0].nPages == 1, NV_ERR_OUT_OF_RANGE); 2624 2625 // Allocate space for PTEs and PDEs. 2626 ptSize = nPages << LIBOS_MEMORY_REGION_RADIX_PAGE_LOG2; 2627 allocSize = ptSize; 2628 2629 if (pData != NULL) 2630 { 2631 // We don't have a separate descriptor for the data. We need PTEs, 2632 // so include space for data in the new descriptor. 2633 allocSize += radix3[3].nPages << LIBOS_MEMORY_REGION_RADIX_PAGE_LOG2; 2634 } 2635 2636 NV_ASSERT_OK_OR_GOTO(status, 2637 memdescCreate(ppMemdescRadix3, pGpu, allocSize, 2638 LIBOS_MEMORY_REGION_RADIX_PAGE_SIZE, 2639 NV_MEMORY_NONCONTIGUOUS, 2640 ADDR_SYSMEM, 2641 NV_MEMORY_CACHED, 2642 MEMDESC_FLAGS_KERNEL_MODE), 2643 done); 2644 2645 NV_ASSERT_OK_OR_GOTO(status, memdescAlloc(*ppMemdescRadix3), error_ret); 2646 2647 // Create kernel mapping. 2648 NV_ASSERT_OK_OR_GOTO(status, 2649 memdescMap(*ppMemdescRadix3, 0, allocSize, NV_TRUE, NV_PROTECT_WRITEABLE, 2650 &pVaKernel, &pPrivKernel), 2651 error_ret); 2652 2653 if (pVaKernel == NvP64_NULL) 2654 { 2655 NV_PRINTF(LEVEL_ERROR, "VA error for radix3 shared buffer\n"); 2656 status = NV_ERR_NO_MEMORY; 2657 goto error_ret; 2658 } 2659 2660 pRadix3Buf = KERNEL_POINTER_FROM_NvP64(NvU8 *, pVaKernel); 2661 2662 // Zap out page table. 2663 portMemSet(pRadix3Buf, 0, ptSize); 2664 2665 // Fill in PDEs. 2666 for (i = 0; i < NV_ARRAY_ELEMENTS(radix3) - 2; i++) 2667 { 2668 memdescGetPhysAddrs(*ppMemdescRadix3, 2669 AT_GPU, // addressTranslation 2670 radix3[i + 1].offset, // offset 2671 RM_PAGE_SIZE, // stride 2672 radix3[i + 1].nPages, // count 2673 (RmPhysAddr *)(pRadix3Buf + radix3[i].offset)); // physical address table 2674 } 2675 2676 if (pData != NULL) 2677 { 2678 dataOffset = radix3[3].offset; 2679 2680 // Optionally copy data into the radix3 buffer. 2681 portMemCopy(pRadix3Buf + dataOffset, size, pData, size); 2682 2683 // If we only have part of the last page, clear the rest. 2684 NvU32 clearSize = allocSize - dataOffset - size; 2685 if (clearSize != 0) 2686 portMemSet(pRadix3Buf + dataOffset + size, 0, clearSize); 2687 2688 pMemdescData = *ppMemdescRadix3; 2689 } 2690 2691 memdescGetPhysAddrs(*ppMemdescRadix3, 2692 AT_GPU, // addressTranslation 2693 dataOffset, // offset 2694 RM_PAGE_SIZE, // stride 2695 radix3[3].nPages, // count 2696 (RmPhysAddr *)(pRadix3Buf + radix3[2].offset)); // physical address table 2697 2698 // 2699 // No reason to keep this memory mapped on the CPU side. Only GSP will 2700 // access it after this point. 2701 // 2702 memdescUnmap(*ppMemdescRadix3, NV_TRUE, osGetCurrentProcess(), 2703 pVaKernel, pPrivKernel); 2704 done: 2705 return status; 2706 2707 error_ret: 2708 if (*ppMemdescRadix3 != NULL) 2709 { 2710 memdescFree(*ppMemdescRadix3); 2711 memdescDestroy(*ppMemdescRadix3); 2712 *ppMemdescRadix3 = NULL; 2713 } 2714 2715 return status; 2716 } 2717 2718 static NV_STATUS 2719 _kgspFwContainerGetSection 2720 ( 2721 OBJGPU *pGpu, 2722 KernelGsp *pKernelGsp, 2723 const void *pElfData, 2724 NvU64 elfDataSize, 2725 const char *pSectionName, 2726 const void **ppSectionData, 2727 NvU64 *pSectionSize 2728 ) 2729 { 2730 const NvU8 *pGspBuf = pElfData; 2731 const LibosElf64Header *pElfHeader; 2732 const LibosElf64SectionHeader *pElfSectionHeader; 2733 NvU64 elfSectionHeaderTableLength; 2734 NvU64 elfSectionHeaderMaxIdx; 2735 NvU64 elfSectionNamesTableOffset; 2736 NvU64 elfSectionNamesTableSize; 2737 NvU64 elfSectionNamesTableMaxIdx; 2738 static const NvU32 elfMagicNumber = 0x464C457F; 2739 static const NvU8 elfClass64 = 0x2; 2740 static const NvU8 elfLittleEndian = 0x1; 2741 const char *pCurrentSectionName; 2742 NvLength sectionNameLength; 2743 NvS16 idx; 2744 2745 NV_CHECK_OR_RETURN(LEVEL_ERROR, pElfData != NULL, NV_ERR_INVALID_ARGUMENT); 2746 NV_CHECK_OR_RETURN(LEVEL_ERROR, elfDataSize > 0, NV_ERR_INVALID_ARGUMENT); 2747 NV_CHECK_OR_RETURN(LEVEL_ERROR, pSectionName != NULL, NV_ERR_INVALID_ARGUMENT); 2748 NV_CHECK_OR_RETURN(LEVEL_ERROR, ppSectionData != NULL, NV_ERR_INVALID_ARGUMENT); 2749 NV_CHECK_OR_RETURN(LEVEL_ERROR, pSectionSize != NULL, NV_ERR_INVALID_ARGUMENT); 2750 NV_CHECK_OR_RETURN(LEVEL_ERROR, elfDataSize >= sizeof(LibosElf64Header), NV_ERR_INVALID_DATA); 2751 2752 sectionNameLength = portStringLength(pSectionName); 2753 2754 pElfHeader = (const LibosElf64Header*) pGspBuf; 2755 2756 // Check for the elf identifier at the beginning of the file 2757 NV_CHECK_OR_RETURN(LEVEL_ERROR, *(NvU32*)&pElfHeader->ident == elfMagicNumber, NV_ERR_INVALID_DATA); 2758 // Make sure the data is formatted as little endian 2759 NV_CHECK_OR_RETURN(LEVEL_ERROR, pElfHeader->ident[5] == elfLittleEndian, NV_ERR_INVALID_DATA); 2760 // Check the class type, only ELFCLASS64 is supported 2761 NV_CHECK_OR_RETURN(LEVEL_ERROR, pElfHeader->ident[4] == elfClass64, NV_ERR_INVALID_DATA); 2762 2763 // Make sure that the elf section header table is valid 2764 NV_CHECK_OR_RETURN(LEVEL_ERROR, pElfHeader->shentsize == sizeof(LibosElf64SectionHeader), NV_ERR_INVALID_DATA); 2765 NV_CHECK_OR_RETURN(LEVEL_ERROR, portSafeMulU64(pElfHeader->shentsize, pElfHeader->shnum, &elfSectionHeaderTableLength), NV_ERR_INVALID_DATA); 2766 NV_CHECK_OR_RETURN(LEVEL_ERROR, portSafeAddU64(pElfHeader->shoff, elfSectionHeaderTableLength - 1, &elfSectionHeaderMaxIdx), NV_ERR_INVALID_DATA); 2767 NV_CHECK_OR_RETURN(LEVEL_ERROR, elfDataSize >= elfSectionHeaderMaxIdx, NV_ERR_INVALID_DATA); 2768 NV_CHECK_OR_RETURN(LEVEL_ERROR, pElfHeader->shstrndx <= pElfHeader->shnum, NV_ERR_INVALID_DATA); 2769 2770 // Get the offset and size of the table that holds the section names and make sure they are valid 2771 pElfSectionHeader = (const LibosElf64SectionHeader*) &pGspBuf[pElfHeader->shoff + (pElfHeader->shstrndx * pElfHeader->shentsize)]; 2772 elfSectionNamesTableOffset = pElfSectionHeader->offset; 2773 elfSectionNamesTableSize = pElfSectionHeader->size; 2774 NV_CHECK_OR_RETURN(LEVEL_ERROR, portSafeAddU64(elfSectionNamesTableOffset, elfSectionNamesTableSize - 1, &elfSectionNamesTableMaxIdx), NV_ERR_INVALID_DATA); 2775 NV_CHECK_OR_RETURN(LEVEL_ERROR, elfDataSize >= elfSectionNamesTableMaxIdx, NV_ERR_INVALID_DATA); 2776 2777 // Iterate through all of the section headers to find the signatures 2778 pElfSectionHeader = (const LibosElf64SectionHeader*) &pGspBuf[elfSectionHeaderMaxIdx + 1 - sizeof(*pElfSectionHeader)]; 2779 2780 for (idx = pElfHeader->shnum - 1; idx >= 0; idx--, pElfSectionHeader--) 2781 { 2782 NvU64 currentSectionNameMaxLength; 2783 NvU64 elfSectionMaxIdx; 2784 2785 // Make sure the header name index fits within the section names table 2786 NV_CHECK_OR_RETURN(LEVEL_ERROR, elfSectionNamesTableSize - 1 >= pElfSectionHeader->name, NV_ERR_INVALID_DATA); 2787 currentSectionNameMaxLength = elfSectionNamesTableSize - pElfSectionHeader->name - 1; 2788 pCurrentSectionName = (const char *) &pGspBuf[elfSectionNamesTableOffset + pElfSectionHeader->name]; 2789 2790 // Make sure the elf section size and offset are valid 2791 if (pElfSectionHeader->size > 0) 2792 { 2793 NV_CHECK_OR_RETURN(LEVEL_ERROR, portSafeAddU64(pElfSectionHeader->offset, pElfSectionHeader->size - 1, &elfSectionMaxIdx), NV_ERR_INVALID_DATA); 2794 } 2795 else 2796 { 2797 elfSectionMaxIdx = pElfSectionHeader->offset; 2798 } 2799 NV_CHECK_OR_RETURN(LEVEL_ERROR, elfDataSize >= elfSectionMaxIdx, NV_ERR_INVALID_DATA); 2800 2801 // Check whether the section name matches the expected section name 2802 if ((sectionNameLength <= currentSectionNameMaxLength) && 2803 (portStringCompare(pCurrentSectionName, pSectionName, sectionNameLength) == 0) && 2804 (pCurrentSectionName[sectionNameLength] == '\0')) 2805 { 2806 *ppSectionData = &pGspBuf[pElfSectionHeader->offset]; 2807 *pSectionSize = pElfSectionHeader->size; 2808 2809 return NV_OK; 2810 } 2811 } 2812 2813 return NV_ERR_OBJECT_NOT_FOUND; 2814 } 2815 2816 /*! 2817 * Setup libos init arguments. 2818 */ 2819 void 2820 kgspSetupLibosInitArgs_IMPL 2821 ( 2822 OBJGPU *pGpu, 2823 KernelGsp *pKernelGsp 2824 ) 2825 { 2826 LibosMemoryRegionInitArgument *pLibosInitArgs = pKernelGsp->pLibosInitArgumentsCached; 2827 NvU8 idx; 2828 2829 portMemSet(pLibosInitArgs, 0, LIBOS_INIT_ARGUMENTS_SIZE); 2830 2831 // Add memory areas for logging each LIBOS task. 2832 // @note LOGINIT must be first for early init logging to work. 2833 // @note: These should be switched to radix regions to remove the need 2834 // for large apertures in the RM task for logging. 2835 for (idx = 0; idx < LOGIDX_SIZE; idx++) 2836 { 2837 pLibosInitArgs[idx].kind = LIBOS_MEMORY_REGION_CONTIGUOUS; 2838 pLibosInitArgs[idx].loc = LIBOS_MEMORY_REGION_LOC_SYSMEM; 2839 pLibosInitArgs[idx].id8 = pKernelGsp->rmLibosLogMem[idx].id8; 2840 pLibosInitArgs[idx].pa = pKernelGsp->rmLibosLogMem[idx].pTaskLogBuffer[1]; 2841 pLibosInitArgs[idx].size = memdescGetSize(pKernelGsp->rmLibosLogMem[idx].pTaskLogDescriptor); 2842 } 2843 2844 // insert GSP-RM ELF args address; id must match libos-config.py entry 2845 pLibosInitArgs[idx].kind = LIBOS_MEMORY_REGION_CONTIGUOUS; 2846 pLibosInitArgs[idx].loc = LIBOS_MEMORY_REGION_LOC_SYSMEM; 2847 pLibosInitArgs[idx].id8 = _kgspGenerateInitArgId("RMARGS"); 2848 pLibosInitArgs[idx].pa = memdescGetPhysAddr(pKernelGsp->pGspArgumentsDescriptor, AT_GPU, 0); 2849 pLibosInitArgs[idx].size = memdescGetSize(pKernelGsp->pGspArgumentsDescriptor); 2850 2851 portAtomicMemoryFenceFull(); 2852 } 2853 2854 /*! 2855 * Receive and process RPC event from GSP-RM. 2856 * 2857 * This function is called from interrupt bottom-half handler (DPC) and 2858 * would race with normal RPC flow, _kgspRpcRecvPoll(). 2859 * This race is currently avoided only because DPC is executed under 2860 * gpus lock, so RPC and Bottom-half handler are mutually exclusive 2861 * control flows. 2862 */ 2863 void 2864 kgspRpcRecvEvents_IMPL 2865 ( 2866 OBJGPU *pGpu, 2867 KernelGsp *pKernelGsp 2868 ) 2869 { 2870 NV_ASSERT(rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 2871 // 2872 // We should never have an event with code NV_VGPU_MSG_FUNCTION_NUM_FUNCTIONS. 2873 // If we do the assert will fail on NV_WARN_MORE_PROCESSING_REQUIRED, 2874 // in addition to general error codes. 2875 // 2876 NV_ASSERT_OK(_kgspRpcDrainEvents(pGpu, pKernelGsp, NV_VGPU_MSG_FUNCTION_NUM_FUNCTIONS)); 2877 } 2878 2879 /*! 2880 * Wait for GSP-RM initialization to complete. 2881 */ 2882 NV_STATUS 2883 kgspWaitForRmInitDone_IMPL 2884 ( 2885 OBJGPU *pGpu, 2886 KernelGsp *pKernelGsp 2887 ) 2888 { 2889 NV_STATUS nvStatus = NV_OK; 2890 2891 // 2892 // Kernel RM can timeout when GSP-RM has an error condition. Give GSP-RM 2893 // a chance to report the error before we pull the rug out from under it. 2894 // 2895 threadStateResetTimeout(pGpu); 2896 2897 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2898 rpcRecvPoll(pGpu, pKernelGsp->pRpc, NV_VGPU_MSG_EVENT_GSP_INIT_DONE)); 2899 2900 // Now check if RPC really succeeded 2901 NV_ASSERT_OK_OR_RETURN(((rpc_message_header_v *)pKernelGsp->pRpc->message_buffer)->rpc_result); 2902 if (nvStatus != NV_VGPU_MSG_RESULT_SUCCESS) 2903 { 2904 NV_ASSERT_OK_FAILED("nvStatus", nvStatus); 2905 } 2906 2907 return nvStatus; 2908 } 2909 2910 /*! 2911 * Execute a sequencer buffer coming from GSP 2912 * 2913 * @param[in] pGpu GPU object pointer 2914 * @param[in] pKernelGsp KernelGsp object pointer 2915 * @param[in] pRunCpuSeqParams Sequence buffer RPC parameters 2916 * 2917 * @return NV_OK if the GSP sequencer buffer has been executed successfully 2918 * NV_ERR_INVALID_STATE if the sequencer buffer is not allocated 2919 * NV_ERR_INVALID_DATA is the sequencer buffer is malformed 2920 */ 2921 NV_STATUS 2922 kgspExecuteSequencerBuffer_IMPL 2923 ( 2924 OBJGPU *pGpu, 2925 KernelGsp *pKernelGsp, 2926 void *pRunCpuSeqParams 2927 ) 2928 { 2929 rpc_run_cpu_sequencer_v17_00 *pParams = (rpc_run_cpu_sequencer_v17_00 *)pRunCpuSeqParams; 2930 NvU32 *pCmd = pParams->commandBuffer; 2931 NvU32 buffer_end = pParams->cmdIndex; 2932 NvU32 current_cmd_index = 0; 2933 NV_STATUS nvStatus = NV_OK; 2934 NvU32 payloadSize; 2935 2936 NV_ASSERT_OR_RETURN(IS_GSP_CLIENT(pGpu), NV_ERR_NOT_SUPPORTED); 2937 NV_ASSERT_OR_RETURN((pParams->bufferSizeDWord != 0), NV_ERR_INVALID_STATE); 2938 NV_ASSERT_OR_RETURN(buffer_end < pParams->bufferSizeDWord, NV_ERR_INVALID_DATA); 2939 2940 while (current_cmd_index < buffer_end) 2941 { 2942 NvU32 opCode = pCmd[current_cmd_index++]; 2943 payloadSize = GSP_SEQUENCER_PAYLOAD_SIZE_DWORDS(opCode); 2944 2945 NV_ASSERT_OR_RETURN(current_cmd_index + payloadSize <= buffer_end, NV_ERR_INVALID_DATA); 2946 2947 // 2948 // Handling of sequencer commands is split between those commands 2949 // that are common to all architectures (handled directly here) and 2950 // those commands that are arch-specific and are handled via the 2951 // kgspExecuteSequencerCommand_HAL() call below. 2952 // 2953 switch (opCode) 2954 { 2955 // 2 arguments 2956 case GSP_SEQ_BUF_OPCODE_REG_WRITE: 2957 { 2958 GSP_SEQ_BUF_PAYLOAD_REG_WRITE regWrite; 2959 portMemCopy(®Write, sizeof(GSP_SEQ_BUF_PAYLOAD_REG_WRITE), &pCmd[current_cmd_index], sizeof(GSP_SEQ_BUF_PAYLOAD_REG_WRITE)); 2960 2961 GPU_REG_WR32(pGpu, regWrite.addr, regWrite.val); 2962 break; 2963 } 2964 2965 // 3 arguments 2966 case GSP_SEQ_BUF_OPCODE_REG_MODIFY: 2967 { 2968 GSP_SEQ_BUF_PAYLOAD_REG_MODIFY regModify; 2969 NvU32 regVal; 2970 2971 portMemCopy(®Modify, sizeof(GSP_SEQ_BUF_PAYLOAD_REG_MODIFY), &pCmd[current_cmd_index], sizeof(GSP_SEQ_BUF_PAYLOAD_REG_MODIFY)); 2972 2973 regVal = GPU_REG_RD32(pGpu, regModify.addr); 2974 regVal = regVal & ~regModify.mask; 2975 regVal = regVal | regModify.val; 2976 GPU_REG_WR32(pGpu, regModify.addr, regVal); 2977 break; 2978 } 2979 2980 // 5 arguments 2981 case GSP_SEQ_BUF_OPCODE_REG_POLL: 2982 { 2983 GSP_SEQ_BUF_PAYLOAD_REG_POLL regPoll; 2984 NvU32 regval; 2985 RMTIMEOUT timeout; 2986 2987 portMemCopy(®Poll, sizeof(GSP_SEQ_BUF_PAYLOAD_REG_POLL), &pCmd[current_cmd_index], sizeof(GSP_SEQ_BUF_PAYLOAD_REG_POLL)); 2988 2989 regval = GPU_REG_RD32(pGpu, regPoll.addr); 2990 2991 gpuSetTimeout(pGpu, regPoll.timeout, &timeout, 0); 2992 while ((regval & regPoll.mask) != regPoll.val) 2993 { 2994 nvStatus = gpuCheckTimeout(pGpu, &timeout); 2995 if (nvStatus == NV_ERR_TIMEOUT) 2996 { 2997 NV_PRINTF(LEVEL_ERROR, "Timeout waiting for register to settle, value = 0x%x, err_code = 0x%x\n", 2998 regval, regPoll.error); 2999 DBG_BREAKPOINT(); 3000 return nvStatus; 3001 } 3002 osSpinLoop(); 3003 regval = GPU_REG_RD32(pGpu, regPoll.addr); 3004 } 3005 break; 3006 } 3007 3008 case GSP_SEQ_BUF_OPCODE_DELAY_US: 3009 { 3010 GSP_SEQ_BUF_PAYLOAD_DELAY_US delayUs; 3011 portMemCopy(&delayUs, sizeof(GSP_SEQ_BUF_PAYLOAD_DELAY_US), &pCmd[current_cmd_index], sizeof(GSP_SEQ_BUF_PAYLOAD_DELAY_US)); 3012 3013 osDelayUs(delayUs.val); 3014 break; 3015 } 3016 3017 case GSP_SEQ_BUF_OPCODE_REG_STORE: 3018 { 3019 GSP_SEQ_BUF_PAYLOAD_REG_STORE regStore; 3020 portMemCopy(®Store, sizeof(GSP_SEQ_BUF_PAYLOAD_REG_STORE), &pCmd[current_cmd_index], sizeof(GSP_SEQ_BUF_PAYLOAD_REG_STORE)); 3021 3022 NV_ASSERT_OR_RETURN(regStore.index < GSP_SEQ_BUF_REG_SAVE_SIZE, NV_ERR_INVALID_ARGUMENT); 3023 3024 pParams->regSaveArea[regStore.index] = GPU_REG_RD32(pGpu, regStore.addr); 3025 break; 3026 } 3027 3028 default: 3029 // 3030 // Route this command to the arch-specific handler. 3031 // 3032 NV_ASSERT_OK_OR_RETURN(kgspExecuteSequencerCommand_HAL(pGpu, pKernelGsp, opCode, &pCmd[current_cmd_index], payloadSize * sizeof (*pCmd))); 3033 break; 3034 } 3035 current_cmd_index += payloadSize; 3036 } 3037 3038 return NV_OK; 3039 } 3040 3041 #if LIBOS_LOG_DECODE_ENABLE 3042 static void 3043 _kgspLogPollingCallback 3044 ( 3045 OBJGPU *pGpu, 3046 void *data 3047 ) 3048 { 3049 KernelGsp *pKernelGsp = GPU_GET_KERNEL_GSP(pGpu); 3050 kgspDumpGspLogs(pGpu, pKernelGsp, NV_FALSE); 3051 } 3052 3053 NV_STATUS 3054 kgspStartLogPolling_IMPL 3055 ( 3056 OBJGPU *pGpu, 3057 KernelGsp *pKernelGsp 3058 ) 3059 { 3060 NV_STATUS status; 3061 status = osSchedule1SecondCallback(pGpu, _kgspLogPollingCallback, NULL, NV_OS_1HZ_REPEAT); 3062 return status; 3063 } 3064 3065 static void 3066 _kgspStopLogPolling 3067 ( 3068 OBJGPU *pGpu, 3069 KernelGsp *pKernelGsp 3070 ) 3071 { 3072 osRemove1SecondRepeatingCallback(pGpu, _kgspLogPollingCallback, NULL); 3073 } 3074 3075 #else // LIBOS_LOG_DECODE_ENABLE 3076 3077 NV_STATUS 3078 kgspStartLogPolling_IMPL 3079 ( 3080 OBJGPU *pGpu, 3081 KernelGsp *pKernelGsp 3082 ) 3083 { 3084 return NV_OK; 3085 } 3086 3087 static void 3088 _kgspStopLogPolling 3089 ( 3090 OBJGPU *pGpu, 3091 KernelGsp *pKernelGsp 3092 ) 3093 { 3094 return; 3095 } 3096 #endif // LIBOS_LOG_DECODE_ENABLE 3097 3098 /*! 3099 * Provides an opportunity to register some IntrService during intrStateInit. 3100 */ 3101 void 3102 kgspRegisterIntrService_IMPL 3103 ( 3104 OBJGPU *pGpu, 3105 KernelGsp *pKernelGsp, 3106 IntrServiceRecord pRecords[MC_ENGINE_IDX_MAX] 3107 ) 3108 { 3109 NvU32 engineIdx = MC_ENGINE_IDX_GSP; 3110 3111 if (!IS_GSP_CLIENT(pGpu)) 3112 return; 3113 3114 NV_ASSERT(pRecords[engineIdx].pInterruptService == NULL); 3115 pRecords[engineIdx].pInterruptService = staticCast(pKernelGsp, IntrService); 3116 } 3117 3118 /*! 3119 * Service GSP interrupts. 3120 * 3121 * @returns Zero, or any implementation-chosen nonzero value. If the same nonzero value is returned enough 3122 * times the interrupt is considered stuck. 3123 */ 3124 NvU32 3125 kgspServiceInterrupt_IMPL 3126 ( 3127 OBJGPU *pGpu, 3128 KernelGsp *pKernelGsp, 3129 IntrServiceServiceInterruptArguments *pParams 3130 ) 3131 { 3132 NV_ASSERT_OR_RETURN(pParams != NULL, 0); 3133 NV_ASSERT_OR_RETURN(pParams->engineIdx == MC_ENGINE_IDX_GSP, 0); 3134 3135 return kgspService_HAL(pGpu, pKernelGsp); 3136 } 3137