1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 1999-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include "kernel/gpu/deferred_api.h" 25 26 #include "kernel/core/locks.h" 27 #include "kernel/gpu/device/device.h" 28 #include "kernel/gpu/subdevice/subdevice.h" 29 #include "kernel/mem_mgr/vaspace.h" 30 #include "kernel/rmapi/control.h" 31 #include "kernel/rmapi/rs_utils.h" 32 #include "kernel/virtualization/hypervisor/hypervisor.h" 33 #include "kernel/gpu/fifo/kernel_channel.h" 34 35 #include "class/cl5080.h" 36 37 #include "ctrl/ctrl2080.h" 38 39 #include "libraries/resserv/rs_server.h" 40 #include "vgpu/rpc.h" 41 42 43 static NV_STATUS _Class5080DelDeferredApi(DeferredApiObject *pDeferredApiObject, 44 NvHandle hDeferredApi); 45 46 static NV_STATUS _class5080DeferredApiV2(OBJGPU *pGpu, 47 ChannelDescendant *Object, 48 NvU32 Offset, 49 NvU32 Data); 50 51 static NV_STATUS 52 _Class5080UpdateTLBFlushState(DeferredApiObject *pDeferredApiObject); 53 54 static NV_STATUS 55 _Class5080GetDeferredApiInfo(DeferredApiObject *pDeferredApiObject, 56 NvHandle hDeferredApi, 57 DEFERRED_API_INFO **ppCliDeferredApi); 58 59 static NV_STATUS 60 _Class5080AddDeferredApi(DeferredApiObject *pDeferredApiObject, 61 NvHandle hClient, 62 NvHandle hDeferredApi, 63 NV5080_CTRL_DEFERRED_API_PARAMS *pDeferredApi, 64 NvU32 size, 65 NvBool bUserModeArgs); 66 67 static NV_STATUS 68 _Class5080AddDeferredApiV2(DeferredApiObject *pDeferredApiObject, 69 NvHandle hClient, 70 NvHandle hDeferredApi, 71 NV5080_CTRL_DEFERRED_API_V2_PARAMS *pDeferredApi, 72 NvU32 size); 73 74 75 static NV_STATUS 76 _Class5080AddDeferredApi 77 ( 78 DeferredApiObject *pDeferredApiObject, 79 NvHandle hClient, 80 NvHandle hDeferredApi, 81 NV5080_CTRL_DEFERRED_API_PARAMS *pDeferredApi, 82 NvU32 size, 83 NvBool bUserModeArgs 84 ) 85 { 86 NV_STATUS rmStatus = NV_OK; 87 DEFERRED_API_INFO *pCliDeferredApi; 88 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 89 90 NV_ASSERT_OR_RETURN(pCallContext != NULL, NV_ERR_INVALID_STATE); 91 92 if (NV_OK != serverGetClientUnderLock(&g_resServ, hClient, NULL)) 93 return NV_ERR_INVALID_CLIENT; 94 95 // validate handle 96 if (!serverutilValidateNewResourceHandle(hClient, hDeferredApi)) 97 { 98 return NV_ERR_INVALID_OBJECT_HANDLE; 99 } 100 101 // allocate a new Deferred Api and add to the client 102 pCliDeferredApi = portMemAllocNonPaged(sizeof(DEFERRED_API_INFO) + size); 103 if (NULL != pCliDeferredApi) 104 { 105 // initialize the entry 106 pCliDeferredApi->Client = hClient; 107 pCliDeferredApi->Handle = hDeferredApi; 108 pCliDeferredApi->pDeferredApiInfo = (NvU8 *)pCliDeferredApi + sizeof(DEFERRED_API_INFO); 109 pCliDeferredApi->Node.keyStart = pCliDeferredApi->Handle; 110 pCliDeferredApi->Node.keyEnd = pCliDeferredApi->Handle; 111 pCliDeferredApi->Node.Data = pCliDeferredApi; 112 pCliDeferredApi->privLevel = pCallContext->secInfo.privLevel; 113 pCliDeferredApi->Flags = 0; 114 portMemCopy(pCliDeferredApi->pDeferredApiInfo, size, pDeferredApi, size); 115 116 // link in the new entry 117 rmStatus = btreeInsert(&pCliDeferredApi->Node, &pDeferredApiObject->DeferredApiList); 118 if (rmStatus != NV_OK) 119 { 120 portMemFree(pCliDeferredApi); 121 pCliDeferredApi = NULL; 122 } 123 } 124 else 125 return NV_ERR_NO_MEMORY; 126 127 return rmStatus; 128 } 129 130 131 static NV_STATUS 132 _Class5080AddDeferredApiV2 133 ( 134 DeferredApiObject *pDeferredApiObject, 135 NvHandle hClient, 136 NvHandle hDeferredApi, 137 NV5080_CTRL_DEFERRED_API_V2_PARAMS *pDeferredApi, 138 NvU32 size 139 ) 140 { 141 NV_STATUS rmStatus = NV_OK; 142 DEFERRED_API_INFO *pCliDeferredApi; 143 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 144 145 NV_ASSERT_OR_RETURN(pCallContext != NULL, NV_ERR_INVALID_STATE); 146 147 if (NV_OK != serverGetClientUnderLock(&g_resServ, hClient, NULL)) 148 return NV_ERR_INVALID_CLIENT; 149 150 // validate handle 151 if (!serverutilValidateNewResourceHandle(hClient, hDeferredApi)) 152 { 153 return NV_ERR_INVALID_OBJECT_HANDLE; 154 } 155 156 rmStatus = _Class5080GetDeferredApiInfo(pDeferredApiObject, hDeferredApi, &pCliDeferredApi); 157 158 // Object already exists 159 if (NV_OK == rmStatus) 160 { 161 return NV_ERR_INVALID_OBJECT_HANDLE; 162 } 163 164 // allocate a new Deferred Api and add to the client 165 pCliDeferredApi = portMemAllocNonPaged(sizeof(DEFERRED_API_INFO) + size); 166 if (NULL != pCliDeferredApi) 167 { 168 // initialize the entry 169 pCliDeferredApi->Client = hClient; 170 pCliDeferredApi->Handle = hDeferredApi; 171 pCliDeferredApi->pDeferredApiInfo = (NvU8 *)pCliDeferredApi + sizeof(DEFERRED_API_INFO); 172 pCliDeferredApi->Node.keyStart = pCliDeferredApi->Handle; 173 pCliDeferredApi->Node.keyEnd = pCliDeferredApi->Handle; 174 pCliDeferredApi->Node.Data = pCliDeferredApi; 175 pCliDeferredApi->Flags = 0; 176 pCliDeferredApi->privLevel = pCallContext->secInfo.privLevel; 177 portMemCopy(pCliDeferredApi->pDeferredApiInfo, size, pDeferredApi, size); 178 179 // link in the new entry 180 rmStatus = btreeInsert(&pCliDeferredApi->Node, &pDeferredApiObject->DeferredApiList); 181 182 if (rmStatus != NV_OK) 183 { 184 portMemFree(pCliDeferredApi); 185 pCliDeferredApi = NULL; 186 } 187 } 188 else 189 return NV_ERR_NO_MEMORY; 190 191 return rmStatus; 192 } 193 194 195 static NV_STATUS 196 _Class5080GetDeferredApiInfo 197 ( 198 DeferredApiObject *pDeferredApiObject, 199 NvHandle hDeferredApi, 200 DEFERRED_API_INFO **ppCliDeferredApi 201 ) 202 { 203 NODE *pNode; 204 205 if (btreeSearch(hDeferredApi, &pNode, pDeferredApiObject->DeferredApiList) == NV_OK) 206 { 207 *ppCliDeferredApi = pNode->Data; 208 return NV_OK; 209 } 210 211 return NV_ERR_INVALID_DATA; 212 213 } 214 215 216 static NV_STATUS _Class5080DelDeferredApi 217 ( 218 DeferredApiObject *pDeferredApiObject, 219 NvHandle hDeferredApi 220 ) 221 { 222 DEFERRED_API_INFO *pDeferredApi = NULL; 223 NV_STATUS status; 224 NODE *pNode; 225 226 // remove the event from the client database 227 if (NV_OK == _Class5080GetDeferredApiInfo(pDeferredApiObject, 228 hDeferredApi, &pDeferredApi)) 229 { 230 status = btreeSearch(hDeferredApi, &pNode, pDeferredApiObject->DeferredApiList); 231 if (status != NV_OK) 232 return NV_ERR_GENERIC; 233 234 status = btreeUnlink(pNode, &pDeferredApiObject->DeferredApiList); 235 if (status == NV_OK) 236 { 237 NV5080_CTRL_DEFERRED_API_PARAMS *pDeferredApiParams; 238 pDeferredApiParams = (NV5080_CTRL_DEFERRED_API_PARAMS *)pDeferredApi->pDeferredApiInfo; 239 240 if (pDeferredApi->Flags & DEFERRED_API_INFO_FLAGS_HAS_PRIVATE_DATA_ALLOC) 241 { 242 portMemFree((void *)NvP64_VALUE(pDeferredApi->pDeferredPrivateData)); 243 } 244 245 if (DRF_VAL(5080_CTRL, _CMD_DEFERRED_API, _FLAGS_WAIT_FOR_TLB_FLUSH, pDeferredApiParams->flags) == 246 NV5080_CTRL_CMD_DEFERRED_API_FLAGS_WAIT_FOR_TLB_FLUSH_TRUE) 247 { 248 // decrement count, if API was waiting on a TLB flush, but never saw one 249 if ((pDeferredApi->Flags & DEFERRED_API_INFO_FLAGS_HAS_EXECUTED) && 250 !(pDeferredApi->Flags & DEFERRED_API_INFO_FLAGS_HAS_TLB_FLUSHED)) 251 { 252 pDeferredApiObject->NumWaitingOnTLBFlush--; 253 } 254 } 255 256 // free the list element 257 portMemFree(pDeferredApi); 258 return NV_OK; 259 } 260 } 261 262 return NV_ERR_GENERIC; 263 } 264 265 266 static NV_STATUS _Class5080UpdateTLBFlushState 267 ( 268 DeferredApiObject *pDeferredApiObject 269 ) 270 { 271 NODE *pNode; 272 DEFERRED_API_INFO *pCliDeferredApi; 273 NV5080_CTRL_DEFERRED_API_PARAMS *pDeferredApi; 274 275 btreeEnumStart(0, &pNode, pDeferredApiObject->DeferredApiList); 276 while (pNode && 277 pDeferredApiObject->NumWaitingOnTLBFlush) 278 { 279 pCliDeferredApi = pNode->Data; 280 pDeferredApi = (NV5080_CTRL_DEFERRED_API_PARAMS *) pCliDeferredApi->pDeferredApiInfo; 281 282 // update any APIs with WAIT_FOR_TLB_FLUSH set 283 if (DRF_VAL(5080_CTRL, _CMD_DEFERRED_API, _FLAGS_WAIT_FOR_TLB_FLUSH, pDeferredApi->flags) == 284 NV5080_CTRL_CMD_DEFERRED_API_FLAGS_WAIT_FOR_TLB_FLUSH_TRUE) 285 { 286 // check if API has EXECUTED and newly TLB_FLUSHED 287 if ((pCliDeferredApi->Flags & DEFERRED_API_INFO_FLAGS_HAS_EXECUTED) && 288 !(pCliDeferredApi->Flags & DEFERRED_API_INFO_FLAGS_HAS_TLB_FLUSHED)) 289 { 290 pCliDeferredApi->Flags |= DEFERRED_API_INFO_FLAGS_HAS_TLB_FLUSHED; 291 pDeferredApiObject->NumWaitingOnTLBFlush--; 292 293 btreeEnumNext(&pNode, pDeferredApiObject->DeferredApiList); 294 295 // check if API can now be implicitly deleted 296 if (DRF_VAL(5080_CTRL, _CMD_DEFERRED_API, _FLAGS_DELETE, pDeferredApi->flags) == 297 NV5080_CTRL_CMD_DEFERRED_API_FLAGS_DELETE_IMPLICIT) 298 { 299 _Class5080DelDeferredApi(pDeferredApiObject, pCliDeferredApi->Handle); 300 } 301 continue; 302 } 303 } 304 btreeEnumNext(&pNode, pDeferredApiObject->DeferredApiList); 305 } 306 307 return NV_OK; 308 } 309 310 311 //--------------------------------------------------------------------------- 312 // 313 // Class object creation and destruction 314 // 315 //--------------------------------------------------------------------------- 316 317 NV_STATUS 318 defapiConstruct_IMPL 319 ( 320 DeferredApiObject *pDeferredApi, 321 CALL_CONTEXT *pCallContext, 322 RS_RES_ALLOC_PARAMS_INTERNAL *pParams 323 ) 324 { 325 if (pParams->pAllocParams != NULL) 326 { 327 NV5080_ALLOC_PARAMS *pAllocParams = pParams->pAllocParams; 328 if (pAllocParams->notifyCompletion) 329 { 330 staticCast(pDeferredApi, ChannelDescendant)->notifyAction = NV_OS_WRITE_THEN_AWAKEN; 331 staticCast(pDeferredApi, ChannelDescendant)->bNotifyTrigger = NV_TRUE; 332 } 333 } 334 335 return NV_OK; 336 } 337 338 void 339 defapiDestruct_IMPL 340 ( 341 DeferredApiObject *pDeferredApi 342 ) 343 { 344 ChannelDescendant *pChannelDescendant = staticCast(pDeferredApi, ChannelDescendant); 345 NODE *pNode; 346 DEFERRED_API_INFO *pCliDeferredApi; 347 348 chandesIsolateOnDestruct(pChannelDescendant); 349 350 // Free All Outstanding API on the btree 351 btreeEnumStart(0, &pNode, pDeferredApi->DeferredApiList); 352 while (pNode) 353 { 354 pCliDeferredApi = pNode->Data; 355 356 btreeEnumNext(&pNode, pDeferredApi->DeferredApiList); 357 _Class5080DelDeferredApi(pDeferredApi, pCliDeferredApi->Handle); 358 } 359 } 360 361 NV_STATUS 362 defapiCtrlCmdDeferredApi_IMPL 363 ( 364 DeferredApiObject *pDeferredApiObj, 365 NV5080_CTRL_DEFERRED_API_PARAMS *pDeferredApi 366 ) 367 { 368 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 369 OBJGPU *pGpu = GPU_RES_GET_GPU(pDeferredApiObj); 370 NV_STATUS status = NV_OK; 371 372 // 373 // vGPU: 374 // 375 // Since vGPU does all real hardware management in the 376 // host, there is nothing to do at this point in the 377 // guest OS (where IS_VIRTUAL(pGpu) is true). 378 // 379 if (IS_VIRTUAL(pGpu)) 380 { 381 NV_RM_RPC_DEFERRED_API_CONTROL(pGpu, 382 RES_GET_CLIENT_HANDLE(pDeferredApiObj), 383 RES_GET_PARENT_HANDLE(pDeferredApiObj), 384 RES_GET_HANDLE(pDeferredApiObj), 385 (void *)pDeferredApi, 386 sizeof(NV5080_CTRL_DEFERRED_API_PARAMS), 387 status); 388 return status; 389 } 390 391 return _Class5080AddDeferredApi(pDeferredApiObj, 392 RES_GET_CLIENT_HANDLE(pDeferredApiObj), 393 pDeferredApi->hApiHandle, 394 pDeferredApi, 395 sizeof(NV5080_CTRL_DEFERRED_API_PARAMS), 396 (pCallContext->secInfo.paramLocation != PARAM_LOCATION_KERNEL)); 397 } 398 399 NV_STATUS 400 defapiCtrlCmdDeferredApiV2_IMPL 401 ( 402 DeferredApiObject *pDeferredApiObj, 403 NV5080_CTRL_DEFERRED_API_V2_PARAMS *pDeferredApi 404 ) 405 { 406 NV_STATUS status = NV_OK; 407 OBJGPU *pGpu = GPU_RES_GET_GPU(pDeferredApiObj); 408 409 // 410 // vGPU: 411 // 412 // Since vGPU does all real hardware management in the 413 // host, there is nothing to do at this point in the 414 // guest OS (where IS_VIRTUAL(pGpu) is true). 415 // 416 if (IS_VIRTUAL(pGpu)) 417 { 418 NV_RM_RPC_DEFERRED_API_CONTROL(pGpu, 419 RES_GET_CLIENT_HANDLE(pDeferredApiObj), 420 RES_GET_PARENT_HANDLE(pDeferredApiObj), 421 RES_GET_HANDLE(pDeferredApiObj), 422 (void *)pDeferredApi, 423 sizeof(NV5080_CTRL_DEFERRED_API_V2_PARAMS), 424 status); 425 return status; 426 } 427 428 return _Class5080AddDeferredApiV2(pDeferredApiObj, 429 RES_GET_CLIENT_HANDLE(pDeferredApiObj), 430 pDeferredApi->hApiHandle, 431 pDeferredApi, 432 sizeof(NV5080_CTRL_DEFERRED_API_V2_PARAMS)); 433 } 434 435 NV_STATUS 436 defapiCtrlCmdRemoveApi_IMPL 437 ( 438 DeferredApiObject *pDeferredApiObj, 439 NV5080_CTRL_REMOVE_API_PARAMS *pRemoveApi 440 ) 441 { 442 NV_STATUS status = NV_OK; 443 OBJGPU *pGpu = GPU_RES_GET_GPU(pDeferredApiObj); 444 445 // 446 // vGPU: 447 // 448 // Since vGPU does all real hardware management in the 449 // host, there is nothing to do at this point in the 450 // guest OS (where IS_VIRTUAL(pGpu) is true). 451 // 452 if (IS_VIRTUAL(pGpu)) 453 { 454 NV_RM_RPC_REMOVE_DEFERRED_API(pGpu, 455 RES_GET_CLIENT_HANDLE(pDeferredApiObj), 456 RES_GET_PARENT_HANDLE(pDeferredApiObj), 457 RES_GET_HANDLE(pDeferredApiObj), 458 pRemoveApi->hApiHandle, status); 459 return status; 460 } 461 462 return _Class5080DelDeferredApi(pDeferredApiObj, 463 pRemoveApi->hApiHandle); 464 } 465 466 static NV_STATUS 467 _class5080DeferredApiV2 468 ( 469 OBJGPU *pGpu, 470 ChannelDescendant *Object, 471 NvU32 Offset, 472 NvU32 Data 473 ) 474 { 475 DeferredApiObject *pDeferredApiObject = dynamicCast(Object, DeferredApiObject); 476 DEFERRED_API_INFO *pCliDeferredApi = NULL; 477 NV5080_CTRL_DEFERRED_API_PARAMS *pDeferredApi; 478 NV_STATUS rmStatus = NV_OK; 479 NvU32 paramSize = 0; 480 NvBool bIsCtrlCall = NV_TRUE; 481 482 rmStatus = _Class5080GetDeferredApiInfo(pDeferredApiObject, 483 Data, &pCliDeferredApi); 484 if (rmStatus == NV_OK) 485 { 486 pDeferredApi = pCliDeferredApi->pDeferredApiInfo; 487 488 switch (pDeferredApi->cmd) 489 { 490 case NV2080_CTRL_CMD_GPU_INITIALIZE_CTX: 491 paramSize = sizeof(NV2080_CTRL_GPU_INITIALIZE_CTX_PARAMS); 492 break; 493 494 case NV2080_CTRL_CMD_GPU_PROMOTE_CTX: 495 paramSize = sizeof(NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS); 496 break; 497 498 case NV2080_CTRL_CMD_GPU_EVICT_CTX: 499 paramSize = sizeof(NV2080_CTRL_GPU_EVICT_CTX_PARAMS); 500 break; 501 502 case NV2080_CTRL_CMD_FIFO_UPDATE_CHANNEL_INFO: 503 paramSize = sizeof(NV2080_CTRL_FIFO_UPDATE_CHANNEL_INFO_PARAMS); 504 break; 505 506 case NV2080_CTRL_CMD_DMA_INVALIDATE_TLB: 507 { 508 OBJGPU *pTgtGpu; 509 RsClient *pClientVA; 510 Subdevice *pSubdevice; 511 512 bIsCtrlCall = NV_FALSE; 513 514 rmStatus = serverGetClientUnderLock(&g_resServ, pDeferredApi->hClientVA, 515 &pClientVA); 516 if (rmStatus != NV_OK) 517 break; 518 519 rmStatus = subdeviceGetByHandle(pClientVA, pDeferredApi->hDeviceVA, 520 &pSubdevice); 521 522 if (rmStatus != NV_OK) 523 { 524 NV_PRINTF(LEVEL_ERROR, 525 "Unable to find target gpu from hClient(%x), hDevice(%x)\n", 526 pDeferredApi->hClientVA, pDeferredApi->hDeviceVA); 527 } 528 else 529 { 530 NvHandle hDevice; 531 OBJVASPACE *pVAS = NULL; 532 533 // Fetch target GPU and set threadstate 534 pTgtGpu = GPU_RES_GET_GPU(pSubdevice); 535 536 hDevice = RES_GET_HANDLE(pSubdevice->pDevice); 537 538 GPU_RES_SET_THREAD_BC_STATE(pSubdevice); 539 540 rmStatus = vaspaceGetByHandleOrDeviceDefault(pClientVA, 541 hDevice, 542 pDeferredApi->api_bundle.InvalidateTlb.hVASpace, 543 &pVAS); 544 if (NV_OK == rmStatus) 545 { 546 vaspaceInvalidateTlb(pVAS, pTgtGpu, PTE_UPGRADE); 547 548 if (pDeferredApiObject->NumWaitingOnTLBFlush) 549 rmStatus = _Class5080UpdateTLBFlushState(pDeferredApiObject); 550 } 551 } 552 break; 553 } 554 555 case NV2080_CTRL_CMD_FB_SET_GPU_CACHE_ALLOC_POLICY_V2: 556 paramSize = sizeof(NV2080_CTRL_FB_GPU_CACHE_ALLOC_POLICY_V2_PARAMS); 557 break; 558 559 case NV2080_CTRL_CMD_GR_CTXSW_ZCULL_BIND: 560 paramSize = sizeof(NV2080_CTRL_GR_CTXSW_ZCULL_BIND_PARAMS); 561 break; 562 563 case NV2080_CTRL_CMD_GR_CTXSW_PM_BIND: 564 paramSize = sizeof(NV2080_CTRL_GR_CTXSW_PM_BIND_PARAMS); 565 break; 566 567 case NV2080_CTRL_CMD_GR_CTXSW_PREEMPTION_BIND: 568 paramSize = sizeof(NV2080_CTRL_GR_CTXSW_PREEMPTION_BIND_PARAMS); 569 break; 570 571 case NV2080_CTRL_CMD_FB_SET_GPU_CACHE_PROMOTION_POLICY: 572 paramSize = sizeof(NV2080_CTRL_FB_GPU_CACHE_PROMOTION_POLICY_PARAMS); 573 break; 574 575 default: 576 bIsCtrlCall = NV_FALSE; 577 paramSize = 0; 578 NV_PRINTF(LEVEL_ERROR, "Unknown or Unimplemented Command %x\n", 579 pDeferredApi->cmd); 580 NV_ASSERT(0); 581 rmStatus = NV_ERR_INVALID_ARGUMENT; 582 break; 583 } 584 585 if (bIsCtrlCall) 586 { 587 RmCtrlParams rmCtrlParams; 588 RmCtrlExecuteCookie rmCtrlExecuteCookie = {0}; 589 RS_LOCK_INFO lockInfo = {0}; 590 Subdevice *pSubdevice; 591 RsClient *pRsClient; 592 const struct NVOC_EXPORTED_METHOD_DEF *pEntry; 593 LOCK_ACCESS_TYPE access; 594 NvU32 releaseFlags = 0; 595 CALL_CONTEXT callContext; 596 CALL_CONTEXT *pOldContext = NULL; 597 598 portMemSet(&rmCtrlParams, 0, sizeof(RmCtrlParams)); 599 rmCtrlParams.hClient = pCliDeferredApi->Client; 600 rmCtrlParams.pGpu = pGpu; 601 rmCtrlParams.cmd = pDeferredApi->cmd; 602 rmCtrlParams.flags = 0; 603 rmCtrlParams.pParams = &pDeferredApi->api_bundle; 604 rmCtrlParams.paramsSize = paramSize; 605 rmCtrlParams.secInfo.privLevel = pCliDeferredApi->privLevel; 606 rmCtrlParams.secInfo.paramLocation = PARAM_LOCATION_KERNEL; 607 rmCtrlParams.pCookie = &rmCtrlExecuteCookie; 608 rmCtrlParams.pLockInfo = &lockInfo; 609 rmCtrlParams.bDeferredApi = NV_TRUE; 610 611 lockInfo.flags |= RM_LOCK_FLAGS_NO_GPUS_LOCK | 612 RM_LOCK_FLAGS_NO_CLIENT_LOCK; 613 614 rmCtrlParams.flags |= NVOS54_FLAGS_LOCK_BYPASS; 615 616 // In case of deferred API, the parameters are already copied 617 // from user space to kernel space when the deferred API is registered 618 // So the IRQL_RAISED flag is set to avoid to second copy of paramaters 619 if ((RMCFG_FEATURE_RM_BASIC_LOCK_MODEL && osIsRaisedIRQL()) || 620 hypervisorIsVgxHyper()) 621 { 622 rmCtrlParams.flags |= NVOS54_FLAGS_IRQL_RAISED; 623 } 624 625 rmStatus = serverGetClientUnderLock(&g_resServ, pCliDeferredApi->Client, &pRsClient); 626 if (rmStatus != NV_OK) 627 { 628 goto cleanup; 629 } 630 631 rmStatus = subdeviceGetByGpu(pRsClient, pGpu, &pSubdevice); 632 if (rmStatus != NV_OK) 633 { 634 goto cleanup; 635 } 636 637 rmStatus = resControlLookup(staticCast(pSubdevice, RsResource), &rmCtrlParams, &pEntry); 638 if (rmStatus != NV_OK) 639 { 640 goto cleanup; 641 } 642 643 NV_ASSERT(pEntry != NULL); 644 // Initialize the execution cookie 645 serverControl_InitCookie(pEntry, &rmCtrlExecuteCookie); 646 647 // Set the call context as we use that to validate call permissions 648 // in some cases 649 portMemSet(&callContext, 0, sizeof(callContext)); 650 callContext.pResourceRef = RES_GET_REF(pSubdevice); 651 callContext.pClient = pRsClient; 652 callContext.secInfo = rmCtrlParams.secInfo; 653 callContext.pServer = &g_resServ; 654 callContext.pControlParams = &rmCtrlParams; 655 callContext.pLockInfo = rmCtrlParams.pLockInfo; 656 657 if (RMCFG_FEATURE_PLATFORM_GSP && IS_VGPU_GSP_PLUGIN_OFFLOAD_ENABLED(pGpu)) 658 { 659 NvU32 gfid = kchannelGetGfid(Object->pKernelChannel); 660 callContext.secInfo.pProcessToken = (void *)(NvU64) gfid; 661 } 662 663 resservSwapTlsCallContext(&pOldContext, &callContext); 664 665 rmStatus = serverControl_Prologue(&g_resServ, &rmCtrlParams, &access, &releaseFlags); 666 667 if (rmStatus == NV_OK) 668 { 669 if (pEntry->paramSize == 0) 670 { 671 typedef NV_STATUS (*CONTROL_EXPORT_FNPTR_NO_PARAMS)(void*); 672 CONTROL_EXPORT_FNPTR_NO_PARAMS pFunc = ((CONTROL_EXPORT_FNPTR_NO_PARAMS) pEntry->pFunc); 673 rmStatus = pFunc((void*)pSubdevice); 674 } 675 else 676 { 677 typedef NV_STATUS (*CONTROL_EXPORT_FNPTR)(void*, void*); 678 CONTROL_EXPORT_FNPTR pFunc = ((CONTROL_EXPORT_FNPTR) pEntry->pFunc); 679 rmStatus = pFunc((void*)pSubdevice, rmCtrlParams.pParams); 680 } 681 } 682 683 resservRestoreTlsCallContext(pOldContext); 684 rmStatus = serverControl_Epilogue(&g_resServ, &rmCtrlParams, access, &releaseFlags, rmStatus); 685 } 686 687 cleanup: 688 689 pCliDeferredApi->Flags |= DEFERRED_API_INFO_FLAGS_HAS_EXECUTED; 690 691 if (DRF_VAL(5080_CTRL, _CMD_DEFERRED_API, _FLAGS_DELETE, pDeferredApi->flags) == 692 NV5080_CTRL_CMD_DEFERRED_API_FLAGS_DELETE_IMPLICIT) 693 { 694 // delete implicitly, unless WAIT_FOR_TLB_FLUSH is also NV_TRUE 695 if (DRF_VAL(5080_CTRL, _CMD_DEFERRED_API, _FLAGS_WAIT_FOR_TLB_FLUSH, pDeferredApi->flags) == 696 NV5080_CTRL_CMD_DEFERRED_API_FLAGS_WAIT_FOR_TLB_FLUSH_TRUE) 697 { 698 pDeferredApiObject->NumWaitingOnTLBFlush++; 699 } 700 else 701 { 702 _Class5080DelDeferredApi(pDeferredApiObject, Data); 703 } 704 } 705 } 706 707 return rmStatus; 708 } 709 710 static const METHOD Nv50DeferredApi[] = 711 { 712 { mthdNoOperation, 0x0100, 0x0103 }, 713 { _class5080DeferredApiV2, 0x0200, 0x0203 }, 714 }; 715 716 NV_STATUS defapiGetSwMethods_IMPL 717 ( 718 DeferredApiObject *pDeferredApi, 719 const METHOD **ppMethods, 720 NvU32 *pNumMethods 721 ) 722 { 723 *ppMethods = Nv50DeferredApi; 724 *pNumMethods = NV_ARRAY_ELEMENTS(Nv50DeferredApi); 725 return NV_OK; 726 } 727 728 NvBool defapiIsSwMethodStalling_IMPL 729 ( 730 DeferredApiObject *pDeferredApi, 731 NvU32 hDeferredApi 732 ) 733 { 734 DEFERRED_API_INFO *pCliDeferredApi = NULL; 735 NV5080_CTRL_DEFERRED_API_PARAMS *pDeferredApiParams; 736 737 NV_STATUS rmStatus = _Class5080GetDeferredApiInfo(pDeferredApi, 738 hDeferredApi, 739 &pCliDeferredApi); 740 if (rmStatus == NV_OK) 741 { 742 pDeferredApiParams = pCliDeferredApi->pDeferredApiInfo; 743 744 // Clear the PBDMA interrupt before executing the software method. 745 if (pDeferredApiParams->cmd == NV2080_CTRL_CMD_FIFO_UPDATE_CHANNEL_INFO) 746 { 747 return NV_FALSE; 748 } 749 } 750 751 return NV_TRUE; 752 } 753 754