1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 // FIXME XXX 25 #define NVOC_KERNEL_GRAPHICS_OBJECT_H_PRIVATE_ACCESS_ALLOWED 26 27 #define NVOC_KERNEL_GRAPHICS_CONTEXT_H_PRIVATE_ACCESS_ALLOWED 28 29 #include "kernel/gpu/gr/kernel_graphics_context.h" 30 #include "kernel/gpu/gr/kernel_graphics_manager.h" 31 #include "kernel/gpu/gr/kernel_graphics.h" 32 #include "kernel/gpu/fifo/kernel_channel.h" 33 #include "kernel/gpu/fifo/kernel_channel_group.h" 34 #include "kernel/gpu/fifo/kernel_channel_group_api.h" 35 #include "kernel/mem_mgr/gpu_vaspace.h" 36 #include "kernel/gpu/mem_mgr/mem_mgr.h" 37 #include "kernel/gpu/mem_sys/kern_mem_sys.h" 38 #include "kernel/core/locks.h" 39 #include "kernel/gpu/nvlink/kernel_nvlink.h" 40 #include "vgpu/rpc.h" 41 #include "kernel/gpu/subdevice/subdevice.h" 42 #include "kernel/virtualization/hypervisor/hypervisor.h" 43 #include "gpu/mem_mgr/virt_mem_allocator.h" 44 #include "gpu/mmu/kern_gmmu.h" 45 #include "rmapi/client.h" 46 47 /*! 48 * @brief Retrieve the context object from a KernelChannel. 49 * 50 * @return NV_ERR_OBJECT_NOT_FOUND if context object is missing 51 */ 52 NV_STATUS 53 kgrctxFromKernelChannel_IMPL 54 ( 55 KernelChannel *pKernelChannel, 56 KernelGraphicsContext **ppKernelGraphicsContext 57 ) 58 { 59 RsResourceRef *pResourceRef; 60 61 NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_ARGUMENT); 62 63 if (pKernelChannel->hKernelGraphicsContext != NV01_NULL_OBJECT) 64 { 65 NV_ASSERT_OK_OR_RETURN( 66 clientGetResourceRefByType(RES_GET_CLIENT(pKernelChannel), 67 pKernelChannel->hKernelGraphicsContext, 68 classId(KernelGraphicsContext), 69 &pResourceRef)); 70 } 71 else if ((pKernelChannel->pKernelChannelGroupApi != NULL) && 72 (pKernelChannel->pKernelChannelGroupApi->hKernelGraphicsContext != NV01_NULL_OBJECT)) 73 { 74 NV_ASSERT_OK_OR_RETURN( 75 clientGetResourceRefByType(RES_GET_CLIENT(pKernelChannel), 76 pKernelChannel->pKernelChannelGroupApi->hKernelGraphicsContext, 77 classId(KernelGraphicsContext), 78 &pResourceRef)); 79 } 80 else 81 { 82 return NV_ERR_OBJECT_NOT_FOUND; 83 } 84 85 *ppKernelGraphicsContext = dynamicCast(pResourceRef->pResource, KernelGraphicsContext); 86 NV_ASSERT_OR_RETURN(*ppKernelGraphicsContext != NULL, NV_ERR_INVALID_STATE); 87 88 return NV_OK; 89 } 90 91 /*! 92 * @brief Retrieve the context object from a ChannelGroup. 93 * 94 * @return NV_ERR_OBJECT_NOT_FOUND if context object is missing 95 */ 96 NV_STATUS 97 kgrctxFromKernelChannelGroupApi_IMPL 98 ( 99 KernelChannelGroupApi *pKernelChannelGroupApi, 100 KernelGraphicsContext **ppKernelGraphicsContext 101 ) 102 { 103 RsResourceRef *pResourceRef; 104 105 NV_ASSERT_OR_RETURN(pKernelChannelGroupApi != NULL, NV_ERR_INVALID_ARGUMENT); 106 107 if (pKernelChannelGroupApi->hKernelGraphicsContext == NV01_NULL_OBJECT) 108 return NV_ERR_OBJECT_NOT_FOUND; 109 110 NV_ASSERT_OK_OR_RETURN( 111 clientGetResourceRefByType(RES_GET_CLIENT(pKernelChannelGroupApi), 112 pKernelChannelGroupApi->hKernelGraphicsContext, 113 classId(KernelGraphicsContext), 114 &pResourceRef)); 115 116 *ppKernelGraphicsContext = dynamicCast(pResourceRef->pResource, KernelGraphicsContext); 117 NV_ASSERT_OR_RETURN(*ppKernelGraphicsContext != NULL, NV_ERR_INVALID_STATE); 118 119 return NV_OK; 120 } 121 122 /** 123 * @brief Handle NV0090 ctrl call forwarding. The current control call is 124 * dispatched to the KernelGraphicsContext object provided. 125 */ 126 NV_STATUS kgrctxCtrlHandle 127 ( 128 CALL_CONTEXT *pCallContext, 129 NvHandle hKernelGraphicsContext 130 ) 131 { 132 RsResourceRef *pResourceRef; 133 134 NV_ASSERT_OK_OR_RETURN( 135 clientGetResourceRefByType(pCallContext->pClient, 136 hKernelGraphicsContext, 137 classId(KernelGraphicsContext), 138 &pResourceRef)); 139 140 return resControl(pResourceRef->pResource, pCallContext, pCallContext->pControlParams); 141 } 142 143 /** 144 * @brief Translate global ctx buffer enum to external NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID 145 */ 146 NV_STATUS 147 kgrctxGetGlobalContextBufferExternalId_IMPL 148 ( 149 GR_GLOBALCTX_BUFFER id, 150 NvU32 *pExternalId 151 ) 152 { 153 NV_ASSERT_OR_RETURN(pExternalId != NULL, NV_ERR_INVALID_ARGUMENT); 154 NV_ASSERT_OR_RETURN(NV_ENUM_IS(GR_GLOBALCTX_BUFFER, id), NV_ERR_INVALID_ARGUMENT); 155 156 switch (id) 157 { 158 case GR_GLOBALCTX_BUFFER_BUNDLE_CB: 159 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_BUFFER_BUNDLE_CB; 160 break; 161 case GR_GLOBALCTX_BUFFER_PAGEPOOL: 162 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PAGEPOOL; 163 break; 164 case GR_GLOBALCTX_BUFFER_ATTRIBUTE_CB: 165 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_ATTRIBUTE_CB; 166 break; 167 case GR_GLOBALCTX_BUFFER_RTV_CB: 168 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_RTV_CB_GLOBAL; 169 break; 170 case GR_GLOBALCTX_BUFFER_GFXP_POOL: 171 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_POOL; 172 break; 173 case GR_GLOBALCTX_BUFFER_GFXP_CTRL_BLK: 174 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_CTRL_BLK; 175 break; 176 case GR_GLOBALCTX_BUFFER_FECS_EVENT: 177 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_FECS_EVENT; 178 break; 179 case GR_GLOBALCTX_BUFFER_UNRESTRICTED_PRIV_ACCESS_MAP: 180 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_UNRESTRICTED_PRIV_ACCESS_MAP; 181 break; 182 case GR_GLOBALCTX_BUFFER_PRIV_ACCESS_MAP: 183 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PRIV_ACCESS_MAP; 184 break; 185 case GR_GLOBAL_BUFFER_GLOBAL_PRIV_ACCESS_MAP: 186 // TODO this is not valid, this is not a context buffer 187 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GLOBAL_PRIV_ACCESS_MAP; 188 break; 189 // 190 // No default case: Compiler will enforce that this switch is updated if 191 // new global ctx buffers are added 192 // 193 } 194 195 return NV_OK; 196 } 197 198 /** 199 * @brief Translate NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID to global ctx buffer enum 200 */ 201 NV_STATUS 202 kgrctxGetGlobalContextBufferInternalId_IMPL 203 ( 204 NvU32 externalId, 205 GR_GLOBALCTX_BUFFER *pInternalId 206 ) 207 { 208 NV_ASSERT_OR_RETURN(pInternalId != NULL, NV_ERR_INVALID_ARGUMENT); 209 210 switch (externalId) 211 { 212 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_MAIN: 213 // fall through 214 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PM: 215 // fall through 216 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PATCH: 217 return NV_ERR_INVALID_ARGUMENT; 218 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_BUFFER_BUNDLE_CB: 219 *pInternalId = GR_GLOBALCTX_BUFFER_BUNDLE_CB; 220 break; 221 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PAGEPOOL: 222 *pInternalId = GR_GLOBALCTX_BUFFER_PAGEPOOL; 223 break; 224 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_ATTRIBUTE_CB: 225 *pInternalId = GR_GLOBALCTX_BUFFER_ATTRIBUTE_CB; 226 break; 227 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_RTV_CB_GLOBAL: 228 *pInternalId = GR_GLOBALCTX_BUFFER_RTV_CB; 229 break; 230 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_POOL: 231 *pInternalId = GR_GLOBALCTX_BUFFER_GFXP_POOL; 232 break; 233 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_CTRL_BLK: 234 *pInternalId = GR_GLOBALCTX_BUFFER_GFXP_CTRL_BLK; 235 break; 236 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_FECS_EVENT: 237 *pInternalId = GR_GLOBALCTX_BUFFER_FECS_EVENT; 238 break; 239 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PRIV_ACCESS_MAP: 240 *pInternalId = GR_GLOBALCTX_BUFFER_PRIV_ACCESS_MAP; 241 break; 242 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_UNRESTRICTED_PRIV_ACCESS_MAP: 243 *pInternalId = GR_GLOBALCTX_BUFFER_UNRESTRICTED_PRIV_ACCESS_MAP; 244 break; 245 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GLOBAL_PRIV_ACCESS_MAP: 246 *pInternalId = GR_GLOBAL_BUFFER_GLOBAL_PRIV_ACCESS_MAP; 247 break; 248 default: 249 return NV_ERR_INVALID_ARGUMENT; 250 } 251 252 return NV_OK; 253 } 254 255 /*! Translate GR_CTX_BUFFER to NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID */ 256 NV_STATUS 257 kgrctxCtxBufferToFifoEngineId_IMPL 258 ( 259 GR_CTX_BUFFER buffer, 260 NvU32 *pFifoEngineId 261 ) 262 { 263 NV_ASSERT_OR_RETURN(NV_ENUM_IS(GR_CTX_BUFFER, buffer), NV_ERR_INVALID_ARGUMENT); 264 265 switch (buffer) 266 { 267 case GR_CTX_BUFFER_MAIN: 268 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS; 269 break; 270 case GR_CTX_BUFFER_ZCULL: 271 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_ZCULL; 272 break; 273 case GR_CTX_BUFFER_PM: 274 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PM; 275 break; 276 case GR_CTX_BUFFER_PREEMPT: 277 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PREEMPT; 278 break; 279 case GR_CTX_BUFFER_SPILL: 280 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_SPILL; 281 break; 282 case GR_CTX_BUFFER_BETA_CB: 283 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_BETACB; 284 break; 285 case GR_CTX_BUFFER_PAGEPOOL: 286 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PAGEPOOL; 287 break; 288 case GR_CTX_BUFFER_RTV_CB: 289 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_RTV; 290 break; 291 case GR_CTX_BUFFER_PATCH: 292 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PATCH; 293 break; 294 // No default case - Compiler enforces switch update if enum is changed 295 } 296 297 return NV_OK; 298 } 299 300 /** 301 * @brief Translate global ctx buffer enum to NV0080 FIFO engine context properties index 302 */ 303 NV_STATUS 304 kgrctxGlobalCtxBufferToFifoEngineId_IMPL 305 ( 306 GR_GLOBALCTX_BUFFER buffId, 307 NvU32 *pFifoEngineId 308 ) 309 { 310 NV_ASSERT_OR_RETURN(pFifoEngineId != NULL, NV_ERR_INVALID_ARGUMENT); 311 NV_ASSERT_OR_RETURN(NV_ENUM_IS(GR_GLOBALCTX_BUFFER, buffId), NV_ERR_INVALID_ARGUMENT); 312 313 switch (buffId) 314 { 315 case GR_GLOBALCTX_BUFFER_BUNDLE_CB: 316 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_BUNDLE_CB; 317 break; 318 case GR_GLOBALCTX_BUFFER_PAGEPOOL: 319 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PAGEPOOL_GLOBAL; 320 break; 321 case GR_GLOBALCTX_BUFFER_ATTRIBUTE_CB: 322 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_ATTRIBUTE_CB; 323 break; 324 case GR_GLOBALCTX_BUFFER_RTV_CB: 325 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_RTV_CB_GLOBAL; 326 break; 327 case GR_GLOBALCTX_BUFFER_GFXP_POOL: 328 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_GFXP_POOL; 329 break; 330 case GR_GLOBALCTX_BUFFER_GFXP_CTRL_BLK: 331 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_GFXP_CTRL_BLK; 332 break; 333 case GR_GLOBALCTX_BUFFER_FECS_EVENT: 334 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_FECS_EVENT; 335 break; 336 case GR_GLOBALCTX_BUFFER_PRIV_ACCESS_MAP: 337 // fall-through 338 case GR_GLOBALCTX_BUFFER_UNRESTRICTED_PRIV_ACCESS_MAP: 339 // fall-through 340 case GR_GLOBAL_BUFFER_GLOBAL_PRIV_ACCESS_MAP: 341 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PRIV_ACCESS_MAP; 342 break; 343 // No default case - compiler enforces update if enum changes 344 } 345 346 return NV_OK; 347 } 348 349 NV_STATUS 350 kgrctxGetGidInfoInPlace_IMPL 351 ( 352 OBJGPU *pGpu, 353 NvU8 *pUuidBuffer, 354 NvU32 uuidBufferSize, 355 NvU32 flags 356 ) 357 { 358 NvU8 *pUuid; 359 NvU32 uuidLength; 360 NV_STATUS status; 361 362 // on success, allocates memory for uuid 363 status = gpuGetGidInfo(pGpu, &pUuid, &uuidLength, flags); 364 if (status != NV_OK) 365 { 366 return status; 367 } 368 369 if (uuidLength == uuidBufferSize) 370 { 371 portMemCopy(pUuidBuffer, uuidBufferSize, pUuid, uuidLength); 372 } 373 374 portMemFree(pUuid); 375 376 if (uuidLength != uuidBufferSize) 377 { 378 return NV_ERR_INVALID_ARGUMENT; 379 } 380 381 return NV_OK; 382 } 383 384 NV_STATUS 385 kgrctxFillCtxBufferInfo_IMPL 386 ( 387 MEMORY_DESCRIPTOR *pMemDesc, 388 NvU32 externalId, 389 NvBool bBufferGlobal, 390 NV2080_CTRL_GR_CTX_BUFFER_INFO *pCtxBufferInfo 391 ) 392 { 393 NvU64 pageSize; 394 NV_STATUS status; 395 396 MEMORY_DESCRIPTOR *pRootMemDesc = memdescGetRootMemDesc(pMemDesc, NULL); 397 398 pCtxBufferInfo->bufferHandle = NV_PTR_TO_NvP64(pMemDesc); 399 pCtxBufferInfo->bufferType = externalId; 400 pCtxBufferInfo->bIsContigous = memdescGetContiguity(pMemDesc, AT_GPU); 401 pCtxBufferInfo->aperture = memdescGetAddressSpace(pMemDesc); 402 pCtxBufferInfo->pageCount = pMemDesc->PageCount; 403 pCtxBufferInfo->kind = memdescGetPteKindForGpu(pMemDesc, pMemDesc->pGpu); 404 405 { 406 NvU64 physAddr; 407 GMMU_APERTURE aperture = kgmmuGetExternalAllocAperture(pCtxBufferInfo->aperture); 408 409 memdescGetPhysAddrsForGpu(pMemDesc, pMemDesc->pGpu, 410 AT_GPU, 0, 0, 1, 411 &physAddr); 412 413 pCtxBufferInfo->physAddr = 414 kgmmuEncodePhysAddr(GPU_GET_KERNEL_GMMU(pMemDesc->pGpu), aperture, physAddr, 415 NVLINK_INVALID_FABRIC_ADDR); 416 } 417 418 pageSize = memdescGetPageSize(pMemDesc, AT_GPU); 419 if (pageSize == 0) 420 { 421 status = memmgrSetMemDescPageSize_HAL(pMemDesc->pGpu, 422 GPU_GET_MEMORY_MANAGER(pMemDesc->pGpu), 423 pMemDesc, 424 AT_GPU, 425 RM_ATTR_PAGE_SIZE_DEFAULT); 426 if (status != NV_OK) 427 return status; 428 429 pageSize = memdescGetPageSize(pMemDesc, AT_GPU); 430 NV_ASSERT(pageSize != 0); 431 } 432 433 // 434 // Alignment is used to adjust the mapping VA. Hence, we need to make sure 435 // that at least pageSize to make mapping calculation work correctly. 436 // 437 pCtxBufferInfo->alignment = (pMemDesc->Alignment != 0) ? 438 NV_ALIGN_UP(pMemDesc->Alignment, pageSize) : pageSize; 439 440 pCtxBufferInfo->size = pMemDesc->ActualSize; 441 pCtxBufferInfo->pageSize = pageSize; 442 443 pCtxBufferInfo->bGlobalBuffer = bBufferGlobal; 444 pCtxBufferInfo->bLocalBuffer = !bBufferGlobal; 445 pCtxBufferInfo->bDeviceDescendant = pRootMemDesc->pGpu != NULL; 446 447 if (pCtxBufferInfo->bDeviceDescendant) 448 { 449 status = kgrctxGetGidInfoInPlace(pMemDesc->pGpu, pCtxBufferInfo->uuid, sizeof(pCtxBufferInfo->uuid), 450 DRF_DEF(2080_GPU_CMD, _GPU_GET_GID_FLAGS, _TYPE, _SHA1) | 451 DRF_DEF(2080_GPU_CMD, _GPU_GET_GID_FLAGS, _FORMAT, _BINARY)); 452 if (status != NV_OK) 453 return status; 454 } 455 456 return NV_OK; 457 } 458 459 /*! 460 * @brief Construct dupable kernel graphics context. 461 */ 462 NV_STATUS 463 kgrctxConstruct_IMPL 464 ( 465 KernelGraphicsContext *pKernelGraphicsContext, 466 CALL_CONTEXT *pCallContext, 467 RS_RES_ALLOC_PARAMS_INTERNAL *pParams 468 ) 469 { 470 NV_STATUS status = NV_OK; 471 RsShared *pShared; 472 473 if (RS_IS_COPY_CTOR(pParams)) 474 return kgrctxCopyConstruct_IMPL(pKernelGraphicsContext, pCallContext, pParams); 475 476 NV_ASSERT_OK_OR_GOTO(status, 477 serverAllocShare(&g_resServ, classInfo(KernelGraphicsContextShared), &pShared), 478 cleanup); 479 480 pKernelGraphicsContext->pShared = dynamicCast(pShared, KernelGraphicsContextShared); 481 NV_ASSERT_OK_OR_GOTO(status, 482 shrkgrctxInit(GPU_RES_GET_GPU(pKernelGraphicsContext), 483 pKernelGraphicsContext->pShared, 484 pKernelGraphicsContext), 485 cleanup); 486 487 cleanup: 488 if (status != NV_OK) 489 { 490 if (pKernelGraphicsContext->pShared != NULL) 491 serverFreeShare(&g_resServ, pShared); 492 493 } 494 495 return status; 496 } 497 498 /*! 499 * @brief Copy Construct dupable kernel graphics context. 500 */ 501 NV_STATUS 502 kgrctxCopyConstruct_IMPL 503 ( 504 KernelGraphicsContext *pKernelGraphicsContextDst, 505 CALL_CONTEXT *pCallContext, 506 RS_RES_ALLOC_PARAMS_INTERNAL *pParams 507 ) 508 { 509 RsResourceRef *pSrcRef = pParams->pSrcRef; 510 KernelGraphicsContext *pKernelGraphicsContextSrc = 511 dynamicCast(pSrcRef->pResource, KernelGraphicsContext); 512 513 pKernelGraphicsContextDst->pShared = pKernelGraphicsContextSrc->pShared; 514 serverRefShare(&g_resServ, staticCast(pKernelGraphicsContextDst->pShared, RsShared)); 515 516 return NV_OK; 517 } 518 519 NvHandle 520 kgrctxGetInternalObjectHandle_IMPL(KernelGraphicsContext *pKernelGraphicsContext) 521 { 522 return NV01_NULL_OBJECT; 523 } 524 525 /*! 526 * @brief Destruct dupable kernel graphics context. 527 */ 528 void 529 kgrctxDestruct_IMPL 530 ( 531 KernelGraphicsContext *pKernelGraphicsContext 532 ) 533 { 534 RsShared *pShared = staticCast(pKernelGraphicsContext->pShared, RsShared); 535 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelGraphicsContext); 536 537 if (serverGetShareRefCount(&g_resServ, pShared) == 1) 538 { 539 shrkgrctxTeardown_IMPL(pGpu, pKernelGraphicsContext->pShared, pKernelGraphicsContext); 540 } 541 542 serverFreeShare(&g_resServ, pShared); 543 } 544 545 /*! 546 * @brief Retrieve unicast context state 547 */ 548 NV_STATUS kgrctxGetUnicast_IMPL 549 ( 550 OBJGPU *pGpu, 551 KernelGraphicsContext *pKernelGraphicsContext, 552 KernelGraphicsContextUnicast **ppKernelGraphicsContextUnicast 553 ) 554 { 555 *ppKernelGraphicsContextUnicast = NULL; 556 NV_ASSERT_OR_RETURN(pKernelGraphicsContext->pShared != NULL, NV_ERR_INVALID_STATE); 557 *ppKernelGraphicsContextUnicast = &pKernelGraphicsContext->pShared->kernelGraphicsContextUnicast; 558 return NV_OK; 559 } 560 561 /*! 562 * @brief Query the details of MMU faults caused by this context 563 */ 564 NV_STATUS 565 kgrctxLookupMmuFaultInfo_IMPL 566 ( 567 OBJGPU *pGpu, 568 KernelGraphicsContext *pKernelGraphicsContext, 569 NV83DE_CTRL_DEBUG_READ_MMU_FAULT_INFO_PARAMS *pParams 570 ) 571 { 572 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 573 const NvU32 size = NV_ARRAY_ELEMENTS(pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfoList); 574 NvU32 i; 575 576 NV_ASSERT_OK_OR_RETURN( 577 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast)); 578 579 NV_ASSERT_OR_RETURN(pKernelGraphicsContextUnicast->mmuFault.head < size, NV_ERR_INVALID_STATE); 580 NV_ASSERT_OR_RETURN(pKernelGraphicsContextUnicast->mmuFault.tail < size, NV_ERR_INVALID_STATE); 581 582 pParams->count = 0; 583 for (i = pKernelGraphicsContextUnicast->mmuFault.tail; 584 i != pKernelGraphicsContextUnicast->mmuFault.head; 585 i = (i + 1) % size) 586 { 587 pParams->mmuFaultInfoList[pParams->count] = pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfoList[i]; 588 pParams->count++; 589 } 590 591 return NV_OK; 592 } 593 594 /*! 595 * @brief Query the details of MMU faults caused by this context 596 */ 597 NV_STATUS 598 kgrctxLookupMmuFault_IMPL 599 ( 600 OBJGPU *pGpu, 601 KernelGraphicsContext *pKernelGraphicsContext, 602 NV83DE_MMU_FAULT_INFO *pMmuFaultInfo 603 ) 604 { 605 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 606 607 NV_ASSERT_OK_OR_RETURN( 608 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast)); 609 610 *pMmuFaultInfo = pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfo; 611 return NV_OK; 612 } 613 614 /*! 615 * @brief clear the details of MMU faults caused by this context 616 */ 617 NV_STATUS 618 kgrctxClearMmuFault_IMPL 619 ( 620 OBJGPU *pGpu, 621 KernelGraphicsContext *pKernelGraphicsContext 622 ) 623 { 624 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 625 626 NV_ASSERT_OK_OR_RETURN( 627 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast)); 628 629 pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfo.valid = NV_FALSE; 630 pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfo.faultInfo = 0; 631 return NV_OK; 632 } 633 634 /*! 635 * @brief Record the details of an MMU fault caused by this context 636 */ 637 void 638 kgrctxRecordMmuFault_IMPL 639 ( 640 OBJGPU *pGpu, 641 KernelGraphicsContext *pKernelGraphicsContext, 642 NvU32 mmuFaultInfo, 643 NvU64 mmuFaultAddress, 644 NvU32 mmuFaultType, 645 NvU32 mmuFaultAccessType 646 ) 647 { 648 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 649 NV_STATUS status; 650 651 NV_ASSERT_OK_OR_ELSE(status, 652 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast), 653 return;); 654 655 pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfo.valid = NV_TRUE; 656 pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfo.faultInfo = mmuFaultInfo; 657 658 { 659 const NvU32 size = NV_ARRAY_ELEMENTS(pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfoList); 660 661 NV_ASSERT_OR_RETURN_VOID(pKernelGraphicsContextUnicast->mmuFault.head < size); 662 NV_ASSERT_OR_RETURN_VOID(pKernelGraphicsContextUnicast->mmuFault.tail < size); 663 664 pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfoList[pKernelGraphicsContextUnicast->mmuFault.head].faultAddress = mmuFaultAddress; 665 pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfoList[pKernelGraphicsContextUnicast->mmuFault.head].faultType = mmuFaultType; 666 pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfoList[pKernelGraphicsContextUnicast->mmuFault.head].accessType = mmuFaultAccessType; 667 pKernelGraphicsContextUnicast->mmuFault.head = (pKernelGraphicsContextUnicast->mmuFault.head + 1) % size; 668 if (pKernelGraphicsContextUnicast->mmuFault.head == pKernelGraphicsContextUnicast->mmuFault.tail) 669 pKernelGraphicsContextUnicast->mmuFault.tail = (pKernelGraphicsContextUnicast->mmuFault.tail + 1) % size; 670 } 671 } 672 673 /*! 674 * @brief Determine whether channels in this context are associated with GR engine 675 * 676 * @returns NV_TRUE if passed channel is allocated on GR, and is on GR runlist 677 */ 678 NvBool 679 kgrctxIsValid_IMPL 680 ( 681 OBJGPU *pGpu, 682 KernelGraphicsContext *pKernelGraphicsContext, 683 KernelChannel *pKernelChannel 684 ) 685 { 686 NvU32 runlistId; 687 RM_ENGINE_TYPE engineType; 688 689 // TODO remove pKernelChannel from params 690 691 if (RM_ENGINE_TYPE_IS_VALID(kchannelGetEngineType(pKernelChannel)) && 692 !RM_ENGINE_TYPE_IS_GR(kchannelGetEngineType(pKernelChannel))) 693 { 694 return NV_FALSE; 695 } 696 697 NV_CHECK_OR_RETURN(LEVEL_INFO, kchannelIsRunlistSet(pGpu, pKernelChannel), NV_FALSE); 698 699 runlistId = kchannelGetRunlistId(pKernelChannel); 700 NV_ASSERT_OK( 701 kfifoEngineInfoXlate_HAL(pGpu, GPU_GET_KERNEL_FIFO(pGpu), 702 ENGINE_INFO_TYPE_RUNLIST, runlistId, 703 ENGINE_INFO_TYPE_RM_ENGINE_TYPE, (NvU32 *) &engineType)); 704 705 NV_CHECK_OR_RETURN(LEVEL_INFO, RM_ENGINE_TYPE_IS_GR(engineType), NV_FALSE); 706 707 return NV_TRUE; 708 } 709 710 /*! 711 * @brief returns if the main context buffer has been allocated 712 * 713 * @return NV_TRUE if allocated, NV_FALSE otherwise 714 */ 715 NvBool 716 kgrctxIsMainContextAllocated_IMPL 717 ( 718 OBJGPU *pGpu, 719 KernelGraphicsContext *pKernelGraphicsContext 720 ) 721 { 722 MEMORY_DESCRIPTOR *pMemDesc; 723 return (kgrctxGetMainContextBuffer(pGpu, pKernelGraphicsContext, &pMemDesc) == NV_OK) && 724 (pMemDesc != NULL); 725 } 726 727 /*! Retrieve the memdesc containing the main ctx buffer */ 728 NV_STATUS kgrctxGetMainContextBuffer_IMPL 729 ( 730 OBJGPU *pGpu, 731 KernelGraphicsContext *pKernelGraphicsContext, 732 MEMORY_DESCRIPTOR **ppMemDesc 733 ) 734 { 735 ENGINE_CTX_DESCRIPTOR *pEngCtxDesc; 736 NvU32 subDevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu); 737 RsResourceRef *pParentRef = RES_GET_REF(pKernelGraphicsContext)->pParentRef; 738 KernelChannelGroupApi *pKernelChannelGroupApi; 739 KernelChannelGroup *pKernelChannelGroup; 740 741 *ppMemDesc = NULL; 742 pKernelChannelGroupApi = dynamicCast(pParentRef->pResource, KernelChannelGroupApi); 743 pKernelChannelGroup = pKernelChannelGroupApi->pKernelChannelGroup; 744 pEngCtxDesc = pKernelChannelGroup->ppEngCtxDesc[subDevInst]; 745 746 if (pEngCtxDesc != NULL) 747 *ppMemDesc = pEngCtxDesc->pMemDesc; 748 749 return NV_OK; 750 } 751 752 /*! 753 * @brief Retrieve information about the context buffers 754 */ 755 NV_STATUS 756 kgrctxGetCtxBuffers_IMPL 757 ( 758 OBJGPU *pGpu, 759 KernelGraphicsContext *pKernelGraphicsContext, 760 KernelGraphics *pKernelGraphics, 761 NvU32 gfid, 762 NvU32 bufferCount, 763 MEMORY_DESCRIPTOR **ppBuffers, 764 NvU32 *pCtxBufferType, 765 NvU32 *pBufferCountOut, 766 NvU32 *pFirstGlobalBuffer 767 ) 768 { 769 GR_GLOBALCTX_BUFFERS *pGlobalCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid); 770 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 771 MEMORY_DESCRIPTOR *pGrCtxBufferMemDesc; 772 NvU32 bufferCountOut = 0; 773 NvU32 i; 774 775 NV_ASSERT_OK_OR_RETURN( 776 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast)); 777 778 // Get local context buffer memdesc. 779 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 780 kgrctxGetMainContextBuffer(pGpu, pKernelGraphicsContext, &pGrCtxBufferMemDesc)); 781 782 NV_CHECK_OR_RETURN(LEVEL_ERROR, 783 pGrCtxBufferMemDesc != NULL, 784 NV_ERR_INVALID_OBJECT); 785 786 NV_CHECK_OR_RETURN(LEVEL_INFO, bufferCountOut < bufferCount, NV_ERR_INVALID_ARGUMENT); 787 pCtxBufferType[bufferCountOut] = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_MAIN; 788 ppBuffers[bufferCountOut++] = pGrCtxBufferMemDesc; 789 790 // Get context patch buffer memdesc. 791 NV_CHECK_OR_RETURN(LEVEL_SILENT, 792 pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc != NULL, 793 NV_ERR_INVALID_STATE); 794 795 NV_CHECK_OR_RETURN(LEVEL_INFO, bufferCountOut < bufferCount, NV_ERR_INVALID_ARGUMENT); 796 pCtxBufferType[bufferCountOut] = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PATCH; 797 ppBuffers[bufferCountOut++] = pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc; 798 799 // Add PM ctxsw buffer if it's allocated. 800 if (pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc != NULL) 801 { 802 NV_CHECK_OR_RETURN(LEVEL_INFO, bufferCountOut < bufferCount, NV_ERR_INVALID_ARGUMENT); 803 pCtxBufferType[bufferCountOut] = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PM; 804 ppBuffers[bufferCountOut++] = pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc; 805 } 806 807 if (pFirstGlobalBuffer != NULL) 808 { 809 *pFirstGlobalBuffer = bufferCountOut; 810 } 811 812 // Add global buffers. 813 for (i = 0; i < GR_GLOBALCTX_BUFFER_COUNT; i++) 814 { 815 MEMORY_DESCRIPTOR *pMemDesc = pGlobalCtxBuffers->memDesc[i]; 816 if (pMemDesc != NULL) 817 { 818 NV_CHECK_OR_RETURN(LEVEL_INFO, bufferCountOut < bufferCount, NV_ERR_INVALID_ARGUMENT); 819 NV_ASSERT_OK( 820 kgrctxGetGlobalContextBufferExternalId(i, &pCtxBufferType[bufferCountOut])); 821 ppBuffers[bufferCountOut++] = pMemDesc; 822 } 823 } 824 825 *pBufferCountOut = bufferCountOut; 826 827 return NV_OK; 828 } 829 830 /* 831 * @brief Get maximum context buffer count including global and local buffers 832 * 833 * @param[in] pGpu 834 * @param[in] pKernelGraphicsContext 835 * @param[in] pKernelGraphics 836 * @param[out] pBufferCount Max context buffer count 837 */ 838 NV_STATUS 839 kgrctxGetBufferCount_IMPL 840 ( 841 OBJGPU *pGpu, 842 KernelGraphicsContext *pKernelGraphicsContext, 843 KernelGraphics *pKernelGraphics, 844 NvU32 *pBufferCount 845 ) 846 { 847 NvU32 gfid; 848 NvBool bCallingContextPlugin; 849 GR_GLOBALCTX_BUFFERS *pGlobalCtxBuffers; 850 NvU32 i; 851 852 *pBufferCount = 0; 853 854 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &gfid)); 855 NV_ASSERT_OK_OR_RETURN(vgpuIsCallingContextPlugin(pGpu, &bCallingContextPlugin)); 856 857 pGlobalCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid); 858 859 if (bCallingContextPlugin) 860 { 861 gfid = GPU_GFID_PF; 862 } 863 864 for (i = 0; i < GR_GLOBALCTX_BUFFER_COUNT; i++) 865 { 866 if (pGlobalCtxBuffers->memDesc[i] != NULL) 867 (*pBufferCount)++; 868 } 869 870 // Increment by 3 to include local ctx buffer, patch context buffer and PM Ctxsw buffer 871 *pBufferCount += 3; 872 873 return NV_OK; 874 } 875 876 /* 877 * @brief Get context buffer info like size, alignment for global and 878 * local buffers 879 * 880 * @param[in] pGpu 881 * @param[in] pKernelGraphicsContext 882 * @param[in] pKernelGraphics 883 * @param[in] gfid 884 * @param[in] bufferMaxCount Amount of space provided in pCtxBufferInfo 885 * @param[out] bufferCount Number of buffers described 886 * @param[out] pCtxBufferInfo Structure to fill buffer information 887 * 888 * @return NV_OK successfully probed all buffers 889 * @return NV_ERR_INVALID_ARGUMENT not enough space provided for buffers queried 890 */ 891 NV_STATUS 892 kgrctxGetCtxBufferInfo_IMPL 893 ( 894 OBJGPU *pGpu, 895 KernelGraphicsContext *pKernelGraphicsContext, 896 KernelGraphics *pKernelGraphics, 897 NvU32 gfid, 898 NvU32 bufferMaxCount, 899 NvU32 *pBufferCount, 900 NV2080_CTRL_GR_CTX_BUFFER_INFO *pCtxBufferInfo 901 ) 902 { 903 MEMORY_DESCRIPTOR *pMemDescArray[3 + GR_GLOBALCTX_BUFFER_COUNT]; 904 NvU32 bufferExternalId[3 + GR_GLOBALCTX_BUFFER_COUNT]; 905 NvU32 memdescCount; 906 NvU32 firstGlobalBuffer; 907 NvU32 i; 908 NvU32 j; 909 910 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 911 kgrctxGetCtxBuffers(pGpu, pKernelGraphicsContext, pKernelGraphics, 912 gfid, 913 NV_ARRAY_ELEMENTS(pMemDescArray), 914 pMemDescArray, 915 bufferExternalId, 916 &memdescCount, 917 &firstGlobalBuffer)); 918 919 if (bufferMaxCount < memdescCount) 920 return NV_ERR_INVALID_ARGUMENT; 921 922 for (i = 0; i < memdescCount; i++) 923 { 924 NvBool bGlobalBuffer = (i >= firstGlobalBuffer); 925 926 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, 927 kgrctxFillCtxBufferInfo(pMemDescArray[i], 928 bufferExternalId[i], 929 bGlobalBuffer, 930 &pCtxBufferInfo[i])); 931 } 932 933 // 934 // Sort the buffer info in descending order using alignment so that the 935 // VA range calculation can figure out the optimal VA range size. 936 // 937 for (i = 0; i < memdescCount; i++) 938 { 939 for (j = 0; j < memdescCount - 1; j++) 940 { 941 if (pCtxBufferInfo[j].alignment < pCtxBufferInfo[j + 1].alignment) 942 { 943 NV2080_CTRL_GR_CTX_BUFFER_INFO tmp; 944 945 portMemCopy(&tmp, sizeof(tmp), &pCtxBufferInfo[j], sizeof(tmp)); 946 portMemCopy(&pCtxBufferInfo[j], sizeof(tmp), &pCtxBufferInfo[j + 1], sizeof(tmp)); 947 portMemCopy(&pCtxBufferInfo[j + 1], sizeof(tmp), &tmp, sizeof(tmp)); 948 } 949 } 950 } 951 952 *pBufferCount = memdescCount; 953 954 return NV_OK; 955 } 956 957 /* 958 * @brief Return physical addresses of context buffer starting from its 'firstPage' page. 959 * The function is intended to be called repeatedly while advancing 960 * 'firstPage' parameter until all pages of the buffer are queried. 961 * This condition is indicated by *pNoMorePage == NV_TRUE. 962 * If requested buffer is contiguous, address of first page is returned 963 * always. 964 * 965 * @param[in] pGpu 966 * @param[in] pKernelGraphicsContext 967 * @param[in] pKernelGraphics 968 * @param[in] gfid 969 * @param[in] bufferType Requested ctx buffer type 970 * @param[in] firstPage First page of the buffer to be queried 971 * @param[out] pPhysAddrs Array to be filled with page addresses 972 * @param[in] addrsSize Number of elements of pPhysAddrs 973 * @param[out] pNumPages Number of page addresses returned 974 * @param[out] pbNoMorePages End of buffer reached 975 * 976 * @return NV_OK successfully obtained requested addresses 977 * @return NV_ERR_INVALID_ARGUMENT pMemDesc does not specify a context buffer 978 */ 979 NV_STATUS 980 kgrctxGetCtxBufferPtes_IMPL 981 ( 982 OBJGPU *pGpu, 983 KernelGraphicsContext *pKernelGraphicsContext, 984 KernelGraphics *pKernelGraphics, 985 NvU32 gfid, 986 NvU32 bufferType, 987 NvU32 firstPage, 988 NvU64 *pPhysAddrs, 989 NvU32 addrsSize, 990 NvU32 *pNumPages, 991 NvBool *pbNoMorePages 992 ) 993 { 994 NvU64 bufferSize; 995 NvU64 pageSize; 996 NvU32 numPages; 997 NvU32 bufferCount; 998 NvU32 i; 999 MEMORY_DESCRIPTOR *pMemDescArray[3 + GR_GLOBALCTX_BUFFER_COUNT]; 1000 NvU32 bufferExternalId[3 + GR_GLOBALCTX_BUFFER_COUNT]; 1001 1002 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 1003 kgrctxGetCtxBuffers(pGpu, pKernelGraphicsContext, pKernelGraphics, 1004 gfid, 1005 NV_ARRAY_ELEMENTS(pMemDescArray), 1006 pMemDescArray, 1007 bufferExternalId, 1008 &bufferCount, 1009 NULL)); 1010 1011 for (i = 0; i < bufferCount; i++) 1012 { 1013 if (bufferExternalId[i] == bufferType) 1014 break; 1015 } 1016 1017 if (addrsSize == 0) 1018 { 1019 return NV_ERR_INVALID_ARGUMENT; 1020 } 1021 1022 NV_CHECK_OR_RETURN(LEVEL_ERROR, i != bufferCount, NV_ERR_INVALID_ARGUMENT); 1023 1024 bufferSize = memdescGetSize(pMemDescArray[i]); 1025 pageSize = memdescGetPageSize(pMemDescArray[i], AT_GPU); 1026 1027 if (pageSize == 0) 1028 { 1029 return NV_ERR_INVALID_STATE; 1030 } 1031 1032 numPages = NV_ROUNDUP(bufferSize, pageSize) / pageSize; 1033 1034 if (firstPage >= numPages) 1035 { 1036 numPages = 0; 1037 *pbNoMorePages = NV_TRUE; 1038 goto done; 1039 } 1040 1041 if (memdescGetContiguity(pMemDescArray[i], AT_GPU)) 1042 { 1043 firstPage = 0; 1044 numPages = 1; 1045 *pbNoMorePages = NV_TRUE; 1046 } 1047 else 1048 { 1049 numPages -= firstPage; 1050 *pbNoMorePages = (numPages <= addrsSize); 1051 numPages = NV_MIN(numPages, addrsSize); 1052 } 1053 1054 if (numPages > 0) 1055 { 1056 memdescGetPhysAddrs(pMemDescArray[i], 1057 AT_GPU, 1058 firstPage * pageSize, 1059 pageSize, 1060 numPages, 1061 pPhysAddrs); 1062 } 1063 1064 done: 1065 *pNumPages = numPages; 1066 1067 return NV_OK; 1068 } 1069 1070 /*! 1071 * This function does following things 1072 * 1. Allocate main GR context buffer 1073 * 2. Sets up GR context memory descriptor by calling kchannelSetEngineContextMemDesc 1074 */ 1075 NV_STATUS 1076 kgrctxAllocMainCtxBuffer_IMPL 1077 ( 1078 OBJGPU *pGpu, 1079 KernelGraphicsContext *pKernelGraphicsContext, 1080 KernelGraphics *pKernelGraphics, 1081 KernelChannel *pKernelChannel 1082 ) 1083 { 1084 MEMORY_DESCRIPTOR *pGrCtxBufferMemDesc = NULL; 1085 NvU32 ctxSize; 1086 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 1087 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 1088 NvU64 allocFlags = MEMDESC_FLAGS_GPU_PRIVILEGED; 1089 CTX_BUF_POOL_INFO *pCtxBufPool = NULL; 1090 NvBool bIsContiguous = kgraphicsShouldForceMainCtxContiguity_HAL(pGpu, pKernelGraphics); 1091 const GR_BUFFER_ATTR *pAttr = kgraphicsGetContextBufferAttr(pGpu, pKernelGraphics, GR_CTX_BUFFER_MAIN); 1092 1093 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 1094 1095 NV_ASSERT_OK_OR_RETURN( 1096 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast)); 1097 1098 // 1099 // Allocate space for per-subcontext headers in the context buffer. 1100 // With subcontext, per-subcontext context header is programmed in the channel instance block. 1101 // Per-subcontext headers can be seperate from the context buffer. 1102 // For the initial phase, we allocate them at the end of the context buffers for easier tracking. 1103 // This will waste some memory (256 KB), if the number of subcontexts are sparse. 1104 // Will cleanup this up later to be on-demand. 1105 // 1106 // We also need to report the updated context size to KMD for virtual context. 1107 // Bug 1764102 tracks the VC support. 1108 // 1109 NV_ASSERT_OK_OR_RETURN( 1110 kgraphicsGetMainCtxBufferSize(pGpu, pKernelGraphics, NV_TRUE, &ctxSize)); 1111 1112 if (ctxBufPoolIsSupported(pGpu) && 1113 pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->pCtxBufPool != NULL) 1114 { 1115 allocFlags |= MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL; 1116 pCtxBufPool = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->pCtxBufPool; 1117 } 1118 1119 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 1120 memdescCreate(&pGrCtxBufferMemDesc, pGpu, ctxSize, 1121 RM_PAGE_SIZE, bIsContiguous, ADDR_UNKNOWN, 1122 pAttr->cpuAttr, 1123 allocFlags | MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE)); 1124 1125 // 1126 // Force page size to 4KB, we can change this later when RM access method 1127 // support 64k pages 1128 // 1129 NV_ASSERT_OK_OR_RETURN( 1130 memmgrSetMemDescPageSize_HAL(pGpu, pMemoryManager, pGrCtxBufferMemDesc, AT_GPU, RM_ATTR_PAGE_SIZE_4KB)); 1131 1132 NV_ASSERT_OK_OR_RETURN(memdescSetCtxBufPool(pGrCtxBufferMemDesc, pCtxBufPool)); 1133 NV_ASSERT_OK_OR_RETURN( 1134 memdescAllocList(pGrCtxBufferMemDesc, pAttr->pAllocList)); 1135 1136 NV_ASSERT_OK_OR_RETURN( 1137 kchannelSetEngineContextMemDesc(pGpu, pKernelChannel, 1138 ENG_GR(kgraphicsGetInstance(pGpu, pKernelGraphics)), 1139 pGrCtxBufferMemDesc)); 1140 pKernelGraphicsContextUnicast->pMainCtxBuffer = pGrCtxBufferMemDesc; 1141 return NV_OK; 1142 } 1143 1144 /*! 1145 * @brief Allocate and setup the GR ctx patch buffer 1146 */ 1147 NV_STATUS 1148 kgrctxAllocPatchBuffer_IMPL 1149 ( 1150 OBJGPU *pGpu, 1151 KernelGraphicsContext *pKernelGraphicsContext, 1152 KernelGraphics *pKernelGraphics, 1153 KernelChannel *pKernelChannel 1154 ) 1155 { 1156 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 1157 const KGRAPHICS_STATIC_INFO *pStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 1158 NvU32 status = NV_OK; 1159 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 1160 MEMORY_DESCRIPTOR **ppMemDesc; 1161 NvU64 size; 1162 NvU64 flags = MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE; 1163 CTX_BUF_POOL_INFO *pCtxBufPool; 1164 const GR_BUFFER_ATTR *pAttr = kgraphicsGetContextBufferAttr(pGpu, pKernelGraphics, GR_CTX_BUFFER_PATCH); 1165 1166 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 1167 1168 NV_ASSERT_OR_RETURN(pStaticInfo != NULL, NV_ERR_INVALID_STATE); 1169 NV_ASSERT_OR_RETURN(pStaticInfo->pContextBuffersInfo != NULL, NV_ERR_INVALID_STATE); 1170 1171 NV_ASSERT_OK_OR_RETURN( 1172 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast)); 1173 1174 pCtxBufPool = NULL; 1175 if (ctxBufPoolIsSupported(pGpu) && 1176 (pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->pCtxBufPool) != NULL) 1177 { 1178 flags |= MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL; 1179 pCtxBufPool = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->pCtxBufPool; 1180 } 1181 1182 ppMemDesc = &pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc; 1183 size = pStaticInfo->pContextBuffersInfo->engine[NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PATCH].size; 1184 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 1185 memdescCreate(ppMemDesc, pGpu, size, RM_PAGE_SIZE, NV_TRUE, 1186 ADDR_UNKNOWN, 1187 pAttr->cpuAttr, 1188 flags)); 1189 1190 // 1191 // Force page size to 4KB we can change this later when RM access method 1192 // support 64k pages 1193 // 1194 memmgrSetMemDescPageSize_HAL(pGpu, pMemoryManager, *ppMemDesc, AT_GPU, RM_ATTR_PAGE_SIZE_4KB); 1195 NV_ASSERT_OK_OR_GOTO(status, 1196 memdescSetCtxBufPool(*ppMemDesc, pCtxBufPool), 1197 failed); 1198 1199 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, 1200 memdescAllocList(*ppMemDesc, pAttr->pAllocList), 1201 failed); 1202 1203 return NV_OK; 1204 1205 failed: 1206 memdescFree(*ppMemDesc); 1207 memdescDestroy(*ppMemDesc); 1208 *ppMemDesc = NULL; 1209 return status; 1210 } 1211 1212 /*! 1213 * @brief Allocate the local ctx PM buffer 1214 */ 1215 NV_STATUS 1216 kgrctxAllocPmBuffer_IMPL 1217 ( 1218 OBJGPU *pGpu, 1219 KernelGraphicsContext *pKernelGraphicsContext, 1220 KernelGraphics *pKernelGraphics, 1221 KernelChannel *pKernelChannel 1222 ) 1223 { 1224 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 1225 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 1226 NvU32 size; 1227 NV_STATUS status = NV_OK; 1228 NvU64 flags = MEMDESC_FLAGS_GPU_PRIVILEGED; 1229 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 1230 CTX_BUF_POOL_INFO *pCtxBufPool; 1231 MEMORY_DESCRIPTOR **ppMemDesc; 1232 const GR_BUFFER_ATTR *pAttr = kgraphicsGetContextBufferAttr(pGpu, pKernelGraphics, GR_CTX_BUFFER_PM); 1233 1234 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 1235 1236 NV_ASSERT_OK_OR_RETURN( 1237 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast)); 1238 1239 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 1240 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pContextBuffersInfo != NULL, NV_ERR_INVALID_STATE); 1241 size = pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PM].size; 1242 if (size == 0) 1243 return NV_ERR_INVALID_STATE; 1244 1245 pCtxBufPool = NULL; 1246 if (ctxBufPoolIsSupported(pGpu) && 1247 pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->pCtxBufPool != NULL) 1248 { 1249 flags |= MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL; 1250 pCtxBufPool = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->pCtxBufPool; 1251 } 1252 1253 // 1254 // For SRIOV Heavy, the PM ctxsw buffer allocation will be redirected to 1255 // host RM subheap. Subheap is used by host RM to allocate memory 1256 // on behalf of the guest(VF) context. 1257 // 1258 if (gpuIsWarBug200577889SriovHeavyEnabled(pGpu)) 1259 { 1260 flags |= MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE; 1261 } 1262 1263 ppMemDesc = &pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc; 1264 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 1265 memdescCreate(ppMemDesc, pGpu, 1266 size, 1267 RM_PAGE_SIZE, 1268 NV_TRUE, 1269 ADDR_UNKNOWN, 1270 pAttr->cpuAttr, 1271 flags)); 1272 1273 // 1274 // Force page size to 4KB we can change this later when RM access method 1275 // support 64k pages 1276 // 1277 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, 1278 memmgrSetMemDescPageSize_HAL(pGpu, pMemoryManager, *ppMemDesc, AT_GPU, RM_ATTR_PAGE_SIZE_4KB), 1279 error); 1280 1281 NV_ASSERT_OK_OR_GOTO(status, memdescSetCtxBufPool(*ppMemDesc, pCtxBufPool), error); 1282 1283 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, 1284 memdescAllocList(*ppMemDesc, pAttr->pAllocList), 1285 error); 1286 1287 return NV_OK; 1288 1289 error: 1290 memdescFree(*ppMemDesc); 1291 memdescDestroy(*ppMemDesc); 1292 *ppMemDesc = NULL; 1293 1294 return status; 1295 } 1296 1297 /*! 1298 * This function allocates and maps various GR buffers. 1299 */ 1300 NV_STATUS 1301 kgrctxAllocCtxBuffers_IMPL 1302 ( 1303 OBJGPU *pGpu, 1304 KernelGraphicsContext *pKernelGraphicsContext, 1305 KernelGraphics *pKernelGraphics, 1306 KernelGraphicsObject *pKernelGraphicsObject 1307 ) 1308 { 1309 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 1310 ChannelDescendant *pChannelDescendant = staticCast(pKernelGraphicsObject, ChannelDescendant); 1311 NV_STATUS status = NV_OK; 1312 NvU32 classNum = pChannelDescendant->resourceDesc.externalClassId; 1313 NvU32 objType; 1314 NvU32 gfid = kchannelGetGfid(pChannelDescendant->pKernelChannel); 1315 KernelChannel *pKernelChannel = pChannelDescendant->pKernelChannel; 1316 GR_GLOBALCTX_BUFFERS *pGlobalCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid); 1317 1318 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 1319 1320 NV_ASSERT_OR_RETURN(pKernelGraphicsObject->pKernelGraphicsContext != NULL, NV_ERR_INVALID_STATE); 1321 1322 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 1323 kgrctxGetUnicast(pGpu, 1324 pKernelGraphicsObject->pKernelGraphicsContext, 1325 &pKernelGraphicsContextUnicast)); 1326 1327 // Allocate the GR ctx buffer if required 1328 if (!kgrctxIsMainContextAllocated(pGpu, pKernelGraphicsContext) && 1329 !kchannelIsCtxBufferAllocSkipped(pChannelDescendant->pKernelChannel)) 1330 { 1331 if (pKernelGraphicsContextUnicast->pMainCtxBuffer == NULL) 1332 { 1333 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 1334 kgrctxAllocMainCtxBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel)); 1335 } 1336 else 1337 { 1338 // Ctx buffer was uninstalled in FIFO but not freed. Reinstall it 1339 NV_ASSERT_OK_OR_RETURN( 1340 kchannelSetEngineContextMemDesc(pGpu, pKernelChannel, 1341 ENG_GR(kgraphicsGetInstance(pGpu, pKernelGraphics)), 1342 pKernelGraphicsContextUnicast->pMainCtxBuffer)); 1343 } 1344 } 1345 1346 kgrmgrGetGrObjectType(classNum, &objType); 1347 1348 // 1349 // for each channel, we need to allocate the context patch buffer, this memory region 1350 // will be used to override settings in the context after it is restored, things 1351 // like the global ctx buffer addresses, etc. 1352 // 1353 if (pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc == NULL) 1354 { 1355 NV_ASSERT_OK_OR_RETURN( 1356 kgrctxAllocPatchBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel)); 1357 } 1358 1359 // Pre-allocate the PM ctxsw buffer, if required 1360 if (kgrctxShouldPreAllocPmBuffer_HAL(pGpu, pKernelGraphicsContext, pChannelDescendant->pKernelChannel)) 1361 { 1362 NV_ASSERT_OK_OR_RETURN( 1363 kgrctxAllocPmBuffer(pGpu, pKernelGraphicsObject->pKernelGraphicsContext, 1364 pKernelGraphics, 1365 pChannelDescendant->pKernelChannel)); 1366 } 1367 1368 // 1369 // Allocate Ctx Buffers that are local to this channel if required 1370 // and they have yet to be allocated. 1371 // 1372 if ((pKernelGraphicsContextUnicast->bVprChannel 1373 ) && !pKernelGraphicsContextUnicast->localCtxBuffer.bAllocated) 1374 { 1375 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 1376 kgraphicsAllocGrGlobalCtxBuffers_HAL(pGpu, pKernelGraphics, gfid, 1377 pKernelGraphicsObject->pKernelGraphicsContext)); 1378 } 1379 1380 // Allocate global context for this gfid if they haven't been already 1381 if (!pGlobalCtxBuffers->bAllocated) 1382 { 1383 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 1384 kgraphicsAllocGrGlobalCtxBuffers_HAL(pGpu, pKernelGraphics, gfid, NULL)); 1385 } 1386 1387 return status; 1388 } 1389 1390 // Map common buffer to a channel's context helper function 1391 NV_STATUS 1392 kgrctxMapGlobalCtxBuffer_IMPL 1393 ( 1394 OBJGPU *pGpu, 1395 KernelGraphicsContext *pKernelGraphicsContext, 1396 KernelGraphics *pKernelGraphics, 1397 NvU32 gfid, 1398 OBJVASPACE *pVAS, 1399 GR_GLOBALCTX_BUFFER buffId, 1400 NvBool bIsReadOnly 1401 ) 1402 { 1403 GR_GLOBALCTX_BUFFERS *pGlobalCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid); 1404 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 1405 NV_STATUS status = NV_OK; 1406 NvU64 vaddr; 1407 MEMORY_DESCRIPTOR *pMemDesc; 1408 1409 NV_ASSERT_OK_OR_RETURN( 1410 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast)); 1411 1412 // if we have local buffers per-channel allocated, use them, otherwise use the global buffers 1413 pMemDesc = pKernelGraphicsContextUnicast->localCtxBuffer.memDesc[buffId]; 1414 if (pMemDesc == NULL) 1415 { 1416 pMemDesc = pGlobalCtxBuffers->memDesc[buffId]; 1417 } 1418 1419 if (pMemDesc == NULL) 1420 { 1421 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 1422 NvU32 fifoEngineId; 1423 NvU32 buffSize; 1424 1425 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 1426 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 1427 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pContextBuffersInfo != NULL, NV_ERR_INVALID_STATE); 1428 NV_ASSERT_OK_OR_RETURN(kgrctxGlobalCtxBufferToFifoEngineId(buffId, &fifoEngineId)); 1429 buffSize = pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[fifoEngineId].size; 1430 1431 if (buffSize == 0) 1432 { 1433 NV_PRINTF(LEVEL_INFO, 1434 "Could not map %s Buffer as buffer is not supported\n", 1435 NV_ENUM_TO_STRING(GR_GLOBALCTX_BUFFER, buffId)); 1436 return NV_OK; 1437 } 1438 else 1439 { 1440 NV_PRINTF(LEVEL_ERROR, 1441 "Could not map %s Buffer, no memory allocated for it!\n", 1442 NV_ENUM_TO_STRING(GR_GLOBALCTX_BUFFER, buffId)); 1443 return NV_ERR_INVALID_ARGUMENT; 1444 } 1445 } 1446 1447 // Unconditionally call map for refcounting 1448 NV_ASSERT_OK_OR_ELSE(status, 1449 kgraphicsMapCtxBuffer(pGpu, pKernelGraphics, pMemDesc, pVAS, 1450 &pKernelGraphicsContextUnicast->globalCtxBufferVaList[buffId], 1451 kgraphicsIsGlobalCtxBufferSizeAligned(pGpu, pKernelGraphics, buffId), 1452 bIsReadOnly), 1453 NV_PRINTF(LEVEL_ERROR, "%s Buffer could not be mapped\n", 1454 NV_ENUM_TO_STRING(GR_GLOBALCTX_BUFFER, buffId)); 1455 return status; ); 1456 1457 NV_ASSERT_OK_OR_CAPTURE_FIRST_ERROR(status, 1458 vaListFindVa(&pKernelGraphicsContextUnicast->globalCtxBufferVaList[buffId], pVAS, &vaddr)); 1459 1460 NV_PRINTF(LEVEL_INFO, 1461 "GPU:%d %s Buffer PA @ 0x%llx VA @ 0x%llx of Size 0x%llx\n", 1462 pGpu->gpuInstance, NV_ENUM_TO_STRING(GR_GLOBALCTX_BUFFER, buffId), 1463 memdescGetPhysAddr(memdescGetMemDescFromGpu(pMemDesc, pGpu), AT_GPU, 0), 1464 vaddr, pMemDesc->Size); 1465 1466 return status; 1467 } 1468 1469 // 1470 // Map common buffers to a channel's context 1471 // 1472 NV_STATUS 1473 kgrctxMapGlobalCtxBuffers_IMPL 1474 ( 1475 OBJGPU *pGpu, 1476 KernelGraphicsContext *pKernelGraphicsContext, 1477 KernelGraphics *pKernelGraphics, 1478 NvU32 gfid, 1479 KernelChannel *pKernelChannel 1480 ) 1481 { 1482 GR_GLOBALCTX_BUFFERS *pGlobalCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid); 1483 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 1484 1485 NV_ASSERT_OR_RETURN(!gpumgrGetBcEnabledStatus(pGpu), NV_ERR_INVALID_STATE); 1486 1487 NV_ASSERT_OK_OR_RETURN( 1488 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast)); 1489 1490 // if global ctx buffers were never allocated then bail out early 1491 if (!pKernelGraphicsContextUnicast->localCtxBuffer.bAllocated && 1492 !pGlobalCtxBuffers->bAllocated) 1493 { 1494 return NV_ERR_INVALID_STATE; 1495 } 1496 1497 kgrctxMapGlobalCtxBuffer(pGpu, 1498 pKernelGraphicsContext, 1499 pKernelGraphics, 1500 gfid, 1501 pKernelChannel->pVAS, 1502 GR_GLOBALCTX_BUFFER_BUNDLE_CB, 1503 NV_FALSE); 1504 kgrctxMapGlobalCtxBuffer(pGpu, 1505 pKernelGraphicsContext, 1506 pKernelGraphics, 1507 gfid, 1508 pKernelChannel->pVAS, 1509 GR_GLOBALCTX_BUFFER_ATTRIBUTE_CB, 1510 NV_FALSE); 1511 kgrctxMapGlobalCtxBuffer(pGpu, 1512 pKernelGraphicsContext, 1513 pKernelGraphics, 1514 gfid, 1515 pKernelChannel->pVAS, 1516 GR_GLOBALCTX_BUFFER_PAGEPOOL, 1517 NV_FALSE); 1518 1519 if (kgraphicsIsRtvCbSupported(pGpu, pKernelGraphics)) 1520 { 1521 kgrctxMapGlobalCtxBuffer(pGpu, 1522 pKernelGraphicsContext, 1523 pKernelGraphics, 1524 gfid, 1525 pKernelChannel->pVAS, 1526 GR_GLOBALCTX_BUFFER_RTV_CB, 1527 NV_FALSE); 1528 } 1529 1530 return NV_OK; 1531 } 1532 1533 /*! 1534 * @brief This function allocates and maps various GR buffers. 1535 */ 1536 NV_STATUS 1537 kgrctxMapCtxBuffers_IMPL 1538 ( 1539 OBJGPU *pGpu, 1540 KernelGraphicsContext *pKernelGraphicsContext, 1541 KernelGraphics *pKernelGraphics, 1542 KernelGraphicsObject *pKernelGraphicsObject 1543 ) 1544 { 1545 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 1546 ChannelDescendant *pChannelDescendant = staticCast(pKernelGraphicsObject, ChannelDescendant); 1547 KernelChannel *pKernelChannel = pChannelDescendant->pKernelChannel; 1548 NvU32 classNum = pChannelDescendant->resourceDesc.externalClassId; 1549 OBJGVASPACE *pGVAS; 1550 NvU32 objType; 1551 NvU32 gfid = kchannelGetGfid(pKernelChannel); 1552 NvBool bAcquire3d = NV_FALSE; 1553 1554 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 1555 1556 NV_ASSERT_OK_OR_RETURN( 1557 kgrctxGetUnicast(pGpu, pKernelGraphicsObject->pKernelGraphicsContext, &pKernelGraphicsContextUnicast)); 1558 1559 pGVAS = dynamicCast(pKernelChannel->pVAS, OBJGVASPACE); 1560 if (gvaspaceIsExternallyOwned(pGVAS)) 1561 return NV_OK; 1562 1563 // 1564 // in kernel RM we skip the context buffer mapping as part of 1565 // fifoSetUpChannelDma. fifoSetUpChannelDma is not enabled in kernel RM. 1566 // So it's possible for the main ctx buffer to be unmapped by this point. 1567 // 1568 if (!kchannelIsCtxBufferAllocSkipped(pKernelChannel)) 1569 { 1570 ENGINE_CTX_DESCRIPTOR *pEngCtx; 1571 MEMORY_DESCRIPTOR *pMemDesc; 1572 NvU32 subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu); 1573 1574 pEngCtx = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->ppEngCtxDesc[subdevInst]; 1575 1576 NV_ASSERT_OR_RETURN(pEngCtx != NULL, NV_ERR_INVALID_STATE); 1577 NV_ASSERT_OR_RETURN(pEngCtx->pMemDesc != NULL, NV_ERR_INVALID_STATE); 1578 pMemDesc = pEngCtx->pMemDesc; 1579 1580 NV_ASSERT_OK_OR_RETURN( 1581 kgraphicsMapCtxBuffer(pGpu, pKernelGraphics, pMemDesc, pKernelChannel->pVAS, 1582 &pEngCtx->vaList, NV_FALSE, NV_FALSE)); 1583 } 1584 1585 kgrmgrGetGrObjectType(classNum, &objType); 1586 1587 NV_ASSERT_OK_OR_RETURN( 1588 kgraphicsMapCtxBuffer(pGpu, pKernelGraphics, pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc, pKernelChannel->pVAS, 1589 &pKernelGraphicsContextUnicast->ctxPatchBuffer.vAddrList, NV_FALSE, NV_FALSE)); 1590 1591 if (pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc != NULL) 1592 { 1593 NV_ASSERT_OK_OR_RETURN( 1594 kgraphicsMapCtxBuffer(pGpu, pKernelGraphics, pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc, pKernelChannel->pVAS, 1595 &pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList, NV_FALSE, NV_FALSE)); 1596 } 1597 1598 if (kgraphicsDoesUcodeSupportPrivAccessMap(pGpu, pKernelGraphics)) 1599 { 1600 kgrctxMapGlobalCtxBuffer(pGpu, 1601 pKernelGraphicsContext, 1602 pKernelGraphics, 1603 gfid, 1604 pKernelChannel->pVAS, 1605 kgrctxGetRegisterAccessMapId_HAL(pGpu, pKernelGraphicsContext, pChannelDescendant->pKernelChannel), 1606 NV_TRUE); 1607 } 1608 1609 // 1610 // Condition for acquiring 3d context buffer mappings. 1611 // For non-TSG & legacy TSG mode, always map 1612 // For subcontext TSG case, only map on 2D/3D allocations 1613 // 1614 if (pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->bLegacyMode) 1615 { 1616 bAcquire3d = NV_TRUE; 1617 } 1618 else 1619 { 1620 bAcquire3d = ((objType == GR_OBJECT_TYPE_2D) || (objType == GR_OBJECT_TYPE_3D)); 1621 } 1622 1623 if (bAcquire3d) 1624 { 1625 kgrctxMapGlobalCtxBuffers(pGpu, pKernelGraphicsContext, pKernelGraphics, gfid, pKernelChannel); 1626 } 1627 1628 return NV_OK; 1629 } 1630 1631 /*! 1632 * @brief Set parameters for promoting the PA of a ctx buffer to physical RM. 1633 */ 1634 NV_STATUS 1635 kgrctxPrepareInitializeCtxBuffer_IMPL 1636 ( 1637 OBJGPU *pGpu, 1638 KernelGraphicsContext *pKernelGraphicsContext, 1639 KernelGraphics *pKernelGraphics, 1640 KernelChannel *pKernelChannel, 1641 NvU32 externalId, 1642 NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ENTRY *pEntry, 1643 NvBool *pbAddEntry 1644 ) 1645 { 1646 MEMORY_DESCRIPTOR *pMemDesc; 1647 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 1648 NvU32 physAttr; 1649 1650 *pbAddEntry = NV_FALSE; 1651 1652 NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_CHANNEL); 1653 1654 NV_ASSERT_OK_OR_RETURN( 1655 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast)); 1656 1657 switch (externalId) 1658 { 1659 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_MAIN: 1660 { 1661 ENGINE_CTX_DESCRIPTOR *pEngCtx; 1662 NvU32 subdevInst; 1663 if (pKernelGraphicsContextUnicast->bKGrMainCtxBufferInitialized) 1664 return NV_OK; 1665 1666 // Do not try to init a KMD virtual context buffer 1667 if (kchannelIsCtxBufferAllocSkipped(pKernelChannel)) 1668 return NV_OK; 1669 1670 subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu); 1671 pEngCtx = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->ppEngCtxDesc[subdevInst]; 1672 1673 NV_ASSERT_OR_RETURN(pEngCtx != NULL, NV_ERR_INVALID_STATE); 1674 NV_ASSERT_OR_RETURN(pEngCtx->pMemDesc != NULL, NV_ERR_INVALID_STATE); 1675 pMemDesc = pEngCtx->pMemDesc; 1676 break; 1677 } 1678 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PM: 1679 if (pKernelGraphicsContextUnicast->bKGrPmCtxBufferInitialized) 1680 return NV_OK; 1681 1682 if (pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc == NULL) 1683 return NV_OK; 1684 1685 pMemDesc = pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc; 1686 break; 1687 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PATCH: 1688 if (pKernelGraphicsContextUnicast->bKGrPatchCtxBufferInitialized) 1689 return NV_OK; 1690 1691 NV_ASSERT(pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc != NULL); 1692 pMemDesc = pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc; 1693 break; 1694 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_BUFFER_BUNDLE_CB: 1695 // fall-through 1696 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PAGEPOOL: 1697 // fall-through 1698 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_ATTRIBUTE_CB: 1699 // fall-through 1700 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_RTV_CB_GLOBAL: 1701 // fall-through 1702 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_POOL: 1703 // No initialization from kernel RM 1704 return NV_OK; 1705 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_CTRL_BLK: 1706 { 1707 GR_GLOBALCTX_BUFFER internalId; 1708 NvU32 gfid = kchannelGetGfid(pKernelChannel); 1709 GR_GLOBALCTX_BUFFERS *pCtxBuffers; 1710 1711 NV_ASSERT_OK_OR_RETURN( 1712 kgrctxGetGlobalContextBufferInternalId(externalId, &internalId)); 1713 1714 if (pKernelGraphicsContextUnicast->localCtxBuffer.bAllocated) 1715 { 1716 pCtxBuffers = &pKernelGraphicsContextUnicast->localCtxBuffer; 1717 } 1718 else 1719 { 1720 pCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid); 1721 } 1722 1723 if (pCtxBuffers->bInitialized[internalId]) 1724 return NV_OK; 1725 1726 pMemDesc = pCtxBuffers->memDesc[internalId]; 1727 break; 1728 } 1729 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_FECS_EVENT: 1730 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PRIV_ACCESS_MAP: 1731 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_UNRESTRICTED_PRIV_ACCESS_MAP: 1732 { 1733 GR_GLOBALCTX_BUFFER internalId; 1734 NvU32 gfid = kchannelGetGfid(pKernelChannel); 1735 GR_GLOBALCTX_BUFFERS *pCtxBuffers; 1736 1737 NV_ASSERT_OK_OR_RETURN( 1738 kgrctxGetGlobalContextBufferInternalId(externalId, &internalId)); 1739 1740 pCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid); 1741 1742 if (pCtxBuffers->bInitialized[internalId]) 1743 return NV_OK; 1744 1745 pMemDesc = pCtxBuffers->memDesc[internalId]; 1746 1747 break; 1748 } 1749 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GLOBAL_PRIV_ACCESS_MAP: 1750 // No initialization from kernel RM 1751 return NV_OK; 1752 default: 1753 NV_ASSERT_OR_RETURN(!"Unrecognized promote ctx enum", NV_ERR_INVALID_ARGUMENT); 1754 } 1755 1756 // If these buffers are not supported or not allocated, no need to init 1757 if (pMemDesc == NULL) 1758 return NV_OK; 1759 1760 physAttr = 0x0; 1761 switch (memdescGetAddressSpace(pMemDesc)) 1762 { 1763 case ADDR_FBMEM: 1764 physAttr = FLD_SET_DRF(2080, _CTRL_GPU_INITIALIZE_CTX, 1765 _APERTURE, _VIDMEM, physAttr); 1766 break; 1767 1768 case ADDR_SYSMEM: 1769 if (memdescGetCpuCacheAttrib(pMemDesc) == NV_MEMORY_CACHED) 1770 { 1771 physAttr = FLD_SET_DRF(2080, _CTRL_GPU_INITIALIZE_CTX, 1772 _APERTURE, _COH_SYS, physAttr); 1773 } 1774 else if (memdescGetCpuCacheAttrib(pMemDesc) == NV_MEMORY_UNCACHED) 1775 { 1776 physAttr = FLD_SET_DRF(2080, _CTRL_GPU_INITIALIZE_CTX, 1777 _APERTURE, _NCOH_SYS, physAttr); 1778 } 1779 else 1780 { 1781 return NV_ERR_INVALID_STATE; 1782 } 1783 break; 1784 1785 default: 1786 return NV_ERR_INVALID_STATE; 1787 } 1788 1789 physAttr = FLD_SET_DRF(2080, _CTRL_GPU_INITIALIZE_CTX, _GPU_CACHEABLE, _NO, physAttr); 1790 1791 pEntry->gpuPhysAddr = memdescGetPhysAddr(pMemDesc, AT_GPU, 0); 1792 pEntry->size = pMemDesc->Size; 1793 pEntry->physAttr = physAttr; 1794 pEntry->bufferId = externalId; 1795 pEntry->bInitialize = NV_TRUE; 1796 pEntry->bNonmapped = NV_TRUE; 1797 1798 *pbAddEntry = NV_TRUE; 1799 1800 return NV_OK; 1801 } 1802 1803 /*! 1804 * @brief Set parameters for promoting the VA of a ctx buffer to physical RM. 1805 */ 1806 NV_STATUS 1807 kgrctxPreparePromoteCtxBuffer_IMPL 1808 ( 1809 OBJGPU *pGpu, 1810 KernelGraphicsContext *pKernelGraphicsContext, 1811 KernelChannel *pKernelChannel, 1812 NvU32 externalId, 1813 NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ENTRY *pEntry, 1814 NvBool *pbAddEntry 1815 ) 1816 { 1817 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 1818 VA_LIST *pVaList; 1819 NvU64 vaddr; 1820 NvU64 refCount; 1821 OBJGVASPACE *pGVAS = dynamicCast(pKernelChannel->pVAS, OBJGVASPACE); 1822 NV_STATUS status; 1823 1824 NV_ASSERT_OK_OR_RETURN( 1825 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast)); 1826 1827 *pbAddEntry = NV_FALSE; 1828 1829 // RM is not responsible for promoting the buffers when UVM is enabled 1830 if (gvaspaceIsExternallyOwned(pGVAS)) 1831 return NV_OK; 1832 1833 switch (externalId) 1834 { 1835 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_MAIN: 1836 { 1837 ENGINE_CTX_DESCRIPTOR *pEngCtx; 1838 NvU32 subdevInst; 1839 1840 // Do not try to promote a KMD virtual context buffer 1841 if (kchannelIsCtxBufferAllocSkipped(pKernelChannel)) 1842 return NV_OK; 1843 1844 subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu); 1845 pEngCtx = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->ppEngCtxDesc[subdevInst]; 1846 1847 NV_ASSERT_OR_RETURN(pEngCtx != NULL, NV_ERR_INVALID_STATE); 1848 pVaList = &pEngCtx->vaList; 1849 break; 1850 } 1851 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PM: 1852 pVaList = &pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList; 1853 break; 1854 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PATCH: 1855 pVaList = &pKernelGraphicsContextUnicast->ctxPatchBuffer.vAddrList; 1856 break; 1857 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_BUFFER_BUNDLE_CB: 1858 // fall-through 1859 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PAGEPOOL: 1860 // fall-through 1861 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_ATTRIBUTE_CB: 1862 // fall-through 1863 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_RTV_CB_GLOBAL: 1864 // fall-through 1865 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_POOL: 1866 // fall-through 1867 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_CTRL_BLK: 1868 // fall-through 1869 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_FECS_EVENT: 1870 // fall-through 1871 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PRIV_ACCESS_MAP: 1872 // fall-through 1873 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_UNRESTRICTED_PRIV_ACCESS_MAP: 1874 // fall-through 1875 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GLOBAL_PRIV_ACCESS_MAP: 1876 { 1877 GR_GLOBALCTX_BUFFER internalId; 1878 1879 NV_ASSERT_OK_OR_RETURN( 1880 kgrctxGetGlobalContextBufferInternalId(externalId, &internalId)); 1881 1882 pVaList = &pKernelGraphicsContextUnicast->globalCtxBufferVaList[internalId]; 1883 break; 1884 } 1885 default: 1886 NV_ASSERT_OR_RETURN(!"Unrecognized promote ctx enum", NV_ERR_INVALID_ARGUMENT); 1887 } 1888 1889 // If the buffer isn't currently mapped or was already promoted, nothing to do 1890 status = vaListGetRefCount(pVaList, pKernelChannel->pVAS, &refCount); 1891 if ((status != NV_OK) || (refCount > 1)) 1892 return NV_OK; 1893 1894 NV_ASSERT_OK_OR_RETURN(vaListFindVa(pVaList, pKernelChannel->pVAS, &vaddr)); 1895 1896 pEntry->bufferId = externalId; 1897 pEntry->gpuVirtAddr = vaddr; 1898 pEntry->bNonmapped = NV_FALSE; 1899 1900 *pbAddEntry = NV_TRUE; 1901 return NV_OK; 1902 } 1903 1904 /*! Mark the context buffer as initialized, prevent further calls to init */ 1905 void 1906 kgrctxMarkCtxBufferInitialized_IMPL 1907 ( 1908 OBJGPU *pGpu, 1909 KernelGraphicsContext *pKernelGraphicsContext, 1910 KernelGraphics *pKernelGraphics, 1911 KernelChannel *pKernelChannel, 1912 NvU32 externalId 1913 ) 1914 { 1915 NV_STATUS status; 1916 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 1917 1918 NV_ASSERT_OK_OR_ELSE(status, 1919 kgrctxGetUnicast(pGpu, 1920 pKernelGraphicsContext, 1921 &pKernelGraphicsContextUnicast), 1922 return;); 1923 1924 switch (externalId) 1925 { 1926 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_MAIN: 1927 pKernelGraphicsContextUnicast->bKGrMainCtxBufferInitialized = NV_TRUE; 1928 break; 1929 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PM: 1930 pKernelGraphicsContextUnicast->bKGrPmCtxBufferInitialized = NV_TRUE; 1931 break; 1932 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PATCH: 1933 pKernelGraphicsContextUnicast->bKGrPatchCtxBufferInitialized = NV_TRUE; 1934 break; 1935 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_BUFFER_BUNDLE_CB: 1936 // fall-through 1937 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PAGEPOOL: 1938 // fall-through 1939 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_ATTRIBUTE_CB: 1940 // fall-through 1941 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_RTV_CB_GLOBAL: 1942 // fall-through 1943 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_POOL: 1944 // fall-through 1945 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_CTRL_BLK: 1946 { 1947 // If "local" global ctx buffers are allocated, check those first 1948 if (pKernelGraphicsContextUnicast->localCtxBuffer.bAllocated) 1949 { 1950 GR_GLOBALCTX_BUFFER internalId; 1951 GR_GLOBALCTX_BUFFERS *pKCtxBuffers; 1952 1953 NV_ASSERT_OK_OR_ELSE(status, 1954 kgrctxGetGlobalContextBufferInternalId(externalId, &internalId), 1955 return;); 1956 1957 pKCtxBuffers = &pKernelGraphicsContextUnicast->localCtxBuffer; 1958 pKCtxBuffers->bInitialized[internalId] = NV_TRUE; 1959 break; 1960 } 1961 // fall-through 1962 } 1963 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_FECS_EVENT: 1964 // fall-through 1965 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PRIV_ACCESS_MAP: 1966 // fall-through 1967 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_UNRESTRICTED_PRIV_ACCESS_MAP: 1968 // fall-through 1969 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GLOBAL_PRIV_ACCESS_MAP: 1970 { 1971 GR_GLOBALCTX_BUFFER internalId; 1972 NvU32 gfid = kchannelGetGfid(pKernelChannel); 1973 GR_GLOBALCTX_BUFFERS *pKCtxBuffers; 1974 1975 NV_ASSERT_OK_OR_ELSE(status, 1976 kgrctxGetGlobalContextBufferInternalId(externalId, &internalId), 1977 return;); 1978 1979 // TODO XXX Make context buffers available from KGRCTX alone 1980 pKCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid); 1981 1982 pKCtxBuffers->bInitialized[internalId] = NV_TRUE; 1983 break; 1984 } 1985 default: 1986 NV_ASSERT(!"Unrecognized promote ctx enum"); 1987 } 1988 } 1989 1990 /*! Non-UVM late bind PM ctx buffer */ 1991 NV_STATUS 1992 kgrctxSetupDeferredPmBuffer_IMPL 1993 ( 1994 OBJGPU *pGpu, 1995 KernelGraphicsContext *pKernelGraphicsContext, 1996 KernelGraphics *pKernelGraphics, 1997 KernelChannel *pKernelChannel 1998 ) 1999 { 2000 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2001 OBJGVASPACE *pGVAS = dynamicCast(pKernelChannel->pVAS, OBJGVASPACE); 2002 NV_STATUS status = NV_OK; 2003 Subdevice *pSubdevice; 2004 2005 NV_ASSERT_OK_OR_RETURN( 2006 subdeviceGetByGpu(RES_GET_CLIENT(pKernelChannel), pGpu, &pSubdevice)); 2007 2008 GPU_RES_SET_THREAD_BC_STATE(pSubdevice); 2009 2010 NV_ASSERT_OK_OR_RETURN( 2011 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast)); 2012 2013 if (gvaspaceIsExternallyOwned(pGVAS)) 2014 { 2015 // 2016 // The PM Ctxsw buffer is now pre-allocated along with the other context buffers 2017 // This is done as a WAR for the issue tracked under bug 1760699 2018 // Reason: With the enablement of UVM8 by default, the UVM channel registration relies on the 2019 // buffers being allocated at channel allocation time and are not going to work for a buffer 2020 // created later with an rm ctrl 2021 // 2022 NV_ASSERT_OR_RETURN(pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc != NULL, NV_ERR_INVALID_STATE); 2023 return NV_OK; 2024 } 2025 else if (pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc != NULL) 2026 { 2027 return NV_OK; 2028 } 2029 2030 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2031 kgrctxAllocPmBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel)); 2032 2033 // 2034 // !!! 2035 // From this point on, use `goto failed` to exit 2036 // !!! 2037 // 2038 2039 { 2040 RS_ORDERED_ITERATOR it; 2041 RsResourceRef *pScopeRef = RES_GET_REF(pKernelChannel); 2042 2043 // Iterate over all channels in this TSG and map the new buffer 2044 if (!pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->bAllocatedByRm && 2045 kgraphicsIsPerSubcontextContextHeaderSupported(pGpu, pKernelGraphics)) 2046 { 2047 pScopeRef = RES_GET_REF(pKernelChannel->pKernelChannelGroupApi); 2048 } 2049 2050 it = kchannelGetIter(RES_GET_CLIENT(pKernelChannel), pScopeRef); 2051 while (clientRefOrderedIterNext(it.pClient, &it)) 2052 { 2053 NvU64 vaddr; 2054 KernelChannel *pLoopKernelChannel = dynamicCast(it.pResourceRef->pResource, KernelChannel); 2055 2056 NV_ASSERT_OR_ELSE(pLoopKernelChannel != NULL, 2057 status = NV_ERR_INVALID_STATE; 2058 goto failed;); 2059 2060 if ((vaListFindVa(&pKernelGraphicsContextUnicast->ctxPatchBuffer.vAddrList, pLoopKernelChannel->pVAS, &vaddr) == NV_OK) && 2061 (vaListFindVa(&pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList, pLoopKernelChannel->pVAS, &vaddr) != NV_OK)) 2062 { 2063 NvU64 refCount; 2064 NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS params; 2065 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 2066 NvBool bInitialize; 2067 NvBool bPromote; 2068 2069 NV_ASSERT_OK_OR_GOTO(status, 2070 kgraphicsMapCtxBuffer(pGpu, pKernelGraphics, 2071 pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc, 2072 pLoopKernelChannel->pVAS, 2073 &pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList, 2074 NV_FALSE, 2075 NV_FALSE), 2076 failed); 2077 2078 // Promote the vaddr for each unique VAS 2079 portMemSet(¶ms, 0, sizeof(params)); 2080 2081 // Setup parameters to initialize the PA if necessary 2082 NV_ASSERT_OK_OR_GOTO(status, 2083 kgrctxPrepareInitializeCtxBuffer(pGpu, 2084 pKernelGraphicsContext, 2085 pKernelGraphics, 2086 pKernelChannel, 2087 NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PM, 2088 ¶ms.promoteEntry[0], 2089 &bInitialize), 2090 failed); 2091 2092 // Setup parameters to promote the VA if necessary 2093 NV_ASSERT_OK_OR_GOTO(status, 2094 kgrctxPreparePromoteCtxBuffer(pGpu, 2095 pKernelGraphicsContext, 2096 pKernelChannel, 2097 NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PM, 2098 ¶ms.promoteEntry[0], 2099 &bPromote), 2100 failed); 2101 2102 NV_ASSERT_OR_ELSE(bInitialize || bPromote, 2103 status = NV_ERR_INVALID_STATE; 2104 goto failed;); 2105 2106 params.engineType = NV2080_ENGINE_TYPE_GR(kgraphicsGetInstance(pGpu, pKernelGraphics)); 2107 params.hChanClient = RES_GET_CLIENT_HANDLE(pKernelChannel); 2108 params.hObject = RES_GET_HANDLE(pKernelChannel); 2109 params.entryCount = 1; 2110 2111 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, 2112 pRmApi->Control(pRmApi, 2113 RES_GET_CLIENT_HANDLE(pSubdevice), 2114 RES_GET_HANDLE(pSubdevice), 2115 NV2080_CTRL_CMD_GPU_PROMOTE_CTX, 2116 ¶ms, 2117 sizeof(params)), 2118 failed); 2119 2120 // 2121 // If we successfully promoted the PA, flip a flag to ensure we don't 2122 // try to promote it again. The VA_LIST should already track this for 2123 // VA, but we can't rely on it for PA due to UVM. 2124 // 2125 if (params.promoteEntry[0].bInitialize) 2126 { 2127 kgrctxMarkCtxBufferInitialized(pGpu, 2128 pKernelGraphicsContext, 2129 pKernelGraphics, 2130 pKernelChannel, 2131 params.promoteEntry[0].bufferId); 2132 } 2133 2134 // Update the refcount for this buffer now that we've promoted it 2135 NV_ASSERT_OK_OR_GOTO(status, 2136 vaListGetRefCount(&pKernelGraphicsContextUnicast->ctxPatchBuffer.vAddrList, pLoopKernelChannel->pVAS, &refCount), 2137 failed); 2138 NV_ASSERT_OK_OR_GOTO(status, 2139 vaListSetRefCount(&pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList, pLoopKernelChannel->pVAS, refCount), 2140 failed); 2141 } 2142 } 2143 } 2144 2145 return NV_OK; 2146 2147 failed: 2148 2149 if (kgraphicsIsPerSubcontextContextHeaderSupported(pGpu, pKernelGraphics)) 2150 { 2151 RS_ORDERED_ITERATOR it; 2152 RsResourceRef *pScopeRef = RES_GET_REF(pKernelChannel); 2153 2154 // Iterate over all channels in this TSG and try to unmap the PM ctx buffer 2155 if (!pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->bAllocatedByRm && 2156 kgraphicsIsPerSubcontextContextHeaderSupported(pGpu, pKernelGraphics)) 2157 { 2158 pScopeRef = RES_GET_REF(pKernelChannel->pKernelChannelGroupApi); 2159 } 2160 2161 it = kchannelGetIter(RES_GET_CLIENT(pKernelChannel), pScopeRef); 2162 while (clientRefOrderedIterNext(it.pClient, &it)) 2163 { 2164 NvU64 vaddr; 2165 KernelChannel *pLoopKernelChannel = dynamicCast(it.pResourceRef->pResource, KernelChannel); 2166 if (pLoopKernelChannel == NULL) 2167 continue; 2168 2169 while (vaListFindVa(&pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList, pLoopKernelChannel->pVAS, &vaddr) == NV_OK) 2170 { 2171 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, 2172 pLoopKernelChannel->pVAS, 2173 &pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList); 2174 } 2175 } 2176 } 2177 2178 kgrctxFreePmBuffer(pGpu, pKernelGraphicsContext); 2179 2180 return status; 2181 } 2182 2183 /** 2184 * @brief unmap the memory for the pm context buffer associated with a channel 2185 */ 2186 void 2187 kgrctxUnmapCtxPmBuffer_IMPL 2188 ( 2189 OBJGPU *pGpu, 2190 KernelGraphicsContext *pKernelGraphicsContext, 2191 KernelGraphics *pKernelGraphics, 2192 OBJVASPACE *pVAS 2193 ) 2194 { 2195 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2196 NV_STATUS status; 2197 2198 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2199 2200 NV_ASSERT_OK_OR_ELSE(status, 2201 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast), 2202 return;); 2203 2204 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pVAS, &pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList); 2205 } 2206 2207 /** 2208 * @brief unmap the memory for the zcull context buffer 2209 */ 2210 void 2211 kgrctxUnmapCtxZcullBuffer_IMPL 2212 ( 2213 OBJGPU *pGpu, 2214 KernelGraphicsContext *pKernelGraphicsContext, 2215 KernelGraphics *pKernelGraphics, 2216 OBJVASPACE *pVAS 2217 ) 2218 { 2219 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2220 MEMORY_DESCRIPTOR *pMemDesc; 2221 VA_LIST *pVaList; 2222 NvU64 vaddr; 2223 NV_STATUS status; 2224 2225 NV_ASSERT_OK_OR_ELSE(status, 2226 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast), 2227 return;); 2228 2229 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2230 2231 pMemDesc = pKernelGraphicsContextUnicast->zcullCtxswBuffer.pMemDesc; 2232 pVaList = &pKernelGraphicsContextUnicast->zcullCtxswBuffer.vAddrList; 2233 if (pMemDesc != NULL) 2234 { 2235 // 2236 // This func assumes that the buffer was not allocated per subdevice, 2237 // and will leak any mappings performed by the secondaries. 2238 // 2239 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc)); 2240 2241 if (vaListFindVa(pVaList, pVAS, &vaddr) == NV_OK) 2242 { 2243 dmaUnmapBuffer_HAL(pGpu, GPU_GET_DMA(pGpu), pVAS, vaddr); 2244 vaListRemoveVa(pVaList, pVAS); 2245 } 2246 } 2247 else if (vaListFindVa(pVaList, pVAS, &vaddr) == NV_OK) 2248 { 2249 // Zcull buffer mapped by client. Remove the VA here 2250 vaListRemoveVa(pVaList, pVAS); 2251 } 2252 } 2253 2254 /** 2255 * @brief unmap the memory for the preemption context buffers 2256 */ 2257 void 2258 kgrctxUnmapCtxPreemptionBuffers_IMPL 2259 ( 2260 OBJGPU *pGpu, 2261 KernelGraphicsContext *pKernelGraphicsContext, 2262 KernelGraphics *pKernelGraphics, 2263 OBJVASPACE *pVAS 2264 ) 2265 { 2266 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2267 NV_STATUS status; 2268 2269 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2270 2271 NV_ASSERT_OK_OR_ELSE(status, 2272 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast), 2273 return;); 2274 2275 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pVAS, &pKernelGraphicsContextUnicast->preemptCtxswBuffer.vAddrList); 2276 2277 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pVAS, &pKernelGraphicsContextUnicast->spillCtxswBuffer.vAddrList); 2278 2279 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pVAS, &pKernelGraphicsContextUnicast->betaCBCtxswBuffer.vAddrList); 2280 2281 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pVAS, &pKernelGraphicsContextUnicast->pagepoolCtxswBuffer.vAddrList); 2282 2283 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pVAS, &pKernelGraphicsContextUnicast->rtvCbCtxswBuffer.vAddrList); 2284 } 2285 2286 /** 2287 * @brief Unmap associated ctx buffers (main, patch, global buffers etc). 2288 * 2289 * This function is called on every channel free and therefore can't assume any 2290 * graphics objects actually exist on the given channel. 2291 * TODO: Bug 3164256 2292 * This function also unmaps and frees zcull and preempt buffers. Ideally we want to decouple 2293 * unmap and free for all buffers and move free of buffers to kgrctxFreeAssociatedCtxBuffers 2294 */ 2295 void 2296 kgrctxUnmapAssociatedCtxBuffers_IMPL 2297 ( 2298 OBJGPU *pGpu, 2299 KernelGraphicsContext *pKernelGraphicsContext, 2300 KernelGraphics *pKernelGraphics, 2301 KernelChannel *pKernelChannel 2302 ) 2303 { 2304 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2305 NvBool bRelease3d = NV_FALSE; 2306 NvU32 gfid = kchannelGetGfid(pKernelChannel); 2307 NvU32 status; 2308 GR_GLOBALCTX_BUFFER registerMapBufferId = kgrctxGetRegisterAccessMapId_HAL(pGpu, pKernelGraphicsContext, pKernelChannel); 2309 2310 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2311 2312 NV_ASSERT_OK_OR_ELSE(status, 2313 kgrctxGetUnicast(pGpu, 2314 pKernelGraphicsContext, 2315 &pKernelGraphicsContextUnicast), 2316 return;); 2317 2318 // 2319 // Unmap 3D context buffers on the last 3D channel free 2320 // 2321 // Condition for unmapping 3d context buffer mappings: 2322 // For non-TSG & legacy TSG mode, always unmap 3d context buffers. 2323 // For subcontext TSG case, unmap on the last graphics channel free. 2324 // 2325 if (pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->bLegacyMode) 2326 { 2327 bRelease3d = pKernelGraphicsContextUnicast->channelObjects == 0; 2328 } 2329 else 2330 { 2331 bRelease3d = ((pKernelGraphicsContextUnicast->objectCounts[GR_OBJECT_TYPE_3D] == 0) && 2332 (pKernelGraphicsContextUnicast->objectCounts[GR_OBJECT_TYPE_2D] == 0)); 2333 } 2334 2335 if (bRelease3d) 2336 { 2337 // Unmap Circular buffer from the current channel's address space 2338 kgrctxUnmapGlobalCtxBuffers(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel->pVAS, gfid); 2339 } 2340 2341 // 2342 // If multiple channels sharing the same VAS exist, it is possible both 2343 // channels could be using these mappings, and we must wait for both 2344 // channels to be detached before we remove them. 2345 // 2346 if (pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->pChanList != NULL) 2347 { 2348 CHANNEL_NODE *pChanNode; 2349 CHANNEL_LIST *pChanList; 2350 2351 pChanList = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->pChanList; 2352 2353 for (pChanNode = pChanList->pHead; pChanNode; pChanNode = pChanNode->pNext) 2354 { 2355 // Skip the channel we are looking to unmap 2356 if (kchannelGetDebugTag(pKernelChannel) == kchannelGetDebugTag(pChanNode->pKernelChannel)) 2357 continue; 2358 2359 if (pKernelChannel->pVAS == pChanNode->pKernelChannel->pVAS) 2360 { 2361 NV_PRINTF(LEVEL_INFO, "TSG %d Channel %d shares a pVAS with channel %d\n", 2362 pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->grpID, 2363 kchannelGetDebugTag(pKernelChannel), 2364 kchannelGetDebugTag(pChanNode->pKernelChannel)); 2365 return; 2366 } 2367 } 2368 } 2369 2370 // Only unmap once the last channel using this VAS has gone 2371 if (kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid)->memDesc[GR_GLOBALCTX_BUFFER_FECS_EVENT] != NULL) 2372 { 2373 kgrctxUnmapGlobalCtxBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel->pVAS, GR_GLOBALCTX_BUFFER_FECS_EVENT); 2374 } 2375 2376 if (pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc != NULL) 2377 { 2378 kgrctxUnmapCtxPmBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel->pVAS); 2379 } 2380 2381 if (kgraphicsDoesUcodeSupportPrivAccessMap(pGpu, pKernelGraphics)) 2382 { 2383 kgrctxUnmapGlobalCtxBuffer(pGpu, 2384 pKernelGraphicsContext, 2385 pKernelGraphics, 2386 pKernelChannel->pVAS, 2387 registerMapBufferId); 2388 } 2389 2390 if (pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc != NULL) 2391 { 2392 // 2393 // Delay freeing the patch buffer until the last channel free. 2394 // these buffers are accessed even after the last GR object is freed. 2395 // 2396 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pKernelChannel->pVAS, &pKernelGraphicsContextUnicast->ctxPatchBuffer.vAddrList); 2397 } 2398 2399 kgrctxUnmapCtxZcullBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel->pVAS); 2400 2401 // Release all preemption buffers if they were allocated 2402 kgrctxUnmapCtxPreemptionBuffers(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel->pVAS); 2403 2404 // 2405 // Cleanup lingering main ctx buffer mappings for this VAS 2406 // TODO fix main ctx buffer refcounting 2407 // TODO move setEngineContextMemDesc to FreeMainCtxBuffer, move this loop 2408 // inside UnmapMainCtxBuffer 2409 // 2410 if (pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->ppEngCtxDesc[gpumgrGetSubDeviceInstanceFromGpu(pGpu)] != NULL) 2411 { 2412 NvU32 subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu); 2413 VA_LIST *pVAddrList = &pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->ppEngCtxDesc[subdevInst]->vaList; 2414 NvU64 vaddr; 2415 2416 while (vaListFindVa(pVAddrList, pKernelChannel->pVAS, &vaddr) == NV_OK) 2417 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pKernelChannel->pVAS, pVAddrList); 2418 } 2419 2420 // 2421 // When sharing contexts across channels we need to defer this until all 2422 // objects have been freed. 2423 // 2424 NV_CHECK_OR_RETURN_VOID(LEVEL_SILENT, 2425 pKernelGraphicsContextUnicast->channelObjects == 0); 2426 2427 kgrctxUnmapMainCtxBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel); 2428 } 2429 2430 /*! 2431 * @brief is this object responsible for any cleanup tasks i.e. buffer unmapping? 2432 */ 2433 NvBool kgrctxShouldCleanup_KERNEL 2434 ( 2435 OBJGPU *pGpu, 2436 KernelGraphicsContext *pKernelGraphicsContext 2437 ) 2438 { 2439 return gpuIsClientRmAllocatedCtxBufferEnabled(pGpu); 2440 }; 2441 2442 /*! 2443 * This function returns whether PM ctxsw buffer should be pre-allocated or not. 2444 */ 2445 NvBool kgrctxShouldPreAllocPmBuffer_PF 2446 ( 2447 OBJGPU *pGpu, 2448 KernelGraphicsContext *pKernelGraphicsContext, 2449 KernelChannel *pKernelChannel 2450 ) 2451 { 2452 OBJGVASPACE *pGVAS = NULL; 2453 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2454 2455 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2456 NV_ASSERT_OR_RETURN(NULL != pKernelChannel, NV_ERR_INVALID_STATE); 2457 NV_ASSERT_OR_RETURN(pKernelGraphicsContext != NULL, NV_ERR_INVALID_STATE); 2458 2459 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2460 kgrctxGetUnicast(pGpu, 2461 pKernelGraphicsContext, 2462 &pKernelGraphicsContextUnicast)); 2463 2464 pGVAS = dynamicCast(pKernelChannel->pVAS, OBJGVASPACE); 2465 2466 // Do not allocate the buffer, if already allocated 2467 if (pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc != NULL) 2468 { 2469 return NV_FALSE; 2470 } 2471 2472 // 2473 // The PM Ctxsw buffer is now pre-allocated along with the other context buffers 2474 // This is done as a WAR for the issue tracked under bug 1760699 2475 // Reason: With the enablement of UVM8 by default, the UVM channel registration relies on the 2476 // buffers being allocated at channel allocation time and are not going to work for a buffer 2477 // created later with an rm ctrl 2478 // 2479 return gvaspaceIsExternallyOwned(pGVAS); 2480 } 2481 2482 /*! 2483 * This function returns whether PM ctxsw buffer should be pre-allocated 2484 * or not. 2485 */ 2486 NvBool 2487 kgrctxShouldPreAllocPmBuffer_VF 2488 ( 2489 OBJGPU *pGpu, 2490 KernelGraphicsContext *pKernelGraphicsContext, 2491 KernelChannel *pKernelChannel 2492 ) 2493 { 2494 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast = NULL; 2495 2496 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2497 NV_ASSERT_OR_RETURN(NULL != pKernelChannel, NV_ERR_INVALID_STATE); 2498 NV_ASSERT_OR_RETURN(pKernelGraphicsContext != NULL, NV_ERR_INVALID_STATE); 2499 2500 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2501 kgrctxGetUnicast(pGpu, 2502 pKernelGraphicsContext, 2503 &pKernelGraphicsContextUnicast)); 2504 2505 // Do not allocate the buffer, if already allocated 2506 if (pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc != NULL) 2507 { 2508 return NV_FALSE; 2509 } 2510 2511 // 2512 // Pre-allocate the PM Ctxsw buffer along with the other context buffers 2513 // in below scenarios: 2514 // 1. For externally owned VA spaces. This is done as a WAR for the issue 2515 // tracked under bug 1760699. 2516 // Reason: With the enablement of UVM8 by default, the UVM channel registration relies on the 2517 // buffers being allocated at channel allocation time and are not going to work for a buffer 2518 // created later with an rm ctrl. 2519 // 2. For full SRIOV vGPU guests with Profiling capability enabled 2520 // 2521 2522 return kgrctxShouldPreAllocPmBuffer_PF(pGpu, pKernelGraphicsContext, pKernelChannel); 2523 } 2524 2525 /*! 2526 * @brief should this layer manage the ctx buffers? 2527 */ 2528 NvBool 2529 kgrctxShouldManageCtxBuffers_KERNEL 2530 ( 2531 OBJGPU *pGpu, 2532 KernelGraphicsContext *pKernelGraphicsContext, 2533 NvU32 gfid 2534 ) 2535 { 2536 return gpuIsClientRmAllocatedCtxBufferEnabled(pGpu) && !IS_GFID_VF(gfid); 2537 } 2538 2539 /*! 2540 * @brief should this layer manage the ctx buffers? 2541 * If client RM is managing the ctx buffers but the channel is a plugin 2542 * channel, we should still manage them. 2543 */ 2544 NvBool 2545 kgrctxShouldManageCtxBuffers_PHYSICAL 2546 ( 2547 OBJGPU *pGpu, 2548 KernelGraphicsContext *pKernelGraphicsContext, 2549 NvU32 gfid 2550 ) 2551 { 2552 return !gpuIsClientRmAllocatedCtxBufferEnabled(pGpu) || (gpuIsSriovEnabled(pGpu) && IS_GFID_PF(gfid)); 2553 } 2554 2555 /** 2556 * @brief Unmaps buffers associated with this context. 2557 */ 2558 void kgrctxUnmapBuffers_KERNEL 2559 ( 2560 OBJGPU *pGpu, 2561 KernelGraphicsContext *pKernelGraphicsContext, 2562 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast, 2563 KernelChannel *pKernelChannel 2564 ) 2565 { 2566 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 2567 NV_STATUS status; 2568 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo; 2569 KernelGraphics *pKernelGraphics; 2570 NvHandle hClient = RES_GET_CLIENT_HANDLE(pKernelGraphicsContext); 2571 NvHandle hParent = RES_GET_PARENT_HANDLE(pKernelGraphicsContext); 2572 2573 portMemSet(&grRouteInfo, 0, sizeof(grRouteInfo)); 2574 kgrmgrCtrlSetChannelHandle(hParent, &grRouteInfo); 2575 NV_ASSERT_OK_OR_ELSE(status, 2576 kgrmgrCtrlRouteKGR(pGpu, pKernelGraphicsManager, hClient, &grRouteInfo, &pKernelGraphics), 2577 return; ); 2578 2579 kgrctxUnmapAssociatedCtxBuffers(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel); 2580 } 2581 2582 /*! 2583 * @brief Unmap main GR ctx buffer 2584 * 2585 * @param[in] pGpu 2586 * @param[in] pKernelGraphicsContext 2587 * @param[in] pKernelGraphics 2588 * @param[in] pKernelChannel 2589 */ 2590 void 2591 kgrctxUnmapMainCtxBuffer_IMPL 2592 ( 2593 OBJGPU *pGpu, 2594 KernelGraphicsContext *pKernelGraphicsContext, 2595 KernelGraphics *pKernelGraphics, 2596 KernelChannel *pKernelChannel 2597 ) 2598 { 2599 MEMORY_DESCRIPTOR *pCtxBufferMemDesc; 2600 NV_STATUS status; 2601 2602 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2603 2604 status = kchangrpGetEngineContextMemDesc(pGpu, pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup, &pCtxBufferMemDesc); 2605 if ((status == NV_OK) && (pCtxBufferMemDesc != NULL)) 2606 { 2607 // TODO remove Channel, ENG_GR dependencies 2608 kchannelUnmapEngineCtxBuf(pGpu, pKernelChannel, ENG_GR(kgraphicsGetInstance(pGpu, pKernelGraphics))); 2609 kchannelSetEngineContextMemDesc(pGpu, pKernelChannel, ENG_GR(kgraphicsGetInstance(pGpu, pKernelGraphics)), NULL); 2610 } 2611 } 2612 2613 /*! 2614 * @brief Unmap all global ctx buffers from this context 2615 */ 2616 void 2617 kgrctxUnmapGlobalCtxBuffers_IMPL 2618 ( 2619 OBJGPU *pGpu, 2620 KernelGraphicsContext *pKernelGraphicsContext, 2621 KernelGraphics *pKernelGraphics, 2622 OBJVASPACE *pVAS, 2623 NvU32 gfid 2624 ) 2625 { 2626 GR_GLOBALCTX_BUFFERS *pGlobalCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid); 2627 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2628 NV_STATUS status; 2629 2630 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2631 2632 NV_ASSERT_OK_OR_ELSE(status, 2633 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast), 2634 return;); 2635 2636 // if global ctx buffers were never allocated then bail out early 2637 if (!pKernelGraphicsContextUnicast->localCtxBuffer.bAllocated && 2638 !pGlobalCtxBuffers->bAllocated) 2639 { 2640 return; 2641 } 2642 2643 kgrctxUnmapGlobalCtxBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pVAS, GR_GLOBALCTX_BUFFER_BUNDLE_CB); 2644 kgrctxUnmapGlobalCtxBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pVAS, GR_GLOBALCTX_BUFFER_ATTRIBUTE_CB); 2645 kgrctxUnmapGlobalCtxBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pVAS, GR_GLOBALCTX_BUFFER_PAGEPOOL); 2646 2647 if (kgraphicsIsRtvCbSupported(pGpu, pKernelGraphics)) 2648 { 2649 kgrctxUnmapGlobalCtxBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pVAS, GR_GLOBALCTX_BUFFER_RTV_CB); 2650 } 2651 2652 } 2653 2654 /*! 2655 * @brief Unmap specified global ctx buffer from the given VAS 2656 */ 2657 void 2658 kgrctxUnmapGlobalCtxBuffer_IMPL 2659 ( 2660 OBJGPU *pGpu, 2661 KernelGraphicsContext *pKernelGraphicsContext, 2662 KernelGraphics *pKernelGraphics, 2663 OBJVASPACE *pVAS, 2664 GR_GLOBALCTX_BUFFER buffId 2665 ) 2666 { 2667 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2668 NV_STATUS status; 2669 NvU64 vaddr; 2670 2671 NV_ASSERT_OK_OR_ELSE(status, 2672 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast), 2673 return;); 2674 2675 status = vaListFindVa(&pKernelGraphicsContextUnicast->globalCtxBufferVaList[buffId], pVAS, &vaddr); 2676 if (status == NV_OK) 2677 { 2678 NV_PRINTF(LEVEL_INFO, "Unmapping %s from VA @ 0x%llx\n", 2679 NV_ENUM_TO_STRING(GR_GLOBALCTX_BUFFER, buffId), vaddr); 2680 2681 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pVAS, &pKernelGraphicsContextUnicast->globalCtxBufferVaList[buffId]); 2682 } 2683 else 2684 { 2685 NV_PRINTF(LEVEL_INFO, "Buffer for %s already unmapped\n", 2686 NV_ENUM_TO_STRING(GR_GLOBALCTX_BUFFER, buffId)); 2687 } 2688 } 2689 2690 /*! 2691 * @brief Free main GR ctx buffer 2692 */ 2693 void 2694 kgrctxFreeMainCtxBuffer_IMPL 2695 ( 2696 OBJGPU *pGpu, 2697 KernelGraphicsContext *pKernelGraphicsContext 2698 ) 2699 { 2700 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2701 NV_STATUS status; 2702 2703 NV_ASSERT_OK_OR_ELSE(status, 2704 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast), 2705 return;); 2706 2707 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2708 if (pKernelGraphicsContextUnicast->pMainCtxBuffer != NULL) 2709 { 2710 memdescFree(pKernelGraphicsContextUnicast->pMainCtxBuffer); 2711 memdescDestroy(pKernelGraphicsContextUnicast->pMainCtxBuffer); 2712 pKernelGraphicsContextUnicast->pMainCtxBuffer = NULL; 2713 pKernelGraphicsContextUnicast->bKGrMainCtxBufferInitialized = NV_FALSE; 2714 } 2715 } 2716 2717 /** 2718 * @brief free the memory for the zcull context buffer 2719 */ 2720 void 2721 kgrctxFreeZcullBuffer_IMPL 2722 ( 2723 OBJGPU *pGpu, 2724 KernelGraphicsContext *pKernelGraphicsContext 2725 ) 2726 { 2727 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2728 MEMORY_DESCRIPTOR *pMemDesc; 2729 NV_STATUS status; 2730 2731 NV_ASSERT_OK_OR_ELSE(status, 2732 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast), 2733 return;); 2734 2735 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2736 2737 pMemDesc = pKernelGraphicsContextUnicast->zcullCtxswBuffer.pMemDesc; 2738 if (pMemDesc == NULL) 2739 { 2740 NV_PRINTF(LEVEL_INFO, 2741 "call to free zcull ctx buffer not RM managed, skipped!\n"); 2742 } 2743 2744 // buffer can be shared and refcounted -- released on final free 2745 memdescFree(pMemDesc); 2746 memdescDestroy(pMemDesc); 2747 pKernelGraphicsContextUnicast->zcullCtxswBuffer.pMemDesc = NULL; 2748 } 2749 2750 /** 2751 * @brief free the memory for the preemption context buffers 2752 */ 2753 void 2754 kgrctxFreeCtxPreemptionBuffers_IMPL 2755 ( 2756 OBJGPU *pGpu, 2757 KernelGraphicsContext *pKernelGraphicsContext 2758 ) 2759 { 2760 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2761 NV_STATUS status; 2762 2763 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2764 2765 NV_ASSERT_OK_OR_ELSE(status, 2766 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast), 2767 return;); 2768 2769 memdescFree(pKernelGraphicsContextUnicast->preemptCtxswBuffer.pMemDesc); 2770 memdescDestroy(pKernelGraphicsContextUnicast->preemptCtxswBuffer.pMemDesc); 2771 pKernelGraphicsContextUnicast->preemptCtxswBuffer.pMemDesc = NULL; 2772 2773 memdescFree(pKernelGraphicsContextUnicast->spillCtxswBuffer.pMemDesc); 2774 memdescDestroy(pKernelGraphicsContextUnicast->spillCtxswBuffer.pMemDesc); 2775 pKernelGraphicsContextUnicast->spillCtxswBuffer.pMemDesc = NULL; 2776 2777 memdescFree(pKernelGraphicsContextUnicast->betaCBCtxswBuffer.pMemDesc); 2778 memdescDestroy(pKernelGraphicsContextUnicast->betaCBCtxswBuffer.pMemDesc); 2779 pKernelGraphicsContextUnicast->betaCBCtxswBuffer.pMemDesc = NULL; 2780 2781 memdescFree(pKernelGraphicsContextUnicast->pagepoolCtxswBuffer.pMemDesc); 2782 memdescDestroy(pKernelGraphicsContextUnicast->pagepoolCtxswBuffer.pMemDesc); 2783 pKernelGraphicsContextUnicast->pagepoolCtxswBuffer.pMemDesc = NULL; 2784 2785 memdescFree(pKernelGraphicsContextUnicast->rtvCbCtxswBuffer.pMemDesc); 2786 memdescDestroy(pKernelGraphicsContextUnicast->rtvCbCtxswBuffer.pMemDesc); 2787 pKernelGraphicsContextUnicast->rtvCbCtxswBuffer.pMemDesc = NULL; 2788 } 2789 2790 /*! 2791 * @brief Free patch GR ctx buffer 2792 */ 2793 void 2794 kgrctxFreePatchBuffer_IMPL 2795 ( 2796 OBJGPU *pGpu, 2797 KernelGraphicsContext *pKernelGraphicsContext 2798 ) 2799 { 2800 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2801 NV_STATUS status; 2802 2803 NV_ASSERT_OK_OR_ELSE(status, 2804 kgrctxGetUnicast(pGpu, 2805 pKernelGraphicsContext, 2806 &pKernelGraphicsContextUnicast), 2807 return;); 2808 2809 if (pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc == NULL) 2810 { 2811 NV_PRINTF(LEVEL_INFO, 2812 "Attempt to free null ctx patch buffer pointer, skipped!\n"); 2813 return; 2814 } 2815 2816 memdescFree(pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc); 2817 memdescDestroy(pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc); 2818 2819 pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc = NULL; 2820 pKernelGraphicsContextUnicast->bKGrPatchCtxBufferInitialized = NV_FALSE; 2821 } 2822 2823 /** 2824 * @brief Free the memory for the pm context buffer associated with a channel 2825 */ 2826 void 2827 kgrctxFreePmBuffer_IMPL 2828 ( 2829 OBJGPU *pGpu, 2830 KernelGraphicsContext *pKernelGraphicsContext 2831 ) 2832 { 2833 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2834 MEMORY_DESCRIPTOR *pMemDesc; 2835 NV_STATUS status; 2836 2837 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2838 2839 NV_ASSERT_OK_OR_ELSE(status, 2840 kgrctxGetUnicast(pGpu, 2841 pKernelGraphicsContext, 2842 &pKernelGraphicsContextUnicast), 2843 return;); 2844 2845 pMemDesc = pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc; 2846 if (pMemDesc != NULL) 2847 { 2848 // 2849 // This func assumes that the buffer was not allocated per subdevice, 2850 // and will leak any mappings performed by the secondaries. 2851 // 2852 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc)); 2853 2854 memdescFree(pMemDesc); 2855 memdescDestroy(pMemDesc); 2856 } 2857 else 2858 { 2859 NV_PRINTF(LEVEL_INFO, 2860 "Attempt to free null pm ctx buffer pointer??\n"); 2861 } 2862 2863 pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc = NULL; 2864 pKernelGraphicsContextUnicast->bKGrPmCtxBufferInitialized = NV_FALSE; 2865 } 2866 2867 /*! 2868 * @brief Free "local" global context buffers 2869 */ 2870 void kgrctxFreeLocalGlobalCtxBuffers_IMPL 2871 ( 2872 OBJGPU *pGpu, 2873 KernelGraphicsContext *pKernelGraphicsContext 2874 ) 2875 { 2876 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2877 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 2878 MEMORY_DESCRIPTOR *pMemDesc; 2879 GR_GLOBALCTX_BUFFERS *pCtxBuffers; 2880 GR_GLOBALCTX_BUFFER buff; 2881 NV_STATUS status; 2882 2883 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2884 2885 NV_ASSERT_OK_OR_ELSE(status, 2886 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast), 2887 return;); 2888 2889 pCtxBuffers = &pKernelGraphicsContextUnicast->localCtxBuffer; 2890 2891 // no ctx buffers allocated, so get out early 2892 if (!pCtxBuffers->bAllocated) 2893 return; 2894 2895 FOR_EACH_IN_ENUM(GR_GLOBALCTX_BUFFER, buff) 2896 { 2897 pMemDesc = pCtxBuffers->memDesc[buff]; 2898 memdescFree(pMemDesc); 2899 memdescDestroy(pMemDesc); 2900 pCtxBuffers->memDesc[buff] = NULL; 2901 } 2902 FOR_EACH_IN_ENUM_END; 2903 2904 pCtxBuffers->bAllocated = NV_FALSE; 2905 2906 // make sure all L2 cache lines using CB buffers are clear after we free them 2907 NV_ASSERT_OK(kmemsysCacheOp_HAL(pGpu, pKernelMemorySystem, NULL, FB_CACHE_VIDEO_MEMORY, FB_CACHE_EVICT)); 2908 } 2909 2910 /** 2911 * @brief Free all associated ctx buffers (main, patch, PM and private global buffers for cases like VPR). 2912 * 2913 * This function is called on every channel free and thefore can't assume any 2914 * graphics objects actually exist on the given channel. 2915 */ 2916 void kgrctxFreeAssociatedCtxBuffers_IMPL 2917 ( 2918 OBJGPU *pGpu, 2919 KernelGraphicsContext *pKernelGraphicsContext 2920 ) 2921 { 2922 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2923 NV_STATUS status; 2924 2925 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2926 2927 NV_ASSERT_OK_OR_ELSE(status, 2928 kgrctxGetUnicast(pGpu, 2929 pKernelGraphicsContext, 2930 &pKernelGraphicsContextUnicast), 2931 return;); 2932 2933 // 2934 // When sharing contexts across channels we need to defer this until all 2935 // objects have been freed. 2936 // 2937 NV_CHECK_OR_RETURN_VOID(LEVEL_SILENT, pKernelGraphicsContextUnicast->channelObjects == 0); 2938 2939 // if we have a context patch buffer, this will free it up 2940 kgrctxFreePatchBuffer(pGpu, pKernelGraphicsContext); 2941 2942 // if we have a PM ctxsw buffer, this will free it up 2943 kgrctxFreePmBuffer(pGpu, pKernelGraphicsContext); 2944 2945 // if we have a zcull buffer, this will free it up 2946 kgrctxFreeZcullBuffer(pGpu, pKernelGraphicsContext); 2947 2948 // If we have preemption buffers, this will free them up 2949 kgrctxFreeCtxPreemptionBuffers(pGpu, pKernelGraphicsContext); 2950 2951 // Release all common buffers used as part of the gr context. 2952 kgrctxFreeLocalGlobalCtxBuffers(pGpu, pKernelGraphicsContext); 2953 2954 kgrctxFreeMainCtxBuffer(pGpu, pKernelGraphicsContext); 2955 } 2956 2957 /*! 2958 * This function unmaps various GR buffers. 2959 */ 2960 NV_STATUS 2961 kgrctxUnmapCtxBuffers_IMPL 2962 ( 2963 OBJGPU *pGpu, 2964 KernelGraphicsContext *pKernelGraphicsContext, 2965 KernelGraphicsObject *pKernelGraphicsObject, 2966 KernelGraphics *pKernelGraphics, 2967 NvBool bDestructor 2968 ) 2969 { 2970 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2971 ChannelDescendant *pChannelDescendant = staticCast(pKernelGraphicsObject, ChannelDescendant); 2972 KernelChannel *pKernelChannel = pChannelDescendant->pKernelChannel; 2973 NvU32 classNum = pChannelDescendant->resourceDesc.externalClassId; 2974 NvBool bRelease3d = NV_FALSE; 2975 NvU32 objType; 2976 NvU32 gfid; 2977 NvU64 refCount; 2978 GR_GLOBALCTX_BUFFER registerMapBufferId = kgrctxGetRegisterAccessMapId_HAL(pGpu, pKernelGraphicsContext, pChannelDescendant->pKernelChannel); 2979 2980 NV_PRINTF(LEVEL_INFO, "gpu:%d isBC=%d\n", pGpu->gpuInstance, 2981 gpumgrGetBcEnabledStatus(pGpu)); 2982 2983 gfid = kchannelGetGfid(pKernelChannel); 2984 2985 NV_ASSERT_OK_OR_RETURN( 2986 kgrctxGetUnicast(pGpu, 2987 pKernelGraphicsObject->pKernelGraphicsContext, 2988 &pKernelGraphicsContextUnicast)); 2989 2990 kgrmgrGetGrObjectType(classNum, &objType); 2991 2992 if (!bDestructor) 2993 { 2994 // 2995 // If we are cleaning up from the constructor, then we know the main ctx 2996 // buffer is not being used yet, and no other context buffers need to be 2997 // cleaned up, so we can return early here. If we are coming from the 2998 // destructor, the context buffers are not freed until channel 2999 // destruction. 3000 // TODO move this up one stack frame 3001 // 3002 kgrctxUnmapMainCtxBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pChannelDescendant->pKernelChannel); 3003 kgrctxFreeMainCtxBuffer(pGpu, pKernelGraphicsContext); 3004 return NV_OK; 3005 } 3006 3007 if ((pKernelGraphicsContextUnicast->channelObjects != 0) && 3008 (pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc != NULL)) 3009 { 3010 // 3011 // Delay freeing the patch buffer until the last channel free. 3012 // these buffers are accessed even after the last GR object is freed. 3013 // 3014 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pKernelChannel->pVAS, &pKernelGraphicsContextUnicast->ctxPatchBuffer.vAddrList); 3015 } 3016 3017 // Defer releasing mapping if this would cause the buffer to be unmapped 3018 if ((pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc != NULL) && 3019 (vaListGetRefCount(&pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList, pKernelChannel->pVAS, &refCount) == NV_OK) && 3020 (refCount > 1)) 3021 { 3022 kgrctxUnmapCtxPmBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel->pVAS); 3023 } 3024 3025 // Defer releasing mapping if this would cause the buffer to be unmapped 3026 if (kgraphicsDoesUcodeSupportPrivAccessMap(pGpu, pKernelGraphics) && 3027 (vaListGetRefCount(&pKernelGraphicsContextUnicast->globalCtxBufferVaList[registerMapBufferId], pKernelChannel->pVAS, &refCount) == NV_OK) && 3028 (refCount > 1)) 3029 { 3030 kgrctxUnmapGlobalCtxBuffer(pGpu, 3031 pKernelGraphicsContext, 3032 pKernelGraphics, 3033 pKernelChannel->pVAS, 3034 registerMapBufferId); 3035 } 3036 3037 // Defer releasing mapping if this would cause the buffer to be unmapped 3038 if ((kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid)->memDesc[GR_GLOBALCTX_BUFFER_FECS_EVENT] != NULL) && 3039 (vaListGetRefCount(&pKernelGraphicsContextUnicast->globalCtxBufferVaList[GR_GLOBALCTX_BUFFER_FECS_EVENT], pKernelChannel->pVAS, &refCount) == NV_OK) && 3040 (refCount > 1)) 3041 { 3042 kgrctxUnmapGlobalCtxBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel->pVAS, GR_GLOBALCTX_BUFFER_FECS_EVENT); 3043 } 3044 3045 // 3046 // Condition for releasing 3d context buffer mappings: 3047 // For non-TSG & legacy TSG mode, always unmap as long as its not the last 3048 // object in TSG 3049 // If it is the last object, then unmap during channel free as these buffers 3050 // are accessed even after the last 3D object is freed. 3051 // For subcontext TSG case, only unmap on 2D/3D object free as long its not 3052 // the last 2D/3D object 3053 // For compute object with subcontext we never map so we don't need to call 3054 // unmap. 3055 // If we free on last object then buffer gets actually unmapped (as opposed 3056 // to decrement in refcount) and we start hitting page faults 3057 // 3058 if (pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->bLegacyMode) 3059 { 3060 bRelease3d = (pKernelGraphicsContextUnicast->channelObjects > 0); 3061 } 3062 else 3063 { 3064 NvBool bIs3dBuffer = ((objType == GR_OBJECT_TYPE_2D) || (objType == GR_OBJECT_TYPE_3D)); 3065 bRelease3d = (bIs3dBuffer && ((pKernelGraphicsContextUnicast->objectCounts[GR_OBJECT_TYPE_2D] > 0) || 3066 (pKernelGraphicsContextUnicast->objectCounts[GR_OBJECT_TYPE_3D] > 0))); 3067 } 3068 3069 if (bRelease3d) 3070 { 3071 kgrctxUnmapGlobalCtxBuffers(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel->pVAS, gfid); 3072 } 3073 3074 return NV_OK; 3075 } 3076 3077 /*! 3078 * Function to increment the GR channel object count 3079 */ 3080 void 3081 kgrctxIncObjectCount_IMPL 3082 ( 3083 OBJGPU *pGpu, 3084 KernelGraphicsContext *pKernelGraphicsContext, 3085 NvU32 classNum 3086 ) 3087 { 3088 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 3089 NvU32 objType; 3090 NV_STATUS status; 3091 3092 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 3093 3094 kgrmgrGetGrObjectType(classNum, &objType); 3095 3096 NV_ASSERT_OK_OR_ELSE(status, 3097 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast), 3098 return;); 3099 3100 switch (objType) 3101 { 3102 case GR_OBJECT_TYPE_COMPUTE: 3103 pKernelGraphicsContextUnicast->objectCounts[GR_OBJECT_TYPE_COMPUTE]++; 3104 gpuChangeComputeModeRefCount(pGpu, NV_GPU_COMPUTE_REFCOUNT_COMMAND_INCREMENT); 3105 break; 3106 3107 case GR_OBJECT_TYPE_3D: 3108 pKernelGraphicsContextUnicast->objectCounts[GR_OBJECT_TYPE_3D]++; 3109 break; 3110 3111 case GR_OBJECT_TYPE_2D: 3112 pKernelGraphicsContextUnicast->objectCounts[GR_OBJECT_TYPE_2D]++; 3113 break; 3114 3115 case GR_OBJECT_TYPE_MEM: 3116 pKernelGraphicsContextUnicast->objectCounts[GR_OBJECT_TYPE_MEM]++; 3117 break; 3118 3119 default: 3120 NV_PRINTF(LEVEL_ERROR, "Unrecognized graphics class 0x%x\n", 3121 classNum); 3122 DBG_BREAKPOINT(); 3123 return; 3124 } 3125 3126 pKernelGraphicsContextUnicast->channelObjects++; 3127 NV_PRINTF(LEVEL_INFO, 3128 "Class 0x%x allocated. %d objects allocated\n", 3129 classNum, 3130 pKernelGraphicsContextUnicast->channelObjects); 3131 } 3132 3133 /*! 3134 * Function to decrement the GR channel object count of the given class 3135 */ 3136 void 3137 kgrctxDecObjectCount_IMPL 3138 ( 3139 OBJGPU *pGpu, 3140 KernelGraphicsContext *pKernelGraphicsContext, 3141 NvU32 classNum 3142 ) 3143 { 3144 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 3145 NvU32 objType; 3146 NvU32 countIdx; 3147 NV_STATUS status; 3148 3149 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 3150 3151 kgrmgrGetGrObjectType(classNum, &objType); 3152 3153 NV_ASSERT_OK_OR_ELSE(status, 3154 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast), 3155 return;); 3156 3157 NV_ASSERT_OR_ELSE(pKernelGraphicsContextUnicast->channelObjects != 0, 3158 NV_PRINTF(LEVEL_ERROR, "No active GR objects to free for Class 0x%x\n", classNum)); 3159 3160 // 3161 // Handle 2D and Compute class reference counting 3162 // 3163 switch (objType) 3164 { 3165 case GR_OBJECT_TYPE_COMPUTE: 3166 gpuChangeComputeModeRefCount(pGpu, NV_GPU_COMPUTE_REFCOUNT_COMMAND_DECREMENT); 3167 countIdx = GR_OBJECT_TYPE_COMPUTE; 3168 break; 3169 3170 case GR_OBJECT_TYPE_3D: 3171 countIdx = GR_OBJECT_TYPE_3D; 3172 break; 3173 3174 case GR_OBJECT_TYPE_2D: 3175 countIdx = GR_OBJECT_TYPE_2D; 3176 break; 3177 3178 case GR_OBJECT_TYPE_MEM: 3179 countIdx = GR_OBJECT_TYPE_MEM; 3180 break; 3181 3182 default: 3183 NV_PRINTF(LEVEL_ERROR, "Unrecognized graphics class 0x%x\n", 3184 classNum); 3185 DBG_BREAKPOINT(); 3186 return; 3187 } 3188 3189 NV_ASSERT_OR_RETURN_VOID(pKernelGraphicsContextUnicast->objectCounts[countIdx] > 0); 3190 pKernelGraphicsContextUnicast->objectCounts[countIdx]--; 3191 pKernelGraphicsContextUnicast->channelObjects--; 3192 } 3193 3194 /*! 3195 * @brief Return the register access buffer used for a given Channel's permission 3196 * 3197 * The base register access map comes from the architecture team. The user version 3198 * of the buffer removes access to PM regisers. 3199 * 3200 * The policy currently depends on administrator access on the system, except on 3201 * one VGPU configuration. 3202 */ 3203 GR_GLOBALCTX_BUFFER 3204 kgrctxGetRegisterAccessMapId_PF 3205 ( 3206 OBJGPU *pGpu, 3207 KernelGraphicsContext *pKernelGraphicsContext, 3208 KernelChannel *pKernelChannel 3209 ) 3210 { 3211 RmClient *pRmClient = dynamicCast(RES_GET_CLIENT(pKernelChannel), RmClient); 3212 RS_PRIV_LEVEL privLevel = rmclientGetCachedPrivilege(pRmClient); 3213 3214 // Using cached privilege because this function is called at a raised IRQL. 3215 if ((privLevel >= RS_PRIV_LEVEL_USER_ROOT) 3216 && !hypervisorIsVgxHyper() && IS_GFID_PF(kchannelGetGfid(pKernelChannel))) 3217 { 3218 return GR_GLOBALCTX_BUFFER_UNRESTRICTED_PRIV_ACCESS_MAP; 3219 } 3220 3221 return GR_GLOBALCTX_BUFFER_PRIV_ACCESS_MAP; 3222 } 3223 3224 NV_STATUS 3225 kgrctxCtrlGetTpcPartitionMode_IMPL 3226 ( 3227 KernelGraphicsContext *pKernelGraphicsContext, 3228 NV0090_CTRL_TPC_PARTITION_MODE_PARAMS *pParams 3229 ) 3230 { 3231 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelGraphicsContext); 3232 3233 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner()); 3234 3235 if (IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu)) 3236 { 3237 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 3238 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams; 3239 NV_STATUS status = NV_OK; 3240 3241 NV_RM_RPC_CONTROL(pGpu, 3242 pRmCtrlParams->hClient, 3243 pRmCtrlParams->hObject, 3244 pRmCtrlParams->cmd, 3245 pRmCtrlParams->pParams, 3246 pRmCtrlParams->paramsSize, 3247 status); 3248 3249 return status; 3250 } 3251 3252 return gpuresInternalControlForward_IMPL(staticCast(pKernelGraphicsContext, GpuResource), 3253 NV0090_CTRL_CMD_INTERNAL_GET_TPC_PARTITION_MODE, 3254 pParams, 3255 sizeof(*pParams)); 3256 } 3257 3258 NV_STATUS 3259 kgrctxCtrlSetTpcPartitionMode_IMPL 3260 ( 3261 KernelGraphicsContext *pKernelGraphicsContext, 3262 NV0090_CTRL_TPC_PARTITION_MODE_PARAMS *pParams 3263 ) 3264 { 3265 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelGraphicsContext); 3266 3267 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner()); 3268 3269 if (IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu)) 3270 { 3271 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 3272 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams; 3273 NV_STATUS status = NV_OK; 3274 3275 NV_RM_RPC_CONTROL(pGpu, 3276 pRmCtrlParams->hClient, 3277 pRmCtrlParams->hObject, 3278 pRmCtrlParams->cmd, 3279 pRmCtrlParams->pParams, 3280 pRmCtrlParams->paramsSize, 3281 status); 3282 3283 return status; 3284 } 3285 3286 return gpuresInternalControlForward_IMPL(staticCast(pKernelGraphicsContext, GpuResource), 3287 NV0090_CTRL_CMD_INTERNAL_SET_TPC_PARTITION_MODE, 3288 pParams, 3289 sizeof(*pParams)); 3290 } 3291 3292 NV_STATUS 3293 kgrctxCtrlGetMMUDebugMode_IMPL 3294 ( 3295 KernelGraphicsContext *pKernelGraphicsContext, 3296 NV0090_CTRL_GET_MMU_DEBUG_MODE_PARAMS *pParams 3297 ) 3298 { 3299 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelGraphicsContext); 3300 3301 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner()); 3302 3303 if (IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu)) 3304 { 3305 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 3306 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams; 3307 NV_STATUS status = NV_OK; 3308 3309 NV_RM_RPC_CONTROL(pGpu, 3310 pRmCtrlParams->hClient, 3311 pRmCtrlParams->hObject, 3312 pRmCtrlParams->cmd, 3313 pRmCtrlParams->pParams, 3314 pRmCtrlParams->paramsSize, 3315 status); 3316 3317 return status; 3318 } 3319 3320 return gpuresInternalControlForward_IMPL(staticCast(pKernelGraphicsContext, GpuResource), 3321 NV0090_CTRL_CMD_INTERNAL_GET_MMU_DEBUG_MODE, 3322 pParams, 3323 sizeof(*pParams)); 3324 } 3325 3326 NV_STATUS 3327 kgrctxCtrlProgramVidmemPromote_IMPL 3328 ( 3329 KernelGraphicsContext *pKernelGraphicsContext, 3330 NV0090_CTRL_PROGRAM_VIDMEM_PROMOTE_PARAMS *pParams 3331 ) 3332 { 3333 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelGraphicsContext); 3334 3335 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner()); 3336 3337 if (IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu)) 3338 { 3339 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 3340 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams; 3341 NV_STATUS status = NV_OK; 3342 3343 NV_RM_RPC_CONTROL(pGpu, 3344 pRmCtrlParams->hClient, 3345 pRmCtrlParams->hObject, 3346 pRmCtrlParams->cmd, 3347 pRmCtrlParams->pParams, 3348 pRmCtrlParams->paramsSize, 3349 status); 3350 3351 return status; 3352 } 3353 3354 return gpuresInternalControlForward_IMPL(staticCast(pKernelGraphicsContext, GpuResource), 3355 NV0090_CTRL_CMD_INTERNAL_PROGRAM_VIDMEM_PROMOTE, 3356 pParams, 3357 sizeof(*pParams)); 3358 } 3359 3360 /*! 3361 * @brief Construct shared kernel graphics context. (Does nothing) 3362 */ 3363 NV_STATUS 3364 shrkgrctxConstruct_IMPL(KernelGraphicsContextShared *pKernelGraphicsContextShared) 3365 { 3366 return NV_OK; 3367 } 3368 3369 /*! 3370 * @brief Construct shared kernel graphics context 3371 * 3372 * @param[in] pGpu 3373 * @param[in] pKernelGraphicsContextShared 3374 * @param[in] pKernelGraphicsContext 3375 */ 3376 NV_STATUS 3377 shrkgrctxInit_IMPL 3378 ( 3379 OBJGPU *pGpu, 3380 KernelGraphicsContextShared *pKernelGraphicsContextShared, 3381 KernelGraphicsContext *pKernelGraphicsContext 3382 ) 3383 { 3384 NV_STATUS status = NV_OK; 3385 3386 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY) 3387 { 3388 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 3389 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3390 KernelGraphics *pKernelGraphics; 3391 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo; 3392 NvHandle hClient = RES_GET_CLIENT_HANDLE(pKernelGraphicsContext); 3393 NvHandle hParent = RES_GET_PARENT_HANDLE(pKernelGraphicsContext); 3394 3395 portMemSet(&grRouteInfo, 0, sizeof(grRouteInfo)); 3396 kgrmgrCtrlSetChannelHandle(hParent, &grRouteInfo); 3397 3398 NV_CHECK_OK_OR_CAPTURE_FIRST_ERROR(status, LEVEL_ERROR, 3399 kgrmgrCtrlRouteKGR(pGpu, pKernelGraphicsManager, hClient, &grRouteInfo, &pKernelGraphics)); 3400 3401 if (status != NV_OK) 3402 SLI_LOOP_BREAK; 3403 3404 if (kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast) == NV_OK) 3405 { 3406 NV_CHECK_OK_OR_CAPTURE_FIRST_ERROR(status, LEVEL_ERROR, 3407 shrkgrctxConstructUnicast(pGpu, pKernelGraphicsContextShared, pKernelGraphicsContext, pKernelGraphics, pKernelGraphicsContextUnicast)); 3408 3409 if (status != NV_OK) 3410 SLI_LOOP_BREAK; 3411 } 3412 } 3413 SLI_LOOP_END; 3414 3415 if (status != NV_OK) 3416 shrkgrctxTeardown_IMPL(pGpu, pKernelGraphicsContextShared, pKernelGraphicsContext); 3417 3418 return status; 3419 } 3420 3421 /*! 3422 * @brief Construct unicast state for this context 3423 * 3424 * @param[in] pGpu 3425 * @param[in] pKernelGraphicsContextShared 3426 * @param[in] pKernelGraphicsContext 3427 * @param[in] pKernelGraphics 3428 * @param[in] pKernelGraphicsContextUnicast 3429 */ 3430 NV_STATUS 3431 shrkgrctxConstructUnicast_IMPL 3432 ( 3433 OBJGPU *pGpu, 3434 KernelGraphicsContextShared *pKernelGraphicsContextShared, 3435 KernelGraphicsContext *pKernelGraphicsContext, 3436 KernelGraphics *pKernelGraphics, 3437 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast 3438 ) 3439 { 3440 NvU32 i; 3441 3442 for (i = 0; i < GR_GLOBALCTX_BUFFER_COUNT; i++) 3443 NV_ASSERT_OK_OR_RETURN(vaListInit(&pKernelGraphicsContextUnicast->globalCtxBufferVaList[i])); 3444 3445 NV_ASSERT_OK_OR_RETURN(vaListInit(&pKernelGraphicsContextUnicast->ctxPatchBuffer.vAddrList)); 3446 NV_ASSERT_OK_OR_RETURN(vaListInit(&pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList)); 3447 NV_ASSERT_OK_OR_RETURN(vaListInit(&pKernelGraphicsContextUnicast->zcullCtxswBuffer.vAddrList)); 3448 3449 NV_ASSERT_OK_OR_RETURN(vaListInit(&pKernelGraphicsContextUnicast->preemptCtxswBuffer.vAddrList)); 3450 NV_ASSERT_OK_OR_RETURN(vaListInit(&pKernelGraphicsContextUnicast->spillCtxswBuffer.vAddrList)); 3451 NV_ASSERT_OK_OR_RETURN(vaListInit(&pKernelGraphicsContextUnicast->betaCBCtxswBuffer.vAddrList)); 3452 NV_ASSERT_OK_OR_RETURN(vaListInit(&pKernelGraphicsContextUnicast->pagepoolCtxswBuffer.vAddrList)); 3453 NV_ASSERT_OK_OR_RETURN(vaListInit(&pKernelGraphicsContextUnicast->rtvCbCtxswBuffer.vAddrList)); 3454 3455 pKernelGraphicsContextUnicast->bSupportsPerSubctxHeader = 3456 kgraphicsIsPerSubcontextContextHeaderSupported(pGpu, pKernelGraphics); 3457 3458 return NV_OK; 3459 } 3460 3461 /*! 3462 * @brief Destruct shared kernel graphics context. (Does nothing) 3463 */ 3464 void 3465 shrkgrctxDestruct_IMPL(KernelGraphicsContextShared *pKernelGraphicsContextShared) 3466 { 3467 return; 3468 } 3469 3470 /*! 3471 * @brief Destroy the shared context state 3472 */ 3473 void 3474 shrkgrctxTeardown_IMPL 3475 ( 3476 OBJGPU *pGpu, 3477 KernelGraphicsContextShared *pKernelGraphicsContextShared, 3478 KernelGraphicsContext *pKernelGraphicsContext 3479 ) 3480 { 3481 3482 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY) 3483 { 3484 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 3485 3486 if (kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast) == NV_OK) 3487 shrkgrctxDestructUnicast(pGpu, pKernelGraphicsContextShared, pKernelGraphicsContext, pKernelGraphicsContextUnicast); 3488 } 3489 SLI_LOOP_END; 3490 3491 portMemSet(&pKernelGraphicsContext->pShared->kernelGraphicsContextUnicast, 0x0, sizeof(pKernelGraphicsContext->pShared->kernelGraphicsContextUnicast)); 3492 } 3493 3494 /*! 3495 * Destroy the unicast context state 3496 */ 3497 void 3498 shrkgrctxDestructUnicast_IMPL 3499 ( 3500 OBJGPU *pGpu, 3501 KernelGraphicsContextShared *pKernelGraphicsContextShared, 3502 KernelGraphicsContext *pKernelGraphicsContext, 3503 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast 3504 ) 3505 { 3506 GR_GLOBALCTX_BUFFER buff; 3507 3508 if (kgrctxShouldCleanup_HAL(pGpu, pKernelGraphicsContext)) 3509 kgrctxFreeAssociatedCtxBuffers(pGpu, pKernelGraphicsContext); 3510 3511 FOR_EACH_IN_ENUM(GR_GLOBALCTX_BUFFER, buff) 3512 { 3513 vaListDestroy(&pKernelGraphicsContextUnicast->globalCtxBufferVaList[buff]); 3514 } 3515 FOR_EACH_IN_ENUM_END; 3516 vaListDestroy(&pKernelGraphicsContextUnicast->ctxPatchBuffer.vAddrList); 3517 vaListDestroy(&pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList); 3518 vaListDestroy(&pKernelGraphicsContextUnicast->zcullCtxswBuffer.vAddrList); 3519 3520 vaListDestroy(&pKernelGraphicsContextUnicast->preemptCtxswBuffer.vAddrList); 3521 vaListDestroy(&pKernelGraphicsContextUnicast->spillCtxswBuffer.vAddrList); 3522 vaListDestroy(&pKernelGraphicsContextUnicast->betaCBCtxswBuffer.vAddrList); 3523 vaListDestroy(&pKernelGraphicsContextUnicast->pagepoolCtxswBuffer.vAddrList); 3524 vaListDestroy(&pKernelGraphicsContextUnicast->rtvCbCtxswBuffer.vAddrList); 3525 } 3526 3527 /*! 3528 * @brief Perform cleanup tasks run on channel removal from context 3529 */ 3530 void shrkgrctxDetach_IMPL 3531 ( 3532 OBJGPU *pGpu, 3533 KernelGraphicsContextShared *pKernelGraphicsContextShared, 3534 KernelGraphicsContext *pKernelGraphicsContext, 3535 KernelChannel *pKernelChannel 3536 ) 3537 { 3538 RM_ENGINE_TYPE rmEngineType = kchannelGetEngineType(pKernelChannel); 3539 3540 // pre-Ampere chips can have NULL engine types. Find the engine type based on runlistId if set 3541 if (!RM_ENGINE_TYPE_IS_VALID(rmEngineType)) 3542 { 3543 if (kchannelIsRunlistSet(pGpu, pKernelChannel)) 3544 { 3545 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 3546 3547 NV_ASSERT_OK( 3548 kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo, 3549 ENGINE_INFO_TYPE_RUNLIST, kchannelGetRunlistId(pKernelChannel), 3550 ENGINE_INFO_TYPE_RM_ENGINE_TYPE, (NvU32 *)&rmEngineType)); 3551 } 3552 else 3553 { 3554 NV_PRINTF(LEVEL_INFO, "Channel destroyed but never bound, scheduled, or had a descendant object created\n"); 3555 } 3556 } 3557 3558 if (RM_ENGINE_TYPE_IS_GR(rmEngineType)) 3559 { 3560 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannel); 3561 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3562 NvHandle hClient = RES_GET_CLIENT_HANDLE(pKernelChannel); 3563 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo = {0}; 3564 KernelGraphics *pKernelGraphics; 3565 NV_STATUS status; 3566 3567 kgrmgrCtrlSetChannelHandle(RES_GET_HANDLE(pKernelChannel), &grRouteInfo); 3568 NV_ASSERT_OK_OR_ELSE(status, 3569 kgrmgrCtrlRouteKGR(pGpu, pKernelGraphicsManager, hClient, &grRouteInfo, &pKernelGraphics), 3570 return;); 3571 3572 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 3573 { 3574 // Last chance to process FECS ctxsw log before channel goes away 3575 nvEventBufferFecsCallback(pGpu, pKernelGraphics); 3576 } 3577 SLI_LOOP_END 3578 } 3579 3580 // 3581 // If PDB_PROP_GPU_CLIENT_RM_ALLOCATED_CTX_BUFFER is set, then kernel RM 3582 // is responsible for unmapping the context buffers, otherwise it is left to 3583 // physical RM. 3584 // 3585 if (!kgrctxShouldCleanup(pGpu, pKernelGraphicsContext)) 3586 return; 3587 3588 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY) 3589 { 3590 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 3591 3592 if (kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast) == NV_OK) 3593 kgrctxUnmapBuffers_HAL(pGpu, pKernelGraphicsContext, pKernelGraphicsContextUnicast, pKernelChannel); 3594 } 3595 SLI_LOOP_END; 3596 } 3597 3598