1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #define NVOC_KERNEL_GRAPHICS_CONTEXT_H_PRIVATE_ACCESS_ALLOWED 25 26 #include "kernel/gpu/gr/kernel_graphics_context.h" 27 #include "kernel/gpu/gr/kernel_graphics_manager.h" 28 #include "kernel/gpu/gr/kernel_graphics.h" 29 #include "kernel/gpu/fifo/kernel_channel.h" 30 #include "kernel/gpu/fifo/kernel_channel_group.h" 31 #include "kernel/gpu/fifo/kernel_channel_group_api.h" 32 #include "kernel/mem_mgr/gpu_vaspace.h" 33 #include "kernel/gpu/mem_mgr/mem_mgr.h" 34 #include "kernel/gpu/mem_sys/kern_mem_sys.h" 35 #include "kernel/core/locks.h" 36 #include "kernel/gpu/nvlink/kernel_nvlink.h" 37 #include "vgpu/rpc.h" 38 #include "kernel/gpu/subdevice/subdevice.h" 39 #include "kernel/virtualization/hypervisor/hypervisor.h" 40 #include "gpu/mem_mgr/virt_mem_allocator.h" 41 #include "gpu/mmu/kern_gmmu.h" 42 #include "rmapi/client.h" 43 44 /*! 45 * @brief Retrieve the context object from a KernelChannel. 46 * 47 * @return NV_ERR_OBJECT_NOT_FOUND if context object is missing 48 */ 49 NV_STATUS 50 kgrctxFromKernelChannel_IMPL 51 ( 52 KernelChannel *pKernelChannel, 53 KernelGraphicsContext **ppKernelGraphicsContext 54 ) 55 { 56 RsResourceRef *pResourceRef; 57 58 NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_ARGUMENT); 59 60 if (pKernelChannel->hKernelGraphicsContext != NV01_NULL_OBJECT) 61 { 62 NV_ASSERT_OK_OR_RETURN( 63 clientGetResourceRefByType(RES_GET_CLIENT(pKernelChannel), 64 pKernelChannel->hKernelGraphicsContext, 65 classId(KernelGraphicsContext), 66 &pResourceRef)); 67 } 68 else if ((pKernelChannel->pKernelChannelGroupApi != NULL) && 69 (pKernelChannel->pKernelChannelGroupApi->hKernelGraphicsContext != NV01_NULL_OBJECT)) 70 { 71 NV_ASSERT_OK_OR_RETURN( 72 clientGetResourceRefByType(RES_GET_CLIENT(pKernelChannel), 73 pKernelChannel->pKernelChannelGroupApi->hKernelGraphicsContext, 74 classId(KernelGraphicsContext), 75 &pResourceRef)); 76 } 77 else 78 { 79 return NV_ERR_OBJECT_NOT_FOUND; 80 } 81 82 *ppKernelGraphicsContext = dynamicCast(pResourceRef->pResource, KernelGraphicsContext); 83 NV_ASSERT_OR_RETURN(*ppKernelGraphicsContext != NULL, NV_ERR_INVALID_STATE); 84 85 return NV_OK; 86 } 87 88 /*! 89 * @brief Retrieve the context object from a ChannelGroup. 90 * 91 * @return NV_ERR_OBJECT_NOT_FOUND if context object is missing 92 */ 93 NV_STATUS 94 kgrctxFromKernelChannelGroupApi_IMPL 95 ( 96 KernelChannelGroupApi *pKernelChannelGroupApi, 97 KernelGraphicsContext **ppKernelGraphicsContext 98 ) 99 { 100 RsResourceRef *pResourceRef; 101 102 NV_ASSERT_OR_RETURN(pKernelChannelGroupApi != NULL, NV_ERR_INVALID_ARGUMENT); 103 104 if (pKernelChannelGroupApi->hKernelGraphicsContext == NV01_NULL_OBJECT) 105 return NV_ERR_OBJECT_NOT_FOUND; 106 107 NV_ASSERT_OK_OR_RETURN( 108 clientGetResourceRefByType(RES_GET_CLIENT(pKernelChannelGroupApi), 109 pKernelChannelGroupApi->hKernelGraphicsContext, 110 classId(KernelGraphicsContext), 111 &pResourceRef)); 112 113 *ppKernelGraphicsContext = dynamicCast(pResourceRef->pResource, KernelGraphicsContext); 114 NV_ASSERT_OR_RETURN(*ppKernelGraphicsContext != NULL, NV_ERR_INVALID_STATE); 115 116 return NV_OK; 117 } 118 119 /** 120 * @brief Handle NV0090 ctrl call forwarding. The current control call is 121 * dispatched to the KernelGraphicsContext object provided. 122 */ 123 NV_STATUS kgrctxCtrlHandle 124 ( 125 CALL_CONTEXT *pCallContext, 126 NvHandle hKernelGraphicsContext 127 ) 128 { 129 RsResourceRef *pResourceRef; 130 131 NV_ASSERT_OK_OR_RETURN( 132 clientGetResourceRefByType(pCallContext->pClient, 133 hKernelGraphicsContext, 134 classId(KernelGraphicsContext), 135 &pResourceRef)); 136 137 return resControl(pResourceRef->pResource, pCallContext, pCallContext->pControlParams); 138 } 139 140 /** 141 * @brief Translate global ctx buffer enum to external NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID 142 */ 143 NV_STATUS 144 kgrctxGetGlobalContextBufferExternalId_IMPL 145 ( 146 GR_GLOBALCTX_BUFFER id, 147 NvU32 *pExternalId 148 ) 149 { 150 NV_ASSERT_OR_RETURN(pExternalId != NULL, NV_ERR_INVALID_ARGUMENT); 151 NV_ASSERT_OR_RETURN(NV_ENUM_IS(GR_GLOBALCTX_BUFFER, id), NV_ERR_INVALID_ARGUMENT); 152 153 switch (id) 154 { 155 case GR_GLOBALCTX_BUFFER_BUNDLE_CB: 156 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_BUFFER_BUNDLE_CB; 157 break; 158 case GR_GLOBALCTX_BUFFER_PAGEPOOL: 159 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PAGEPOOL; 160 break; 161 case GR_GLOBALCTX_BUFFER_ATTRIBUTE_CB: 162 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_ATTRIBUTE_CB; 163 break; 164 case GR_GLOBALCTX_BUFFER_RTV_CB: 165 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_RTV_CB_GLOBAL; 166 break; 167 case GR_GLOBALCTX_BUFFER_GFXP_POOL: 168 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_POOL; 169 break; 170 case GR_GLOBALCTX_BUFFER_GFXP_CTRL_BLK: 171 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_CTRL_BLK; 172 break; 173 case GR_GLOBALCTX_BUFFER_FECS_EVENT: 174 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_FECS_EVENT; 175 break; 176 case GR_GLOBALCTX_BUFFER_UNRESTRICTED_PRIV_ACCESS_MAP: 177 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_UNRESTRICTED_PRIV_ACCESS_MAP; 178 break; 179 case GR_GLOBALCTX_BUFFER_PRIV_ACCESS_MAP: 180 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PRIV_ACCESS_MAP; 181 break; 182 case GR_GLOBAL_BUFFER_GLOBAL_PRIV_ACCESS_MAP: 183 // TODO this is not valid, this is not a context buffer 184 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GLOBAL_PRIV_ACCESS_MAP; 185 break; 186 // 187 // No default case: Compiler will enforce that this switch is updated if 188 // new global ctx buffers are added 189 // 190 } 191 192 return NV_OK; 193 } 194 195 /** 196 * @brief Translate NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID to global ctx buffer enum 197 */ 198 NV_STATUS 199 kgrctxGetGlobalContextBufferInternalId_IMPL 200 ( 201 NvU32 externalId, 202 GR_GLOBALCTX_BUFFER *pInternalId 203 ) 204 { 205 NV_ASSERT_OR_RETURN(pInternalId != NULL, NV_ERR_INVALID_ARGUMENT); 206 207 switch (externalId) 208 { 209 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_MAIN: 210 // fall through 211 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PM: 212 // fall through 213 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PATCH: 214 return NV_ERR_INVALID_ARGUMENT; 215 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_BUFFER_BUNDLE_CB: 216 *pInternalId = GR_GLOBALCTX_BUFFER_BUNDLE_CB; 217 break; 218 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PAGEPOOL: 219 *pInternalId = GR_GLOBALCTX_BUFFER_PAGEPOOL; 220 break; 221 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_ATTRIBUTE_CB: 222 *pInternalId = GR_GLOBALCTX_BUFFER_ATTRIBUTE_CB; 223 break; 224 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_RTV_CB_GLOBAL: 225 *pInternalId = GR_GLOBALCTX_BUFFER_RTV_CB; 226 break; 227 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_POOL: 228 *pInternalId = GR_GLOBALCTX_BUFFER_GFXP_POOL; 229 break; 230 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_CTRL_BLK: 231 *pInternalId = GR_GLOBALCTX_BUFFER_GFXP_CTRL_BLK; 232 break; 233 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_FECS_EVENT: 234 *pInternalId = GR_GLOBALCTX_BUFFER_FECS_EVENT; 235 break; 236 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PRIV_ACCESS_MAP: 237 *pInternalId = GR_GLOBALCTX_BUFFER_PRIV_ACCESS_MAP; 238 break; 239 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_UNRESTRICTED_PRIV_ACCESS_MAP: 240 *pInternalId = GR_GLOBALCTX_BUFFER_UNRESTRICTED_PRIV_ACCESS_MAP; 241 break; 242 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GLOBAL_PRIV_ACCESS_MAP: 243 *pInternalId = GR_GLOBAL_BUFFER_GLOBAL_PRIV_ACCESS_MAP; 244 break; 245 default: 246 return NV_ERR_INVALID_ARGUMENT; 247 } 248 249 return NV_OK; 250 } 251 252 /*! Translate GR_CTX_BUFFER to NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID */ 253 NV_STATUS 254 kgrctxCtxBufferToFifoEngineId_IMPL 255 ( 256 GR_CTX_BUFFER buffer, 257 NvU32 *pFifoEngineId 258 ) 259 { 260 NV_ASSERT_OR_RETURN(NV_ENUM_IS(GR_CTX_BUFFER, buffer), NV_ERR_INVALID_ARGUMENT); 261 262 switch (buffer) 263 { 264 case GR_CTX_BUFFER_MAIN: 265 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS; 266 break; 267 case GR_CTX_BUFFER_ZCULL: 268 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_ZCULL; 269 break; 270 case GR_CTX_BUFFER_PM: 271 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PM; 272 break; 273 case GR_CTX_BUFFER_PREEMPT: 274 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PREEMPT; 275 break; 276 case GR_CTX_BUFFER_SPILL: 277 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_SPILL; 278 break; 279 case GR_CTX_BUFFER_BETA_CB: 280 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_BETACB; 281 break; 282 case GR_CTX_BUFFER_PAGEPOOL: 283 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PAGEPOOL; 284 break; 285 case GR_CTX_BUFFER_RTV_CB: 286 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_RTV; 287 break; 288 case GR_CTX_BUFFER_PATCH: 289 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PATCH; 290 break; 291 // No default case - Compiler enforces switch update if enum is changed 292 } 293 294 return NV_OK; 295 } 296 297 /** 298 * @brief Translate global ctx buffer enum to NV0080 FIFO engine context properties index 299 */ 300 NV_STATUS 301 kgrctxGlobalCtxBufferToFifoEngineId_IMPL 302 ( 303 GR_GLOBALCTX_BUFFER buffId, 304 NvU32 *pFifoEngineId 305 ) 306 { 307 NV_ASSERT_OR_RETURN(pFifoEngineId != NULL, NV_ERR_INVALID_ARGUMENT); 308 NV_ASSERT_OR_RETURN(NV_ENUM_IS(GR_GLOBALCTX_BUFFER, buffId), NV_ERR_INVALID_ARGUMENT); 309 310 switch (buffId) 311 { 312 case GR_GLOBALCTX_BUFFER_BUNDLE_CB: 313 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_BUNDLE_CB; 314 break; 315 case GR_GLOBALCTX_BUFFER_PAGEPOOL: 316 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PAGEPOOL_GLOBAL; 317 break; 318 case GR_GLOBALCTX_BUFFER_ATTRIBUTE_CB: 319 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_ATTRIBUTE_CB; 320 break; 321 case GR_GLOBALCTX_BUFFER_RTV_CB: 322 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_RTV_CB_GLOBAL; 323 break; 324 case GR_GLOBALCTX_BUFFER_GFXP_POOL: 325 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_GFXP_POOL; 326 break; 327 case GR_GLOBALCTX_BUFFER_GFXP_CTRL_BLK: 328 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_GFXP_CTRL_BLK; 329 break; 330 case GR_GLOBALCTX_BUFFER_FECS_EVENT: 331 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_FECS_EVENT; 332 break; 333 case GR_GLOBALCTX_BUFFER_PRIV_ACCESS_MAP: 334 // fall-through 335 case GR_GLOBALCTX_BUFFER_UNRESTRICTED_PRIV_ACCESS_MAP: 336 // fall-through 337 case GR_GLOBAL_BUFFER_GLOBAL_PRIV_ACCESS_MAP: 338 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PRIV_ACCESS_MAP; 339 break; 340 // No default case - compiler enforces update if enum changes 341 } 342 343 return NV_OK; 344 } 345 346 NV_STATUS 347 kgrctxGetGidInfoInPlace_IMPL 348 ( 349 OBJGPU *pGpu, 350 NvU8 *pUuidBuffer, 351 NvU32 uuidBufferSize, 352 NvU32 flags 353 ) 354 { 355 NvU8 *pUuid; 356 NvU32 uuidLength; 357 NV_STATUS status; 358 359 // on success, allocates memory for uuid 360 status = gpuGetGidInfo(pGpu, &pUuid, &uuidLength, flags); 361 if (status != NV_OK) 362 { 363 return status; 364 } 365 366 if (uuidLength == uuidBufferSize) 367 { 368 portMemCopy(pUuidBuffer, uuidBufferSize, pUuid, uuidLength); 369 } 370 371 portMemFree(pUuid); 372 373 if (uuidLength != uuidBufferSize) 374 { 375 return NV_ERR_INVALID_ARGUMENT; 376 } 377 378 return NV_OK; 379 } 380 381 NV_STATUS 382 kgrctxFillCtxBufferInfo_IMPL 383 ( 384 MEMORY_DESCRIPTOR *pMemDesc, 385 NvU32 externalId, 386 NvBool bBufferGlobal, 387 NV2080_CTRL_GR_CTX_BUFFER_INFO *pCtxBufferInfo 388 ) 389 { 390 NvU64 pageSize; 391 NV_STATUS status; 392 393 MEMORY_DESCRIPTOR *pRootMemDesc = memdescGetRootMemDesc(pMemDesc, NULL); 394 395 pCtxBufferInfo->bufferHandle = NV_PTR_TO_NvP64(pMemDesc); 396 pCtxBufferInfo->bufferType = externalId; 397 pCtxBufferInfo->bIsContigous = memdescGetContiguity(pMemDesc, AT_GPU); 398 pCtxBufferInfo->aperture = memdescGetAddressSpace(pMemDesc); 399 pCtxBufferInfo->pageCount = pMemDesc->PageCount; 400 pCtxBufferInfo->kind = memdescGetPteKindForGpu(pMemDesc, pMemDesc->pGpu); 401 402 { 403 NvU64 physAddr; 404 GMMU_APERTURE aperture = kgmmuGetExternalAllocAperture(pCtxBufferInfo->aperture); 405 406 memdescGetPhysAddrsForGpu(pMemDesc, pMemDesc->pGpu, 407 AT_GPU, 0, 0, 1, 408 &physAddr); 409 410 pCtxBufferInfo->physAddr = 411 kgmmuEncodePhysAddr(GPU_GET_KERNEL_GMMU(pMemDesc->pGpu), aperture, physAddr, 412 NVLINK_INVALID_FABRIC_ADDR); 413 } 414 415 pageSize = memdescGetPageSize(pMemDesc, AT_GPU); 416 if (pageSize == 0) 417 { 418 status = memmgrSetMemDescPageSize_HAL(pMemDesc->pGpu, 419 GPU_GET_MEMORY_MANAGER(pMemDesc->pGpu), 420 pMemDesc, 421 AT_GPU, 422 RM_ATTR_PAGE_SIZE_DEFAULT); 423 if (status != NV_OK) 424 return status; 425 426 pageSize = memdescGetPageSize(pMemDesc, AT_GPU); 427 NV_ASSERT(pageSize != 0); 428 } 429 430 // 431 // Alignment is used to adjust the mapping VA. Hence, we need to make sure 432 // that at least pageSize to make mapping calculation work correctly. 433 // 434 pCtxBufferInfo->alignment = (pMemDesc->Alignment != 0) ? 435 NV_ALIGN_UP(pMemDesc->Alignment, pageSize) : pageSize; 436 437 pCtxBufferInfo->size = pMemDesc->ActualSize; 438 pCtxBufferInfo->pageSize = pageSize; 439 440 pCtxBufferInfo->bGlobalBuffer = bBufferGlobal; 441 pCtxBufferInfo->bLocalBuffer = !bBufferGlobal; 442 pCtxBufferInfo->bDeviceDescendant = pRootMemDesc->pGpu != NULL; 443 444 if (pCtxBufferInfo->bDeviceDescendant) 445 { 446 status = kgrctxGetGidInfoInPlace(pMemDesc->pGpu, pCtxBufferInfo->uuid, sizeof(pCtxBufferInfo->uuid), 447 DRF_DEF(2080_GPU_CMD, _GPU_GET_GID_FLAGS, _TYPE, _SHA1) | 448 DRF_DEF(2080_GPU_CMD, _GPU_GET_GID_FLAGS, _FORMAT, _BINARY)); 449 if (status != NV_OK) 450 return status; 451 } 452 453 return NV_OK; 454 } 455 456 /*! 457 * @brief Construct dupable kernel graphics context. 458 */ 459 NV_STATUS 460 kgrctxConstruct_IMPL 461 ( 462 KernelGraphicsContext *pKernelGraphicsContext, 463 CALL_CONTEXT *pCallContext, 464 RS_RES_ALLOC_PARAMS_INTERNAL *pParams 465 ) 466 { 467 NV_STATUS status = NV_OK; 468 RsShared *pShared; 469 470 if (RS_IS_COPY_CTOR(pParams)) 471 return kgrctxCopyConstruct_IMPL(pKernelGraphicsContext, pCallContext, pParams); 472 473 NV_ASSERT_OK_OR_GOTO(status, 474 serverAllocShare(&g_resServ, classInfo(KernelGraphicsContextShared), &pShared), 475 cleanup); 476 477 pKernelGraphicsContext->pShared = dynamicCast(pShared, KernelGraphicsContextShared); 478 NV_ASSERT_OK_OR_GOTO(status, 479 shrkgrctxInit(GPU_RES_GET_GPU(pKernelGraphicsContext), 480 pKernelGraphicsContext->pShared, 481 pKernelGraphicsContext), 482 cleanup); 483 484 cleanup: 485 if (status != NV_OK) 486 { 487 if (pKernelGraphicsContext->pShared != NULL) 488 serverFreeShare(&g_resServ, pShared); 489 490 } 491 492 return status; 493 } 494 495 /*! 496 * @brief Copy Construct dupable kernel graphics context. 497 */ 498 NV_STATUS 499 kgrctxCopyConstruct_IMPL 500 ( 501 KernelGraphicsContext *pKernelGraphicsContextDst, 502 CALL_CONTEXT *pCallContext, 503 RS_RES_ALLOC_PARAMS_INTERNAL *pParams 504 ) 505 { 506 RsResourceRef *pSrcRef = pParams->pSrcRef; 507 KernelGraphicsContext *pKernelGraphicsContextSrc = 508 dynamicCast(pSrcRef->pResource, KernelGraphicsContext); 509 510 pKernelGraphicsContextDst->pShared = pKernelGraphicsContextSrc->pShared; 511 serverRefShare(&g_resServ, staticCast(pKernelGraphicsContextDst->pShared, RsShared)); 512 513 return NV_OK; 514 } 515 516 NvHandle 517 kgrctxGetInternalObjectHandle_IMPL(KernelGraphicsContext *pKernelGraphicsContext) 518 { 519 return NV01_NULL_OBJECT; 520 } 521 522 /*! 523 * @brief Destruct dupable kernel graphics context. 524 */ 525 void 526 kgrctxDestruct_IMPL 527 ( 528 KernelGraphicsContext *pKernelGraphicsContext 529 ) 530 { 531 RsShared *pShared = staticCast(pKernelGraphicsContext->pShared, RsShared); 532 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelGraphicsContext); 533 534 if (serverGetShareRefCount(&g_resServ, pShared) == 1) 535 { 536 shrkgrctxTeardown_IMPL(pGpu, pKernelGraphicsContext->pShared, pKernelGraphicsContext); 537 } 538 539 serverFreeShare(&g_resServ, pShared); 540 } 541 542 /*! 543 * @brief Retrieve unicast context state 544 */ 545 NV_STATUS kgrctxGetUnicast_IMPL 546 ( 547 OBJGPU *pGpu, 548 KernelGraphicsContext *pKernelGraphicsContext, 549 KernelGraphicsContextUnicast **ppKernelGraphicsContextUnicast 550 ) 551 { 552 *ppKernelGraphicsContextUnicast = NULL; 553 NV_ASSERT_OR_RETURN(pKernelGraphicsContext->pShared != NULL, NV_ERR_INVALID_STATE); 554 *ppKernelGraphicsContextUnicast = &pKernelGraphicsContext->pShared->kernelGraphicsContextUnicast; 555 return NV_OK; 556 } 557 558 /*! 559 * @brief Query the details of MMU faults caused by this context 560 */ 561 NV_STATUS 562 kgrctxLookupMmuFaultInfo_IMPL 563 ( 564 OBJGPU *pGpu, 565 KernelGraphicsContext *pKernelGraphicsContext, 566 NV83DE_CTRL_DEBUG_READ_MMU_FAULT_INFO_PARAMS *pParams 567 ) 568 { 569 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 570 const NvU32 size = NV_ARRAY_ELEMENTS(pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfoList); 571 NvU32 i; 572 573 NV_ASSERT_OK_OR_RETURN( 574 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast)); 575 576 NV_ASSERT_OR_RETURN(pKernelGraphicsContextUnicast->mmuFault.head < size, NV_ERR_INVALID_STATE); 577 NV_ASSERT_OR_RETURN(pKernelGraphicsContextUnicast->mmuFault.tail < size, NV_ERR_INVALID_STATE); 578 579 pParams->count = 0; 580 for (i = pKernelGraphicsContextUnicast->mmuFault.tail; 581 i != pKernelGraphicsContextUnicast->mmuFault.head; 582 i = (i + 1) % size) 583 { 584 pParams->mmuFaultInfoList[pParams->count] = pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfoList[i]; 585 pParams->count++; 586 } 587 588 return NV_OK; 589 } 590 591 /*! 592 * @brief Query the details of MMU faults caused by this context 593 */ 594 NV_STATUS 595 kgrctxLookupMmuFault_IMPL 596 ( 597 OBJGPU *pGpu, 598 KernelGraphicsContext *pKernelGraphicsContext, 599 NV83DE_MMU_FAULT_INFO *pMmuFaultInfo 600 ) 601 { 602 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 603 604 NV_ASSERT_OK_OR_RETURN( 605 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast)); 606 607 *pMmuFaultInfo = pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfo; 608 return NV_OK; 609 } 610 611 /*! 612 * @brief clear the details of MMU faults caused by this context 613 */ 614 NV_STATUS 615 kgrctxClearMmuFault_IMPL 616 ( 617 OBJGPU *pGpu, 618 KernelGraphicsContext *pKernelGraphicsContext 619 ) 620 { 621 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 622 623 NV_ASSERT_OK_OR_RETURN( 624 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast)); 625 626 pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfo.valid = NV_FALSE; 627 pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfo.faultInfo = 0; 628 return NV_OK; 629 } 630 631 /*! 632 * @brief Record the details of an MMU fault caused by this context 633 */ 634 void 635 kgrctxRecordMmuFault_IMPL 636 ( 637 OBJGPU *pGpu, 638 KernelGraphicsContext *pKernelGraphicsContext, 639 NvU32 mmuFaultInfo, 640 NvU64 mmuFaultAddress, 641 NvU32 mmuFaultType, 642 NvU32 mmuFaultAccessType 643 ) 644 { 645 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 646 NV_STATUS status; 647 648 NV_ASSERT_OK_OR_ELSE(status, 649 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast), 650 return;); 651 652 pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfo.valid = NV_TRUE; 653 pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfo.faultInfo = mmuFaultInfo; 654 655 { 656 const NvU32 size = NV_ARRAY_ELEMENTS(pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfoList); 657 658 NV_ASSERT_OR_RETURN_VOID(pKernelGraphicsContextUnicast->mmuFault.head < size); 659 NV_ASSERT_OR_RETURN_VOID(pKernelGraphicsContextUnicast->mmuFault.tail < size); 660 661 pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfoList[pKernelGraphicsContextUnicast->mmuFault.head].faultAddress = mmuFaultAddress; 662 pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfoList[pKernelGraphicsContextUnicast->mmuFault.head].faultType = mmuFaultType; 663 pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfoList[pKernelGraphicsContextUnicast->mmuFault.head].accessType = mmuFaultAccessType; 664 pKernelGraphicsContextUnicast->mmuFault.head = (pKernelGraphicsContextUnicast->mmuFault.head + 1) % size; 665 if (pKernelGraphicsContextUnicast->mmuFault.head == pKernelGraphicsContextUnicast->mmuFault.tail) 666 pKernelGraphicsContextUnicast->mmuFault.tail = (pKernelGraphicsContextUnicast->mmuFault.tail + 1) % size; 667 } 668 } 669 670 /*! 671 * @brief getter for active debugger interator 672 */ 673 KernelSMDebuggerSessionListIter 674 kgrctxGetDebuggerSessionIter_IMPL 675 ( 676 OBJGPU *pGpu, 677 KernelGraphicsContext *pKernelGraphicsContext 678 ) 679 { 680 return listIterAll(&pKernelGraphicsContext->pShared->activeDebuggers); 681 } 682 683 /*! add active debugger session */ 684 NvBool 685 kgrctxRegisterKernelSMDebuggerSession_IMPL 686 ( 687 OBJGPU *pGpu, 688 KernelGraphicsContext *pKernelGraphicsContext, 689 KernelSMDebuggerSession *pKernelSMDebuggerSession 690 ) 691 { 692 return listAppendValue(&pKernelGraphicsContext->pShared->activeDebuggers, &pKernelSMDebuggerSession) != NULL; 693 } 694 695 /*! remove active debugger session */ 696 void 697 kgrctxDeregisterKernelSMDebuggerSession_IMPL 698 ( 699 OBJGPU *pGpu, 700 KernelGraphicsContext *pKernelGraphicsContext, 701 KernelSMDebuggerSession *pKernelSMDebuggerSession 702 ) 703 { 704 listRemoveFirstByValue(&pKernelGraphicsContext->pShared->activeDebuggers, &pKernelSMDebuggerSession); 705 } 706 707 /*! 708 * @brief Determine whether channels in this context are associated with GR engine 709 * 710 * @returns NV_TRUE if passed channel is allocated on GR, and is on GR runlist 711 */ 712 NvBool 713 kgrctxIsValid_IMPL 714 ( 715 OBJGPU *pGpu, 716 KernelGraphicsContext *pKernelGraphicsContext, 717 KernelChannel *pKernelChannel 718 ) 719 { 720 NvU32 runlistId; 721 RM_ENGINE_TYPE engineType; 722 723 // TODO remove pKernelChannel from params 724 725 if (RM_ENGINE_TYPE_IS_VALID(kchannelGetEngineType(pKernelChannel)) && 726 !RM_ENGINE_TYPE_IS_GR(kchannelGetEngineType(pKernelChannel))) 727 { 728 return NV_FALSE; 729 } 730 731 NV_CHECK_OR_RETURN(LEVEL_INFO, kchannelIsRunlistSet(pGpu, pKernelChannel), NV_FALSE); 732 733 runlistId = kchannelGetRunlistId(pKernelChannel); 734 NV_ASSERT_OK( 735 kfifoEngineInfoXlate_HAL(pGpu, GPU_GET_KERNEL_FIFO(pGpu), 736 ENGINE_INFO_TYPE_RUNLIST, runlistId, 737 ENGINE_INFO_TYPE_RM_ENGINE_TYPE, (NvU32 *) &engineType)); 738 739 NV_CHECK_OR_RETURN(LEVEL_INFO, RM_ENGINE_TYPE_IS_GR(engineType), NV_FALSE); 740 741 return NV_TRUE; 742 } 743 744 /*! 745 * @brief returns if the main context buffer has been allocated 746 * 747 * @return NV_TRUE if allocated, NV_FALSE otherwise 748 */ 749 NvBool 750 kgrctxIsMainContextAllocated_IMPL 751 ( 752 OBJGPU *pGpu, 753 KernelGraphicsContext *pKernelGraphicsContext 754 ) 755 { 756 MEMORY_DESCRIPTOR *pMemDesc; 757 return (kgrctxGetMainContextBuffer(pGpu, pKernelGraphicsContext, &pMemDesc) == NV_OK) && 758 (pMemDesc != NULL); 759 } 760 761 /*! Retrieve the memdesc containing the main ctx buffer */ 762 NV_STATUS kgrctxGetMainContextBuffer_IMPL 763 ( 764 OBJGPU *pGpu, 765 KernelGraphicsContext *pKernelGraphicsContext, 766 MEMORY_DESCRIPTOR **ppMemDesc 767 ) 768 { 769 ENGINE_CTX_DESCRIPTOR *pEngCtxDesc; 770 NvU32 subDevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu); 771 RsResourceRef *pParentRef = RES_GET_REF(pKernelGraphicsContext)->pParentRef; 772 KernelChannelGroupApi *pKernelChannelGroupApi; 773 KernelChannelGroup *pKernelChannelGroup; 774 775 *ppMemDesc = NULL; 776 pKernelChannelGroupApi = dynamicCast(pParentRef->pResource, KernelChannelGroupApi); 777 pKernelChannelGroup = pKernelChannelGroupApi->pKernelChannelGroup; 778 pEngCtxDesc = pKernelChannelGroup->ppEngCtxDesc[subDevInst]; 779 780 if (pEngCtxDesc != NULL) 781 *ppMemDesc = pEngCtxDesc->pMemDesc; 782 783 return NV_OK; 784 } 785 786 /*! 787 * @brief Retrieve information about the context buffers 788 */ 789 NV_STATUS 790 kgrctxGetCtxBuffers_IMPL 791 ( 792 OBJGPU *pGpu, 793 KernelGraphicsContext *pKernelGraphicsContext, 794 KernelGraphics *pKernelGraphics, 795 NvU32 gfid, 796 NvU32 bufferCount, 797 MEMORY_DESCRIPTOR **ppBuffers, 798 NvU32 *pCtxBufferType, 799 NvU32 *pBufferCountOut, 800 NvU32 *pFirstGlobalBuffer 801 ) 802 { 803 GR_GLOBALCTX_BUFFERS *pGlobalCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid); 804 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 805 MEMORY_DESCRIPTOR *pGrCtxBufferMemDesc; 806 NvU32 bufferCountOut = 0; 807 NvU32 i; 808 809 NV_ASSERT_OK_OR_RETURN( 810 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast)); 811 812 // Get local context buffer memdesc. 813 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 814 kgrctxGetMainContextBuffer(pGpu, pKernelGraphicsContext, &pGrCtxBufferMemDesc)); 815 816 NV_CHECK_OR_RETURN(LEVEL_ERROR, 817 pGrCtxBufferMemDesc != NULL, 818 NV_ERR_INVALID_OBJECT); 819 820 NV_CHECK_OR_RETURN(LEVEL_INFO, bufferCountOut < bufferCount, NV_ERR_INVALID_ARGUMENT); 821 pCtxBufferType[bufferCountOut] = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_MAIN; 822 ppBuffers[bufferCountOut++] = pGrCtxBufferMemDesc; 823 824 // Get context patch buffer memdesc. 825 NV_CHECK_OR_RETURN(LEVEL_SILENT, 826 pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc != NULL, 827 NV_ERR_INVALID_STATE); 828 829 NV_CHECK_OR_RETURN(LEVEL_INFO, bufferCountOut < bufferCount, NV_ERR_INVALID_ARGUMENT); 830 pCtxBufferType[bufferCountOut] = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PATCH; 831 ppBuffers[bufferCountOut++] = pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc; 832 833 // Add PM ctxsw buffer if it's allocated. 834 if (pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc != NULL) 835 { 836 NV_CHECK_OR_RETURN(LEVEL_INFO, bufferCountOut < bufferCount, NV_ERR_INVALID_ARGUMENT); 837 pCtxBufferType[bufferCountOut] = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PM; 838 ppBuffers[bufferCountOut++] = pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc; 839 } 840 841 if (pFirstGlobalBuffer != NULL) 842 { 843 *pFirstGlobalBuffer = bufferCountOut; 844 } 845 846 // Add global buffers. 847 for (i = 0; i < GR_GLOBALCTX_BUFFER_COUNT; i++) 848 { 849 MEMORY_DESCRIPTOR *pMemDesc = pGlobalCtxBuffers->memDesc[i]; 850 if (pMemDesc != NULL) 851 { 852 NV_CHECK_OR_RETURN(LEVEL_INFO, bufferCountOut < bufferCount, NV_ERR_INVALID_ARGUMENT); 853 NV_ASSERT_OK( 854 kgrctxGetGlobalContextBufferExternalId(i, &pCtxBufferType[bufferCountOut])); 855 ppBuffers[bufferCountOut++] = pMemDesc; 856 } 857 } 858 859 *pBufferCountOut = bufferCountOut; 860 861 return NV_OK; 862 } 863 864 /* 865 * @brief Get maximum context buffer count including global and local buffers 866 * 867 * @param[in] pGpu 868 * @param[in] pKernelGraphicsContext 869 * @param[in] pKernelGraphics 870 * @param[out] pBufferCount Max context buffer count 871 */ 872 NV_STATUS 873 kgrctxGetBufferCount_IMPL 874 ( 875 OBJGPU *pGpu, 876 KernelGraphicsContext *pKernelGraphicsContext, 877 KernelGraphics *pKernelGraphics, 878 NvU32 *pBufferCount 879 ) 880 { 881 NvU32 gfid; 882 NvBool bCallingContextPlugin; 883 GR_GLOBALCTX_BUFFERS *pGlobalCtxBuffers; 884 NvU32 i; 885 886 *pBufferCount = 0; 887 888 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &gfid)); 889 NV_ASSERT_OK_OR_RETURN(vgpuIsCallingContextPlugin(pGpu, &bCallingContextPlugin)); 890 891 pGlobalCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid); 892 893 if (bCallingContextPlugin) 894 { 895 gfid = GPU_GFID_PF; 896 } 897 898 for (i = 0; i < GR_GLOBALCTX_BUFFER_COUNT; i++) 899 { 900 if (pGlobalCtxBuffers->memDesc[i] != NULL) 901 (*pBufferCount)++; 902 } 903 904 // Increment by 3 to include local ctx buffer, patch context buffer and PM Ctxsw buffer 905 *pBufferCount += 3; 906 907 return NV_OK; 908 } 909 910 /* 911 * @brief Get context buffer info like size, alignment for global and 912 * local buffers 913 * 914 * @param[in] pGpu 915 * @param[in] pKernelGraphicsContext 916 * @param[in] pKernelGraphics 917 * @param[in] gfid 918 * @param[in] bufferMaxCount Amount of space provided in pCtxBufferInfo 919 * @param[out] bufferCount Number of buffers described 920 * @param[out] pCtxBufferInfo Structure to fill buffer information 921 * 922 * @return NV_OK successfully probed all buffers 923 * @return NV_ERR_INVALID_ARGUMENT not enough space provided for buffers queried 924 */ 925 NV_STATUS 926 kgrctxGetCtxBufferInfo_IMPL 927 ( 928 OBJGPU *pGpu, 929 KernelGraphicsContext *pKernelGraphicsContext, 930 KernelGraphics *pKernelGraphics, 931 NvU32 gfid, 932 NvU32 bufferMaxCount, 933 NvU32 *pBufferCount, 934 NV2080_CTRL_GR_CTX_BUFFER_INFO *pCtxBufferInfo 935 ) 936 { 937 MEMORY_DESCRIPTOR *pMemDescArray[3 + GR_GLOBALCTX_BUFFER_COUNT]; 938 NvU32 bufferExternalId[3 + GR_GLOBALCTX_BUFFER_COUNT]; 939 NvU32 memdescCount; 940 NvU32 firstGlobalBuffer; 941 NvU32 i; 942 NvU32 j; 943 944 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 945 kgrctxGetCtxBuffers(pGpu, pKernelGraphicsContext, pKernelGraphics, 946 gfid, 947 NV_ARRAY_ELEMENTS(pMemDescArray), 948 pMemDescArray, 949 bufferExternalId, 950 &memdescCount, 951 &firstGlobalBuffer)); 952 953 if (bufferMaxCount < memdescCount) 954 return NV_ERR_INVALID_ARGUMENT; 955 956 for (i = 0; i < memdescCount; i++) 957 { 958 NvBool bGlobalBuffer = (i >= firstGlobalBuffer); 959 960 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, 961 kgrctxFillCtxBufferInfo(pMemDescArray[i], 962 bufferExternalId[i], 963 bGlobalBuffer, 964 &pCtxBufferInfo[i])); 965 } 966 967 // 968 // Sort the buffer info in descending order using alignment so that the 969 // VA range calculation can figure out the optimal VA range size. 970 // 971 for (i = 0; i < memdescCount; i++) 972 { 973 for (j = 0; j < memdescCount - 1; j++) 974 { 975 if (pCtxBufferInfo[j].alignment < pCtxBufferInfo[j + 1].alignment) 976 { 977 NV2080_CTRL_GR_CTX_BUFFER_INFO tmp; 978 979 portMemCopy(&tmp, sizeof(tmp), &pCtxBufferInfo[j], sizeof(tmp)); 980 portMemCopy(&pCtxBufferInfo[j], sizeof(tmp), &pCtxBufferInfo[j + 1], sizeof(tmp)); 981 portMemCopy(&pCtxBufferInfo[j + 1], sizeof(tmp), &tmp, sizeof(tmp)); 982 } 983 } 984 } 985 986 *pBufferCount = memdescCount; 987 988 return NV_OK; 989 } 990 991 /* 992 * @brief Return physical addresses of context buffer starting from its 'firstPage' page. 993 * The function is intended to be called repeatedly while advancing 994 * 'firstPage' parameter until all pages of the buffer are queried. 995 * This condition is indicated by *pNoMorePage == NV_TRUE. 996 * If requested buffer is contiguous, address of first page is returned 997 * always. 998 * 999 * @param[in] pGpu 1000 * @param[in] pKernelGraphicsContext 1001 * @param[in] pKernelGraphics 1002 * @param[in] gfid 1003 * @param[in] bufferType Requested ctx buffer type 1004 * @param[in] firstPage First page of the buffer to be queried 1005 * @param[out] pPhysAddrs Array to be filled with page addresses 1006 * @param[in] addrsSize Number of elements of pPhysAddrs 1007 * @param[out] pNumPages Number of page addresses returned 1008 * @param[out] pbNoMorePages End of buffer reached 1009 * 1010 * @return NV_OK successfully obtained requested addresses 1011 * @return NV_ERR_INVALID_ARGUMENT pMemDesc does not specify a context buffer 1012 */ 1013 NV_STATUS 1014 kgrctxGetCtxBufferPtes_IMPL 1015 ( 1016 OBJGPU *pGpu, 1017 KernelGraphicsContext *pKernelGraphicsContext, 1018 KernelGraphics *pKernelGraphics, 1019 NvU32 gfid, 1020 NvU32 bufferType, 1021 NvU32 firstPage, 1022 NvU64 *pPhysAddrs, 1023 NvU32 addrsSize, 1024 NvU32 *pNumPages, 1025 NvBool *pbNoMorePages 1026 ) 1027 { 1028 NvU64 bufferSize; 1029 NvU64 pageSize; 1030 NvU32 numPages; 1031 NvU32 bufferCount; 1032 NvU32 i; 1033 MEMORY_DESCRIPTOR *pMemDescArray[3 + GR_GLOBALCTX_BUFFER_COUNT]; 1034 NvU32 bufferExternalId[3 + GR_GLOBALCTX_BUFFER_COUNT]; 1035 1036 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 1037 kgrctxGetCtxBuffers(pGpu, pKernelGraphicsContext, pKernelGraphics, 1038 gfid, 1039 NV_ARRAY_ELEMENTS(pMemDescArray), 1040 pMemDescArray, 1041 bufferExternalId, 1042 &bufferCount, 1043 NULL)); 1044 1045 for (i = 0; i < bufferCount; i++) 1046 { 1047 if (bufferExternalId[i] == bufferType) 1048 break; 1049 } 1050 1051 if (addrsSize == 0) 1052 { 1053 return NV_ERR_INVALID_ARGUMENT; 1054 } 1055 1056 NV_CHECK_OR_RETURN(LEVEL_ERROR, i != bufferCount, NV_ERR_INVALID_ARGUMENT); 1057 1058 bufferSize = memdescGetSize(pMemDescArray[i]); 1059 pageSize = memdescGetPageSize(pMemDescArray[i], AT_GPU); 1060 1061 if (pageSize == 0) 1062 { 1063 return NV_ERR_INVALID_STATE; 1064 } 1065 1066 numPages = NV_ROUNDUP(bufferSize, pageSize) / pageSize; 1067 1068 if (firstPage >= numPages) 1069 { 1070 numPages = 0; 1071 *pbNoMorePages = NV_TRUE; 1072 goto done; 1073 } 1074 1075 if (memdescGetContiguity(pMemDescArray[i], AT_GPU)) 1076 { 1077 firstPage = 0; 1078 numPages = 1; 1079 *pbNoMorePages = NV_TRUE; 1080 } 1081 else 1082 { 1083 numPages -= firstPage; 1084 *pbNoMorePages = (numPages <= addrsSize); 1085 numPages = NV_MIN(numPages, addrsSize); 1086 } 1087 1088 if (numPages > 0) 1089 { 1090 memdescGetPhysAddrs(pMemDescArray[i], 1091 AT_GPU, 1092 firstPage * pageSize, 1093 pageSize, 1094 numPages, 1095 pPhysAddrs); 1096 } 1097 1098 done: 1099 *pNumPages = numPages; 1100 1101 return NV_OK; 1102 } 1103 1104 /*! 1105 * This function does following things 1106 * 1. Allocate main GR context buffer 1107 * 2. Sets up GR context memory descriptor by calling kchannelSetEngineContextMemDesc 1108 */ 1109 NV_STATUS 1110 kgrctxAllocMainCtxBuffer_IMPL 1111 ( 1112 OBJGPU *pGpu, 1113 KernelGraphicsContext *pKernelGraphicsContext, 1114 KernelGraphics *pKernelGraphics, 1115 KernelChannel *pKernelChannel 1116 ) 1117 { 1118 MEMORY_DESCRIPTOR *pGrCtxBufferMemDesc = NULL; 1119 NvU32 ctxSize; 1120 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 1121 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 1122 NvU64 allocFlags = MEMDESC_FLAGS_GPU_PRIVILEGED; 1123 CTX_BUF_POOL_INFO *pCtxBufPool = NULL; 1124 NvBool bIsContiguous = kgraphicsShouldForceMainCtxContiguity_HAL(pGpu, pKernelGraphics); 1125 const GR_BUFFER_ATTR *pAttr = kgraphicsGetContextBufferAttr(pGpu, pKernelGraphics, GR_CTX_BUFFER_MAIN); 1126 1127 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 1128 1129 NV_ASSERT_OK_OR_RETURN( 1130 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast)); 1131 1132 // 1133 // Allocate space for per-subcontext headers in the context buffer. 1134 // With subcontext, per-subcontext context header is programmed in the channel instance block. 1135 // Per-subcontext headers can be seperate from the context buffer. 1136 // For the initial phase, we allocate them at the end of the context buffers for easier tracking. 1137 // This will waste some memory (256 KB), if the number of subcontexts are sparse. 1138 // Will cleanup this up later to be on-demand. 1139 // 1140 // We also need to report the updated context size to KMD for virtual context. 1141 // Bug 1764102 tracks the VC support. 1142 // 1143 NV_ASSERT_OK_OR_RETURN( 1144 kgraphicsGetMainCtxBufferSize(pGpu, pKernelGraphics, NV_TRUE, &ctxSize)); 1145 1146 if (ctxBufPoolIsSupported(pGpu) && 1147 pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->pCtxBufPool != NULL) 1148 { 1149 allocFlags |= MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL; 1150 pCtxBufPool = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->pCtxBufPool; 1151 } 1152 1153 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 1154 memdescCreate(&pGrCtxBufferMemDesc, pGpu, ctxSize, 1155 RM_PAGE_SIZE, bIsContiguous, ADDR_UNKNOWN, 1156 pAttr->cpuAttr, 1157 allocFlags | MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE)); 1158 1159 // 1160 // Force page size to 4KB, we can change this later when RM access method 1161 // support 64k pages 1162 // 1163 NV_ASSERT_OK_OR_RETURN( 1164 memmgrSetMemDescPageSize_HAL(pGpu, pMemoryManager, pGrCtxBufferMemDesc, AT_GPU, RM_ATTR_PAGE_SIZE_4KB)); 1165 1166 NV_ASSERT_OK_OR_RETURN(memdescSetCtxBufPool(pGrCtxBufferMemDesc, pCtxBufPool)); 1167 1168 NV_STATUS status; 1169 memdescTagAllocList(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_CONTEXT_BUFFER, pGrCtxBufferMemDesc, pAttr->pAllocList); 1170 NV_ASSERT_OK_OR_RETURN(status); 1171 1172 NV_ASSERT_OK_OR_RETURN( 1173 kchannelSetEngineContextMemDesc(pGpu, pKernelChannel, 1174 ENG_GR(kgraphicsGetInstance(pGpu, pKernelGraphics)), 1175 pGrCtxBufferMemDesc)); 1176 pKernelGraphicsContextUnicast->pMainCtxBuffer = pGrCtxBufferMemDesc; 1177 return NV_OK; 1178 } 1179 1180 /*! 1181 * @brief Allocate and setup the GR ctx patch buffer 1182 */ 1183 NV_STATUS 1184 kgrctxAllocPatchBuffer_IMPL 1185 ( 1186 OBJGPU *pGpu, 1187 KernelGraphicsContext *pKernelGraphicsContext, 1188 KernelGraphics *pKernelGraphics, 1189 KernelChannel *pKernelChannel 1190 ) 1191 { 1192 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 1193 const KGRAPHICS_STATIC_INFO *pStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 1194 NvU32 status = NV_OK; 1195 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 1196 MEMORY_DESCRIPTOR **ppMemDesc; 1197 NvU64 size; 1198 NvU64 flags = MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE; 1199 CTX_BUF_POOL_INFO *pCtxBufPool; 1200 const GR_BUFFER_ATTR *pAttr = kgraphicsGetContextBufferAttr(pGpu, pKernelGraphics, GR_CTX_BUFFER_PATCH); 1201 1202 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 1203 1204 NV_ASSERT_OR_RETURN(pStaticInfo != NULL, NV_ERR_INVALID_STATE); 1205 NV_ASSERT_OR_RETURN(pStaticInfo->pContextBuffersInfo != NULL, NV_ERR_INVALID_STATE); 1206 1207 NV_ASSERT_OK_OR_RETURN( 1208 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast)); 1209 1210 pCtxBufPool = NULL; 1211 if (ctxBufPoolIsSupported(pGpu) && 1212 (pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->pCtxBufPool) != NULL) 1213 { 1214 flags |= MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL; 1215 pCtxBufPool = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->pCtxBufPool; 1216 } 1217 1218 ppMemDesc = &pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc; 1219 size = pStaticInfo->pContextBuffersInfo->engine[NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PATCH].size; 1220 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 1221 memdescCreate(ppMemDesc, pGpu, size, RM_PAGE_SIZE, NV_TRUE, 1222 ADDR_UNKNOWN, 1223 pAttr->cpuAttr, 1224 flags)); 1225 1226 // 1227 // Force page size to 4KB we can change this later when RM access method 1228 // support 64k pages 1229 // 1230 memmgrSetMemDescPageSize_HAL(pGpu, pMemoryManager, *ppMemDesc, AT_GPU, RM_ATTR_PAGE_SIZE_4KB); 1231 NV_ASSERT_OK_OR_GOTO(status, 1232 memdescSetCtxBufPool(*ppMemDesc, pCtxBufPool), 1233 failed); 1234 1235 memdescTagAllocList(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_117, 1236 (*ppMemDesc), pAttr->pAllocList); 1237 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, 1238 status, 1239 failed); 1240 1241 return NV_OK; 1242 1243 failed: 1244 memdescFree(*ppMemDesc); 1245 memdescDestroy(*ppMemDesc); 1246 *ppMemDesc = NULL; 1247 return status; 1248 } 1249 1250 /*! 1251 * @brief Allocate the local ctx PM buffer 1252 */ 1253 NV_STATUS 1254 kgrctxAllocPmBuffer_IMPL 1255 ( 1256 OBJGPU *pGpu, 1257 KernelGraphicsContext *pKernelGraphicsContext, 1258 KernelGraphics *pKernelGraphics, 1259 KernelChannel *pKernelChannel 1260 ) 1261 { 1262 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 1263 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 1264 NvU32 size; 1265 NV_STATUS status = NV_OK; 1266 NvU64 flags = MEMDESC_FLAGS_GPU_PRIVILEGED; 1267 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 1268 CTX_BUF_POOL_INFO *pCtxBufPool; 1269 MEMORY_DESCRIPTOR **ppMemDesc; 1270 const GR_BUFFER_ATTR *pAttr = kgraphicsGetContextBufferAttr(pGpu, pKernelGraphics, GR_CTX_BUFFER_PM); 1271 1272 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 1273 1274 NV_ASSERT_OK_OR_RETURN( 1275 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast)); 1276 1277 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 1278 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pContextBuffersInfo != NULL, NV_ERR_INVALID_STATE); 1279 size = pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PM].size; 1280 if (size == 0) 1281 return NV_ERR_INVALID_STATE; 1282 1283 pCtxBufPool = NULL; 1284 if (ctxBufPoolIsSupported(pGpu) && 1285 pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->pCtxBufPool != NULL) 1286 { 1287 flags |= MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL; 1288 pCtxBufPool = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->pCtxBufPool; 1289 } 1290 1291 // 1292 // For SRIOV Heavy, the PM ctxsw buffer allocation will be redirected to 1293 // host RM subheap. Subheap is used by host RM to allocate memory 1294 // on behalf of the guest(VF) context. 1295 // 1296 if (gpuIsWarBug200577889SriovHeavyEnabled(pGpu)) 1297 { 1298 flags |= MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE; 1299 } 1300 1301 ppMemDesc = &pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc; 1302 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 1303 memdescCreate(ppMemDesc, pGpu, 1304 size, 1305 RM_PAGE_SIZE, 1306 NV_TRUE, 1307 ADDR_UNKNOWN, 1308 pAttr->cpuAttr, 1309 flags)); 1310 1311 // 1312 // Force page size to 4KB we can change this later when RM access method 1313 // support 64k pages 1314 // 1315 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, 1316 memmgrSetMemDescPageSize_HAL(pGpu, pMemoryManager, *ppMemDesc, AT_GPU, RM_ATTR_PAGE_SIZE_4KB), 1317 error); 1318 1319 NV_ASSERT_OK_OR_GOTO(status, memdescSetCtxBufPool(*ppMemDesc, pCtxBufPool), error); 1320 1321 memdescTagAllocList(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_118, 1322 (*ppMemDesc), pAttr->pAllocList); 1323 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, 1324 status, 1325 error); 1326 1327 return NV_OK; 1328 1329 error: 1330 memdescFree(*ppMemDesc); 1331 memdescDestroy(*ppMemDesc); 1332 *ppMemDesc = NULL; 1333 1334 return status; 1335 } 1336 1337 /*! 1338 * This function allocates and maps various GR buffers. 1339 */ 1340 NV_STATUS 1341 kgrctxAllocCtxBuffers_IMPL 1342 ( 1343 OBJGPU *pGpu, 1344 KernelGraphicsContext *pKernelGraphicsContext, 1345 KernelGraphics *pKernelGraphics, 1346 KernelGraphicsObject *pKernelGraphicsObject 1347 ) 1348 { 1349 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 1350 ChannelDescendant *pChannelDescendant = staticCast(pKernelGraphicsObject, ChannelDescendant); 1351 NV_STATUS status = NV_OK; 1352 NvU32 classNum = pChannelDescendant->resourceDesc.externalClassId; 1353 NvU32 objType; 1354 NvU32 gfid = kchannelGetGfid(pChannelDescendant->pKernelChannel); 1355 KernelChannel *pKernelChannel = pChannelDescendant->pKernelChannel; 1356 GR_GLOBALCTX_BUFFERS *pGlobalCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid); 1357 1358 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 1359 1360 NV_ASSERT_OR_RETURN(kgrobjGetKernelGraphicsContext(pGpu, pKernelGraphicsObject) != NULL, NV_ERR_INVALID_STATE); 1361 1362 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 1363 kgrctxGetUnicast(pGpu, 1364 kgrobjGetKernelGraphicsContext(pGpu, pKernelGraphicsObject), 1365 &pKernelGraphicsContextUnicast)); 1366 1367 // Allocate the GR ctx buffer if required 1368 if (!kgrctxIsMainContextAllocated(pGpu, pKernelGraphicsContext) && 1369 !kchannelIsCtxBufferAllocSkipped(pChannelDescendant->pKernelChannel)) 1370 { 1371 if (pKernelGraphicsContextUnicast->pMainCtxBuffer == NULL) 1372 { 1373 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 1374 kgrctxAllocMainCtxBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel)); 1375 } 1376 else 1377 { 1378 // Ctx buffer was uninstalled in FIFO but not freed. Reinstall it 1379 NV_ASSERT_OK_OR_RETURN( 1380 kchannelSetEngineContextMemDesc(pGpu, pKernelChannel, 1381 ENG_GR(kgraphicsGetInstance(pGpu, pKernelGraphics)), 1382 pKernelGraphicsContextUnicast->pMainCtxBuffer)); 1383 } 1384 } 1385 1386 kgrmgrGetGrObjectType(classNum, &objType); 1387 1388 // 1389 // for each channel, we need to allocate the context patch buffer, this memory region 1390 // will be used to override settings in the context after it is restored, things 1391 // like the global ctx buffer addresses, etc. 1392 // 1393 if (pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc == NULL) 1394 { 1395 NV_ASSERT_OK_OR_RETURN( 1396 kgrctxAllocPatchBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel)); 1397 } 1398 1399 // Pre-allocate the PM ctxsw buffer, if required 1400 if (kgrctxShouldPreAllocPmBuffer_HAL(pGpu, pKernelGraphicsContext, pChannelDescendant->pKernelChannel)) 1401 { 1402 NV_ASSERT_OK_OR_RETURN( 1403 kgrctxAllocPmBuffer(pGpu, kgrobjGetKernelGraphicsContext(pGpu, pKernelGraphicsObject), 1404 pKernelGraphics, 1405 pChannelDescendant->pKernelChannel)); 1406 } 1407 1408 // 1409 // Allocate Ctx Buffers that are local to this channel if required 1410 // and they have yet to be allocated. 1411 // 1412 if ((pKernelGraphicsContextUnicast->bVprChannel 1413 ) && !pKernelGraphicsContextUnicast->localCtxBuffer.bAllocated) 1414 { 1415 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 1416 kgraphicsAllocGrGlobalCtxBuffers_HAL(pGpu, pKernelGraphics, gfid, 1417 kgrobjGetKernelGraphicsContext(pGpu, pKernelGraphicsObject))); 1418 } 1419 1420 // Allocate global context for this gfid if they haven't been already 1421 if (!pGlobalCtxBuffers->bAllocated) 1422 { 1423 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 1424 kgraphicsAllocGrGlobalCtxBuffers_HAL(pGpu, pKernelGraphics, gfid, NULL)); 1425 } 1426 1427 return status; 1428 } 1429 1430 // Map common buffer to a channel's context helper function 1431 NV_STATUS 1432 kgrctxMapGlobalCtxBuffer_IMPL 1433 ( 1434 OBJGPU *pGpu, 1435 KernelGraphicsContext *pKernelGraphicsContext, 1436 KernelGraphics *pKernelGraphics, 1437 NvU32 gfid, 1438 OBJVASPACE *pVAS, 1439 GR_GLOBALCTX_BUFFER buffId, 1440 NvBool bIsReadOnly 1441 ) 1442 { 1443 GR_GLOBALCTX_BUFFERS *pGlobalCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid); 1444 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 1445 NV_STATUS status = NV_OK; 1446 NvU64 vaddr; 1447 MEMORY_DESCRIPTOR *pMemDesc; 1448 1449 NV_ASSERT_OK_OR_RETURN( 1450 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast)); 1451 1452 // if we have local buffers per-channel allocated, use them, otherwise use the global buffers 1453 pMemDesc = pKernelGraphicsContextUnicast->localCtxBuffer.memDesc[buffId]; 1454 if (pMemDesc == NULL) 1455 { 1456 pMemDesc = pGlobalCtxBuffers->memDesc[buffId]; 1457 } 1458 1459 if (pMemDesc == NULL) 1460 { 1461 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 1462 NvU32 fifoEngineId; 1463 NvU32 buffSize; 1464 1465 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 1466 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 1467 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pContextBuffersInfo != NULL, NV_ERR_INVALID_STATE); 1468 NV_ASSERT_OK_OR_RETURN(kgrctxGlobalCtxBufferToFifoEngineId(buffId, &fifoEngineId)); 1469 buffSize = pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[fifoEngineId].size; 1470 1471 if (buffSize == 0) 1472 { 1473 NV_PRINTF(LEVEL_INFO, 1474 "Could not map %s Buffer as buffer is not supported\n", 1475 NV_ENUM_TO_STRING(GR_GLOBALCTX_BUFFER, buffId)); 1476 return NV_OK; 1477 } 1478 else 1479 { 1480 NV_PRINTF(LEVEL_ERROR, 1481 "Could not map %s Buffer, no memory allocated for it!\n", 1482 NV_ENUM_TO_STRING(GR_GLOBALCTX_BUFFER, buffId)); 1483 return NV_ERR_INVALID_ARGUMENT; 1484 } 1485 } 1486 1487 // Unconditionally call map for refcounting 1488 NV_ASSERT_OK_OR_ELSE(status, 1489 kgraphicsMapCtxBuffer(pGpu, pKernelGraphics, pMemDesc, pVAS, 1490 &pKernelGraphicsContextUnicast->globalCtxBufferVaList[buffId], 1491 kgraphicsIsGlobalCtxBufferSizeAligned(pGpu, pKernelGraphics, buffId), 1492 bIsReadOnly), 1493 NV_PRINTF(LEVEL_ERROR, "%s Buffer could not be mapped\n", 1494 NV_ENUM_TO_STRING(GR_GLOBALCTX_BUFFER, buffId)); 1495 return status; ); 1496 1497 NV_ASSERT_OK_OR_CAPTURE_FIRST_ERROR(status, 1498 vaListFindVa(&pKernelGraphicsContextUnicast->globalCtxBufferVaList[buffId], pVAS, &vaddr)); 1499 1500 NV_PRINTF(LEVEL_INFO, 1501 "GPU:%d %s Buffer PA @ 0x%llx VA @ 0x%llx of Size 0x%llx\n", 1502 pGpu->gpuInstance, NV_ENUM_TO_STRING(GR_GLOBALCTX_BUFFER, buffId), 1503 memdescGetPhysAddr(memdescGetMemDescFromGpu(pMemDesc, pGpu), AT_GPU, 0), 1504 vaddr, pMemDesc->Size); 1505 1506 return status; 1507 } 1508 1509 // 1510 // Map common buffers to a channel's context 1511 // 1512 NV_STATUS 1513 kgrctxMapGlobalCtxBuffers_IMPL 1514 ( 1515 OBJGPU *pGpu, 1516 KernelGraphicsContext *pKernelGraphicsContext, 1517 KernelGraphics *pKernelGraphics, 1518 NvU32 gfid, 1519 KernelChannel *pKernelChannel 1520 ) 1521 { 1522 GR_GLOBALCTX_BUFFERS *pGlobalCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid); 1523 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 1524 1525 NV_ASSERT_OR_RETURN(!gpumgrGetBcEnabledStatus(pGpu), NV_ERR_INVALID_STATE); 1526 1527 NV_ASSERT_OK_OR_RETURN( 1528 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast)); 1529 1530 // if global ctx buffers were never allocated then bail out early 1531 if (!pKernelGraphicsContextUnicast->localCtxBuffer.bAllocated && 1532 !pGlobalCtxBuffers->bAllocated) 1533 { 1534 return NV_ERR_INVALID_STATE; 1535 } 1536 1537 kgrctxMapGlobalCtxBuffer(pGpu, 1538 pKernelGraphicsContext, 1539 pKernelGraphics, 1540 gfid, 1541 pKernelChannel->pVAS, 1542 GR_GLOBALCTX_BUFFER_BUNDLE_CB, 1543 NV_FALSE); 1544 kgrctxMapGlobalCtxBuffer(pGpu, 1545 pKernelGraphicsContext, 1546 pKernelGraphics, 1547 gfid, 1548 pKernelChannel->pVAS, 1549 GR_GLOBALCTX_BUFFER_ATTRIBUTE_CB, 1550 NV_FALSE); 1551 kgrctxMapGlobalCtxBuffer(pGpu, 1552 pKernelGraphicsContext, 1553 pKernelGraphics, 1554 gfid, 1555 pKernelChannel->pVAS, 1556 GR_GLOBALCTX_BUFFER_PAGEPOOL, 1557 NV_FALSE); 1558 1559 if (kgraphicsIsRtvCbSupported(pGpu, pKernelGraphics)) 1560 { 1561 kgrctxMapGlobalCtxBuffer(pGpu, 1562 pKernelGraphicsContext, 1563 pKernelGraphics, 1564 gfid, 1565 pKernelChannel->pVAS, 1566 GR_GLOBALCTX_BUFFER_RTV_CB, 1567 NV_FALSE); 1568 } 1569 1570 return NV_OK; 1571 } 1572 1573 /*! 1574 * @brief This function allocates and maps various GR buffers. 1575 */ 1576 NV_STATUS 1577 kgrctxMapCtxBuffers_IMPL 1578 ( 1579 OBJGPU *pGpu, 1580 KernelGraphicsContext *pKernelGraphicsContext, 1581 KernelGraphics *pKernelGraphics, 1582 KernelGraphicsObject *pKernelGraphicsObject 1583 ) 1584 { 1585 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 1586 ChannelDescendant *pChannelDescendant = staticCast(pKernelGraphicsObject, ChannelDescendant); 1587 KernelChannel *pKernelChannel = pChannelDescendant->pKernelChannel; 1588 NvU32 classNum = pChannelDescendant->resourceDesc.externalClassId; 1589 OBJGVASPACE *pGVAS; 1590 NvU32 objType; 1591 NvU32 gfid = kchannelGetGfid(pKernelChannel); 1592 NvBool bAcquire3d = NV_FALSE; 1593 1594 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 1595 1596 NV_ASSERT_OK_OR_RETURN( 1597 kgrctxGetUnicast(pGpu, kgrobjGetKernelGraphicsContext(pGpu, pKernelGraphicsObject), &pKernelGraphicsContextUnicast)); 1598 1599 pGVAS = dynamicCast(pKernelChannel->pVAS, OBJGVASPACE); 1600 if (gvaspaceIsExternallyOwned(pGVAS)) 1601 return NV_OK; 1602 1603 // 1604 // in kernel RM we skip the context buffer mapping as part of 1605 // fifoSetUpChannelDma. fifoSetUpChannelDma is not enabled in kernel RM. 1606 // So it's possible for the main ctx buffer to be unmapped by this point. 1607 // 1608 if (!kchannelIsCtxBufferAllocSkipped(pKernelChannel)) 1609 { 1610 ENGINE_CTX_DESCRIPTOR *pEngCtx; 1611 MEMORY_DESCRIPTOR *pMemDesc; 1612 NvU32 subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu); 1613 1614 pEngCtx = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->ppEngCtxDesc[subdevInst]; 1615 1616 NV_ASSERT_OR_RETURN(pEngCtx != NULL, NV_ERR_INVALID_STATE); 1617 NV_ASSERT_OR_RETURN(pEngCtx->pMemDesc != NULL, NV_ERR_INVALID_STATE); 1618 pMemDesc = pEngCtx->pMemDesc; 1619 1620 NV_ASSERT_OK_OR_RETURN( 1621 kgraphicsMapCtxBuffer(pGpu, pKernelGraphics, pMemDesc, pKernelChannel->pVAS, 1622 &pEngCtx->vaList, NV_FALSE, NV_FALSE)); 1623 } 1624 1625 kgrmgrGetGrObjectType(classNum, &objType); 1626 1627 NV_ASSERT_OK_OR_RETURN( 1628 kgraphicsMapCtxBuffer(pGpu, pKernelGraphics, pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc, pKernelChannel->pVAS, 1629 &pKernelGraphicsContextUnicast->ctxPatchBuffer.vAddrList, NV_FALSE, NV_FALSE)); 1630 1631 if (pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc != NULL) 1632 { 1633 NV_ASSERT_OK_OR_RETURN( 1634 kgraphicsMapCtxBuffer(pGpu, pKernelGraphics, pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc, pKernelChannel->pVAS, 1635 &pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList, NV_FALSE, NV_FALSE)); 1636 } 1637 1638 if (kgraphicsDoesUcodeSupportPrivAccessMap(pGpu, pKernelGraphics)) 1639 { 1640 kgrctxMapGlobalCtxBuffer(pGpu, 1641 pKernelGraphicsContext, 1642 pKernelGraphics, 1643 gfid, 1644 pKernelChannel->pVAS, 1645 kgrctxGetRegisterAccessMapId_HAL(pGpu, pKernelGraphicsContext, pChannelDescendant->pKernelChannel), 1646 NV_TRUE); 1647 } 1648 1649 // 1650 // Condition for acquiring 3d context buffer mappings. 1651 // For non-TSG & legacy TSG mode, always map 1652 // For subcontext TSG case, only map on 2D/3D allocations 1653 // 1654 if (pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->bLegacyMode) 1655 { 1656 bAcquire3d = NV_TRUE; 1657 } 1658 else 1659 { 1660 bAcquire3d = ((objType == GR_OBJECT_TYPE_2D) || (objType == GR_OBJECT_TYPE_3D)); 1661 } 1662 1663 if (bAcquire3d) 1664 { 1665 kgrctxMapGlobalCtxBuffers(pGpu, pKernelGraphicsContext, pKernelGraphics, gfid, pKernelChannel); 1666 } 1667 1668 return NV_OK; 1669 } 1670 1671 /*! 1672 * @brief Set parameters for promoting the PA of a ctx buffer to physical RM. 1673 */ 1674 NV_STATUS 1675 kgrctxPrepareInitializeCtxBuffer_IMPL 1676 ( 1677 OBJGPU *pGpu, 1678 KernelGraphicsContext *pKernelGraphicsContext, 1679 KernelGraphics *pKernelGraphics, 1680 KernelChannel *pKernelChannel, 1681 NvU32 externalId, 1682 NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ENTRY *pEntry, 1683 NvBool *pbAddEntry 1684 ) 1685 { 1686 MEMORY_DESCRIPTOR *pMemDesc; 1687 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 1688 NvU32 physAttr; 1689 1690 *pbAddEntry = NV_FALSE; 1691 1692 NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_CHANNEL); 1693 1694 NV_ASSERT_OK_OR_RETURN( 1695 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast)); 1696 1697 switch (externalId) 1698 { 1699 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_MAIN: 1700 { 1701 ENGINE_CTX_DESCRIPTOR *pEngCtx; 1702 NvU32 subdevInst; 1703 if (pKernelGraphicsContextUnicast->bKGrMainCtxBufferInitialized) 1704 return NV_OK; 1705 1706 // Do not try to init a KMD virtual context buffer 1707 if (kchannelIsCtxBufferAllocSkipped(pKernelChannel)) 1708 return NV_OK; 1709 1710 subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu); 1711 pEngCtx = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->ppEngCtxDesc[subdevInst]; 1712 1713 NV_ASSERT_OR_RETURN(pEngCtx != NULL, NV_ERR_INVALID_STATE); 1714 NV_ASSERT_OR_RETURN(pEngCtx->pMemDesc != NULL, NV_ERR_INVALID_STATE); 1715 pMemDesc = pEngCtx->pMemDesc; 1716 break; 1717 } 1718 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PM: 1719 if (pKernelGraphicsContextUnicast->bKGrPmCtxBufferInitialized) 1720 return NV_OK; 1721 1722 if (pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc == NULL) 1723 return NV_OK; 1724 1725 pMemDesc = pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc; 1726 break; 1727 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PATCH: 1728 if (pKernelGraphicsContextUnicast->bKGrPatchCtxBufferInitialized) 1729 return NV_OK; 1730 1731 NV_ASSERT(pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc != NULL); 1732 pMemDesc = pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc; 1733 break; 1734 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_BUFFER_BUNDLE_CB: 1735 // fall-through 1736 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PAGEPOOL: 1737 // fall-through 1738 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_ATTRIBUTE_CB: 1739 // fall-through 1740 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_RTV_CB_GLOBAL: 1741 // fall-through 1742 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_POOL: 1743 // No initialization from kernel RM 1744 return NV_OK; 1745 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_CTRL_BLK: 1746 { 1747 GR_GLOBALCTX_BUFFER internalId; 1748 NvU32 gfid = kchannelGetGfid(pKernelChannel); 1749 GR_GLOBALCTX_BUFFERS *pCtxBuffers; 1750 1751 NV_ASSERT_OK_OR_RETURN( 1752 kgrctxGetGlobalContextBufferInternalId(externalId, &internalId)); 1753 1754 if (pKernelGraphicsContextUnicast->localCtxBuffer.bAllocated) 1755 { 1756 pCtxBuffers = &pKernelGraphicsContextUnicast->localCtxBuffer; 1757 } 1758 else 1759 { 1760 pCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid); 1761 } 1762 1763 if (pCtxBuffers->bInitialized[internalId]) 1764 return NV_OK; 1765 1766 pMemDesc = pCtxBuffers->memDesc[internalId]; 1767 break; 1768 } 1769 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_FECS_EVENT: 1770 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PRIV_ACCESS_MAP: 1771 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_UNRESTRICTED_PRIV_ACCESS_MAP: 1772 { 1773 GR_GLOBALCTX_BUFFER internalId; 1774 NvU32 gfid = kchannelGetGfid(pKernelChannel); 1775 GR_GLOBALCTX_BUFFERS *pCtxBuffers; 1776 1777 NV_ASSERT_OK_OR_RETURN( 1778 kgrctxGetGlobalContextBufferInternalId(externalId, &internalId)); 1779 1780 pCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid); 1781 1782 if (pCtxBuffers->bInitialized[internalId]) 1783 return NV_OK; 1784 1785 pMemDesc = pCtxBuffers->memDesc[internalId]; 1786 1787 break; 1788 } 1789 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GLOBAL_PRIV_ACCESS_MAP: 1790 // No initialization from kernel RM 1791 return NV_OK; 1792 default: 1793 NV_ASSERT_OR_RETURN(!"Unrecognized promote ctx enum", NV_ERR_INVALID_ARGUMENT); 1794 } 1795 1796 // If these buffers are not supported or not allocated, no need to init 1797 if (pMemDesc == NULL) 1798 return NV_OK; 1799 1800 physAttr = 0x0; 1801 switch (memdescGetAddressSpace(pMemDesc)) 1802 { 1803 case ADDR_FBMEM: 1804 physAttr = FLD_SET_DRF(2080, _CTRL_GPU_INITIALIZE_CTX, 1805 _APERTURE, _VIDMEM, physAttr); 1806 break; 1807 1808 case ADDR_SYSMEM: 1809 if (memdescGetCpuCacheAttrib(pMemDesc) == NV_MEMORY_CACHED) 1810 { 1811 physAttr = FLD_SET_DRF(2080, _CTRL_GPU_INITIALIZE_CTX, 1812 _APERTURE, _COH_SYS, physAttr); 1813 } 1814 else if (memdescGetCpuCacheAttrib(pMemDesc) == NV_MEMORY_UNCACHED) 1815 { 1816 physAttr = FLD_SET_DRF(2080, _CTRL_GPU_INITIALIZE_CTX, 1817 _APERTURE, _NCOH_SYS, physAttr); 1818 } 1819 else 1820 { 1821 return NV_ERR_INVALID_STATE; 1822 } 1823 break; 1824 1825 default: 1826 return NV_ERR_INVALID_STATE; 1827 } 1828 1829 physAttr = FLD_SET_DRF(2080, _CTRL_GPU_INITIALIZE_CTX, _GPU_CACHEABLE, _NO, physAttr); 1830 1831 pEntry->gpuPhysAddr = memdescGetPhysAddr(pMemDesc, AT_GPU, 0); 1832 pEntry->size = pMemDesc->Size; 1833 pEntry->physAttr = physAttr; 1834 pEntry->bufferId = externalId; 1835 pEntry->bInitialize = NV_TRUE; 1836 pEntry->bNonmapped = NV_TRUE; 1837 1838 *pbAddEntry = NV_TRUE; 1839 1840 return NV_OK; 1841 } 1842 1843 /*! 1844 * @brief Set parameters for promoting the VA of a ctx buffer to physical RM. 1845 */ 1846 NV_STATUS 1847 kgrctxPreparePromoteCtxBuffer_IMPL 1848 ( 1849 OBJGPU *pGpu, 1850 KernelGraphicsContext *pKernelGraphicsContext, 1851 KernelChannel *pKernelChannel, 1852 NvU32 externalId, 1853 NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ENTRY *pEntry, 1854 NvBool *pbAddEntry 1855 ) 1856 { 1857 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 1858 VA_LIST *pVaList; 1859 NvU64 vaddr; 1860 NvU64 refCount; 1861 OBJGVASPACE *pGVAS = dynamicCast(pKernelChannel->pVAS, OBJGVASPACE); 1862 NV_STATUS status; 1863 1864 NV_ASSERT_OK_OR_RETURN( 1865 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast)); 1866 1867 *pbAddEntry = NV_FALSE; 1868 1869 // RM is not responsible for promoting the buffers when UVM is enabled 1870 if (gvaspaceIsExternallyOwned(pGVAS)) 1871 return NV_OK; 1872 1873 switch (externalId) 1874 { 1875 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_MAIN: 1876 { 1877 ENGINE_CTX_DESCRIPTOR *pEngCtx; 1878 NvU32 subdevInst; 1879 1880 // Do not try to promote a KMD virtual context buffer 1881 if (kchannelIsCtxBufferAllocSkipped(pKernelChannel)) 1882 return NV_OK; 1883 1884 subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu); 1885 pEngCtx = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->ppEngCtxDesc[subdevInst]; 1886 1887 NV_ASSERT_OR_RETURN(pEngCtx != NULL, NV_ERR_INVALID_STATE); 1888 pVaList = &pEngCtx->vaList; 1889 break; 1890 } 1891 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PM: 1892 pVaList = &pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList; 1893 break; 1894 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PATCH: 1895 pVaList = &pKernelGraphicsContextUnicast->ctxPatchBuffer.vAddrList; 1896 break; 1897 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_BUFFER_BUNDLE_CB: 1898 // fall-through 1899 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PAGEPOOL: 1900 // fall-through 1901 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_ATTRIBUTE_CB: 1902 // fall-through 1903 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_RTV_CB_GLOBAL: 1904 // fall-through 1905 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_POOL: 1906 // fall-through 1907 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_CTRL_BLK: 1908 // fall-through 1909 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_FECS_EVENT: 1910 // fall-through 1911 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PRIV_ACCESS_MAP: 1912 // fall-through 1913 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_UNRESTRICTED_PRIV_ACCESS_MAP: 1914 // fall-through 1915 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GLOBAL_PRIV_ACCESS_MAP: 1916 { 1917 GR_GLOBALCTX_BUFFER internalId; 1918 1919 NV_ASSERT_OK_OR_RETURN( 1920 kgrctxGetGlobalContextBufferInternalId(externalId, &internalId)); 1921 1922 pVaList = &pKernelGraphicsContextUnicast->globalCtxBufferVaList[internalId]; 1923 break; 1924 } 1925 default: 1926 NV_ASSERT_OR_RETURN(!"Unrecognized promote ctx enum", NV_ERR_INVALID_ARGUMENT); 1927 } 1928 1929 // If the buffer isn't currently mapped or was already promoted, nothing to do 1930 status = vaListGetRefCount(pVaList, pKernelChannel->pVAS, &refCount); 1931 if ((status != NV_OK) || (refCount > 1)) 1932 return NV_OK; 1933 1934 NV_ASSERT_OK_OR_RETURN(vaListFindVa(pVaList, pKernelChannel->pVAS, &vaddr)); 1935 1936 pEntry->bufferId = externalId; 1937 pEntry->gpuVirtAddr = vaddr; 1938 pEntry->bNonmapped = NV_FALSE; 1939 1940 *pbAddEntry = NV_TRUE; 1941 return NV_OK; 1942 } 1943 1944 /*! Mark the context buffer as initialized, prevent further calls to init */ 1945 void 1946 kgrctxMarkCtxBufferInitialized_IMPL 1947 ( 1948 OBJGPU *pGpu, 1949 KernelGraphicsContext *pKernelGraphicsContext, 1950 KernelGraphics *pKernelGraphics, 1951 KernelChannel *pKernelChannel, 1952 NvU32 externalId 1953 ) 1954 { 1955 NV_STATUS status; 1956 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 1957 1958 NV_ASSERT_OK_OR_ELSE(status, 1959 kgrctxGetUnicast(pGpu, 1960 pKernelGraphicsContext, 1961 &pKernelGraphicsContextUnicast), 1962 return;); 1963 1964 switch (externalId) 1965 { 1966 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_MAIN: 1967 pKernelGraphicsContextUnicast->bKGrMainCtxBufferInitialized = NV_TRUE; 1968 break; 1969 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PM: 1970 pKernelGraphicsContextUnicast->bKGrPmCtxBufferInitialized = NV_TRUE; 1971 break; 1972 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PATCH: 1973 pKernelGraphicsContextUnicast->bKGrPatchCtxBufferInitialized = NV_TRUE; 1974 break; 1975 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_BUFFER_BUNDLE_CB: 1976 // fall-through 1977 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PAGEPOOL: 1978 // fall-through 1979 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_ATTRIBUTE_CB: 1980 // fall-through 1981 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_RTV_CB_GLOBAL: 1982 // fall-through 1983 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_POOL: 1984 // fall-through 1985 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_CTRL_BLK: 1986 { 1987 // If "local" global ctx buffers are allocated, check those first 1988 if (pKernelGraphicsContextUnicast->localCtxBuffer.bAllocated) 1989 { 1990 GR_GLOBALCTX_BUFFER internalId; 1991 GR_GLOBALCTX_BUFFERS *pKCtxBuffers; 1992 1993 NV_ASSERT_OK_OR_ELSE(status, 1994 kgrctxGetGlobalContextBufferInternalId(externalId, &internalId), 1995 return;); 1996 1997 pKCtxBuffers = &pKernelGraphicsContextUnicast->localCtxBuffer; 1998 pKCtxBuffers->bInitialized[internalId] = NV_TRUE; 1999 break; 2000 } 2001 // fall-through 2002 } 2003 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_FECS_EVENT: 2004 // fall-through 2005 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PRIV_ACCESS_MAP: 2006 // fall-through 2007 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_UNRESTRICTED_PRIV_ACCESS_MAP: 2008 // fall-through 2009 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GLOBAL_PRIV_ACCESS_MAP: 2010 { 2011 GR_GLOBALCTX_BUFFER internalId; 2012 NvU32 gfid = kchannelGetGfid(pKernelChannel); 2013 GR_GLOBALCTX_BUFFERS *pKCtxBuffers; 2014 2015 NV_ASSERT_OK_OR_ELSE(status, 2016 kgrctxGetGlobalContextBufferInternalId(externalId, &internalId), 2017 return;); 2018 2019 // TODO XXX Make context buffers available from KGRCTX alone 2020 pKCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid); 2021 2022 pKCtxBuffers->bInitialized[internalId] = NV_TRUE; 2023 break; 2024 } 2025 default: 2026 NV_ASSERT(!"Unrecognized promote ctx enum"); 2027 } 2028 } 2029 2030 /*! Non-UVM late bind PM ctx buffer */ 2031 NV_STATUS 2032 kgrctxSetupDeferredPmBuffer_IMPL 2033 ( 2034 OBJGPU *pGpu, 2035 KernelGraphicsContext *pKernelGraphicsContext, 2036 KernelGraphics *pKernelGraphics, 2037 KernelChannel *pKernelChannel 2038 ) 2039 { 2040 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2041 OBJGVASPACE *pGVAS = dynamicCast(pKernelChannel->pVAS, OBJGVASPACE); 2042 NV_STATUS status = NV_OK; 2043 Device *pDevice = GPU_RES_GET_DEVICE(pKernelChannel); 2044 Subdevice *pSubdevice; 2045 2046 NV_ASSERT_OK_OR_RETURN( 2047 subdeviceGetByInstance( 2048 RES_GET_CLIENT(pKernelChannel), 2049 RES_GET_HANDLE(pDevice), 2050 gpumgrGetSubDeviceInstanceFromGpu(pGpu), 2051 &pSubdevice)); 2052 2053 GPU_RES_SET_THREAD_BC_STATE(pSubdevice); 2054 2055 NV_ASSERT_OK_OR_RETURN( 2056 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast)); 2057 2058 if (gvaspaceIsExternallyOwned(pGVAS)) 2059 { 2060 // 2061 // The PM Ctxsw buffer is now pre-allocated along with the other context buffers 2062 // This is done as a WAR for the issue tracked under bug 1760699 2063 // Reason: With the enablement of UVM8 by default, the UVM channel registration relies on the 2064 // buffers being allocated at channel allocation time and are not going to work for a buffer 2065 // created later with an rm ctrl 2066 // 2067 NV_ASSERT_OR_RETURN(pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc != NULL, NV_ERR_INVALID_STATE); 2068 return NV_OK; 2069 } 2070 else if (pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc != NULL) 2071 { 2072 return NV_OK; 2073 } 2074 2075 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2076 kgrctxAllocPmBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel)); 2077 2078 // 2079 // !!! 2080 // From this point on, use `goto failed` to exit 2081 // !!! 2082 // 2083 2084 { 2085 RS_ORDERED_ITERATOR it; 2086 RsResourceRef *pScopeRef = RES_GET_REF(pKernelChannel); 2087 2088 // Iterate over all channels in this TSG and map the new buffer 2089 if (!pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->bAllocatedByRm && 2090 kgraphicsIsPerSubcontextContextHeaderSupported(pGpu, pKernelGraphics)) 2091 { 2092 pScopeRef = RES_GET_REF(pKernelChannel->pKernelChannelGroupApi); 2093 } 2094 2095 it = kchannelGetIter(RES_GET_CLIENT(pKernelChannel), pScopeRef); 2096 while (clientRefOrderedIterNext(it.pClient, &it)) 2097 { 2098 NvU64 vaddr; 2099 KernelChannel *pLoopKernelChannel = dynamicCast(it.pResourceRef->pResource, KernelChannel); 2100 2101 NV_ASSERT_OR_ELSE(pLoopKernelChannel != NULL, 2102 status = NV_ERR_INVALID_STATE; 2103 goto failed;); 2104 2105 if ((vaListFindVa(&pKernelGraphicsContextUnicast->ctxPatchBuffer.vAddrList, pLoopKernelChannel->pVAS, &vaddr) == NV_OK) && 2106 (vaListFindVa(&pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList, pLoopKernelChannel->pVAS, &vaddr) != NV_OK)) 2107 { 2108 NvU64 refCount; 2109 NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS params; 2110 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 2111 NvBool bInitialize; 2112 NvBool bPromote; 2113 2114 NV_ASSERT_OK_OR_GOTO(status, 2115 kgraphicsMapCtxBuffer(pGpu, pKernelGraphics, 2116 pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc, 2117 pLoopKernelChannel->pVAS, 2118 &pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList, 2119 NV_FALSE, 2120 NV_FALSE), 2121 failed); 2122 2123 // Promote the vaddr for each unique VAS 2124 portMemSet(¶ms, 0, sizeof(params)); 2125 2126 // Setup parameters to initialize the PA if necessary 2127 NV_ASSERT_OK_OR_GOTO(status, 2128 kgrctxPrepareInitializeCtxBuffer(pGpu, 2129 pKernelGraphicsContext, 2130 pKernelGraphics, 2131 pKernelChannel, 2132 NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PM, 2133 ¶ms.promoteEntry[0], 2134 &bInitialize), 2135 failed); 2136 2137 // Setup parameters to promote the VA if necessary 2138 NV_ASSERT_OK_OR_GOTO(status, 2139 kgrctxPreparePromoteCtxBuffer(pGpu, 2140 pKernelGraphicsContext, 2141 pKernelChannel, 2142 NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PM, 2143 ¶ms.promoteEntry[0], 2144 &bPromote), 2145 failed); 2146 2147 NV_ASSERT_OR_ELSE(bInitialize || bPromote, 2148 status = NV_ERR_INVALID_STATE; 2149 goto failed;); 2150 2151 params.engineType = NV2080_ENGINE_TYPE_GR(kgraphicsGetInstance(pGpu, pKernelGraphics)); 2152 params.hChanClient = RES_GET_CLIENT_HANDLE(pKernelChannel); 2153 params.hObject = RES_GET_HANDLE(pKernelChannel); 2154 params.entryCount = 1; 2155 2156 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, 2157 pRmApi->Control(pRmApi, 2158 RES_GET_CLIENT_HANDLE(pSubdevice), 2159 RES_GET_HANDLE(pSubdevice), 2160 NV2080_CTRL_CMD_GPU_PROMOTE_CTX, 2161 ¶ms, 2162 sizeof(params)), 2163 failed); 2164 2165 // 2166 // If we successfully promoted the PA, flip a flag to ensure we don't 2167 // try to promote it again. The VA_LIST should already track this for 2168 // VA, but we can't rely on it for PA due to UVM. 2169 // 2170 if (params.promoteEntry[0].bInitialize) 2171 { 2172 kgrctxMarkCtxBufferInitialized(pGpu, 2173 pKernelGraphicsContext, 2174 pKernelGraphics, 2175 pKernelChannel, 2176 params.promoteEntry[0].bufferId); 2177 } 2178 2179 // Update the refcount for this buffer now that we've promoted it 2180 NV_ASSERT_OK_OR_GOTO(status, 2181 vaListGetRefCount(&pKernelGraphicsContextUnicast->ctxPatchBuffer.vAddrList, pLoopKernelChannel->pVAS, &refCount), 2182 failed); 2183 NV_ASSERT_OK_OR_GOTO(status, 2184 vaListSetRefCount(&pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList, pLoopKernelChannel->pVAS, refCount), 2185 failed); 2186 } 2187 } 2188 } 2189 2190 return NV_OK; 2191 2192 failed: 2193 2194 if (kgraphicsIsPerSubcontextContextHeaderSupported(pGpu, pKernelGraphics)) 2195 { 2196 RS_ORDERED_ITERATOR it; 2197 RsResourceRef *pScopeRef = RES_GET_REF(pKernelChannel); 2198 2199 // Iterate over all channels in this TSG and try to unmap the PM ctx buffer 2200 if (!pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->bAllocatedByRm && 2201 kgraphicsIsPerSubcontextContextHeaderSupported(pGpu, pKernelGraphics)) 2202 { 2203 pScopeRef = RES_GET_REF(pKernelChannel->pKernelChannelGroupApi); 2204 } 2205 2206 it = kchannelGetIter(RES_GET_CLIENT(pKernelChannel), pScopeRef); 2207 while (clientRefOrderedIterNext(it.pClient, &it)) 2208 { 2209 NvU64 vaddr; 2210 KernelChannel *pLoopKernelChannel = dynamicCast(it.pResourceRef->pResource, KernelChannel); 2211 if (pLoopKernelChannel == NULL) 2212 continue; 2213 2214 while (vaListFindVa(&pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList, pLoopKernelChannel->pVAS, &vaddr) == NV_OK) 2215 { 2216 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, 2217 pLoopKernelChannel->pVAS, 2218 &pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList); 2219 } 2220 } 2221 } 2222 2223 kgrctxFreePmBuffer(pGpu, pKernelGraphicsContext); 2224 2225 return status; 2226 } 2227 2228 /** 2229 * @brief unmap the memory for the pm context buffer associated with a channel 2230 */ 2231 void 2232 kgrctxUnmapCtxPmBuffer_IMPL 2233 ( 2234 OBJGPU *pGpu, 2235 KernelGraphicsContext *pKernelGraphicsContext, 2236 KernelGraphics *pKernelGraphics, 2237 OBJVASPACE *pVAS 2238 ) 2239 { 2240 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2241 NV_STATUS status; 2242 2243 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2244 2245 NV_ASSERT_OK_OR_ELSE(status, 2246 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast), 2247 return;); 2248 2249 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pVAS, &pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList); 2250 } 2251 2252 /** 2253 * @brief unmap the memory for the zcull context buffer 2254 */ 2255 void 2256 kgrctxUnmapCtxZcullBuffer_IMPL 2257 ( 2258 OBJGPU *pGpu, 2259 KernelGraphicsContext *pKernelGraphicsContext, 2260 KernelGraphics *pKernelGraphics, 2261 OBJVASPACE *pVAS 2262 ) 2263 { 2264 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2265 MEMORY_DESCRIPTOR *pMemDesc; 2266 VA_LIST *pVaList; 2267 NvU64 vaddr; 2268 NV_STATUS status; 2269 2270 NV_ASSERT_OK_OR_ELSE(status, 2271 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast), 2272 return;); 2273 2274 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2275 2276 pMemDesc = pKernelGraphicsContextUnicast->zcullCtxswBuffer.pMemDesc; 2277 pVaList = &pKernelGraphicsContextUnicast->zcullCtxswBuffer.vAddrList; 2278 if (pMemDesc != NULL) 2279 { 2280 // 2281 // This func assumes that the buffer was not allocated per subdevice, 2282 // and will leak any mappings performed by the secondaries. 2283 // 2284 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc)); 2285 2286 if (vaListFindVa(pVaList, pVAS, &vaddr) == NV_OK) 2287 { 2288 dmaUnmapBuffer_HAL(pGpu, GPU_GET_DMA(pGpu), pVAS, vaddr); 2289 vaListRemoveVa(pVaList, pVAS); 2290 } 2291 } 2292 else if (vaListFindVa(pVaList, pVAS, &vaddr) == NV_OK) 2293 { 2294 // Zcull buffer mapped by client. Remove the VA here 2295 vaListRemoveVa(pVaList, pVAS); 2296 } 2297 } 2298 2299 /** 2300 * @brief unmap the memory for the preemption context buffers 2301 */ 2302 void 2303 kgrctxUnmapCtxPreemptionBuffers_IMPL 2304 ( 2305 OBJGPU *pGpu, 2306 KernelGraphicsContext *pKernelGraphicsContext, 2307 KernelGraphics *pKernelGraphics, 2308 OBJVASPACE *pVAS 2309 ) 2310 { 2311 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2312 NV_STATUS status; 2313 2314 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2315 2316 NV_ASSERT_OK_OR_ELSE(status, 2317 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast), 2318 return;); 2319 2320 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pVAS, &pKernelGraphicsContextUnicast->preemptCtxswBuffer.vAddrList); 2321 2322 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pVAS, &pKernelGraphicsContextUnicast->spillCtxswBuffer.vAddrList); 2323 2324 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pVAS, &pKernelGraphicsContextUnicast->betaCBCtxswBuffer.vAddrList); 2325 2326 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pVAS, &pKernelGraphicsContextUnicast->pagepoolCtxswBuffer.vAddrList); 2327 2328 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pVAS, &pKernelGraphicsContextUnicast->rtvCbCtxswBuffer.vAddrList); 2329 } 2330 2331 /** 2332 * @brief Unmap associated ctx buffers (main, patch, global buffers etc). 2333 * 2334 * This function is called on every channel free and therefore can't assume any 2335 * graphics objects actually exist on the given channel. 2336 * TODO: Bug 3164256 2337 * This function also unmaps and frees zcull and preempt buffers. Ideally we want to decouple 2338 * unmap and free for all buffers and move free of buffers to kgrctxFreeAssociatedCtxBuffers 2339 */ 2340 void 2341 kgrctxUnmapAssociatedCtxBuffers_IMPL 2342 ( 2343 OBJGPU *pGpu, 2344 KernelGraphicsContext *pKernelGraphicsContext, 2345 KernelGraphics *pKernelGraphics, 2346 KernelChannel *pKernelChannel 2347 ) 2348 { 2349 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2350 NvBool bRelease3d = NV_FALSE; 2351 NvU32 gfid = kchannelGetGfid(pKernelChannel); 2352 NvU32 status; 2353 GR_GLOBALCTX_BUFFER registerMapBufferId = kgrctxGetRegisterAccessMapId_HAL(pGpu, pKernelGraphicsContext, pKernelChannel); 2354 2355 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2356 2357 NV_ASSERT_OK_OR_ELSE(status, 2358 kgrctxGetUnicast(pGpu, 2359 pKernelGraphicsContext, 2360 &pKernelGraphicsContextUnicast), 2361 return;); 2362 2363 // 2364 // Unmap 3D context buffers on the last 3D channel free 2365 // 2366 // Condition for unmapping 3d context buffer mappings: 2367 // For non-TSG & legacy TSG mode, always unmap 3d context buffers. 2368 // For subcontext TSG case, unmap on the last graphics channel free. 2369 // 2370 if (pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->bLegacyMode) 2371 { 2372 bRelease3d = pKernelGraphicsContextUnicast->channelObjects == 0; 2373 } 2374 else 2375 { 2376 bRelease3d = ((pKernelGraphicsContextUnicast->objectCounts[GR_OBJECT_TYPE_3D] == 0) && 2377 (pKernelGraphicsContextUnicast->objectCounts[GR_OBJECT_TYPE_2D] == 0)); 2378 } 2379 2380 if (bRelease3d) 2381 { 2382 // Unmap Circular buffer from the current channel's address space 2383 kgrctxUnmapGlobalCtxBuffers(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel->pVAS, gfid); 2384 } 2385 2386 // 2387 // If multiple channels sharing the same VAS exist, it is possible both 2388 // channels could be using these mappings, and we must wait for both 2389 // channels to be detached before we remove them. 2390 // 2391 { 2392 RS_ORDERED_ITERATOR it; 2393 RsResourceRef *pScopeRef = RES_GET_REF(pKernelChannel); 2394 2395 // Iterate over all channels in this TSG and check for duplicate VAS 2396 if (!pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->bAllocatedByRm) 2397 pScopeRef = RES_GET_REF(pKernelChannel->pKernelChannelGroupApi); 2398 2399 it = kchannelGetIter(RES_GET_CLIENT(pKernelChannel), pScopeRef); 2400 while (clientRefOrderedIterNext(it.pClient, &it)) 2401 { 2402 KernelChannel *pLoopKernelChannel = dynamicCast(it.pResourceRef->pResource, KernelChannel); 2403 NV_ASSERT_OR_RETURN_VOID(pLoopKernelChannel != NULL); 2404 2405 if (pLoopKernelChannel == pKernelChannel) 2406 continue; 2407 2408 NV_CHECK_OR_RETURN_VOID(LEVEL_SILENT, pLoopKernelChannel->pVAS != pKernelChannel->pVAS); 2409 } 2410 } 2411 2412 // Only unmap once the last channel using this VAS has gone 2413 if (kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid)->memDesc[GR_GLOBALCTX_BUFFER_FECS_EVENT] != NULL) 2414 { 2415 kgrctxUnmapGlobalCtxBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel->pVAS, GR_GLOBALCTX_BUFFER_FECS_EVENT); 2416 } 2417 2418 if (pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc != NULL) 2419 { 2420 kgrctxUnmapCtxPmBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel->pVAS); 2421 } 2422 2423 if (kgraphicsDoesUcodeSupportPrivAccessMap(pGpu, pKernelGraphics)) 2424 { 2425 kgrctxUnmapGlobalCtxBuffer(pGpu, 2426 pKernelGraphicsContext, 2427 pKernelGraphics, 2428 pKernelChannel->pVAS, 2429 registerMapBufferId); 2430 } 2431 2432 if (pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc != NULL) 2433 { 2434 // 2435 // Delay freeing the patch buffer until the last channel free. 2436 // these buffers are accessed even after the last GR object is freed. 2437 // 2438 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pKernelChannel->pVAS, &pKernelGraphicsContextUnicast->ctxPatchBuffer.vAddrList); 2439 } 2440 2441 kgrctxUnmapCtxZcullBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel->pVAS); 2442 2443 // Release all preemption buffers if they were allocated 2444 kgrctxUnmapCtxPreemptionBuffers(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel->pVAS); 2445 2446 // 2447 // Cleanup lingering main ctx buffer mappings for this VAS 2448 // TODO fix main ctx buffer refcounting 2449 // TODO move setEngineContextMemDesc to FreeMainCtxBuffer, move this loop 2450 // inside UnmapMainCtxBuffer 2451 // 2452 if (pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->ppEngCtxDesc[gpumgrGetSubDeviceInstanceFromGpu(pGpu)] != NULL) 2453 { 2454 NvU32 subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu); 2455 VA_LIST *pVAddrList = &pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->ppEngCtxDesc[subdevInst]->vaList; 2456 NvU64 vaddr; 2457 2458 while (vaListFindVa(pVAddrList, pKernelChannel->pVAS, &vaddr) == NV_OK) 2459 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pKernelChannel->pVAS, pVAddrList); 2460 } 2461 2462 // 2463 // When sharing contexts across channels we need to defer this until all 2464 // objects have been freed. 2465 // 2466 NV_CHECK_OR_RETURN_VOID(LEVEL_SILENT, 2467 pKernelGraphicsContextUnicast->channelObjects == 0); 2468 2469 kgrctxUnmapMainCtxBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel); 2470 } 2471 2472 /*! 2473 * @brief is this object responsible for any cleanup tasks i.e. buffer unmapping? 2474 */ 2475 NvBool kgrctxShouldCleanup_KERNEL 2476 ( 2477 OBJGPU *pGpu, 2478 KernelGraphicsContext *pKernelGraphicsContext 2479 ) 2480 { 2481 return gpuIsClientRmAllocatedCtxBufferEnabled(pGpu); 2482 }; 2483 2484 /*! 2485 * This function returns whether PM ctxsw buffer should be pre-allocated or not. 2486 */ 2487 NvBool kgrctxShouldPreAllocPmBuffer_PF 2488 ( 2489 OBJGPU *pGpu, 2490 KernelGraphicsContext *pKernelGraphicsContext, 2491 KernelChannel *pKernelChannel 2492 ) 2493 { 2494 OBJGVASPACE *pGVAS = NULL; 2495 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2496 2497 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2498 NV_ASSERT_OR_RETURN(NULL != pKernelChannel, NV_ERR_INVALID_STATE); 2499 NV_ASSERT_OR_RETURN(pKernelGraphicsContext != NULL, NV_ERR_INVALID_STATE); 2500 2501 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2502 kgrctxGetUnicast(pGpu, 2503 pKernelGraphicsContext, 2504 &pKernelGraphicsContextUnicast)); 2505 2506 pGVAS = dynamicCast(pKernelChannel->pVAS, OBJGVASPACE); 2507 2508 // Do not allocate the buffer, if already allocated 2509 if (pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc != NULL) 2510 { 2511 return NV_FALSE; 2512 } 2513 2514 // 2515 // The PM Ctxsw buffer is now pre-allocated along with the other context buffers 2516 // This is done as a WAR for the issue tracked under bug 1760699 2517 // Reason: With the enablement of UVM8 by default, the UVM channel registration relies on the 2518 // buffers being allocated at channel allocation time and are not going to work for a buffer 2519 // created later with an rm ctrl 2520 // 2521 return gvaspaceIsExternallyOwned(pGVAS); 2522 } 2523 2524 /*! 2525 * This function returns whether PM ctxsw buffer should be pre-allocated 2526 * or not. 2527 */ 2528 NvBool 2529 kgrctxShouldPreAllocPmBuffer_VF 2530 ( 2531 OBJGPU *pGpu, 2532 KernelGraphicsContext *pKernelGraphicsContext, 2533 KernelChannel *pKernelChannel 2534 ) 2535 { 2536 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast = NULL; 2537 2538 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2539 NV_ASSERT_OR_RETURN(NULL != pKernelChannel, NV_ERR_INVALID_STATE); 2540 NV_ASSERT_OR_RETURN(pKernelGraphicsContext != NULL, NV_ERR_INVALID_STATE); 2541 2542 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2543 kgrctxGetUnicast(pGpu, 2544 pKernelGraphicsContext, 2545 &pKernelGraphicsContextUnicast)); 2546 2547 // Do not allocate the buffer, if already allocated 2548 if (pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc != NULL) 2549 { 2550 return NV_FALSE; 2551 } 2552 2553 // 2554 // Pre-allocate the PM Ctxsw buffer along with the other context buffers 2555 // in below scenarios: 2556 // 1. For externally owned VA spaces. This is done as a WAR for the issue 2557 // tracked under bug 1760699. 2558 // Reason: With the enablement of UVM8 by default, the UVM channel registration relies on the 2559 // buffers being allocated at channel allocation time and are not going to work for a buffer 2560 // created later with an rm ctrl. 2561 // 2. For full SRIOV vGPU guests with Profiling capability enabled 2562 // 2563 2564 return kgrctxShouldPreAllocPmBuffer_PF(pGpu, pKernelGraphicsContext, pKernelChannel); 2565 } 2566 2567 /*! 2568 * @brief should this layer manage the ctx buffers? 2569 */ 2570 NvBool 2571 kgrctxShouldManageCtxBuffers_KERNEL 2572 ( 2573 OBJGPU *pGpu, 2574 KernelGraphicsContext *pKernelGraphicsContext, 2575 NvU32 gfid 2576 ) 2577 { 2578 return gpuIsClientRmAllocatedCtxBufferEnabled(pGpu) && !IS_GFID_VF(gfid); 2579 } 2580 2581 /*! 2582 * @brief should this layer manage the ctx buffers? 2583 * If client RM is managing the ctx buffers but the channel is a plugin 2584 * channel, we should still manage them. 2585 */ 2586 NvBool 2587 kgrctxShouldManageCtxBuffers_PHYSICAL 2588 ( 2589 OBJGPU *pGpu, 2590 KernelGraphicsContext *pKernelGraphicsContext, 2591 NvU32 gfid 2592 ) 2593 { 2594 return !gpuIsClientRmAllocatedCtxBufferEnabled(pGpu) || (gpuIsSriovEnabled(pGpu) && IS_GFID_PF(gfid)); 2595 } 2596 2597 /** 2598 * @brief Unmaps buffers associated with this context. 2599 */ 2600 void kgrctxUnmapBuffers_KERNEL 2601 ( 2602 OBJGPU *pGpu, 2603 KernelGraphicsContext *pKernelGraphicsContext, 2604 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast, 2605 KernelChannel *pKernelChannel 2606 ) 2607 { 2608 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 2609 NV_STATUS status; 2610 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo; 2611 KernelGraphics *pKernelGraphics; 2612 Device *pDevice = GPU_RES_GET_DEVICE(pKernelGraphicsContext); 2613 NvHandle hParent = RES_GET_PARENT_HANDLE(pKernelGraphicsContext); 2614 2615 portMemSet(&grRouteInfo, 0, sizeof(grRouteInfo)); 2616 kgrmgrCtrlSetChannelHandle(hParent, &grRouteInfo); 2617 NV_ASSERT_OK_OR_ELSE(status, 2618 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &grRouteInfo, &pKernelGraphics), 2619 return; ); 2620 2621 kgrctxUnmapAssociatedCtxBuffers(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel); 2622 } 2623 2624 /*! 2625 * @brief Unmap main GR ctx buffer 2626 * 2627 * @param[in] pGpu 2628 * @param[in] pKernelGraphicsContext 2629 * @param[in] pKernelGraphics 2630 * @param[in] pKernelChannel 2631 */ 2632 void 2633 kgrctxUnmapMainCtxBuffer_IMPL 2634 ( 2635 OBJGPU *pGpu, 2636 KernelGraphicsContext *pKernelGraphicsContext, 2637 KernelGraphics *pKernelGraphics, 2638 KernelChannel *pKernelChannel 2639 ) 2640 { 2641 MEMORY_DESCRIPTOR *pCtxBufferMemDesc; 2642 NV_STATUS status; 2643 2644 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2645 2646 status = kchangrpGetEngineContextMemDesc(pGpu, pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup, &pCtxBufferMemDesc); 2647 if ((status == NV_OK) && (pCtxBufferMemDesc != NULL)) 2648 { 2649 // TODO remove Channel, ENG_GR dependencies 2650 kchannelUnmapEngineCtxBuf(pGpu, pKernelChannel, ENG_GR(kgraphicsGetInstance(pGpu, pKernelGraphics))); 2651 NV_ASSERT_OR_RETURN_VOID( 2652 kchannelSetEngineContextMemDesc(pGpu, pKernelChannel, ENG_GR(kgraphicsGetInstance(pGpu, pKernelGraphics)), NULL) == NV_OK); 2653 } 2654 } 2655 2656 /*! 2657 * @brief Unmap all global ctx buffers from this context 2658 */ 2659 void 2660 kgrctxUnmapGlobalCtxBuffers_IMPL 2661 ( 2662 OBJGPU *pGpu, 2663 KernelGraphicsContext *pKernelGraphicsContext, 2664 KernelGraphics *pKernelGraphics, 2665 OBJVASPACE *pVAS, 2666 NvU32 gfid 2667 ) 2668 { 2669 GR_GLOBALCTX_BUFFERS *pGlobalCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid); 2670 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2671 NV_STATUS status; 2672 2673 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2674 2675 NV_ASSERT_OK_OR_ELSE(status, 2676 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast), 2677 return;); 2678 2679 // if global ctx buffers were never allocated then bail out early 2680 if (!pKernelGraphicsContextUnicast->localCtxBuffer.bAllocated && 2681 !pGlobalCtxBuffers->bAllocated) 2682 { 2683 return; 2684 } 2685 2686 kgrctxUnmapGlobalCtxBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pVAS, GR_GLOBALCTX_BUFFER_BUNDLE_CB); 2687 kgrctxUnmapGlobalCtxBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pVAS, GR_GLOBALCTX_BUFFER_ATTRIBUTE_CB); 2688 kgrctxUnmapGlobalCtxBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pVAS, GR_GLOBALCTX_BUFFER_PAGEPOOL); 2689 2690 if (kgraphicsIsRtvCbSupported(pGpu, pKernelGraphics)) 2691 { 2692 kgrctxUnmapGlobalCtxBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pVAS, GR_GLOBALCTX_BUFFER_RTV_CB); 2693 } 2694 2695 } 2696 2697 /*! 2698 * @brief Unmap specified global ctx buffer from the given VAS 2699 */ 2700 void 2701 kgrctxUnmapGlobalCtxBuffer_IMPL 2702 ( 2703 OBJGPU *pGpu, 2704 KernelGraphicsContext *pKernelGraphicsContext, 2705 KernelGraphics *pKernelGraphics, 2706 OBJVASPACE *pVAS, 2707 GR_GLOBALCTX_BUFFER buffId 2708 ) 2709 { 2710 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2711 NV_STATUS status; 2712 NvU64 vaddr; 2713 2714 NV_ASSERT_OK_OR_ELSE(status, 2715 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast), 2716 return;); 2717 2718 status = vaListFindVa(&pKernelGraphicsContextUnicast->globalCtxBufferVaList[buffId], pVAS, &vaddr); 2719 if (status == NV_OK) 2720 { 2721 NV_PRINTF(LEVEL_INFO, "Unmapping %s from VA @ 0x%llx\n", 2722 NV_ENUM_TO_STRING(GR_GLOBALCTX_BUFFER, buffId), vaddr); 2723 2724 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pVAS, &pKernelGraphicsContextUnicast->globalCtxBufferVaList[buffId]); 2725 } 2726 else 2727 { 2728 NV_PRINTF(LEVEL_INFO, "Buffer for %s already unmapped\n", 2729 NV_ENUM_TO_STRING(GR_GLOBALCTX_BUFFER, buffId)); 2730 } 2731 } 2732 2733 /*! 2734 * @brief Free main GR ctx buffer 2735 */ 2736 void 2737 kgrctxFreeMainCtxBuffer_IMPL 2738 ( 2739 OBJGPU *pGpu, 2740 KernelGraphicsContext *pKernelGraphicsContext 2741 ) 2742 { 2743 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2744 NV_STATUS status; 2745 2746 NV_ASSERT_OK_OR_ELSE(status, 2747 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast), 2748 return;); 2749 2750 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2751 if (pKernelGraphicsContextUnicast->pMainCtxBuffer != NULL) 2752 { 2753 memdescFree(pKernelGraphicsContextUnicast->pMainCtxBuffer); 2754 memdescDestroy(pKernelGraphicsContextUnicast->pMainCtxBuffer); 2755 pKernelGraphicsContextUnicast->pMainCtxBuffer = NULL; 2756 pKernelGraphicsContextUnicast->bKGrMainCtxBufferInitialized = NV_FALSE; 2757 } 2758 } 2759 2760 /** 2761 * @brief free the memory for the zcull context buffer 2762 */ 2763 void 2764 kgrctxFreeZcullBuffer_IMPL 2765 ( 2766 OBJGPU *pGpu, 2767 KernelGraphicsContext *pKernelGraphicsContext 2768 ) 2769 { 2770 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2771 MEMORY_DESCRIPTOR *pMemDesc; 2772 NV_STATUS status; 2773 2774 NV_ASSERT_OK_OR_ELSE(status, 2775 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast), 2776 return;); 2777 2778 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2779 2780 pMemDesc = pKernelGraphicsContextUnicast->zcullCtxswBuffer.pMemDesc; 2781 if (pMemDesc == NULL) 2782 { 2783 NV_PRINTF(LEVEL_INFO, 2784 "call to free zcull ctx buffer not RM managed, skipped!\n"); 2785 } 2786 2787 // buffer can be shared and refcounted -- released on final free 2788 memdescFree(pMemDesc); 2789 memdescDestroy(pMemDesc); 2790 pKernelGraphicsContextUnicast->zcullCtxswBuffer.pMemDesc = NULL; 2791 } 2792 2793 /** 2794 * @brief free the memory for the preemption context buffers 2795 */ 2796 void 2797 kgrctxFreeCtxPreemptionBuffers_IMPL 2798 ( 2799 OBJGPU *pGpu, 2800 KernelGraphicsContext *pKernelGraphicsContext 2801 ) 2802 { 2803 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2804 NV_STATUS status; 2805 2806 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2807 2808 NV_ASSERT_OK_OR_ELSE(status, 2809 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast), 2810 return;); 2811 2812 memdescFree(pKernelGraphicsContextUnicast->preemptCtxswBuffer.pMemDesc); 2813 memdescDestroy(pKernelGraphicsContextUnicast->preemptCtxswBuffer.pMemDesc); 2814 pKernelGraphicsContextUnicast->preemptCtxswBuffer.pMemDesc = NULL; 2815 2816 memdescFree(pKernelGraphicsContextUnicast->spillCtxswBuffer.pMemDesc); 2817 memdescDestroy(pKernelGraphicsContextUnicast->spillCtxswBuffer.pMemDesc); 2818 pKernelGraphicsContextUnicast->spillCtxswBuffer.pMemDesc = NULL; 2819 2820 memdescFree(pKernelGraphicsContextUnicast->betaCBCtxswBuffer.pMemDesc); 2821 memdescDestroy(pKernelGraphicsContextUnicast->betaCBCtxswBuffer.pMemDesc); 2822 pKernelGraphicsContextUnicast->betaCBCtxswBuffer.pMemDesc = NULL; 2823 2824 memdescFree(pKernelGraphicsContextUnicast->pagepoolCtxswBuffer.pMemDesc); 2825 memdescDestroy(pKernelGraphicsContextUnicast->pagepoolCtxswBuffer.pMemDesc); 2826 pKernelGraphicsContextUnicast->pagepoolCtxswBuffer.pMemDesc = NULL; 2827 2828 memdescFree(pKernelGraphicsContextUnicast->rtvCbCtxswBuffer.pMemDesc); 2829 memdescDestroy(pKernelGraphicsContextUnicast->rtvCbCtxswBuffer.pMemDesc); 2830 pKernelGraphicsContextUnicast->rtvCbCtxswBuffer.pMemDesc = NULL; 2831 } 2832 2833 /*! 2834 * @brief Free patch GR ctx buffer 2835 */ 2836 void 2837 kgrctxFreePatchBuffer_IMPL 2838 ( 2839 OBJGPU *pGpu, 2840 KernelGraphicsContext *pKernelGraphicsContext 2841 ) 2842 { 2843 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2844 NV_STATUS status; 2845 2846 NV_ASSERT_OK_OR_ELSE(status, 2847 kgrctxGetUnicast(pGpu, 2848 pKernelGraphicsContext, 2849 &pKernelGraphicsContextUnicast), 2850 return;); 2851 2852 if (pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc == NULL) 2853 { 2854 NV_PRINTF(LEVEL_INFO, 2855 "Attempt to free null ctx patch buffer pointer, skipped!\n"); 2856 return; 2857 } 2858 2859 memdescFree(pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc); 2860 memdescDestroy(pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc); 2861 2862 pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc = NULL; 2863 pKernelGraphicsContextUnicast->bKGrPatchCtxBufferInitialized = NV_FALSE; 2864 } 2865 2866 /** 2867 * @brief Free the memory for the pm context buffer associated with a channel 2868 */ 2869 void 2870 kgrctxFreePmBuffer_IMPL 2871 ( 2872 OBJGPU *pGpu, 2873 KernelGraphicsContext *pKernelGraphicsContext 2874 ) 2875 { 2876 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2877 MEMORY_DESCRIPTOR *pMemDesc; 2878 NV_STATUS status; 2879 2880 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2881 2882 NV_ASSERT_OK_OR_ELSE(status, 2883 kgrctxGetUnicast(pGpu, 2884 pKernelGraphicsContext, 2885 &pKernelGraphicsContextUnicast), 2886 return;); 2887 2888 pMemDesc = pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc; 2889 if (pMemDesc != NULL) 2890 { 2891 // 2892 // This func assumes that the buffer was not allocated per subdevice, 2893 // and will leak any mappings performed by the secondaries. 2894 // 2895 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc)); 2896 2897 memdescFree(pMemDesc); 2898 memdescDestroy(pMemDesc); 2899 } 2900 else 2901 { 2902 NV_PRINTF(LEVEL_INFO, 2903 "Attempt to free null pm ctx buffer pointer??\n"); 2904 } 2905 2906 pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc = NULL; 2907 pKernelGraphicsContextUnicast->bKGrPmCtxBufferInitialized = NV_FALSE; 2908 } 2909 2910 /*! 2911 * @brief Free "local" global context buffers 2912 */ 2913 void kgrctxFreeLocalGlobalCtxBuffers_IMPL 2914 ( 2915 OBJGPU *pGpu, 2916 KernelGraphicsContext *pKernelGraphicsContext 2917 ) 2918 { 2919 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2920 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 2921 MEMORY_DESCRIPTOR *pMemDesc; 2922 GR_GLOBALCTX_BUFFERS *pCtxBuffers; 2923 GR_GLOBALCTX_BUFFER buff; 2924 NV_STATUS status; 2925 2926 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2927 2928 NV_ASSERT_OK_OR_ELSE(status, 2929 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast), 2930 return;); 2931 2932 pCtxBuffers = &pKernelGraphicsContextUnicast->localCtxBuffer; 2933 2934 // no ctx buffers allocated, so get out early 2935 if (!pCtxBuffers->bAllocated) 2936 return; 2937 2938 FOR_EACH_IN_ENUM(GR_GLOBALCTX_BUFFER, buff) 2939 { 2940 pMemDesc = pCtxBuffers->memDesc[buff]; 2941 memdescFree(pMemDesc); 2942 memdescDestroy(pMemDesc); 2943 pCtxBuffers->memDesc[buff] = NULL; 2944 } 2945 FOR_EACH_IN_ENUM_END; 2946 2947 pCtxBuffers->bAllocated = NV_FALSE; 2948 2949 // make sure all L2 cache lines using CB buffers are clear after we free them 2950 NV_ASSERT_OK(kmemsysCacheOp_HAL(pGpu, pKernelMemorySystem, NULL, FB_CACHE_VIDEO_MEMORY, FB_CACHE_EVICT)); 2951 } 2952 2953 /** 2954 * @brief Free all associated ctx buffers (main, patch, PM and private global buffers for cases like VPR). 2955 * 2956 * This function is called on every channel free and thefore can't assume any 2957 * graphics objects actually exist on the given channel. 2958 */ 2959 void kgrctxFreeAssociatedCtxBuffers_IMPL 2960 ( 2961 OBJGPU *pGpu, 2962 KernelGraphicsContext *pKernelGraphicsContext 2963 ) 2964 { 2965 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 2966 NV_STATUS status; 2967 2968 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2969 2970 NV_ASSERT_OK_OR_ELSE(status, 2971 kgrctxGetUnicast(pGpu, 2972 pKernelGraphicsContext, 2973 &pKernelGraphicsContextUnicast), 2974 return;); 2975 2976 // 2977 // When sharing contexts across channels we need to defer this until all 2978 // objects have been freed. 2979 // 2980 NV_CHECK_OR_RETURN_VOID(LEVEL_SILENT, pKernelGraphicsContextUnicast->channelObjects == 0); 2981 2982 // if we have a context patch buffer, this will free it up 2983 kgrctxFreePatchBuffer(pGpu, pKernelGraphicsContext); 2984 2985 // if we have a PM ctxsw buffer, this will free it up 2986 kgrctxFreePmBuffer(pGpu, pKernelGraphicsContext); 2987 2988 // if we have a zcull buffer, this will free it up 2989 kgrctxFreeZcullBuffer(pGpu, pKernelGraphicsContext); 2990 2991 // If we have preemption buffers, this will free them up 2992 kgrctxFreeCtxPreemptionBuffers(pGpu, pKernelGraphicsContext); 2993 2994 // Release all common buffers used as part of the gr context. 2995 kgrctxFreeLocalGlobalCtxBuffers(pGpu, pKernelGraphicsContext); 2996 2997 kgrctxFreeMainCtxBuffer(pGpu, pKernelGraphicsContext); 2998 } 2999 3000 /*! 3001 * This function unmaps various GR buffers. 3002 */ 3003 NV_STATUS 3004 kgrctxUnmapCtxBuffers_IMPL 3005 ( 3006 OBJGPU *pGpu, 3007 KernelGraphicsContext *pKernelGraphicsContext, 3008 KernelGraphicsObject *pKernelGraphicsObject, 3009 KernelGraphics *pKernelGraphics, 3010 NvBool bDestructor 3011 ) 3012 { 3013 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 3014 ChannelDescendant *pChannelDescendant = staticCast(pKernelGraphicsObject, ChannelDescendant); 3015 KernelChannel *pKernelChannel = pChannelDescendant->pKernelChannel; 3016 NvU32 classNum = pChannelDescendant->resourceDesc.externalClassId; 3017 NvBool bRelease3d = NV_FALSE; 3018 NvU32 objType; 3019 NvU32 gfid; 3020 NvU64 refCount; 3021 GR_GLOBALCTX_BUFFER registerMapBufferId = kgrctxGetRegisterAccessMapId_HAL(pGpu, pKernelGraphicsContext, pChannelDescendant->pKernelChannel); 3022 3023 NV_PRINTF(LEVEL_INFO, "gpu:%d isBC=%d\n", pGpu->gpuInstance, 3024 gpumgrGetBcEnabledStatus(pGpu)); 3025 3026 gfid = kchannelGetGfid(pKernelChannel); 3027 3028 NV_ASSERT_OK_OR_RETURN( 3029 kgrctxGetUnicast(pGpu, 3030 kgrobjGetKernelGraphicsContext(pGpu, pKernelGraphicsObject), 3031 &pKernelGraphicsContextUnicast)); 3032 3033 kgrmgrGetGrObjectType(classNum, &objType); 3034 3035 if (!bDestructor) 3036 { 3037 // 3038 // If we are cleaning up from the constructor, then we know the main ctx 3039 // buffer is not being used yet, and no other context buffers need to be 3040 // cleaned up, so we can return early here. If we are coming from the 3041 // destructor, the context buffers are not freed until channel 3042 // destruction. 3043 // TODO move this up one stack frame 3044 // 3045 kgrctxUnmapMainCtxBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pChannelDescendant->pKernelChannel); 3046 kgrctxFreeMainCtxBuffer(pGpu, pKernelGraphicsContext); 3047 return NV_OK; 3048 } 3049 3050 if ((pKernelGraphicsContextUnicast->channelObjects != 0) && 3051 (pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc != NULL)) 3052 { 3053 // 3054 // Delay freeing the patch buffer until the last channel free. 3055 // these buffers are accessed even after the last GR object is freed. 3056 // 3057 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pKernelChannel->pVAS, &pKernelGraphicsContextUnicast->ctxPatchBuffer.vAddrList); 3058 } 3059 3060 // Defer releasing mapping if this would cause the buffer to be unmapped 3061 if ((pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc != NULL) && 3062 (vaListGetRefCount(&pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList, pKernelChannel->pVAS, &refCount) == NV_OK) && 3063 (refCount > 1)) 3064 { 3065 kgrctxUnmapCtxPmBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel->pVAS); 3066 } 3067 3068 // Defer releasing mapping if this would cause the buffer to be unmapped 3069 if (kgraphicsDoesUcodeSupportPrivAccessMap(pGpu, pKernelGraphics) && 3070 (vaListGetRefCount(&pKernelGraphicsContextUnicast->globalCtxBufferVaList[registerMapBufferId], pKernelChannel->pVAS, &refCount) == NV_OK) && 3071 (refCount > 1)) 3072 { 3073 kgrctxUnmapGlobalCtxBuffer(pGpu, 3074 pKernelGraphicsContext, 3075 pKernelGraphics, 3076 pKernelChannel->pVAS, 3077 registerMapBufferId); 3078 } 3079 3080 // Defer releasing mapping if this would cause the buffer to be unmapped 3081 if ((kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid)->memDesc[GR_GLOBALCTX_BUFFER_FECS_EVENT] != NULL) && 3082 (vaListGetRefCount(&pKernelGraphicsContextUnicast->globalCtxBufferVaList[GR_GLOBALCTX_BUFFER_FECS_EVENT], pKernelChannel->pVAS, &refCount) == NV_OK) && 3083 (refCount > 1)) 3084 { 3085 kgrctxUnmapGlobalCtxBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel->pVAS, GR_GLOBALCTX_BUFFER_FECS_EVENT); 3086 } 3087 3088 // 3089 // Condition for releasing 3d context buffer mappings: 3090 // For non-TSG & legacy TSG mode, always unmap as long as its not the last 3091 // object in TSG 3092 // If it is the last object, then unmap during channel free as these buffers 3093 // are accessed even after the last 3D object is freed. 3094 // For subcontext TSG case, only unmap on 2D/3D object free as long its not 3095 // the last 2D/3D object 3096 // For compute object with subcontext we never map so we don't need to call 3097 // unmap. 3098 // If we free on last object then buffer gets actually unmapped (as opposed 3099 // to decrement in refcount) and we start hitting page faults 3100 // 3101 if (pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->bLegacyMode) 3102 { 3103 bRelease3d = (pKernelGraphicsContextUnicast->channelObjects > 0); 3104 } 3105 else 3106 { 3107 NvBool bIs3dBuffer = ((objType == GR_OBJECT_TYPE_2D) || (objType == GR_OBJECT_TYPE_3D)); 3108 bRelease3d = (bIs3dBuffer && ((pKernelGraphicsContextUnicast->objectCounts[GR_OBJECT_TYPE_2D] > 0) || 3109 (pKernelGraphicsContextUnicast->objectCounts[GR_OBJECT_TYPE_3D] > 0))); 3110 } 3111 3112 if (bRelease3d) 3113 { 3114 kgrctxUnmapGlobalCtxBuffers(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel->pVAS, gfid); 3115 } 3116 3117 return NV_OK; 3118 } 3119 3120 /*! 3121 * Function to increment the GR channel object count 3122 */ 3123 void 3124 kgrctxIncObjectCount_IMPL 3125 ( 3126 OBJGPU *pGpu, 3127 KernelGraphicsContext *pKernelGraphicsContext, 3128 NvU32 classNum 3129 ) 3130 { 3131 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 3132 NvU32 objType; 3133 NV_STATUS status; 3134 3135 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 3136 3137 kgrmgrGetGrObjectType(classNum, &objType); 3138 3139 NV_ASSERT_OK_OR_ELSE(status, 3140 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast), 3141 return;); 3142 3143 switch (objType) 3144 { 3145 case GR_OBJECT_TYPE_COMPUTE: 3146 pKernelGraphicsContextUnicast->objectCounts[GR_OBJECT_TYPE_COMPUTE]++; 3147 gpuChangeComputeModeRefCount(pGpu, NV_GPU_COMPUTE_REFCOUNT_COMMAND_INCREMENT); 3148 break; 3149 3150 case GR_OBJECT_TYPE_3D: 3151 pKernelGraphicsContextUnicast->objectCounts[GR_OBJECT_TYPE_3D]++; 3152 break; 3153 3154 case GR_OBJECT_TYPE_2D: 3155 pKernelGraphicsContextUnicast->objectCounts[GR_OBJECT_TYPE_2D]++; 3156 break; 3157 3158 case GR_OBJECT_TYPE_MEM: 3159 pKernelGraphicsContextUnicast->objectCounts[GR_OBJECT_TYPE_MEM]++; 3160 break; 3161 3162 default: 3163 NV_PRINTF(LEVEL_ERROR, "Unrecognized graphics class 0x%x\n", 3164 classNum); 3165 DBG_BREAKPOINT(); 3166 return; 3167 } 3168 3169 pKernelGraphicsContextUnicast->channelObjects++; 3170 NV_PRINTF(LEVEL_INFO, 3171 "Class 0x%x allocated. %d objects allocated\n", 3172 classNum, 3173 pKernelGraphicsContextUnicast->channelObjects); 3174 } 3175 3176 /*! 3177 * Function to decrement the GR channel object count of the given class 3178 */ 3179 void 3180 kgrctxDecObjectCount_IMPL 3181 ( 3182 OBJGPU *pGpu, 3183 KernelGraphicsContext *pKernelGraphicsContext, 3184 NvU32 classNum 3185 ) 3186 { 3187 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 3188 NvU32 objType; 3189 NvU32 countIdx; 3190 NV_STATUS status; 3191 3192 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 3193 3194 kgrmgrGetGrObjectType(classNum, &objType); 3195 3196 NV_ASSERT_OK_OR_ELSE(status, 3197 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast), 3198 return;); 3199 3200 NV_ASSERT_OR_ELSE(pKernelGraphicsContextUnicast->channelObjects != 0, 3201 NV_PRINTF(LEVEL_ERROR, "No active GR objects to free for Class 0x%x\n", classNum)); 3202 3203 // 3204 // Handle 2D and Compute class reference counting 3205 // 3206 switch (objType) 3207 { 3208 case GR_OBJECT_TYPE_COMPUTE: 3209 gpuChangeComputeModeRefCount(pGpu, NV_GPU_COMPUTE_REFCOUNT_COMMAND_DECREMENT); 3210 countIdx = GR_OBJECT_TYPE_COMPUTE; 3211 break; 3212 3213 case GR_OBJECT_TYPE_3D: 3214 countIdx = GR_OBJECT_TYPE_3D; 3215 break; 3216 3217 case GR_OBJECT_TYPE_2D: 3218 countIdx = GR_OBJECT_TYPE_2D; 3219 break; 3220 3221 case GR_OBJECT_TYPE_MEM: 3222 countIdx = GR_OBJECT_TYPE_MEM; 3223 break; 3224 3225 default: 3226 NV_PRINTF(LEVEL_ERROR, "Unrecognized graphics class 0x%x\n", 3227 classNum); 3228 DBG_BREAKPOINT(); 3229 return; 3230 } 3231 3232 NV_ASSERT_OR_RETURN_VOID(pKernelGraphicsContextUnicast->objectCounts[countIdx] > 0); 3233 pKernelGraphicsContextUnicast->objectCounts[countIdx]--; 3234 pKernelGraphicsContextUnicast->channelObjects--; 3235 } 3236 3237 /*! 3238 * @brief Return the register access buffer used for a given Channel's permission 3239 * 3240 * The base register access map comes from the architecture team. The user version 3241 * of the buffer removes access to PM regisers. 3242 * 3243 * The policy currently depends on administrator access on the system, except on 3244 * one VGPU configuration. 3245 */ 3246 GR_GLOBALCTX_BUFFER 3247 kgrctxGetRegisterAccessMapId_PF 3248 ( 3249 OBJGPU *pGpu, 3250 KernelGraphicsContext *pKernelGraphicsContext, 3251 KernelChannel *pKernelChannel 3252 ) 3253 { 3254 RmClient *pRmClient = dynamicCast(RES_GET_CLIENT(pKernelChannel), RmClient); 3255 RS_PRIV_LEVEL privLevel = rmclientGetCachedPrivilege(pRmClient); 3256 3257 // Using cached privilege because this function is called at a raised IRQL. 3258 if ((privLevel >= RS_PRIV_LEVEL_USER_ROOT) 3259 && !hypervisorIsVgxHyper() && IS_GFID_PF(kchannelGetGfid(pKernelChannel))) 3260 { 3261 return GR_GLOBALCTX_BUFFER_UNRESTRICTED_PRIV_ACCESS_MAP; 3262 } 3263 3264 return GR_GLOBALCTX_BUFFER_PRIV_ACCESS_MAP; 3265 } 3266 3267 NV_STATUS 3268 kgrctxCtrlGetTpcPartitionMode_IMPL 3269 ( 3270 KernelGraphicsContext *pKernelGraphicsContext, 3271 NV0090_CTRL_TPC_PARTITION_MODE_PARAMS *pParams 3272 ) 3273 { 3274 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelGraphicsContext); 3275 3276 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner()); 3277 3278 if (IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu)) 3279 { 3280 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 3281 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams; 3282 NV_STATUS status = NV_OK; 3283 3284 NV_RM_RPC_CONTROL(pGpu, 3285 pRmCtrlParams->hClient, 3286 pRmCtrlParams->hObject, 3287 pRmCtrlParams->cmd, 3288 pRmCtrlParams->pParams, 3289 pRmCtrlParams->paramsSize, 3290 status); 3291 3292 return status; 3293 } 3294 3295 return gpuresInternalControlForward_IMPL(staticCast(pKernelGraphicsContext, GpuResource), 3296 NV0090_CTRL_CMD_INTERNAL_GET_TPC_PARTITION_MODE, 3297 pParams, 3298 sizeof(*pParams)); 3299 } 3300 3301 NV_STATUS 3302 kgrctxCtrlSetTpcPartitionMode_IMPL 3303 ( 3304 KernelGraphicsContext *pKernelGraphicsContext, 3305 NV0090_CTRL_TPC_PARTITION_MODE_PARAMS *pParams 3306 ) 3307 { 3308 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelGraphicsContext); 3309 3310 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner()); 3311 3312 if (IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu)) 3313 { 3314 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 3315 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams; 3316 NV_STATUS status = NV_OK; 3317 3318 NV_RM_RPC_CONTROL(pGpu, 3319 pRmCtrlParams->hClient, 3320 pRmCtrlParams->hObject, 3321 pRmCtrlParams->cmd, 3322 pRmCtrlParams->pParams, 3323 pRmCtrlParams->paramsSize, 3324 status); 3325 3326 return status; 3327 } 3328 3329 return gpuresInternalControlForward_IMPL(staticCast(pKernelGraphicsContext, GpuResource), 3330 NV0090_CTRL_CMD_INTERNAL_SET_TPC_PARTITION_MODE, 3331 pParams, 3332 sizeof(*pParams)); 3333 } 3334 3335 NV_STATUS 3336 kgrctxCtrlGetMMUDebugMode_IMPL 3337 ( 3338 KernelGraphicsContext *pKernelGraphicsContext, 3339 NV0090_CTRL_GET_MMU_DEBUG_MODE_PARAMS *pParams 3340 ) 3341 { 3342 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelGraphicsContext); 3343 3344 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner()); 3345 3346 if (IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu)) 3347 { 3348 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 3349 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams; 3350 NV_STATUS status = NV_OK; 3351 3352 NV_RM_RPC_CONTROL(pGpu, 3353 pRmCtrlParams->hClient, 3354 pRmCtrlParams->hObject, 3355 pRmCtrlParams->cmd, 3356 pRmCtrlParams->pParams, 3357 pRmCtrlParams->paramsSize, 3358 status); 3359 3360 return status; 3361 } 3362 3363 return gpuresInternalControlForward_IMPL(staticCast(pKernelGraphicsContext, GpuResource), 3364 NV0090_CTRL_CMD_INTERNAL_GET_MMU_DEBUG_MODE, 3365 pParams, 3366 sizeof(*pParams)); 3367 } 3368 3369 NV_STATUS 3370 kgrctxCtrlProgramVidmemPromote_IMPL 3371 ( 3372 KernelGraphicsContext *pKernelGraphicsContext, 3373 NV0090_CTRL_PROGRAM_VIDMEM_PROMOTE_PARAMS *pParams 3374 ) 3375 { 3376 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelGraphicsContext); 3377 3378 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner()); 3379 3380 if (IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu)) 3381 { 3382 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 3383 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams; 3384 NV_STATUS status = NV_OK; 3385 3386 NV_RM_RPC_CONTROL(pGpu, 3387 pRmCtrlParams->hClient, 3388 pRmCtrlParams->hObject, 3389 pRmCtrlParams->cmd, 3390 pRmCtrlParams->pParams, 3391 pRmCtrlParams->paramsSize, 3392 status); 3393 3394 return status; 3395 } 3396 3397 return gpuresInternalControlForward_IMPL(staticCast(pKernelGraphicsContext, GpuResource), 3398 NV0090_CTRL_CMD_INTERNAL_PROGRAM_VIDMEM_PROMOTE, 3399 pParams, 3400 sizeof(*pParams)); 3401 } 3402 3403 /*! 3404 * @brief Construct shared kernel graphics context. (Does nothing) 3405 */ 3406 NV_STATUS 3407 shrkgrctxConstruct_IMPL(KernelGraphicsContextShared *pKernelGraphicsContextShared) 3408 { 3409 return NV_OK; 3410 } 3411 3412 /*! 3413 * @brief Construct shared kernel graphics context 3414 * 3415 * @param[in] pGpu 3416 * @param[in] pKernelGraphicsContextShared 3417 * @param[in] pKernelGraphicsContext 3418 */ 3419 NV_STATUS 3420 shrkgrctxInit_IMPL 3421 ( 3422 OBJGPU *pGpu, 3423 KernelGraphicsContextShared *pKernelGraphicsContextShared, 3424 KernelGraphicsContext *pKernelGraphicsContext 3425 ) 3426 { 3427 NV_STATUS status = NV_OK; 3428 3429 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY) 3430 { 3431 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 3432 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3433 KernelGraphics *pKernelGraphics; 3434 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo; 3435 Device *pDevice = GPU_RES_GET_DEVICE(pKernelGraphicsContext); 3436 NvHandle hParent = RES_GET_PARENT_HANDLE(pKernelGraphicsContext); 3437 3438 portMemSet(&grRouteInfo, 0, sizeof(grRouteInfo)); 3439 kgrmgrCtrlSetChannelHandle(hParent, &grRouteInfo); 3440 3441 NV_CHECK_OK_OR_CAPTURE_FIRST_ERROR(status, LEVEL_ERROR, 3442 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &grRouteInfo, &pKernelGraphics)); 3443 3444 if (status != NV_OK) 3445 SLI_LOOP_BREAK; 3446 3447 if (kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast) == NV_OK) 3448 { 3449 NV_CHECK_OK_OR_CAPTURE_FIRST_ERROR(status, LEVEL_ERROR, 3450 shrkgrctxConstructUnicast(pGpu, pKernelGraphicsContextShared, pKernelGraphicsContext, pKernelGraphics, pKernelGraphicsContextUnicast)); 3451 3452 if (status != NV_OK) 3453 SLI_LOOP_BREAK; 3454 } 3455 } 3456 SLI_LOOP_END; 3457 3458 listInit(&pKernelGraphicsContextShared->activeDebuggers, portMemAllocatorGetGlobalNonPaged()); 3459 3460 if (status != NV_OK) 3461 shrkgrctxTeardown_IMPL(pGpu, pKernelGraphicsContextShared, pKernelGraphicsContext); 3462 3463 return status; 3464 } 3465 3466 /*! 3467 * @brief Construct unicast state for this context 3468 * 3469 * @param[in] pGpu 3470 * @param[in] pKernelGraphicsContextShared 3471 * @param[in] pKernelGraphicsContext 3472 * @param[in] pKernelGraphics 3473 * @param[in] pKernelGraphicsContextUnicast 3474 */ 3475 NV_STATUS 3476 shrkgrctxConstructUnicast_IMPL 3477 ( 3478 OBJGPU *pGpu, 3479 KernelGraphicsContextShared *pKernelGraphicsContextShared, 3480 KernelGraphicsContext *pKernelGraphicsContext, 3481 KernelGraphics *pKernelGraphics, 3482 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast 3483 ) 3484 { 3485 NvU32 i; 3486 3487 for (i = 0; i < GR_GLOBALCTX_BUFFER_COUNT; i++) 3488 NV_ASSERT_OK_OR_RETURN(vaListInit(&pKernelGraphicsContextUnicast->globalCtxBufferVaList[i])); 3489 3490 NV_ASSERT_OK_OR_RETURN(vaListInit(&pKernelGraphicsContextUnicast->ctxPatchBuffer.vAddrList)); 3491 NV_ASSERT_OK_OR_RETURN(vaListInit(&pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList)); 3492 NV_ASSERT_OK_OR_RETURN(vaListInit(&pKernelGraphicsContextUnicast->zcullCtxswBuffer.vAddrList)); 3493 3494 NV_ASSERT_OK_OR_RETURN(vaListInit(&pKernelGraphicsContextUnicast->preemptCtxswBuffer.vAddrList)); 3495 NV_ASSERT_OK_OR_RETURN(vaListInit(&pKernelGraphicsContextUnicast->spillCtxswBuffer.vAddrList)); 3496 NV_ASSERT_OK_OR_RETURN(vaListInit(&pKernelGraphicsContextUnicast->betaCBCtxswBuffer.vAddrList)); 3497 NV_ASSERT_OK_OR_RETURN(vaListInit(&pKernelGraphicsContextUnicast->pagepoolCtxswBuffer.vAddrList)); 3498 NV_ASSERT_OK_OR_RETURN(vaListInit(&pKernelGraphicsContextUnicast->rtvCbCtxswBuffer.vAddrList)); 3499 3500 pKernelGraphicsContextUnicast->bSupportsPerSubctxHeader = 3501 kgraphicsIsPerSubcontextContextHeaderSupported(pGpu, pKernelGraphics); 3502 3503 return NV_OK; 3504 } 3505 3506 /*! 3507 * @brief Destruct shared kernel graphics context. 3508 */ 3509 void 3510 shrkgrctxDestruct_IMPL(KernelGraphicsContextShared *pKernelGraphicsContextShared) 3511 { 3512 NV_ASSERT(listCount(&pKernelGraphicsContextShared->activeDebuggers) == 0); 3513 listDestroy(&pKernelGraphicsContextShared->activeDebuggers); 3514 } 3515 3516 /*! 3517 * @brief Destroy the shared context state 3518 */ 3519 void 3520 shrkgrctxTeardown_IMPL 3521 ( 3522 OBJGPU *pGpu, 3523 KernelGraphicsContextShared *pKernelGraphicsContextShared, 3524 KernelGraphicsContext *pKernelGraphicsContext 3525 ) 3526 { 3527 3528 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY) 3529 { 3530 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 3531 3532 if (kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast) == NV_OK) 3533 shrkgrctxDestructUnicast(pGpu, pKernelGraphicsContextShared, pKernelGraphicsContext, pKernelGraphicsContextUnicast); 3534 } 3535 SLI_LOOP_END; 3536 3537 portMemSet(&pKernelGraphicsContext->pShared->kernelGraphicsContextUnicast, 0x0, sizeof(pKernelGraphicsContext->pShared->kernelGraphicsContextUnicast)); 3538 } 3539 3540 /*! 3541 * Destroy the unicast context state 3542 */ 3543 void 3544 shrkgrctxDestructUnicast_IMPL 3545 ( 3546 OBJGPU *pGpu, 3547 KernelGraphicsContextShared *pKernelGraphicsContextShared, 3548 KernelGraphicsContext *pKernelGraphicsContext, 3549 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast 3550 ) 3551 { 3552 GR_GLOBALCTX_BUFFER buff; 3553 3554 if (kgrctxShouldCleanup_HAL(pGpu, pKernelGraphicsContext)) 3555 kgrctxFreeAssociatedCtxBuffers(pGpu, pKernelGraphicsContext); 3556 3557 FOR_EACH_IN_ENUM(GR_GLOBALCTX_BUFFER, buff) 3558 { 3559 vaListDestroy(&pKernelGraphicsContextUnicast->globalCtxBufferVaList[buff]); 3560 } 3561 FOR_EACH_IN_ENUM_END; 3562 vaListDestroy(&pKernelGraphicsContextUnicast->ctxPatchBuffer.vAddrList); 3563 vaListDestroy(&pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList); 3564 vaListDestroy(&pKernelGraphicsContextUnicast->zcullCtxswBuffer.vAddrList); 3565 3566 vaListDestroy(&pKernelGraphicsContextUnicast->preemptCtxswBuffer.vAddrList); 3567 vaListDestroy(&pKernelGraphicsContextUnicast->spillCtxswBuffer.vAddrList); 3568 vaListDestroy(&pKernelGraphicsContextUnicast->betaCBCtxswBuffer.vAddrList); 3569 vaListDestroy(&pKernelGraphicsContextUnicast->pagepoolCtxswBuffer.vAddrList); 3570 vaListDestroy(&pKernelGraphicsContextUnicast->rtvCbCtxswBuffer.vAddrList); 3571 } 3572 3573 /*! 3574 * @brief Perform cleanup tasks run on channel removal from context 3575 */ 3576 void shrkgrctxDetach_IMPL 3577 ( 3578 OBJGPU *pGpu, 3579 KernelGraphicsContextShared *pKernelGraphicsContextShared, 3580 KernelGraphicsContext *pKernelGraphicsContext, 3581 KernelChannel *pKernelChannel 3582 ) 3583 { 3584 RM_ENGINE_TYPE rmEngineType = kchannelGetEngineType(pKernelChannel); 3585 3586 // pre-Ampere chips can have NULL engine types. Find the engine type based on runlistId if set 3587 if (!RM_ENGINE_TYPE_IS_VALID(rmEngineType)) 3588 { 3589 if (kchannelIsRunlistSet(pGpu, pKernelChannel)) 3590 { 3591 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 3592 3593 NV_ASSERT_OK( 3594 kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo, 3595 ENGINE_INFO_TYPE_RUNLIST, kchannelGetRunlistId(pKernelChannel), 3596 ENGINE_INFO_TYPE_RM_ENGINE_TYPE, (NvU32 *)&rmEngineType)); 3597 } 3598 else 3599 { 3600 NV_PRINTF(LEVEL_INFO, "Channel destroyed but never bound, scheduled, or had a descendant object created\n"); 3601 } 3602 } 3603 3604 if (RM_ENGINE_TYPE_IS_GR(rmEngineType)) 3605 { 3606 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannel); 3607 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3608 Device *pDevice = GPU_RES_GET_DEVICE(pKernelChannel); 3609 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo = {0}; 3610 KernelGraphics *pKernelGraphics; 3611 NV_STATUS status; 3612 3613 kgrmgrCtrlSetChannelHandle(RES_GET_HANDLE(pKernelChannel), &grRouteInfo); 3614 NV_ASSERT_OK_OR_ELSE(status, 3615 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &grRouteInfo, &pKernelGraphics), 3616 return;); 3617 3618 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 3619 { 3620 // Last chance to process FECS ctxsw log before channel goes away 3621 nvEventBufferFecsCallback(pGpu, pKernelGraphics); 3622 } 3623 SLI_LOOP_END 3624 } 3625 3626 // 3627 // If PDB_PROP_GPU_CLIENT_RM_ALLOCATED_CTX_BUFFER is set, then kernel RM 3628 // is responsible for unmapping the context buffers, otherwise it is left to 3629 // physical RM. 3630 // 3631 if (!kgrctxShouldCleanup(pGpu, pKernelGraphicsContext)) 3632 return; 3633 3634 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY) 3635 { 3636 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 3637 3638 if (kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast) == NV_OK) 3639 kgrctxUnmapBuffers_HAL(pGpu, pKernelGraphicsContext, pKernelGraphicsContextUnicast, pKernelChannel); 3640 } 3641 SLI_LOOP_END; 3642 } 3643 3644