1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include "kernel/gpu/gr/kernel_graphics_manager.h" 25 #include "kernel/gpu/gr/kernel_graphics.h" 26 27 #include "kernel/gpu/fifo/kernel_channel_group_api.h" 28 #include "kernel/gpu/fifo/kernel_channel_group.h" 29 30 #include "kernel/rmapi/client.h" 31 #include "kernel/rmapi/client_resource.h" 32 33 // COMPUTE 34 #include "class/clb0c0.h" 35 #include "class/clb1c0.h" 36 #include "class/clc0c0.h" 37 #include "class/clc1c0.h" 38 #include "class/clc3c0.h" 39 #include "class/clc5c0.h" 40 #include "class/clc6c0.h" 41 #include "class/clc7c0.h" 42 #include "class/clc9c0.h" 43 #include "class/clcbc0.h" 44 // GFX 45 #include "class/clb097.h" 46 #include "class/clb197.h" 47 #include "class/clc097.h" 48 #include "class/clc197.h" 49 #include "class/clc397.h" 50 #include "class/clc597.h" 51 #include "class/clc697.h" 52 #include "class/clc797.h" 53 #include "class/clc997.h" 54 #include "class/clcb97.h" 55 // TWOD 56 #include "class/cl902d.h" 57 58 // MEM2MEM 59 #include "class/cla140.h" 60 61 static NvBool 62 _kgrmgrGPUInstanceHasComputeInstances 63 ( 64 OBJGPU *pGpu, 65 KernelGraphicsManager *pKernelGraphicsManager, 66 KERNEL_MIG_GPU_INSTANCE *pKernelMIGGpuInstance 67 ) 68 { 69 NV_ASSERT_OR_RETURN(pKernelMIGGpuInstance != NULL, NV_FALSE); 70 NvU32 computeInstanceIdx; 71 72 for (computeInstanceIdx = 0; 73 computeInstanceIdx < NV_ARRAY_ELEMENTS(pKernelMIGGpuInstance->MIGComputeInstance); 74 ++computeInstanceIdx) 75 { 76 if (pKernelMIGGpuInstance->MIGComputeInstance[computeInstanceIdx].bValid) 77 return NV_TRUE; 78 } 79 80 return NV_FALSE; 81 } 82 83 /*! 84 * @brief Get GR object type from the class number 85 * 86 * @param[IN] classNum external class number 87 * @param[OUT] pObjectType GR class subtype 88 */ 89 void 90 kgrmgrGetGrObjectType_IMPL 91 ( 92 NvU32 classNum, 93 NvU32 *pObjectType 94 ) 95 { 96 switch (classNum) 97 { 98 case MAXWELL_COMPUTE_A: 99 case MAXWELL_COMPUTE_B: 100 case PASCAL_COMPUTE_A: 101 case PASCAL_COMPUTE_B: 102 case VOLTA_COMPUTE_A: 103 case TURING_COMPUTE_A: 104 case AMPERE_COMPUTE_A: 105 case AMPERE_COMPUTE_B: 106 case ADA_COMPUTE_A: 107 case HOPPER_COMPUTE_A: 108 *pObjectType = GR_OBJECT_TYPE_COMPUTE; 109 break; 110 case MAXWELL_A: 111 case MAXWELL_B: 112 case PASCAL_A: 113 case PASCAL_B: 114 case VOLTA_A: 115 case TURING_A: 116 case AMPERE_A: 117 case AMPERE_B: 118 case ADA_A: 119 case HOPPER_A: 120 *pObjectType = GR_OBJECT_TYPE_3D; 121 break; 122 case FERMI_TWOD_A: 123 *pObjectType = GR_OBJECT_TYPE_2D; 124 break; 125 case KEPLER_INLINE_TO_MEMORY_B: 126 *pObjectType = GR_OBJECT_TYPE_MEM; 127 break; 128 default: 129 *pObjectType = GR_OBJECT_TYPE_INVALID; 130 break; 131 } 132 } 133 134 /*! 135 * @brief Is local ctx buffer supported 136 * 137 * @param[IN] bufId buffer Id 138 * @param[IN] bClassSupported2D Is 2D class supported 139 */ 140 NvBool 141 kgrmgrIsCtxBufSupported_IMPL 142 ( 143 GR_CTX_BUFFER bufId, 144 NvBool bClassSupported2D 145 ) 146 { 147 NvBool bSupported = NV_FALSE; 148 149 NV_ASSERT_OR_RETURN(NV_ENUM_IS(GR_CTX_BUFFER, bufId), NV_FALSE); 150 151 // All buffers are supported when 2D class is supported 152 if (bClassSupported2D) 153 { 154 return NV_TRUE; 155 } 156 157 switch (bufId) 158 { 159 case GR_CTX_BUFFER_ZCULL: 160 // fall-through 161 case GR_CTX_BUFFER_PREEMPT: 162 // fall-through 163 case GR_CTX_BUFFER_SPILL: 164 // fall-through 165 case GR_CTX_BUFFER_BETA_CB: 166 // fall-through 167 case GR_CTX_BUFFER_PAGEPOOL: 168 // fall-through 169 case GR_CTX_BUFFER_RTV_CB: 170 bSupported = NV_FALSE; 171 break; 172 173 case GR_CTX_BUFFER_PM: 174 // fall-through 175 case GR_CTX_BUFFER_MAIN: 176 // fall-through 177 case GR_CTX_BUFFER_PATCH: 178 bSupported = NV_TRUE; 179 break; 180 181 // No default case - compiler enforces update if enum changes 182 } 183 return bSupported; 184 } 185 186 /*! 187 * @brief Is globalctx buffer supported 188 * 189 * @param[IN] bufId buffer Id 190 * @param[IN] bClassSupported2D Is 2D class supported 191 */ 192 NvBool 193 kgrmgrIsGlobalCtxBufSupported_IMPL 194 ( 195 GR_GLOBALCTX_BUFFER bufId, 196 NvBool bClassSupported2D 197 ) 198 { 199 NvBool bSupported = NV_FALSE; 200 201 NV_ASSERT_OR_RETURN(NV_ENUM_IS(GR_GLOBALCTX_BUFFER, bufId), NV_FALSE); 202 203 // All buffers are supported when 2D class is supported 204 if (bClassSupported2D) 205 { 206 return NV_TRUE; 207 } 208 209 switch (bufId) 210 { 211 case GR_GLOBALCTX_BUFFER_BUNDLE_CB: 212 // fall-through 213 case GR_GLOBALCTX_BUFFER_PAGEPOOL: 214 // fall-through 215 case GR_GLOBALCTX_BUFFER_ATTRIBUTE_CB: 216 // fall-through 217 case GR_GLOBALCTX_BUFFER_RTV_CB: 218 // fall-through 219 case GR_GLOBALCTX_BUFFER_GFXP_POOL: 220 // fall-through 221 case GR_GLOBALCTX_BUFFER_GFXP_CTRL_BLK: 222 bSupported = NV_FALSE; 223 break; 224 225 case GR_GLOBALCTX_BUFFER_PRIV_ACCESS_MAP: 226 // fall-through 227 case GR_GLOBALCTX_BUFFER_UNRESTRICTED_PRIV_ACCESS_MAP: 228 // fall-through 229 case GR_GLOBALCTX_BUFFER_FECS_EVENT: 230 // fall-through 231 case GR_GLOBAL_BUFFER_GLOBAL_PRIV_ACCESS_MAP: 232 bSupported = NV_TRUE; 233 break; 234 235 // No default case - compiler enforces update if enum changes 236 } 237 238 return bSupported; 239 } 240 241 /*! 242 * @brief Sets swizzID and engineID on routing info if they don't already exist. 243 * 244 * @param[in] engID GR engine ID 245 * @param[in, out] pRouteInfo Client provided routing info 246 */ 247 void 248 kgrmgrCtrlSetEngineID_IMPL 249 ( 250 NvU32 engID, 251 NV2080_CTRL_GR_ROUTE_INFO *pRouteInfo 252 ) 253 { 254 if (NULL == pRouteInfo) 255 { 256 return; 257 } 258 else if (NV2080_CTRL_GR_ROUTE_INFO_FLAGS_TYPE_NONE == 259 DRF_VAL(2080_CTRL_GR, _ROUTE_INFO_FLAGS, _TYPE, pRouteInfo->flags)) 260 { 261 pRouteInfo->flags = DRF_DEF(2080_CTRL_GR, _ROUTE_INFO_FLAGS, _TYPE, _ENGID); 262 pRouteInfo->route = DRF_NUM64(2080_CTRL_GR, _ROUTE_INFO_DATA, _ENGID, engID); 263 } 264 } 265 266 /*! 267 * @brief Sets channel handle on routing info if it doesn't already exist. 268 * 269 * @param[in] hChannel Channel handle 270 * @param[in, out] pRouteInfo Client provided routing info 271 */ 272 void 273 kgrmgrCtrlSetChannelHandle_IMPL 274 ( 275 NvHandle hChannel, 276 NV2080_CTRL_GR_ROUTE_INFO *pRouteInfo 277 ) 278 { 279 if (NULL == pRouteInfo) 280 { 281 return; 282 } 283 else if (NV2080_CTRL_GR_ROUTE_INFO_FLAGS_TYPE_NONE == 284 DRF_VAL(2080_CTRL_GR, _ROUTE_INFO_FLAGS, _TYPE, pRouteInfo->flags)) 285 { 286 pRouteInfo->flags = DRF_DEF(2080_CTRL_GR, _ROUTE_INFO_FLAGS, _TYPE, _CHANNEL); 287 pRouteInfo->route = DRF_NUM64(2080_CTRL_GR, _ROUTE_INFO_DATA, _CHANNEL_HANDLE, hChannel); 288 } 289 } 290 291 NV_STATUS 292 kgrmgrConstructEngine_IMPL 293 ( 294 OBJGPU *pGpu, 295 KernelGraphicsManager *pKernelGraphicsManager, 296 ENGDESCRIPTOR engDesc 297 ) 298 { 299 return NV_OK; 300 } 301 302 void 303 kgrmgrDestruct_IMPL 304 ( 305 KernelGraphicsManager *pKernelGraphicsManager 306 ) 307 { 308 portMemSet(&pKernelGraphicsManager->legacyKgraphicsStaticInfo.floorsweepingMasks, 0, 309 sizeof(pKernelGraphicsManager->legacyKgraphicsStaticInfo.floorsweepingMasks)); 310 portMemFree(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pPpcMasks); 311 pKernelGraphicsManager->legacyKgraphicsStaticInfo.pPpcMasks = NULL; 312 portMemFree(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo); 313 pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo = NULL; 314 pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized = NV_FALSE; 315 } 316 317 /*! 318 * @brief Set legacy Kgraphics Static Info (i.e. state of GR0) 319 */ 320 void 321 kgrmgrSetLegacyKgraphicsStaticInfo_IMPL 322 ( 323 OBJGPU *pGpu, 324 KernelGraphicsManager *pKernelGraphicsManager, 325 KernelGraphics *pKernelGraphics 326 ) 327 { 328 NV_CHECK_OR_RETURN_VOID(LEVEL_INFO, !pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized); 329 NV_ASSERT_OR_RETURN_VOID((pKernelGraphics != NULL) && (kgraphicsGetInstance(pGpu, pKernelGraphics) == 0)); 330 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 331 NV_ASSERT_OR_RETURN_VOID(pKernelGraphicsStaticInfo != NULL); 332 333 portMemCopy(&pKernelGraphicsManager->legacyKgraphicsStaticInfo.floorsweepingMasks, sizeof(pKernelGraphicsStaticInfo->floorsweepingMasks), 334 &pKernelGraphicsStaticInfo->floorsweepingMasks, sizeof(pKernelGraphicsStaticInfo->floorsweepingMasks)); 335 336 if (pKernelGraphicsStaticInfo->pPpcMasks != NULL) 337 { 338 pKernelGraphicsManager->legacyKgraphicsStaticInfo.pPpcMasks = portMemAllocNonPaged(sizeof(*pKernelGraphicsStaticInfo->pPpcMasks)); 339 NV_ASSERT_OR_GOTO(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pPpcMasks != NULL, cleanup); 340 341 portMemCopy(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pPpcMasks, sizeof(*pKernelGraphicsStaticInfo->pPpcMasks), 342 pKernelGraphicsStaticInfo->pPpcMasks, sizeof(*pKernelGraphicsStaticInfo->pPpcMasks)); 343 } 344 345 if (pKernelGraphicsStaticInfo->pGrInfo != NULL) 346 { 347 pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo = portMemAllocNonPaged(sizeof(*pKernelGraphicsStaticInfo->pGrInfo)); 348 NV_ASSERT_OR_GOTO(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo != NULL, cleanup); 349 350 portMemCopy(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo, sizeof(*pKernelGraphicsStaticInfo->pGrInfo), 351 pKernelGraphicsStaticInfo->pGrInfo, sizeof(*pKernelGraphicsStaticInfo->pGrInfo)); 352 } 353 354 pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized = NV_TRUE; 355 return; 356 357 cleanup: 358 portMemFree(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pPpcMasks); 359 pKernelGraphicsManager->legacyKgraphicsStaticInfo.pPpcMasks = NULL; 360 portMemFree(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo); 361 pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo = NULL; 362 } 363 364 /*! 365 * @brief Retrieves associated KernelGraphics engine for given client / route info 366 * 367 * @param[in] pGpu 368 * @param[in] pKernelGraphicsManager 369 * @param[in] hClient Client handle 370 * @param[in] grRouteInfo Client-provided info to direct GR accesses 371 * @param[out] ppKernelGraphics (Optional) Ptr to store appropriate KernelGraphics *, if desired. 372 */ 373 NV_STATUS 374 kgrmgrCtrlRouteKGR_IMPL 375 ( 376 OBJGPU *pGpu, 377 KernelGraphicsManager *pKernelGraphicsManager, 378 NvHandle hClient, 379 const NV2080_CTRL_GR_ROUTE_INFO *pGrRouteInfo, 380 KernelGraphics **ppKernelGraphics 381 ) 382 { 383 MIG_INSTANCE_REF ref; 384 KernelGraphics *pKernelGraphics; 385 NvU32 type; 386 NV_STATUS status = NV_OK; 387 NvU32 grIdx; 388 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo = *pGrRouteInfo; 389 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 390 391 if (!IS_MIG_IN_USE(pGpu)) 392 { 393 grIdx = 0; 394 goto done; 395 } 396 397 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 398 kmigmgrGetInstanceRefFromClient(pGpu, pKernelMIGManager, hClient, &ref)); 399 400 // 401 // Compute instances always have 1 GR engine, so automatically fill in 402 // the route info when subscribed to a compute instance 403 // 404 if (ref.pMIGComputeInstance != NULL) 405 { 406 portMemSet(&grRouteInfo, 0, sizeof(NV2080_CTRL_GR_ROUTE_INFO)); 407 kgrmgrCtrlSetEngineID(0, &grRouteInfo); 408 } 409 else 410 { 411 RS_PRIV_LEVEL privLevel = rmclientGetCachedPrivilegeByHandle(hClient); 412 if (!rmclientIsAdminByHandle(hClient, privLevel) && 413 _kgrmgrGPUInstanceHasComputeInstances(pGpu, pKernelGraphicsManager, ref.pKernelMIGGpuInstance)) 414 { 415 return NV_ERR_INSUFFICIENT_PERMISSIONS; 416 } 417 } 418 419 type = DRF_VAL(2080_CTRL_GR, _ROUTE_INFO_FLAGS, _TYPE, grRouteInfo.flags); 420 switch (type) 421 { 422 case NV2080_CTRL_GR_ROUTE_INFO_FLAGS_TYPE_NONE: 423 NV_PRINTF(LEVEL_ERROR, 424 "Cannot give GR Route flag of TYPE_NONE with MIG enabled!\n"); 425 return NV_ERR_INVALID_ARGUMENT; 426 427 case NV2080_CTRL_GR_ROUTE_INFO_FLAGS_TYPE_ENGID: 428 { 429 NvU32 localGrIdx = DRF_VAL64(2080_CTRL_GR, _ROUTE_INFO_DATA, 430 _ENGID, grRouteInfo.route); 431 RM_ENGINE_TYPE globalRmEngType; 432 433 NV_CHECK_OK_OR_RETURN( 434 LEVEL_ERROR, 435 kmigmgrGetLocalToGlobalEngineType(pGpu, pKernelMIGManager, ref, 436 RM_ENGINE_TYPE_GR(localGrIdx), 437 &globalRmEngType)); 438 NV_ASSERT_OR_RETURN(RM_ENGINE_TYPE_IS_GR(globalRmEngType), NV_ERR_INVALID_STATE); 439 grIdx = RM_ENGINE_TYPE_GR_IDX(globalRmEngType); 440 441 break; 442 } 443 444 case NV2080_CTRL_GR_ROUTE_INFO_FLAGS_TYPE_CHANNEL: 445 { 446 KernelChannel *pKernelChannel; 447 RM_ENGINE_TYPE rmEngineType; 448 NvHandle hChannel = DRF_VAL64(2080_CTRL_GR, _ROUTE_INFO_DATA, 449 _CHANNEL_HANDLE, grRouteInfo.route); 450 451 status = CliGetKernelChannel(hClient, hChannel, &pKernelChannel); 452 if (status != NV_OK) 453 { 454 RsResourceRef *pChanGrpRef; 455 KernelChannelGroupApi *pKernelChannelGroupApi = NULL; 456 KernelChannelGroup *pKernelChannelGroup = NULL; 457 458 // 459 // If retrieving a channel with the given hChannel doesn't work, 460 // try interpreting it as a handle to a channel group instead. 461 // 462 status = CliGetChannelGroup(hClient, hChannel, &pChanGrpRef, NULL); 463 if (NV_OK != status) 464 { 465 NV_PRINTF(LEVEL_ERROR, 466 "Failed to find a channel or TSG with given handle 0x%08x associated with hClient=0x%08x\n", 467 hChannel, hClient); 468 return NV_ERR_INVALID_ARGUMENT; 469 } 470 471 pKernelChannelGroupApi = dynamicCast(pChanGrpRef->pResource, 472 KernelChannelGroupApi); 473 NV_ASSERT_OR_RETURN( 474 (pKernelChannelGroupApi != NULL && 475 pKernelChannelGroupApi->pKernelChannelGroup != NULL), 476 NV_ERR_INVALID_STATE); 477 pKernelChannelGroup = 478 pKernelChannelGroupApi->pKernelChannelGroup; 479 480 NV_PRINTF(LEVEL_INFO, 481 "Found TSG with given handle 0x%08x, using this to determine GR engine ID\n", 482 hChannel); 483 rmEngineType = pKernelChannelGroup->engineType; 484 } 485 else 486 { 487 NV_PRINTF(LEVEL_INFO, 488 "Found channel with given handle 0x%08x, using this to determine GR engine ID\n", 489 hChannel); 490 rmEngineType = kchannelGetEngineType(pKernelChannel); 491 } 492 493 if (!RM_ENGINE_TYPE_IS_GR(rmEngineType)) 494 { 495 NV_PRINTF(LEVEL_ERROR, 496 "Failed to route GR using non-GR engine type 0x%x (0x%x)\n", 497 gpuGetNv2080EngineType(rmEngineType), rmEngineType); 498 return NV_ERR_INVALID_ARGUMENT; 499 } 500 501 grIdx = RM_ENGINE_TYPE_GR_IDX(rmEngineType); 502 break; 503 } 504 default: 505 NV_PRINTF(LEVEL_ERROR, 506 "Unrecognized GR Route flag type 0x%x!\n", type); 507 return NV_ERR_INVALID_ARGUMENT; 508 } 509 510 done: 511 pKernelGraphics = GPU_GET_KERNEL_GRAPHICS(pGpu, grIdx); 512 NV_ASSERT_OR_RETURN(pKernelGraphics != NULL, NV_ERR_INVALID_STATE); 513 514 if (ppKernelGraphics != NULL) 515 *ppKernelGraphics = pKernelGraphics; 516 517 return status; 518 } 519 520 /*! 521 * @return legacy GPC mask enumerated by this chip 522 */ 523 NvU32 524 kgrmgrGetLegacyGpcMask_IMPL 525 ( 526 OBJGPU *pGpu, 527 KernelGraphicsManager *pKernelGraphicsManager 528 ) 529 { 530 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized, 0); 531 532 return pKernelGraphicsManager->legacyKgraphicsStaticInfo.floorsweepingMasks.gpcMask; 533 } 534 535 /*! 536 * @return legacy TPC mask for certain GPC 537 * 538 * @param[in] pGpu 539 * @param[in] KernelGraphicsManager 540 * @param[in] gpcId Indicates logical GPC ID when MIG enabled or physical 541 * GPC ID when MIG disabled 542 */ 543 NvU32 544 kgrmgrGetLegacyTpcMask_IMPL 545 ( 546 OBJGPU *pGpu, 547 KernelGraphicsManager *pKernelGraphicsManager, 548 NvU32 gpcId 549 ) 550 { 551 NvU32 maxNumGpcs; 552 553 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized, 0); 554 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo != NULL, 0); 555 556 maxNumGpcs = pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo->infoList[NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_GPCS].data; 557 NV_CHECK_OR_RETURN(LEVEL_ERROR, (gpcId < maxNumGpcs), 0); 558 559 return pKernelGraphicsManager->legacyKgraphicsStaticInfo.floorsweepingMasks.tpcMask[gpcId]; 560 } 561 562 /*! 563 * @brief Get legacy PPC mask for certain GPC 564 */ 565 NV_STATUS 566 kgrmgrGetLegacyPpcMask_IMPL 567 ( 568 OBJGPU *pGpu, 569 KernelGraphicsManager *pKernelGraphicsManager, 570 NvU32 physGpcId, 571 NvU32 *pPpcMask 572 ) 573 { 574 NvU32 maxNumGpcs; 575 576 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized, NV_ERR_INVALID_STATE); 577 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo != NULL, NV_ERR_INVALID_STATE); 578 maxNumGpcs = pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo->infoList[NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_GPCS].data; 579 NV_CHECK_OR_RETURN(LEVEL_ERROR, (physGpcId < maxNumGpcs), NV_ERR_INVALID_ARGUMENT); 580 NV_ASSERT_OR_RETURN((pPpcMask != NULL), NV_ERR_INVALID_ARGUMENT); 581 NV_ASSERT_OR_RETURN((pKernelGraphicsManager->legacyKgraphicsStaticInfo.pPpcMasks != NULL), NV_ERR_NOT_SUPPORTED); 582 583 *pPpcMask = pKernelGraphicsManager->legacyKgraphicsStaticInfo.pPpcMasks->mask[physGpcId]; 584 585 return NV_OK; 586 } 587 588 /*! 589 * @brief Returns legacy zcull mask for specific gpc 590 */ 591 NvU32 592 kgrmgrGetLegacyZcullMask_IMPL 593 ( 594 OBJGPU *pGpu, 595 KernelGraphicsManager *pKernelGraphicsManager, 596 NvU32 physGpcId 597 ) 598 { 599 NvU32 maxNumGpcs; 600 601 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized, 0); 602 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo != NULL, NV_ERR_INVALID_STATE); 603 maxNumGpcs = pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo->infoList[NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_GPCS].data; 604 NV_CHECK_OR_RETURN(LEVEL_ERROR, (physGpcId < maxNumGpcs), NV_ERR_INVALID_ARGUMENT); 605 606 return pKernelGraphicsManager->legacyKgraphicsStaticInfo.floorsweepingMasks.zcullMask[physGpcId]; 607 } 608 609 /*! 610 * @brief Function to Alloc VEIDs for a GR engine 611 * 612 * @param[IN] pGpu 613 * @param[IN] pKernelGraphicsManager 614 * @param[IN] grIdx phys gr idx 615 * @param[IN] veidSpanOffset Offset of spans localized to a GPU instance to make the VEID request from 616 * @param[IN] veidCount Total VEIDs connected to this GR engine 617 * @param[IN] pKernelMIGGPUInstance 618 */ 619 NV_STATUS 620 kgrmgrAllocVeidsForGrIdx_IMPL 621 ( 622 OBJGPU *pGpu, 623 KernelGraphicsManager *pKernelGraphicsManager, 624 NvU32 grIdx, 625 NvU32 veidSpanOffset, 626 NvU32 veidCount, 627 KERNEL_MIG_GPU_INSTANCE *pKernelMIGGPUInstance 628 ) 629 { 630 NvU32 maxVeidsPerGpc; 631 NvU32 veidStart = 0; 632 NvU32 veidEnd = 0; 633 NvU32 GPUInstanceVeidEnd; 634 NvU64 GPUInstanceVeidMask; 635 NvU64 GPUInstanceFreeVeidMask; 636 NvU64 reqVeidMask; 637 638 // This GR should not be already configured to use any VEIDs 639 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->grIdxVeidMask[grIdx] == 0, NV_ERR_INVALID_STATE); 640 641 NV_ASSERT_OK_OR_RETURN( 642 kgrmgrGetMaxVeidsPerGpc(pGpu, pKernelGraphicsManager, &maxVeidsPerGpc)); 643 644 // We statically assign VEIDs to a GR based on the number of GPCs connected to it 645 if (veidCount % maxVeidsPerGpc != 0) 646 { 647 NV_PRINTF(LEVEL_ERROR, "veidCount %d is not aligned to maxVeidsPerGpc=%d\n", veidCount, maxVeidsPerGpc); 648 return NV_ERR_INVALID_ARGUMENT; 649 } 650 651 NV_ASSERT_OR_RETURN(veidSpanOffset != KMIGMGR_SPAN_OFFSET_INVALID, NV_ERR_INVALID_ARGUMENT); 652 653 veidStart = (veidSpanOffset * maxVeidsPerGpc) + pKernelMIGGPUInstance->resourceAllocation.veidOffset; 654 veidEnd = veidStart + veidCount - 1; 655 656 NV_ASSERT_OR_RETURN(veidStart < veidEnd, NV_ERR_INVALID_STATE); 657 NV_ASSERT_OR_RETURN(veidStart < 64, NV_ERR_INVALID_ARGUMENT); 658 NV_ASSERT_OR_RETURN(veidEnd < 64, NV_ERR_INVALID_ARGUMENT); 659 660 reqVeidMask = DRF_SHIFTMASK64(veidEnd:veidStart); 661 662 NV_ASSERT_OR_RETURN(reqVeidMask != 0x0, NV_ERR_INVALID_STATE); 663 664 // Create a mask for VEIDs associated with this GPU instance 665 GPUInstanceVeidEnd = pKernelMIGGPUInstance->resourceAllocation.veidOffset + pKernelMIGGPUInstance->resourceAllocation.veidCount - 1; 666 GPUInstanceVeidMask = DRF_SHIFTMASK64(GPUInstanceVeidEnd:pKernelMIGGPUInstance->resourceAllocation.veidOffset); 667 668 NV_ASSERT_OR_RETURN(GPUInstanceVeidMask != 0x0, NV_ERR_INVALID_STATE); 669 670 GPUInstanceFreeVeidMask = ~(pKernelGraphicsManager->veidInUseMask) & GPUInstanceVeidMask; 671 672 // VEID range should not overlap with existing VEIDs in use 673 NV_ASSERT_OR_RETURN((pKernelGraphicsManager->veidInUseMask & reqVeidMask) == 0, NV_ERR_STATE_IN_USE); 674 NV_ASSERT_OR_RETURN((GPUInstanceFreeVeidMask & reqVeidMask) == reqVeidMask, NV_ERR_INSUFFICIENT_RESOURCES); 675 676 // mark each VEID in the range as "in use" 677 pKernelGraphicsManager->veidInUseMask |= reqVeidMask; 678 pKernelGraphicsManager->grIdxVeidMask[grIdx] |= reqVeidMask; 679 680 return NV_OK; 681 } 682 683 /*! 684 * @brief Function to Clear Gr Engine to VEIDs mapping 685 */ 686 void 687 kgrmgrClearVeidsForGrIdx_IMPL 688 ( 689 OBJGPU *pGpu, 690 KernelGraphicsManager *pKernelGraphicsManager, 691 NvU32 grIdx 692 ) 693 { 694 NvU64 veidMask = pKernelGraphicsManager->grIdxVeidMask[grIdx]; 695 696 // mark all VEIDs of this GR engine as "not in use" 697 NV_ASSERT((pKernelGraphicsManager->veidInUseMask & veidMask) == veidMask); 698 pKernelGraphicsManager->veidInUseMask &= ~veidMask; 699 pKernelGraphicsManager->grIdxVeidMask[grIdx] = 0; 700 } 701 702 /*! 703 * @brief Function to get max VEID count per GPC 704 */ 705 NV_STATUS 706 kgrmgrGetMaxVeidsPerGpc_IMPL 707 ( 708 OBJGPU *pGpu, 709 KernelGraphicsManager *pKernelGraphicsManager, 710 NvU32 *pMaxVeidsPerGpc 711 ) 712 { 713 NvU32 maxVeids; 714 NvU32 maxGpcCount; 715 716 NV_ASSERT_OR_RETURN(pMaxVeidsPerGpc != NULL, NV_ERR_INVALID_ARGUMENT); 717 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized, NV_ERR_INVALID_STATE); 718 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo != NULL, NV_ERR_INVALID_STATE); 719 720 maxVeids = pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo->infoList[NV0080_CTRL_GR_INFO_INDEX_MAX_SUBCONTEXT_COUNT].data; 721 722 maxGpcCount = gpuGetLitterValues_HAL(pGpu, NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_GPCS); 723 NV_ASSERT_OR_RETURN(maxGpcCount != 0, NV_ERR_INSUFFICIENT_RESOURCES); 724 725 *pMaxVeidsPerGpc = (maxVeids / maxGpcCount); 726 727 return NV_OK; 728 } 729 730 /*! 731 * @brief Function to get starting VEID for a Gr Engine 732 */ 733 NV_STATUS 734 kgrmgrGetVeidBaseForGrIdx_IMPL 735 ( 736 OBJGPU *pGpu, 737 KernelGraphicsManager *pKernelGraphicsManager, 738 NvU32 grIdx, 739 NvU32 *pVeidStart 740 ) 741 { 742 NvU64 veidMask; 743 NV_ASSERT_OR_RETURN(pVeidStart != NULL, NV_ERR_INVALID_ARGUMENT); 744 NV_ASSERT_OR_RETURN(grIdx != KGRMGR_MAX_GR, NV_ERR_INVALID_ARGUMENT); 745 746 *pVeidStart = 0; 747 748 veidMask = pKernelGraphicsManager->grIdxVeidMask[grIdx]; 749 750 // 751 // If a GR is not configured, VEID mask for it will be "0" and counting 752 // "0" in a zero-based mask will result in max bit-width size. 753 // 754 if (veidMask != 0x0) 755 *pVeidStart = portUtilCountTrailingZeros64(veidMask); 756 757 return NV_OK; 758 } 759 760 /*! 761 * @brief Function to get GR index for a VEID 762 */ 763 NV_STATUS 764 kgrmgrGetGrIdxForVeid_IMPL 765 ( 766 OBJGPU *pGpu, 767 KernelGraphicsManager *pKernelGraphicsManager, 768 NvU32 veid, 769 NvU32 *pGrIdx 770 ) 771 { 772 NvU32 i; 773 NvU64 veidMask = NVBIT64(veid); 774 775 NV_ASSERT_OR_RETURN(pGrIdx != NULL, NV_ERR_INVALID_ARGUMENT); 776 for (i = 0; i < KGRMGR_MAX_GR; ++i) 777 { 778 if ((pKernelGraphicsManager->grIdxVeidMask[i] & veidMask) != 0) 779 break; 780 } 781 NV_ASSERT_OR_RETURN(i != KGRMGR_MAX_GR, NV_ERR_OBJECT_NOT_FOUND); 782 *pGrIdx = i; 783 return NV_OK; 784 } 785 786 /*! 787 * @brief discovers maximum size of local ctx buffers. 788 */ 789 NV_STATUS 790 kgrmgrDiscoverMaxLocalCtxBufInfo_IMPL 791 ( 792 OBJGPU *pGpu, 793 KernelGraphicsManager *pKernelGraphicsManager, 794 KernelGraphics *pKernelGraphics, 795 NvU32 swizzId 796 ) 797 { 798 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 799 800 // 801 // Most of GR is stub'd so context related things are not needed in AMODEL. 802 // But this function can be called in some MODS test, so return OK directly 803 // to avoid failing the test. 804 // 805 if (IS_MODS_AMODEL(pGpu)) 806 return NV_OK; 807 808 NV_CHECK_OR_RETURN(LEVEL_SILENT, 809 kmigmgrIsMemoryPartitioningNeeded_HAL(pGpu, pKernelMIGManager, swizzId), NV_OK); 810 811 return kgraphicsDiscoverMaxLocalCtxBufferSize(pGpu, pKernelGraphics); 812 } 813 814 /*! 815 * @brief Get ctxbufpool info for the global ctx buffer 816 */ 817 const CTX_BUF_INFO * 818 kgrmgrGetGlobalCtxBufInfo_IMPL 819 ( 820 OBJGPU *pGpu, 821 KernelGraphicsManager *pKernelGraphicsManager, 822 GR_GLOBALCTX_BUFFER buf 823 ) 824 { 825 NV_ASSERT_OR_RETURN(NV_ENUM_IS(GR_GLOBALCTX_BUFFER, buf), NULL); 826 return &pKernelGraphicsManager->globalCtxBufInfo[buf]; 827 } 828 829 /*! 830 * @brief Set ctxbufpool parameters for the global ctx buffer 831 */ 832 void 833 kgrmgrSetGlobalCtxBufInfo_IMPL 834 ( 835 OBJGPU *pGpu, 836 KernelGraphicsManager *pKernelGraphicsManager, 837 GR_GLOBALCTX_BUFFER buf, 838 NvU64 size, 839 NvU64 align, 840 RM_ATTR_PAGE_SIZE attr, 841 NvBool bContiguous 842 ) 843 { 844 CTX_BUF_INFO *pInfo; 845 NV_ASSERT_OR_RETURN_VOID(NV_ENUM_IS(GR_GLOBALCTX_BUFFER, buf)); 846 847 pInfo = &pKernelGraphicsManager->globalCtxBufInfo[buf]; 848 pInfo->size = size; 849 pInfo->align = align; 850 pInfo->attr = attr; 851 pInfo->bContig = bContiguous; 852 } 853 854 /*! 855 * @brief Gets maximum size of GR global ctx buffers. 856 * These are sizes of buffer for GR0 in legacy mode with all GPCs 857 * connected to GR0. 858 */ 859 NV_STATUS 860 kgrmgrDiscoverMaxGlobalCtxBufSizes_IMPL 861 ( 862 OBJGPU *pGpu, 863 KernelGraphicsManager *pKernelGraphicsManager, 864 KernelGraphics *pKernelGraphics, 865 NvBool bMemoryPartitioningNeeded 866 ) 867 { 868 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 869 GR_GLOBALCTX_BUFFER bufId; 870 871 // 872 // Most of GR is stub'd so context related things are not needed in AMODEL. 873 // But this function can be called in some MODS test, so return OK directly 874 // to avoid failing the test. 875 // 876 if (IS_MODS_AMODEL(pGpu)) 877 return NV_OK; 878 879 NV_ASSERT_OR_RETURN(!IS_MIG_IN_USE(pGpu), NV_ERR_INVALID_STATE); 880 881 // 882 // Bug 2915422: Eventually we expect this check to be replaced by ctxBufPoolIsSupported 883 // we can't use that check today because PDB_PROP_GPU_MOVE_CTX_BUFFERS_TO_PMA is not enabled 884 // when this function is called because 885 // kgraphicsInitializeDeferredStaticData below will eventually lead to 886 // global ctx buffer allocation from ctxBufPools even before these pools are 887 // populated which happens later during GPU instance creation. Once we are 888 // able to rip out global buffer alloc from 889 // kgraphicsInitializeDeferredStaticData, we can enable the above property 890 // early. 891 // 892 NV_CHECK_OR_RETURN(LEVEL_SILENT, bMemoryPartitioningNeeded, NV_OK); 893 894 // Make sure sizes of all buffers are setup 895 NV_ASSERT_OK_OR_RETURN( 896 kgraphicsInitializeDeferredStaticData(pGpu, pKernelGraphics, NV01_NULL_OBJECT, NV01_NULL_OBJECT)); 897 898 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 899 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 900 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pContextBuffersInfo != NULL, NV_ERR_INVALID_STATE); 901 902 // Get sizes of global ctx buffers 903 FOR_EACH_IN_ENUM(GR_GLOBALCTX_BUFFER, bufId) 904 { 905 NvU32 fifoEngineId; 906 NV_ASSERT_OK_OR_RETURN( 907 kgrctxGlobalCtxBufferToFifoEngineId(bufId, &fifoEngineId)); 908 909 // 910 // contiguity is determined later before reservation as it depends on settings 911 // that take effect after this point. 912 // 913 kgrmgrSetGlobalCtxBufInfo(pGpu, pKernelGraphicsManager, bufId, 914 pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[fifoEngineId].size, 915 pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[fifoEngineId].alignment, 916 RM_ATTR_PAGE_SIZE_4KB, NV_FALSE); 917 } 918 FOR_EACH_IN_ENUM_END; 919 920 return NV_OK; 921 } 922 923 /*! 924 * @return legacy TPC count for certain GPC 925 * 926 * @param[in] pGpu 927 * @param[in] KernelGraphicsManager 928 * @param[in] gpcId Indicates logical GPC ID 929 */ 930 NvU32 931 kgrmgrGetLegacyGpcTpcCount_IMPL 932 ( 933 OBJGPU *pGpu, 934 KernelGraphicsManager *pKernelGraphicsManager, 935 NvU32 gpcId 936 ) 937 { 938 NvU32 maxNumGpcs; 939 940 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized, 0); 941 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo != NULL, 0); 942 943 maxNumGpcs = nvPopCount32(pKernelGraphicsManager->legacyKgraphicsStaticInfo.floorsweepingMasks.gpcMask); 944 NV_CHECK_OR_RETURN(LEVEL_ERROR, (gpcId < maxNumGpcs), 0); 945 946 return pKernelGraphicsManager->legacyKgraphicsStaticInfo.floorsweepingMasks.tpcCount[gpcId]; 947 } 948 949 /*! 950 * @brief Function to Alloc VEIDs with either a specific placement or first-fit placement 951 * depending upon the value passed into pSpanStart 952 * 953 * @param[IN] pGpu 954 * @param[IN] pKernelGraphicsManager 955 * @param[IN/OUT] pInUseMask Mask of VEIDs in-use, updated with newly set VEIDs 956 * @param[IN] veidCount Total VEIDs connected to this GR engine 957 * @param[IN/OUT] pSpanStart GPU Instance local span offset to begin assigning VEIDs from 958 * @param[IN] pKernelMIGGPUInstance 959 */ 960 NV_STATUS 961 kgrmgrCheckVeidsRequest_IMPL 962 ( 963 OBJGPU *pGpu, 964 KernelGraphicsManager *pKernelGraphicsManager, 965 NvU64 *pInUseMask, 966 NvU32 veidCount, 967 NvU32 *pSpanStart, 968 KERNEL_MIG_GPU_INSTANCE *pKernelMIGGPUInstance 969 ) 970 { 971 NvU32 maxVeidsPerGpc; 972 NvU32 veidStart = 0; 973 NvU32 veidEnd = 0; 974 NvU32 GPUInstanceVeidEnd; 975 NvU64 GPUInstanceVeidMask; 976 NvU64 GPUInstanceFreeVeidMask; 977 NvU64 veidMask; 978 979 NV_ASSERT_OK_OR_RETURN( 980 kgrmgrGetMaxVeidsPerGpc(pGpu, pKernelGraphicsManager, &maxVeidsPerGpc)); 981 982 NV_ASSERT_OR_RETURN(pSpanStart != NULL, NV_ERR_INVALID_ARGUMENT); 983 NV_ASSERT_OR_RETURN(pInUseMask != NULL, NV_ERR_INVALID_ARGUMENT); 984 985 // We statically assign VEIDs to a GR based on the number of GPCs connected to it 986 if (veidCount % maxVeidsPerGpc != 0) 987 { 988 NV_PRINTF(LEVEL_ERROR, "veidCount %d is not aligned to maxVeidsPerGpc=%d\n", veidCount, maxVeidsPerGpc); 989 return NV_ERR_INVALID_ARGUMENT; 990 } 991 992 // Create a mask for VEIDs associated with this GPU instance 993 GPUInstanceVeidEnd = pKernelMIGGPUInstance->resourceAllocation.veidOffset + pKernelMIGGPUInstance->resourceAllocation.veidCount - 1; 994 GPUInstanceVeidMask = DRF_SHIFTMASK64(GPUInstanceVeidEnd:pKernelMIGGPUInstance->resourceAllocation.veidOffset); 995 996 NV_ASSERT_OR_RETURN(GPUInstanceVeidMask != 0x0, NV_ERR_INVALID_STATE); 997 998 GPUInstanceFreeVeidMask = ~(*pInUseMask) & GPUInstanceVeidMask; 999 1000 if (*pSpanStart != KMIGMGR_SPAN_OFFSET_INVALID) 1001 { 1002 veidStart = (*pSpanStart * maxVeidsPerGpc) + pKernelMIGGPUInstance->resourceAllocation.veidOffset; 1003 veidEnd = veidStart + veidCount - 1; 1004 1005 NV_ASSERT_OR_RETURN(veidStart < veidEnd, NV_ERR_INVALID_STATE); 1006 NV_ASSERT_OR_RETURN(veidStart < 64, NV_ERR_INVALID_ARGUMENT); 1007 NV_ASSERT_OR_RETURN(veidEnd < 64, NV_ERR_INVALID_ARGUMENT); 1008 } 1009 else 1010 { 1011 NvU64 reqVeidMask = DRF_SHIFTMASK64(veidCount - 1:0); 1012 NvU32 i; 1013 1014 for (i = pKernelMIGGPUInstance->resourceAllocation.veidOffset; i <= GPUInstanceVeidEnd; i += maxVeidsPerGpc) 1015 { 1016 // See if requested slots are available within this range 1017 if (((GPUInstanceFreeVeidMask >> i) & reqVeidMask) == reqVeidMask) 1018 { 1019 veidStart = i; 1020 veidEnd = veidStart + veidCount - 1; 1021 break; 1022 } 1023 } 1024 1025 NV_CHECK_OR_RETURN(LEVEL_SILENT, i <= GPUInstanceVeidEnd, 1026 NV_ERR_INSUFFICIENT_RESOURCES); 1027 } 1028 1029 veidMask = DRF_SHIFTMASK64(veidEnd:veidStart); 1030 NV_ASSERT_OR_RETURN(veidMask != 0x0, NV_ERR_INVALID_STATE); 1031 1032 NV_CHECK_OR_RETURN(LEVEL_ERROR, (GPUInstanceVeidMask & veidMask) == veidMask, NV_ERR_INSUFFICIENT_RESOURCES); 1033 1034 // VEID range should not overlap with existing VEIDs in use 1035 NV_ASSERT_OR_RETURN((*pInUseMask & veidMask) == 0, NV_ERR_STATE_IN_USE); 1036 1037 NV_ASSERT(veidStart >= pKernelMIGGPUInstance->resourceAllocation.veidOffset); 1038 1039 *pSpanStart = (veidStart - pKernelMIGGPUInstance->resourceAllocation.veidOffset) / maxVeidsPerGpc; 1040 *pInUseMask |= veidMask; 1041 return NV_OK; 1042 } 1043