1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #define NVOC_KERNEL_GRAPHICS_H_PRIVATE_ACCESS_ALLOWED 25 26 27 #include "rmconfig.h" 28 29 #include "kernel/gpu/gr/kernel_graphics_manager.h" 30 #include "kernel/gpu/gr/kernel_graphics.h" 31 #include "kernel/gpu/mig_mgr/kernel_mig_manager.h" 32 #include "kernel/gpu/device/device.h" 33 #include "kernel/gpu/subdevice/subdevice.h" 34 #include "kernel/rmapi/rmapi_utils.h" 35 #include "kernel/core/locks.h" 36 #include "kernel/gpu/mem_sys/kern_mem_sys.h" 37 #include "kernel/mem_mgr/gpu_vaspace.h" 38 #include "virtualization/hypervisor/hypervisor.h" 39 #include "nvRmReg.h" 40 #include "kernel/gpu/mem_mgr/mem_mgr.h" 41 #include "kernel/gpu/mem_mgr/heap.h" 42 #include "kernel/gpu/intr/engine_idx.h" 43 #include "gpu/mem_mgr/virt_mem_allocator.h" 44 #include "gpu/mmu/kern_gmmu.h" 45 #include "platform/sli/sli.h" 46 #include "rmapi/rs_utils.h" 47 #include "rmapi/client.h" 48 49 #include "vgpu/vgpu_events.h" 50 #include "vgpu/rpc.h" 51 52 #include "class/clb0c0.h" 53 #include "class/clb1c0.h" 54 #include "class/clc0c0.h" 55 #include "class/clc1c0.h" 56 #include "class/clc3c0.h" 57 #include "class/clc5c0.h" 58 #include "class/clc6c0.h" 59 #include "class/clc7c0.h" 60 #include "class/clcbc0.h" 61 62 #include "class/cl0080.h" 63 #include "class/cl2080.h" 64 #include "class/cla06f.h" 65 #include "class/cla06fsubch.h" 66 #include "class/cl90f1.h" // FERMI_VASPACE_A 67 #include "class/cl003e.h" // NV01_MEMORY_SYSTEM 68 #include "class/cl50a0.h" // NV50_MEMORY_VIRTUAL 69 #include "class/cl0040.h" // NV01_MEMORY_LOCAL_USER 70 #include "class/clc36f.h" // VOLTA_CHANNEL_GPFIFO_A 71 #include "class/clc46f.h" // TURING_CHANNEL_GPFIFO_A 72 #include "class/clc56f.h" // AMPERE_CHANNEL_GPFIFO_A 73 #include "class/clc86f.h" // HOPPER_CHANNEL_GPFIFO_A 74 #include "class/clc637.h" 75 #include "class/clc638.h" 76 77 // 78 // We use NV2080_CTRL_INTERNAL_GR_MAX_GPC to statically allocate certain 79 // GPC related array in ctrl call header file. We will need to adjust 80 // NV2080_CTRL_INTERNAL_GR_MAX_GPC if some day KGRMGR_MAX_GPC gets changed 81 // 82 ct_assert(NV2080_CTRL_INTERNAL_GR_MAX_GPC == KGRMGR_MAX_GPC); 83 84 // 85 // Ensure the external and internal ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_COUNT 86 // will always in sync 87 // 88 ct_assert(NV2080_CTRL_INTERNAL_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_COUNT == 89 NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_COUNT); 90 91 typedef struct KGRAPHICS_PRIVATE_DATA 92 { 93 NvBool bInitialized; 94 KGRAPHICS_STATIC_INFO staticInfo; 95 } KGRAPHICS_PRIVATE_DATA; 96 static NV_STATUS _kgraphicsMapGlobalCtxBuffer(OBJGPU *pGpu, KernelGraphics *pKernelGraphics, NvU32 gfid, OBJVASPACE *, 97 KernelGraphicsContext *, GR_GLOBALCTX_BUFFER, NvBool bIsReadOnly); 98 static NV_STATUS _kgraphicsPostSchedulingEnableHandler(OBJGPU *, void *); 99 100 static void 101 _kgraphicsInitRegistryOverrides(OBJGPU *pGpu, KernelGraphics *pKernelGraphics) 102 { 103 return; 104 } 105 106 NV_STATUS 107 kgraphicsConstructEngine_IMPL 108 ( 109 OBJGPU *pGpu, 110 KernelGraphics *pKernelGraphics, 111 ENGDESCRIPTOR engDesc 112 ) 113 { 114 KGRAPHICS_PRIVATE_DATA *pPrivate; 115 NvU32 idx; 116 GR_GLOBALCTX_BUFFER buf; 117 GR_CTX_BUFFER localBuf; 118 119 pKernelGraphics->instance = ENGDESC_FIELD(engDesc, _INST); 120 121 pPrivate = portMemAllocNonPaged(sizeof(*pPrivate)); 122 if (pPrivate == NULL) 123 return NV_ERR_NO_MEMORY; 124 portMemSet(pPrivate, 0, sizeof(*pPrivate)); 125 pKernelGraphics->pPrivate = pPrivate; 126 127 // All local ctx buffers default to uncached FB preferred 128 FOR_EACH_IN_ENUM(GR_CTX_BUFFER, localBuf) 129 { 130 pKernelGraphics->ctxAttr[localBuf].pAllocList = ADDRLIST_FBMEM_PREFERRED; 131 pKernelGraphics->ctxAttr[localBuf].cpuAttr = NV_MEMORY_UNCACHED; 132 } 133 FOR_EACH_IN_ENUM_END; 134 135 // Process instloc overrides 136 { 137 struct 138 { 139 GR_CTX_BUFFER buf; 140 NvU32 override; 141 } instlocOverrides[] = 142 { 143 { GR_CTX_BUFFER_MAIN, DRF_VAL(_REG_STR_RM, _INST_LOC, _GRCTX, pGpu->instLocOverrides) }, 144 { GR_CTX_BUFFER_PATCH, DRF_VAL(_REG_STR_RM, _INST_LOC_2, _CTX_PATCH, pGpu->instLocOverrides2) }, 145 { GR_CTX_BUFFER_ZCULL, DRF_VAL(_REG_STR_RM, _INST_LOC_2, _ZCULLCTX, pGpu->instLocOverrides2) }, 146 { GR_CTX_BUFFER_PM, DRF_VAL(_REG_STR_RM, _INST_LOC_2, _PMCTX, pGpu->instLocOverrides2) }, 147 { GR_CTX_BUFFER_PREEMPT, DRF_VAL(_REG_STR_RM, _INST_LOC_3, _PREEMPT_BUFFER, pGpu->instLocOverrides3) }, 148 { GR_CTX_BUFFER_BETA_CB, DRF_VAL(_REG_STR_RM, _INST_LOC_3, _GFXP_BETACB_BUFFER, pGpu->instLocOverrides3) }, 149 { GR_CTX_BUFFER_PAGEPOOL, DRF_VAL(_REG_STR_RM, _INST_LOC_3, _GFXP_PAGEPOOL_BUFFER, pGpu->instLocOverrides3) }, 150 { GR_CTX_BUFFER_SPILL, DRF_VAL(_REG_STR_RM, _INST_LOC_3, _GFXP_SPILL_BUFFER, pGpu->instLocOverrides3) }, 151 { GR_CTX_BUFFER_RTV_CB, DRF_VAL(_REG_STR_RM, _INST_LOC_3, _GFXP_RTVCB_BUFFER, pGpu->instLocOverrides3) }, 152 { GR_CTX_BUFFER_SETUP, DRF_VAL(_REG_STR_RM, _INST_LOC_4, _GFXP_SETUP_BUFFER, pGpu->instLocOverrides4) } 153 }; 154 155 for (idx = 0; idx < NV_ARRAY_ELEMENTS(instlocOverrides); ++idx) 156 { 157 memdescOverrideInstLocList(instlocOverrides[idx].override, 158 NV_ENUM_TO_STRING(GR_CTX_BUFFER, instlocOverrides[idx].buf), 159 &pKernelGraphics->ctxAttr[instlocOverrides[idx].buf].pAllocList, 160 &pKernelGraphics->ctxAttr[instlocOverrides[idx].buf].cpuAttr); 161 } 162 } 163 164 // Most global ctx buffers default to uncached FB preferred 165 FOR_EACH_IN_ENUM(GR_GLOBALCTX_BUFFER, buf) 166 { 167 pKernelGraphics->globalCtxBuffersInfo.globalCtxAttr[buf].pAllocList = ADDRLIST_FBMEM_PREFERRED; 168 pKernelGraphics->globalCtxBuffersInfo.globalCtxAttr[buf].cpuAttr = NV_MEMORY_UNCACHED; 169 } 170 FOR_EACH_IN_ENUM_END; 171 172 // FECS event buffer defaults to cached SYSMEM 173 pKernelGraphics->globalCtxBuffersInfo.globalCtxAttr[GR_GLOBALCTX_BUFFER_FECS_EVENT].pAllocList = ADDRLIST_SYSMEM_ONLY; 174 pKernelGraphics->globalCtxBuffersInfo.globalCtxAttr[GR_GLOBALCTX_BUFFER_FECS_EVENT].cpuAttr = NV_MEMORY_CACHED; 175 176 // Process instloc overrides 177 { 178 struct 179 { 180 GR_GLOBALCTX_BUFFER buf; 181 NvU32 override; 182 } instlocOverrides[] = 183 { 184 { GR_GLOBALCTX_BUFFER_FECS_EVENT, DRF_VAL(_REG_STR_RM, _INST_LOC_4, _FECS_EVENT_BUF, pGpu->instLocOverrides4) }, 185 { GR_GLOBALCTX_BUFFER_ATTRIBUTE_CB, DRF_VAL(_REG_STR_RM, _INST_LOC_2, _ATTR_CB, pGpu->instLocOverrides2) }, 186 { GR_GLOBALCTX_BUFFER_BUNDLE_CB, DRF_VAL(_REG_STR_RM, _INST_LOC_2, _BUNDLE_CB, pGpu->instLocOverrides2) }, 187 { GR_GLOBALCTX_BUFFER_PAGEPOOL, DRF_VAL(_REG_STR_RM, _INST_LOC_2, _PAGEPOOL, pGpu->instLocOverrides2) }, 188 { GR_GLOBALCTX_BUFFER_PRIV_ACCESS_MAP, DRF_VAL(_REG_STR_RM, _INST_LOC_3, _PRIV_ACCESS_MAP, pGpu->instLocOverrides3) }, 189 { GR_GLOBALCTX_BUFFER_UNRESTRICTED_PRIV_ACCESS_MAP, DRF_VAL(_REG_STR_RM, _INST_LOC_3, _PRIV_ACCESS_MAP, pGpu->instLocOverrides3) }, 190 { GR_GLOBAL_BUFFER_GLOBAL_PRIV_ACCESS_MAP, DRF_VAL(_REG_STR_RM, _INST_LOC_3, _PRIV_ACCESS_MAP, pGpu->instLocOverrides3) }, 191 { GR_GLOBALCTX_BUFFER_RTV_CB, DRF_VAL(_REG_STR_RM, _INST_LOC_3, _RTVCB_BUFFER, pGpu->instLocOverrides3) } 192 }; 193 194 for (idx = 0; idx < NV_ARRAY_ELEMENTS(instlocOverrides); ++idx) 195 { 196 memdescOverrideInstLocList(instlocOverrides[idx].override, 197 NV_ENUM_TO_STRING(GR_GLOBALCTX_BUFFER, instlocOverrides[idx].buf), 198 &pKernelGraphics->globalCtxBuffersInfo.globalCtxAttr[instlocOverrides[idx].buf].pAllocList, 199 &pKernelGraphics->globalCtxBuffersInfo.globalCtxAttr[instlocOverrides[idx].buf].cpuAttr); 200 } 201 } 202 203 // Copy final global buffer attributes for local versions 204 FOR_EACH_IN_ENUM(GR_GLOBALCTX_BUFFER, buf) 205 { 206 // Host RM cannot allocate system memory on behalf of the VF RM, so force FB. 207 pKernelGraphics->globalCtxBuffersInfo.vfGlobalCtxAttr[buf].pAllocList = ADDRLIST_FBMEM_ONLY; 208 pKernelGraphics->globalCtxBuffersInfo.vfGlobalCtxAttr[buf].cpuAttr = NV_MEMORY_UNCACHED; 209 210 // Local context allocation 211 pKernelGraphics->globalCtxBuffersInfo.localCtxAttr[buf] = pKernelGraphics->globalCtxBuffersInfo.globalCtxAttr[buf]; 212 } 213 FOR_EACH_IN_ENUM_END; 214 215 // 216 // Default context buffers to non size aligned. The attribute buffer is 217 // required to be mapped with an offset naturally aligned to the size. 218 // 219 for (idx = 0; idx < GR_GLOBALCTX_BUFFER_COUNT; idx++) 220 pKernelGraphics->globalCtxBuffersInfo.bSizeAligned[idx] = NV_FALSE; 221 pKernelGraphics->globalCtxBuffersInfo.bSizeAligned[GR_GLOBALCTX_BUFFER_ATTRIBUTE_CB] = NV_TRUE; 222 223 NV_ASSERT_OK_OR_RETURN(fecsCtxswLoggingInit(pGpu, pKernelGraphics, &pKernelGraphics->pFecsTraceInfo)); 224 225 _kgraphicsInitRegistryOverrides(pGpu, pKernelGraphics); 226 return NV_OK; 227 } 228 229 void 230 kgraphicsDestruct_IMPL 231 ( 232 KernelGraphics *pKernelGraphics 233 ) 234 { 235 OBJGPU *pGpu = ENG_GET_GPU(pKernelGraphics); 236 237 fecsCtxswLoggingTeardown(pGpu, pKernelGraphics); 238 pKernelGraphics->pFecsTraceInfo = NULL; 239 kgraphicsInvalidateStaticInfo(pGpu, pKernelGraphics); 240 241 portMemFree(pKernelGraphics->pPrivate); 242 pKernelGraphics->pPrivate = NULL; 243 } 244 245 NV_STATUS 246 kgraphicsStateInitLocked_IMPL 247 ( 248 OBJGPU *pGpu, 249 KernelGraphics *pKernelGraphics 250 ) 251 { 252 NvU32 nGlobalCtx = 1; 253 NvU32 numClasses; 254 255 NV_ASSERT_OK_OR_RETURN(gpuGetClassList(pGpu, &numClasses, NULL, ENG_GR(pKernelGraphics->instance))); 256 257 // 258 // Number of supported class can be zero when Graphics engine is disabled, but we still 259 // need those classes in ClassDB for KernelGraphics engine operation, thus here we are adding 260 // the ENG_GR(X) supported classes back to ClassDB 261 // 262 if (numClasses == 0) 263 { 264 PGPU_ENGINE_ORDER pEngineOrder = &pGpu->engineOrder; 265 const CLASSDESCRIPTOR *pClassDesc = &pEngineOrder->pClassDescriptors[0]; 266 NvU32 i; 267 NvU32 classNum; 268 269 for (i = 0; i < pEngineOrder->numClassDescriptors; i++) 270 { 271 classNum = pClassDesc[i].externalClassId; 272 if (classNum == (NvU32)~0) 273 continue; 274 275 if (ENG_GR(pKernelGraphics->instance) == pClassDesc[i].engDesc) 276 { 277 NV_PRINTF(LEVEL_INFO, "Adding class ID 0x%x to ClassDB\n", classNum); 278 NV_ASSERT_OK_OR_RETURN( 279 gpuAddClassToClassDBByEngTagClassId(pGpu, ENG_GR(pKernelGraphics->instance), classNum)); 280 } 281 } 282 } 283 284 // 285 // Allocate guest context db array 286 // 287 if (gpuIsSriovEnabled(pGpu)) 288 { 289 nGlobalCtx = VMMU_MAX_GFID; 290 } 291 292 pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers = portMemAllocNonPaged(sizeof(*pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers) * nGlobalCtx); 293 if (pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers == NULL) 294 { 295 return NV_ERR_NO_MEMORY; 296 } 297 portMemSet(pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers, 0, 298 sizeof(*pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers) * nGlobalCtx); 299 300 if (pKernelGraphics->instance == 0) 301 { 302 // 303 // GSP_CLIENT creates the golden context channel GR post load. However, 304 // if PMA scrubber is enabled, a scrubber channel must be constructed 305 // first as a part of Fifo post load. Hence, add the golden channel 306 // creation as a fifo post-scheduling-enablement callback. 307 // 308 NV_ASSERT_OK_OR_RETURN( 309 kfifoAddSchedulingHandler(pGpu, GPU_GET_KERNEL_FIFO(pGpu), 310 _kgraphicsPostSchedulingEnableHandler, 311 (void *)((NvUPtr)(pKernelGraphics->instance)), 312 NULL, NULL)); 313 } 314 315 return NV_OK; 316 } 317 318 NV_STATUS 319 kgraphicsStateUnload_IMPL 320 ( 321 OBJGPU *pGpu, 322 KernelGraphics *pKernelGraphics, 323 NvU32 flags 324 325 ) 326 { 327 if (pKernelGraphics->instance != 0) 328 return NV_OK; 329 330 kfifoRemoveSchedulingHandler(pGpu, GPU_GET_KERNEL_FIFO(pGpu), 331 _kgraphicsPostSchedulingEnableHandler, 332 (void *)((NvUPtr)(pKernelGraphics->instance)), 333 NULL, NULL); 334 335 return NV_OK; 336 } 337 338 NV_STATUS 339 kgraphicsStateLoad_IMPL 340 ( 341 OBJGPU *pGpu, 342 KernelGraphics *pKernelGraphics, 343 NvU32 flags 344 ) 345 { 346 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 347 348 if (fecsGetCtxswLogConsumerCount(pGpu, pKernelGraphicsManager) > 0) 349 { 350 fecsBufferMap(pGpu, pKernelGraphics); 351 fecsBufferReset(pGpu, pKernelGraphics); 352 } 353 354 return NV_OK; 355 } 356 357 NV_STATUS 358 kgraphicsStatePreUnload_IMPL 359 ( 360 OBJGPU *pGpu, 361 KernelGraphics *pKernelGraphics, 362 NvU32 flags 363 ) 364 { 365 366 fecsBufferUnmap(pGpu, pKernelGraphics); 367 368 // Release global buffers used as part of the gr context, when not in S/R 369 if (!(flags & GPU_STATE_FLAGS_PRESERVING)) 370 kgraphicsFreeGlobalCtxBuffers(pGpu, pKernelGraphics, GPU_GFID_PF); 371 372 return NV_OK; 373 } 374 375 void 376 kgraphicsStateDestroy_IMPL 377 ( 378 OBJGPU *pGpu, 379 KernelGraphics *pKernelGraphics 380 ) 381 { 382 fecsBufferTeardown(pGpu, pKernelGraphics); 383 384 portMemFree(pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers); 385 pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers = NULL; 386 } 387 388 NvBool kgraphicsIsPresent_IMPL 389 ( 390 OBJGPU *pGpu, 391 KernelGraphics *pKernelGraphics 392 ) 393 { 394 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 395 NvU32 unused; 396 397 if (IsDFPGA(pGpu)) 398 return NV_FALSE; 399 400 if (IS_MODS_AMODEL(pGpu)) 401 return NV_TRUE; 402 403 return kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo, 404 ENGINE_INFO_TYPE_RM_ENGINE_TYPE, (NvU32)RM_ENGINE_TYPE_GR(pKernelGraphics->instance), 405 ENGINE_INFO_TYPE_INVALID, &unused) == NV_OK; 406 } 407 408 NV_STATUS 409 kgraphicsStatePostLoad_IMPL 410 ( 411 OBJGPU *pGpu, 412 KernelGraphics *pKernelGraphics, 413 NvU32 flags 414 ) 415 { 416 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 417 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, kgraphicsLoadStaticInfo(pGpu, pKernelGraphics, KMIGMGR_SWIZZID_INVALID)); 418 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 419 420 if ((!IS_VIRTUAL(pGpu)) && 421 (pKernelGraphicsStaticInfo != NULL) && 422 (pKernelGraphicsStaticInfo->pContextBuffersInfo != NULL) && 423 (kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, GPU_GFID_PF)->memDesc[GR_GLOBALCTX_BUFFER_FECS_EVENT] == NULL)) 424 { 425 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 426 kgraphicsAllocGlobalCtxBuffers_HAL(pGpu, pKernelGraphics, GPU_GFID_PF)); 427 } 428 429 return NV_OK; 430 } 431 432 /*! 433 * @brief Create a golden image channel after Fifo post load 434 * Instead of lazily waiting until first client request, we proactively create a 435 * golden channel here. 436 */ 437 static NV_STATUS 438 _kgraphicsPostSchedulingEnableHandler 439 ( 440 OBJGPU *pGpu, 441 void *pGrIndex 442 ) 443 { 444 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 445 KernelGraphics *pKernelGraphics = GPU_GET_KERNEL_GRAPHICS(pGpu, ((NvU32)(NvUPtr)pGrIndex)); 446 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 447 448 // Nothing to do for non-GSPCLIENT 449 if (!IS_GSP_CLIENT(pGpu)) 450 return NV_OK; 451 452 // Defer golden context channel creation to GPU instance configuration 453 if (IS_MIG_ENABLED(pGpu)) 454 return NV_OK; 455 456 // Skip for MIG engines with 0 GPCs 457 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 458 if (pKernelGraphicsStaticInfo->floorsweepingMasks.gpcMask == 0x0) 459 return NV_OK; 460 461 if (memmgrIsPmaInitialized(pMemoryManager)) 462 { 463 Heap *pHeap = GPU_GET_HEAP(pGpu); 464 NvU32 pmaConfig = PMA_QUERY_SCRUB_ENABLED | PMA_QUERY_SCRUB_VALID; 465 466 NV_ASSERT_OK_OR_RETURN(pmaQueryConfigs(&pHeap->pmaObject, &pmaConfig)); 467 468 // 469 // Scrubber is also constructed from the same Fifo post scheduling 470 // enable callback queue. This check enforces the dependency that 471 // scrubber must be initialized first 472 // 473 if ((pmaConfig & PMA_QUERY_SCRUB_ENABLED) && 474 !(pmaConfig & PMA_QUERY_SCRUB_VALID)) 475 { 476 return NV_WARN_MORE_PROCESSING_REQUIRED; 477 } 478 } 479 480 return kgraphicsCreateGoldenImageChannel(pGpu, pKernelGraphics); 481 } 482 483 void 484 kgraphicsInvalidateStaticInfo_IMPL 485 ( 486 OBJGPU *pGpu, 487 KernelGraphics *pKernelGraphics 488 ) 489 { 490 portMemFree(pKernelGraphics->pPrivate->staticInfo.pGrInfo); 491 pKernelGraphics->pPrivate->staticInfo.pGrInfo = NULL; 492 493 portMemFree(pKernelGraphics->pPrivate->staticInfo.pPpcMasks); 494 pKernelGraphics->pPrivate->staticInfo.pPpcMasks = NULL; 495 496 portMemFree(pKernelGraphics->pPrivate->staticInfo.pZcullInfo); 497 pKernelGraphics->pPrivate->staticInfo.pZcullInfo = NULL; 498 499 portMemFree(pKernelGraphics->pPrivate->staticInfo.pRopInfo); 500 pKernelGraphics->pPrivate->staticInfo.pRopInfo = NULL; 501 502 portMemFree(pKernelGraphics->pPrivate->staticInfo.pContextBuffersInfo); 503 pKernelGraphics->pPrivate->staticInfo.pContextBuffersInfo = NULL; 504 505 portMemFree(pKernelGraphics->pPrivate->staticInfo.pSmIssueRateModifier); 506 pKernelGraphics->pPrivate->staticInfo.pSmIssueRateModifier = NULL; 507 508 portMemFree(pKernelGraphics->pPrivate->staticInfo.pFecsTraceDefines); 509 pKernelGraphics->pPrivate->staticInfo.pFecsTraceDefines = NULL; 510 511 portMemSet(&pKernelGraphics->pPrivate->staticInfo, 0, sizeof(pKernelGraphics->pPrivate->staticInfo)); 512 pKernelGraphics->pPrivate->bInitialized = NV_FALSE; 513 } 514 515 const KGRAPHICS_STATIC_INFO * 516 kgraphicsGetStaticInfo_IMPL 517 ( 518 OBJGPU *pGpu, 519 KernelGraphics *pKernelGraphics 520 ) 521 { 522 KGRAPHICS_PRIVATE_DATA *pPrivate = pKernelGraphics->pPrivate; 523 return ((pPrivate != NULL) && pPrivate->bInitialized) ? &pPrivate->staticInfo : NULL; 524 } 525 526 static NV_STATUS 527 _kgraphicsInternalClientAlloc 528 ( 529 OBJGPU *pGpu, 530 KernelGraphics *pKernelGraphics, 531 NvU32 swizzId, 532 NvHandle *phClient, 533 NvHandle *phDevice, 534 NvHandle *phSubdevice 535 ) 536 { 537 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); //FIXME = GPU_GET_PHYSICAL_RMAPI(pGpu); 538 NvU32 grIdx = pKernelGraphics->instance; 539 540 NV_ASSERT_OR_RETURN(phClient != NULL, NV_ERR_INVALID_ARGUMENT); 541 NV_ASSERT_OR_RETURN(phDevice != NULL, NV_ERR_INVALID_ARGUMENT); 542 NV_ASSERT_OR_RETURN(phSubdevice != NULL, NV_ERR_INVALID_ARGUMENT); 543 544 if (IS_MIG_IN_USE(pGpu)) 545 { 546 NvHandle hSubscription; 547 548 // Delay initialization to GPU instance configuration 549 if (swizzId == KMIGMGR_SWIZZID_INVALID) 550 return NV_ERR_NOT_READY; 551 552 // With MIG enabled, we need to use a client subscribed to the correct GPU instance. 553 NV_ASSERT_OK_OR_RETURN( 554 rmapiutilAllocClientAndDeviceHandles(pRmApi, pGpu, phClient, phDevice, phSubdevice)); 555 556 { 557 NVC637_ALLOCATION_PARAMETERS params; 558 NV_ASSERT_OK( 559 serverutilGenResourceHandle(*phClient, &hSubscription)); 560 portMemSet(¶ms, 0, sizeof(params)); 561 params.swizzId = swizzId; 562 NV_ASSERT_OK( 563 pRmApi->AllocWithHandle(pRmApi, *phClient, *phSubdevice, hSubscription, AMPERE_SMC_PARTITION_REF, ¶ms, sizeof(params))); 564 } 565 566 } 567 else if (grIdx != 0) 568 { 569 // Static data is only defined for GR0 in legacy mode 570 return NV_ERR_NOT_READY; 571 } 572 else 573 { 574 NV_ASSERT_OK_OR_RETURN( 575 rmapiutilAllocClientAndDeviceHandles(pRmApi, pGpu, phClient, phDevice, phSubdevice)); 576 } 577 578 return NV_OK; 579 } 580 581 /*! 582 * @brief Initialize static data that isn't collected right away 583 */ 584 NV_STATUS 585 kgraphicsInitializeDeferredStaticData_IMPL 586 ( 587 OBJGPU *pGpu, 588 KernelGraphics *pKernelGraphics, 589 NvHandle hClient, 590 NvHandle hSubdevice 591 ) 592 { 593 NV2080_CTRL_INTERNAL_STATIC_GR_GET_CONTEXT_BUFFERS_INFO_PARAMS *pParams; 594 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 595 KGRAPHICS_PRIVATE_DATA *pPrivate = pKernelGraphics->pPrivate; 596 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 597 NvU32 grIdx = pKernelGraphics->instance; 598 NV_STATUS status = NV_OK; 599 NvBool bInternalClientAllocated = NV_FALSE; 600 NvU32 gfid; 601 NvBool bCallingContextPlugin; 602 603 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &gfid)); 604 NV_ASSERT_OK_OR_RETURN(vgpuIsCallingContextPlugin(pGpu, &bCallingContextPlugin)); 605 606 if (bCallingContextPlugin) 607 { 608 gfid = GPU_GFID_PF; 609 } 610 611 // 612 // Most of GR is stub'd so context related things are not needed in AMODEL. 613 // But this function can be called in some MODS test, so return OK directly 614 // to avoid failing the test. 615 // 616 if (IS_MODS_AMODEL(pGpu)) 617 return NV_OK; 618 619 // Not ready 620 if (!pPrivate->bInitialized) 621 return NV_OK; 622 623 // Already done 624 if (pPrivate->staticInfo.pContextBuffersInfo != NULL) 625 return NV_OK; 626 627 // In progress 628 if (pKernelGraphics->bCollectingDeferredStaticData) 629 return NV_OK; 630 631 if (hClient == NV01_NULL_OBJECT) 632 { 633 NvHandle hDevice = NV01_NULL_OBJECT; 634 NvU32 swizzId = KMIGMGR_SWIZZID_INVALID; 635 636 if (IS_MIG_IN_USE(pGpu)) 637 { 638 MIG_INSTANCE_REF ref; 639 640 NV_ASSERT_OK_OR_RETURN( 641 kmigmgrGetMIGReferenceFromEngineType(pGpu, pKernelMIGManager, 642 RM_ENGINE_TYPE_GR(pKernelGraphics->instance), &ref)); 643 644 swizzId = ref.pKernelMIGGpuInstance->swizzId; 645 } 646 647 status = _kgraphicsInternalClientAlloc(pGpu, pKernelGraphics, swizzId, &hClient, &hDevice, &hSubdevice); 648 if (status == NV_ERR_NOT_READY) 649 { 650 return NV_OK; 651 } 652 NV_ASSERT_OR_RETURN(status == NV_OK, status); 653 NV_ASSERT_OR_RETURN(hClient != NV01_NULL_OBJECT, NV_ERR_INVALID_STATE); 654 bInternalClientAllocated = NV_TRUE; 655 } 656 657 // Prevent recursion when deferred static data collection is ON 658 pKernelGraphics->bCollectingDeferredStaticData = NV_TRUE; 659 660 if (IS_MIG_IN_USE(pGpu)) 661 { 662 MIG_INSTANCE_REF ref; 663 RM_ENGINE_TYPE localRmEngineType; 664 Subdevice *pSubdevice; 665 RsClient *pClient; 666 667 NV_CHECK_OK_OR_GOTO( 668 status, 669 LEVEL_ERROR, 670 serverGetClientUnderLock(&g_resServ, hClient, &pClient), 671 cleanup); 672 673 NV_CHECK_OK_OR_GOTO( 674 status, 675 LEVEL_ERROR, 676 subdeviceGetByHandle(pClient, hSubdevice, &pSubdevice), 677 cleanup); 678 679 // Physical RM will fill with local indices, so localize the index 680 NV_CHECK_OK_OR_GOTO( 681 status, 682 LEVEL_ERROR, 683 kmigmgrGetInstanceRefFromDevice(pGpu, pKernelMIGManager, 684 GPU_RES_GET_DEVICE(pSubdevice), 685 &ref), 686 cleanup); 687 NV_CHECK_OK_OR_GOTO( 688 status, 689 LEVEL_ERROR, 690 kmigmgrGetGlobalToLocalEngineType(pGpu, pKernelMIGManager, ref, 691 RM_ENGINE_TYPE_GR(grIdx), 692 &localRmEngineType), 693 cleanup); 694 grIdx = RM_ENGINE_TYPE_GR_IDX(localRmEngineType); 695 } 696 697 pParams = portMemAllocNonPaged(sizeof(*pParams)); 698 if (pParams == NULL) 699 { 700 return NV_ERR_NO_MEMORY; 701 } 702 portMemSet(pParams, 0, sizeof(*pParams)); 703 NV_CHECK_OK_OR_GOTO( 704 status, 705 LEVEL_ERROR, 706 pRmApi->Control(pRmApi, 707 hClient, 708 hSubdevice, 709 NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_CONTEXT_BUFFERS_INFO, 710 pParams, 711 sizeof(*pParams)), 712 cleanup_context_buffers_info); 713 714 pPrivate->staticInfo.pContextBuffersInfo = 715 portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pContextBuffersInfo)); 716 717 if (pPrivate->staticInfo.pContextBuffersInfo == NULL) 718 { 719 status = NV_ERR_NO_MEMORY; 720 goto cleanup_context_buffers_info; 721 } 722 723 portMemCopy(pPrivate->staticInfo.pContextBuffersInfo, 724 sizeof(*pPrivate->staticInfo.pContextBuffersInfo), 725 &pParams->engineContextBuffersInfo[grIdx], 726 sizeof(pParams->engineContextBuffersInfo[grIdx])); 727 728 cleanup_context_buffers_info: 729 portMemFree(pParams); 730 731 // 732 // We are not cleaning pContextBuffersInfo here since it's used after this 733 // function so has to be cleaned after used. 734 // 735 736 cleanup: 737 if (bInternalClientAllocated) 738 { 739 pRmApi->Free(pRmApi, hClient, hClient); 740 } 741 742 pKernelGraphics->bCollectingDeferredStaticData = NV_FALSE; 743 744 if (status == NV_OK) 745 { 746 // 747 // Allocate Ctx Buffers that are global to all channels if they have yet 748 // to be allocated. We delay them until now to save memory when runs 749 // are done without using graphics contexts! 750 // 751 if (!pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers[gfid].bAllocated && 752 (!gpuIsClientRmAllocatedCtxBufferEnabled(pGpu) || 753 (gpuIsSriovEnabled(pGpu) && IS_GFID_PF(gfid)))) 754 { 755 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 756 kgraphicsAllocGrGlobalCtxBuffers_HAL(pGpu, pKernelGraphics, gfid, NULL)); 757 } 758 } 759 760 return status; 761 } 762 763 NV_STATUS 764 kgraphicsLoadStaticInfo_VF 765 ( 766 OBJGPU *pGpu, 767 KernelGraphics *pKernelGraphics, 768 NvU32 swizzId 769 ) 770 { 771 KGRAPHICS_PRIVATE_DATA *pPrivate = pKernelGraphics->pPrivate; 772 VGPU_STATIC_INFO *pVSI = GPU_GET_STATIC_INFO(pGpu); 773 NvU32 grIdx = pKernelGraphics->instance; 774 NVOS_STATUS status = NV_OK; 775 776 NV_ASSERT_OR_RETURN(pVSI != NULL, NV_ERR_INVALID_STATE); 777 NV_ASSERT_OR_RETURN(pPrivate != NULL, NV_ERR_INVALID_STATE); 778 779 if (pPrivate->bInitialized) 780 return status; 781 782 if (IS_MIG_IN_USE(pGpu)) 783 { 784 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 785 786 // 787 // Delay initialization to GPU instance configuration, unless MODS is using 788 // legacy VGPU mode, in which case the guest never receives a 789 // configuration call 790 // 791 if ((swizzId == KMIGMGR_SWIZZID_INVALID) && !kmigmgrUseLegacyVgpuPolicy(pGpu, pKernelMIGManager)) 792 return status; 793 794 pPrivate->staticInfo.pGrInfo = portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pGrInfo)); 795 if (pPrivate->staticInfo.pGrInfo == NULL) 796 { 797 status = NV_ERR_NO_MEMORY; 798 goto cleanup; 799 } 800 801 portMemCopy(pPrivate->staticInfo.pGrInfo->infoList, 802 NV0080_CTRL_GR_INFO_MAX_SIZE * sizeof(*pPrivate->staticInfo.pGrInfo->infoList), 803 pVSI->grInfoParams.engineInfo[grIdx].infoList, 804 NV0080_CTRL_GR_INFO_MAX_SIZE * sizeof(*pVSI->grInfoParams.engineInfo[grIdx].infoList)); 805 806 portMemCopy(&pPrivate->staticInfo.globalSmOrder, sizeof(pPrivate->staticInfo.globalSmOrder), 807 &pVSI->globalSmOrder.globalSmOrder[grIdx], sizeof(pVSI->globalSmOrder.globalSmOrder[grIdx])); 808 809 // grCaps are the same for all GR and can be copied from VGPU static info 810 portMemCopy(pPrivate->staticInfo.grCaps.capsTbl, sizeof(pPrivate->staticInfo.grCaps.capsTbl), 811 pVSI->grCapsBits, sizeof(pVSI->grCapsBits)); 812 813 // Initialize PDB properties synchronized with physical RM 814 pPrivate->staticInfo.pdbTable.bPerSubCtxheaderSupported = pVSI->bPerSubCtxheaderSupported; 815 kgraphicsSetPerSubcontextContextHeaderSupported(pGpu, pKernelGraphics, pPrivate->staticInfo.pdbTable.bPerSubCtxheaderSupported); 816 817 pPrivate->staticInfo.pSmIssueRateModifier = 818 portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pSmIssueRateModifier)); 819 if (pPrivate->staticInfo.pSmIssueRateModifier == NULL) 820 { 821 status = NV_ERR_NO_MEMORY; 822 goto cleanup; 823 } 824 825 portMemCopy(pPrivate->staticInfo.pSmIssueRateModifier, sizeof(*pPrivate->staticInfo.pSmIssueRateModifier), 826 &pVSI->smIssueRateModifier.smIssueRateModifier[grIdx], sizeof(pVSI->smIssueRateModifier.smIssueRateModifier[grIdx])); 827 828 pPrivate->staticInfo.pPpcMasks = portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pPpcMasks)); 829 if (pPrivate->staticInfo.pPpcMasks == NULL) 830 { 831 status = NV_ERR_NO_MEMORY; 832 goto cleanup; 833 } 834 835 portMemCopy(pPrivate->staticInfo.pPpcMasks, sizeof(*pPrivate->staticInfo.pPpcMasks), 836 &pVSI->ppcMaskParams.enginePpcMasks[grIdx], sizeof(pVSI->ppcMaskParams.enginePpcMasks[grIdx])); 837 838 portMemCopy(&pPrivate->staticInfo.floorsweepingMasks, sizeof(pPrivate->staticInfo.floorsweepingMasks), 839 &pVSI->floorsweepMaskParams.floorsweepingMasks[grIdx], sizeof(pVSI->floorsweepMaskParams.floorsweepingMasks[grIdx])); 840 841 pPrivate->staticInfo.pContextBuffersInfo = 842 portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pContextBuffersInfo)); 843 844 if (pPrivate->staticInfo.pContextBuffersInfo == NULL) 845 { 846 status = NV_ERR_NO_MEMORY; 847 goto cleanup; 848 } 849 850 portMemCopy(pPrivate->staticInfo.pContextBuffersInfo, 851 sizeof(*pPrivate->staticInfo.pContextBuffersInfo), 852 &pVSI->ctxBuffInfo.engineContextBuffersInfo[grIdx], 853 sizeof(pVSI->ctxBuffInfo.engineContextBuffersInfo[grIdx])); 854 855 pPrivate->staticInfo.fecsRecordSize.fecsRecordSize = pVSI->fecsRecordSize.fecsRecordSize[grIdx].fecsRecordSize; 856 857 pPrivate->staticInfo.pFecsTraceDefines = 858 portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pFecsTraceDefines)); 859 860 if (pPrivate->staticInfo.pFecsTraceDefines == NULL) 861 { 862 status = NV_ERR_NO_MEMORY; 863 goto cleanup; 864 } 865 866 portMemCopy(pPrivate->staticInfo.pFecsTraceDefines, 867 sizeof(*pPrivate->staticInfo.pFecsTraceDefines), 868 &pVSI->fecsTraceDefines.fecsTraceDefines[grIdx], 869 sizeof(pVSI->fecsTraceDefines.fecsTraceDefines[grIdx])); 870 871 portMemCopy(&pPrivate->staticInfo.pdbTable, sizeof(pPrivate->staticInfo.pdbTable), 872 &pVSI->pdbTableParams.pdbTable[grIdx], sizeof(pVSI->pdbTableParams.pdbTable[grIdx])); 873 } 874 else if (grIdx == 0) 875 { 876 portMemCopy(pPrivate->staticInfo.grCaps.capsTbl, sizeof(pPrivate->staticInfo.grCaps.capsTbl), 877 pVSI->grCapsBits, sizeof(pVSI->grCapsBits)); 878 879 pPrivate->staticInfo.pGrInfo = portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pGrInfo)); 880 if (pPrivate->staticInfo.pGrInfo == NULL) 881 { 882 status = NV_ERR_NO_MEMORY; 883 goto cleanup; 884 } 885 886 portMemCopy(pPrivate->staticInfo.pGrInfo->infoList, 887 NV0080_CTRL_GR_INFO_MAX_SIZE * sizeof(*pPrivate->staticInfo.pGrInfo->infoList), 888 pVSI->grInfoParams.engineInfo[grIdx].infoList, 889 NV0080_CTRL_GR_INFO_MAX_SIZE * sizeof(*pVSI->grInfoParams.engineInfo[grIdx].infoList)); 890 891 // Initialize PDB properties synchronized with physical RM 892 pPrivate->staticInfo.pdbTable.bPerSubCtxheaderSupported = pVSI->bPerSubCtxheaderSupported; 893 kgraphicsSetPerSubcontextContextHeaderSupported(pGpu, pKernelGraphics, pPrivate->staticInfo.pdbTable.bPerSubCtxheaderSupported); 894 895 portMemCopy(&pPrivate->staticInfo.globalSmOrder, sizeof(pPrivate->staticInfo.globalSmOrder), 896 &pVSI->globalSmOrder.globalSmOrder[grIdx], sizeof(pVSI->globalSmOrder.globalSmOrder[grIdx])); 897 898 pPrivate->staticInfo.pSmIssueRateModifier = 899 portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pSmIssueRateModifier)); 900 if (pPrivate->staticInfo.pSmIssueRateModifier == NULL) 901 { 902 status = NV_ERR_NO_MEMORY; 903 goto cleanup; 904 } 905 906 portMemCopy(pPrivate->staticInfo.pSmIssueRateModifier, sizeof(*pPrivate->staticInfo.pSmIssueRateModifier), 907 &pVSI->smIssueRateModifier.smIssueRateModifier[grIdx], sizeof(pVSI->smIssueRateModifier.smIssueRateModifier[grIdx])); 908 909 pPrivate->staticInfo.pPpcMasks = portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pPpcMasks)); 910 if (pPrivate->staticInfo.pPpcMasks == NULL) 911 { 912 status = NV_ERR_NO_MEMORY; 913 goto cleanup; 914 } 915 916 portMemCopy(pPrivate->staticInfo.pPpcMasks, sizeof(*pPrivate->staticInfo.pPpcMasks), 917 &pVSI->ppcMaskParams.enginePpcMasks[grIdx], sizeof(pVSI->ppcMaskParams.enginePpcMasks[grIdx])); 918 919 pPrivate->staticInfo.pContextBuffersInfo = 920 portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pContextBuffersInfo)); 921 922 if (pPrivate->staticInfo.pContextBuffersInfo == NULL) 923 { 924 status = NV_ERR_NO_MEMORY; 925 goto cleanup; 926 } 927 928 portMemCopy(pPrivate->staticInfo.pContextBuffersInfo, 929 sizeof(*pPrivate->staticInfo.pContextBuffersInfo), 930 &pVSI->ctxBuffInfo.engineContextBuffersInfo[grIdx], 931 sizeof(pVSI->ctxBuffInfo.engineContextBuffersInfo[grIdx])); 932 933 portMemCopy(&pPrivate->staticInfo.floorsweepingMasks, sizeof(pPrivate->staticInfo.floorsweepingMasks), 934 &pVSI->floorsweepMaskParams.floorsweepingMasks[grIdx], sizeof(pVSI->floorsweepMaskParams.floorsweepingMasks[grIdx])); 935 936 pPrivate->staticInfo.pRopInfo = portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pRopInfo)); 937 if (pPrivate->staticInfo.pRopInfo == NULL) 938 { 939 status = NV_ERR_NO_MEMORY; 940 goto cleanup; 941 } 942 943 portMemCopy(pPrivate->staticInfo.pRopInfo, sizeof(*pPrivate->staticInfo.pRopInfo), 944 &pVSI->ropInfoParams.engineRopInfo[grIdx], sizeof(pVSI->ropInfoParams.engineRopInfo[grIdx])); 945 946 pPrivate->staticInfo.pZcullInfo = portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pZcullInfo)); 947 if (pPrivate->staticInfo.pZcullInfo == NULL) 948 { 949 status = NV_ERR_NO_MEMORY; 950 goto cleanup; 951 } 952 953 portMemCopy(pPrivate->staticInfo.pZcullInfo, sizeof(*pPrivate->staticInfo.pZcullInfo), 954 &pVSI->zcullInfoParams.engineZcullInfo[grIdx], sizeof(pVSI->zcullInfoParams.engineZcullInfo[grIdx])); 955 956 pPrivate->staticInfo.fecsRecordSize.fecsRecordSize = pVSI->fecsRecordSize.fecsRecordSize[grIdx].fecsRecordSize; 957 958 pPrivate->staticInfo.pFecsTraceDefines = 959 portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pFecsTraceDefines)); 960 if (pPrivate->staticInfo.pFecsTraceDefines == NULL) 961 { 962 status = NV_ERR_NO_MEMORY; 963 goto cleanup; 964 } 965 966 portMemCopy(pPrivate->staticInfo.pFecsTraceDefines, 967 sizeof(*pPrivate->staticInfo.pFecsTraceDefines), 968 &pVSI->fecsTraceDefines.fecsTraceDefines[grIdx], 969 sizeof(pVSI->fecsTraceDefines.fecsTraceDefines[grIdx])); 970 971 portMemCopy(&pPrivate->staticInfo.pdbTable, sizeof(pPrivate->staticInfo.pdbTable), 972 &pVSI->pdbTableParams.pdbTable[grIdx], sizeof(pVSI->pdbTableParams.pdbTable[grIdx])); 973 } 974 else 975 { 976 // if MIG disabled, only GR0 static data needs to be published 977 return status; 978 } 979 980 if (status == NV_OK) 981 { 982 // Publish static configuration 983 pPrivate->bInitialized = NV_TRUE; 984 } 985 986 if (!IS_MIG_IN_USE(pGpu) && (grIdx == 0)) 987 { 988 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 989 990 // Cache legacy GR mask info (i.e. GR0 with MIG disabled) to pKernelGraphicsManager->legacyFsMaskState 991 kgrmgrSetLegacyKgraphicsStaticInfo(pGpu, pKernelGraphicsManager, pKernelGraphics); 992 } 993 994 cleanup : 995 996 if (status != NV_OK) 997 { 998 portMemFree(pPrivate->staticInfo.pGrInfo); 999 pPrivate->staticInfo.pGrInfo = NULL; 1000 1001 portMemFree(pPrivate->staticInfo.pPpcMasks); 1002 pPrivate->staticInfo.pPpcMasks = NULL; 1003 1004 portMemFree(pPrivate->staticInfo.pZcullInfo); 1005 pPrivate->staticInfo.pZcullInfo = NULL; 1006 1007 portMemFree(pPrivate->staticInfo.pRopInfo); 1008 pPrivate->staticInfo.pRopInfo = NULL; 1009 1010 portMemFree(pPrivate->staticInfo.pContextBuffersInfo); 1011 pPrivate->staticInfo.pContextBuffersInfo = NULL; 1012 1013 portMemFree(pPrivate->staticInfo.pSmIssueRateModifier); 1014 pPrivate->staticInfo.pSmIssueRateModifier = NULL; 1015 1016 portMemFree(pPrivate->staticInfo.pFecsTraceDefines); 1017 pPrivate->staticInfo.pFecsTraceDefines = NULL; 1018 } 1019 1020 return status; 1021 } 1022 1023 NV_STATUS 1024 kgraphicsLoadStaticInfo_KERNEL 1025 ( 1026 OBJGPU *pGpu, 1027 KernelGraphics *pKernelGraphics, 1028 NvU32 swizzId 1029 ) 1030 { 1031 KGRAPHICS_PRIVATE_DATA *pPrivate = pKernelGraphics->pPrivate; 1032 NvHandle hClient = NV01_NULL_OBJECT; 1033 NvHandle hDevice; 1034 NvHandle hSubdevice; 1035 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 1036 NvU32 grIdx = pKernelGraphics->instance; 1037 NV_STATUS status = NV_OK; 1038 NvBool bBcState = gpumgrGetBcEnabledStatus(pGpu); 1039 union 1040 { 1041 NV2080_CTRL_INTERNAL_STATIC_GR_GET_CAPS_PARAMS caps; 1042 NV2080_CTRL_INTERNAL_STATIC_GR_GET_INFO_PARAMS info; 1043 NV2080_CTRL_INTERNAL_STATIC_GR_GET_GLOBAL_SM_ORDER_PARAMS globalSmOrder; 1044 NV2080_CTRL_INTERNAL_STATIC_GR_GET_FLOORSWEEPING_MASKS_PARAMS floorsweepingMasks; 1045 NV2080_CTRL_INTERNAL_STATIC_GR_GET_PPC_MASKS_PARAMS ppcMasks; 1046 NV2080_CTRL_INTERNAL_STATIC_GR_GET_ZCULL_INFO_PARAMS zcullInfo; 1047 NV2080_CTRL_INTERNAL_STATIC_GR_GET_ROP_INFO_PARAMS ropInfo; 1048 NV2080_CTRL_INTERNAL_STATIC_GR_GET_SM_ISSUE_RATE_MODIFIER_PARAMS smIssueRateModifier; 1049 NV2080_CTRL_INTERNAL_STATIC_GR_GET_FECS_RECORD_SIZE_PARAMS fecsRecordSize; 1050 NV2080_CTRL_INTERNAL_STATIC_GR_GET_FECS_TRACE_DEFINES_PARAMS fecsTraceDefines; 1051 NV2080_CTRL_INTERNAL_STATIC_GR_GET_PDB_PROPERTIES_PARAMS pdbProperties; 1052 } *pParams = NULL; 1053 1054 NV_ASSERT_OR_RETURN(pPrivate != NULL, NV_ERR_INVALID_STATE); 1055 1056 if (pPrivate->bInitialized) 1057 return NV_OK; 1058 1059 status = _kgraphicsInternalClientAlloc(pGpu, pKernelGraphics, swizzId, &hClient, &hDevice, &hSubdevice); 1060 1061 if (status == NV_ERR_NOT_READY) 1062 { 1063 return NV_OK; 1064 } 1065 NV_ASSERT_OR_RETURN(status == NV_OK, status); 1066 NV_ASSERT_OR_RETURN(hClient != NV01_NULL_OBJECT, NV_ERR_INVALID_STATE); 1067 1068 pParams = portMemAllocNonPaged(sizeof(*pParams)); 1069 if (pParams == NULL) 1070 { 1071 status = NV_ERR_NO_MEMORY; 1072 goto cleanup; 1073 } 1074 1075 if (IS_MIG_IN_USE(pGpu)) 1076 { 1077 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 1078 MIG_INSTANCE_REF ref; 1079 RM_ENGINE_TYPE localRmEngineType; 1080 RsClient *pClient; 1081 Device *pDevice; 1082 1083 NV_CHECK_OK_OR_GOTO( 1084 status, 1085 LEVEL_ERROR, 1086 serverGetClientUnderLock(&g_resServ, hClient, &pClient), 1087 cleanup); 1088 1089 NV_CHECK_OK_OR_GOTO( 1090 status, 1091 LEVEL_ERROR, 1092 deviceGetByHandle(pClient, hDevice, &pDevice), 1093 cleanup); 1094 1095 // Physical RM will fill with local indices, so localize the index 1096 NV_CHECK_OK_OR_GOTO( 1097 status, 1098 LEVEL_ERROR, 1099 kmigmgrGetInstanceRefFromDevice(pGpu, pKernelMIGManager, pDevice, &ref), 1100 cleanup); 1101 NV_CHECK_OK_OR_GOTO( 1102 status, 1103 LEVEL_ERROR, 1104 kmigmgrGetGlobalToLocalEngineType(pGpu, pKernelMIGManager, ref, 1105 RM_ENGINE_TYPE_GR(grIdx), 1106 &localRmEngineType), 1107 cleanup); 1108 grIdx = RM_ENGINE_TYPE_GR_IDX(localRmEngineType); 1109 } 1110 1111 // GR Caps 1112 portMemSet(pParams, 0, sizeof(*pParams)); 1113 NV_CHECK_OK_OR_GOTO( 1114 status, 1115 LEVEL_ERROR, 1116 pRmApi->Control(pRmApi, 1117 hClient, 1118 hSubdevice, 1119 NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_CAPS, 1120 pParams, 1121 sizeof(pParams->caps)), 1122 cleanup); 1123 1124 portMemCopy(&pPrivate->staticInfo.grCaps, sizeof(pPrivate->staticInfo.grCaps), 1125 &pParams->caps.engineCaps[grIdx], sizeof(pParams->caps.engineCaps[grIdx])); 1126 1127 // GR Info 1128 portMemSet(pParams, 0, sizeof(*pParams)); 1129 status = pRmApi->Control(pRmApi, 1130 hClient, 1131 hSubdevice, 1132 NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_INFO, 1133 pParams, 1134 sizeof(pParams->info)); 1135 1136 if (status == NV_OK) 1137 { 1138 pPrivate->staticInfo.pGrInfo = portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pGrInfo)); 1139 if (pPrivate->staticInfo.pGrInfo == NULL) 1140 { 1141 status = NV_ERR_NO_MEMORY; 1142 goto cleanup; 1143 } 1144 1145 portMemCopy(pPrivate->staticInfo.pGrInfo->infoList, 1146 NV0080_CTRL_GR_INFO_MAX_SIZE * sizeof(*pPrivate->staticInfo.pGrInfo->infoList), 1147 pParams->info.engineInfo[grIdx].infoList, 1148 NV0080_CTRL_GR_INFO_MAX_SIZE * sizeof(*pParams->info.engineInfo[grIdx].infoList)); 1149 1150 } 1151 1152 // Floorsweeping masks 1153 portMemSet(pParams, 0, sizeof(*pParams)); 1154 NV_CHECK_OK_OR_GOTO( 1155 status, 1156 LEVEL_ERROR, 1157 pRmApi->Control(pRmApi, 1158 hClient, 1159 hSubdevice, 1160 NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_FLOORSWEEPING_MASKS, 1161 pParams, 1162 sizeof(pParams->floorsweepingMasks)), 1163 cleanup); 1164 1165 portMemCopy(&pPrivate->staticInfo.floorsweepingMasks, sizeof(pPrivate->staticInfo.floorsweepingMasks), 1166 &pParams->floorsweepingMasks.floorsweepingMasks[grIdx], sizeof(pParams->floorsweepingMasks.floorsweepingMasks[grIdx])); 1167 1168 // 1169 // Most of GR is stub'd in AMODEL. However, some tests still need the CAPS/INFO data, 1170 // so we still need to generate CAPS/INFO data for AMODEL 1171 // 1172 if (IS_MODS_AMODEL(pGpu)) 1173 { 1174 pPrivate->bInitialized = NV_TRUE; 1175 if (!IS_MIG_IN_USE(pGpu) && (grIdx == 0)) 1176 { 1177 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 1178 1179 // Cache legacy GR mask info (i.e. GR0 with MIG disabled) to pKernelGraphicsManager->legacyFsMaskState 1180 kgrmgrSetLegacyKgraphicsStaticInfo(pGpu, pKernelGraphicsManager, pKernelGraphics); 1181 } 1182 status = NV_OK; 1183 goto cleanup; 1184 } 1185 1186 // GR Global SM Order 1187 portMemSet(pParams, 0, sizeof(*pParams)); 1188 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, 1189 pRmApi->Control(pRmApi, 1190 hClient, 1191 hSubdevice, 1192 NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_GLOBAL_SM_ORDER, 1193 pParams, 1194 sizeof(pParams->globalSmOrder)), 1195 cleanup); 1196 1197 portMemCopy(&pPrivate->staticInfo.globalSmOrder, sizeof(pPrivate->staticInfo.globalSmOrder), 1198 &pParams->globalSmOrder.globalSmOrder[grIdx], sizeof(pParams->globalSmOrder.globalSmOrder[grIdx])); 1199 1200 // PPC Mask 1201 portMemSet(pParams, 0, sizeof(*pParams)); 1202 status = pRmApi->Control(pRmApi, 1203 hClient, 1204 hSubdevice, 1205 NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_PPC_MASKS, 1206 pParams, 1207 sizeof(pParams->ppcMasks)); 1208 1209 if (status == NV_OK) 1210 { 1211 pPrivate->staticInfo.pPpcMasks = portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pPpcMasks)); 1212 if (pPrivate->staticInfo.pPpcMasks == NULL) 1213 { 1214 status = NV_ERR_NO_MEMORY; 1215 goto cleanup; 1216 } 1217 1218 portMemCopy(pPrivate->staticInfo.pPpcMasks, sizeof(*pPrivate->staticInfo.pPpcMasks), 1219 &pParams->ppcMasks.enginePpcMasks[grIdx], sizeof(pParams->ppcMasks.enginePpcMasks[grIdx])); 1220 } 1221 else if (status == NV_ERR_NOT_SUPPORTED) 1222 { 1223 // 1224 // Some chips don't support this call, so just keep the pPpcMasks 1225 // pointer as NULL, but don't return error 1226 // 1227 status = NV_OK; 1228 } 1229 1230 portMemSet(pParams, 0, sizeof(*pParams)); 1231 status = pRmApi->Control(pRmApi, 1232 hClient, 1233 hSubdevice, 1234 NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_ZCULL_INFO, 1235 pParams, 1236 sizeof(pParams->zcullInfo)); 1237 1238 if (status == NV_OK) 1239 { 1240 pPrivate->staticInfo.pZcullInfo = portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pZcullInfo)); 1241 if (pPrivate->staticInfo.pZcullInfo == NULL) 1242 { 1243 status = NV_ERR_NO_MEMORY; 1244 goto cleanup; 1245 } 1246 1247 portMemCopy(pPrivate->staticInfo.pZcullInfo, sizeof(*pPrivate->staticInfo.pZcullInfo), 1248 &pParams->zcullInfo.engineZcullInfo[grIdx], sizeof(pParams->zcullInfo.engineZcullInfo[grIdx])); 1249 } 1250 else if (status == NV_ERR_NOT_SUPPORTED) 1251 { 1252 // It's expected to get this error when MIG is enabled, thus don't return error 1253 if (IS_MIG_ENABLED(pGpu)) 1254 { 1255 status = NV_OK; 1256 } 1257 } 1258 1259 // ROP Info 1260 portMemSet(pParams, 0, sizeof(*pParams)); 1261 status = pRmApi->Control(pRmApi, 1262 hClient, 1263 hSubdevice, 1264 NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_ROP_INFO, 1265 pParams, 1266 sizeof(pParams->ropInfo)); 1267 1268 if (status == NV_OK) 1269 { 1270 pPrivate->staticInfo.pRopInfo = portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pRopInfo)); 1271 if (pPrivate->staticInfo.pRopInfo == NULL) 1272 { 1273 status = NV_ERR_NO_MEMORY; 1274 goto cleanup; 1275 } 1276 1277 portMemCopy(pPrivate->staticInfo.pRopInfo, sizeof(*pPrivate->staticInfo.pRopInfo), 1278 &pParams->ropInfo.engineRopInfo[grIdx], sizeof(pParams->ropInfo.engineRopInfo[grIdx])); 1279 } 1280 else if (status == NV_ERR_NOT_SUPPORTED) 1281 { 1282 // It's expected to get this error when MIG is enabled, thus don't return error 1283 if (IS_MIG_ENABLED(pGpu)) 1284 { 1285 status = NV_OK; 1286 } 1287 } 1288 1289 // SM Issue Rate Modifier 1290 portMemSet(pParams, 0, sizeof(*pParams)); 1291 status = pRmApi->Control(pRmApi, 1292 hClient, 1293 hSubdevice, 1294 NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_SM_ISSUE_RATE_MODIFIER, 1295 pParams, 1296 sizeof(pParams->smIssueRateModifier)); 1297 1298 if (status == NV_OK) 1299 { 1300 pPrivate->staticInfo.pSmIssueRateModifier = portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pSmIssueRateModifier)); 1301 if (pPrivate->staticInfo.pSmIssueRateModifier == NULL) 1302 { 1303 status = NV_ERR_NO_MEMORY; 1304 goto cleanup; 1305 } 1306 1307 portMemCopy(pPrivate->staticInfo.pSmIssueRateModifier, sizeof(*pPrivate->staticInfo.pSmIssueRateModifier), 1308 &pParams->smIssueRateModifier.smIssueRateModifier[grIdx], sizeof(pParams->smIssueRateModifier.smIssueRateModifier[grIdx])); 1309 } 1310 else if (status == NV_ERR_NOT_SUPPORTED) 1311 { 1312 status = NV_OK; 1313 } 1314 1315 // FECS Record Size 1316 portMemSet(pParams, 0, sizeof(*pParams)); 1317 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, 1318 pRmApi->Control(pRmApi, 1319 hClient, 1320 hSubdevice, 1321 NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_FECS_RECORD_SIZE, 1322 pParams, 1323 sizeof(pParams->fecsRecordSize)), 1324 cleanup); 1325 1326 pPrivate->staticInfo.fecsRecordSize.fecsRecordSize = pParams->fecsRecordSize.fecsRecordSize[grIdx].fecsRecordSize; 1327 1328 // FECS Trace Defines 1329 portMemSet(pParams, 0, sizeof(*pParams)); 1330 status = pRmApi->Control(pRmApi, 1331 hClient, 1332 hSubdevice, 1333 NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_FECS_TRACE_DEFINES, 1334 pParams, 1335 sizeof(pParams->fecsTraceDefines)); 1336 if (status == NV_OK) 1337 { 1338 pPrivate->staticInfo.pFecsTraceDefines = portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pFecsTraceDefines)); 1339 if (pPrivate->staticInfo.pFecsTraceDefines == NULL) 1340 { 1341 status = NV_ERR_NO_MEMORY; 1342 goto cleanup; 1343 } 1344 portMemCopy(pPrivate->staticInfo.pFecsTraceDefines, sizeof(*pPrivate->staticInfo.pFecsTraceDefines), 1345 &pParams->fecsTraceDefines.fecsTraceDefines[grIdx], sizeof(pParams->fecsTraceDefines.fecsTraceDefines[grIdx])); 1346 } 1347 else if (status == NV_ERR_NOT_SUPPORTED) 1348 { 1349 status = NV_OK; 1350 } 1351 1352 // PDB Properties 1353 portMemSet(pParams, 0, sizeof(*pParams)); 1354 NV_CHECK_OK_OR_GOTO( 1355 status, 1356 LEVEL_ERROR, 1357 pRmApi->Control(pRmApi, 1358 hClient, 1359 hSubdevice, 1360 NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_PDB_PROPERTIES, 1361 pParams, 1362 sizeof(pParams->pdbProperties)), 1363 cleanup); 1364 1365 portMemCopy(&pPrivate->staticInfo.pdbTable, sizeof(pPrivate->staticInfo.pdbTable), 1366 &pParams->pdbProperties.pdbTable[grIdx], sizeof(pParams->pdbProperties.pdbTable[grIdx])); 1367 kgraphicsSetPerSubcontextContextHeaderSupported(pGpu, pKernelGraphics, pPrivate->staticInfo.pdbTable.bPerSubCtxheaderSupported); 1368 1369 // Publish static configuration 1370 pPrivate->bInitialized = NV_TRUE; 1371 1372 // The deferred data is ready after MIG is enabled, so no need to defer the initialization 1373 if (IS_MIG_IN_USE(pGpu) || 1374 !kgraphicsShouldDeferContextInit(pGpu, pKernelGraphics)) 1375 { 1376 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, 1377 kgraphicsInitializeDeferredStaticData(pGpu, pKernelGraphics, hClient, hSubdevice), cleanup); 1378 } 1379 1380 if (!IS_MIG_IN_USE(pGpu) && (grIdx == 0)) 1381 { 1382 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 1383 1384 // Cache legacy GR mask info (i.e. GR0 with MIG disabled) to pKernelGraphicsManager->legacyFsMaskState 1385 kgrmgrSetLegacyKgraphicsStaticInfo(pGpu, pKernelGraphicsManager, pKernelGraphics); 1386 } 1387 1388 cleanup: 1389 if (status != NV_OK) 1390 { 1391 // Redact static configuration 1392 pPrivate->bInitialized = NV_FALSE; 1393 1394 portMemFree(pPrivate->staticInfo.pGrInfo); 1395 pPrivate->staticInfo.pGrInfo = NULL; 1396 1397 portMemFree(pPrivate->staticInfo.pPpcMasks); 1398 pPrivate->staticInfo.pPpcMasks = NULL; 1399 1400 portMemFree(pPrivate->staticInfo.pZcullInfo); 1401 pPrivate->staticInfo.pZcullInfo = NULL; 1402 1403 portMemFree(pPrivate->staticInfo.pRopInfo); 1404 pPrivate->staticInfo.pRopInfo = NULL; 1405 1406 portMemFree(pPrivate->staticInfo.pContextBuffersInfo); 1407 pPrivate->staticInfo.pContextBuffersInfo = NULL; 1408 1409 portMemFree(pPrivate->staticInfo.pSmIssueRateModifier); 1410 pPrivate->staticInfo.pSmIssueRateModifier = NULL; 1411 1412 portMemFree(pPrivate->staticInfo.pFecsTraceDefines); 1413 pPrivate->staticInfo.pFecsTraceDefines = NULL; 1414 } 1415 1416 // If we had to subscribe specifically, free the hclient we allocated 1417 if (hClient != NV01_NULL_OBJECT) 1418 pRmApi->Free(pRmApi, hClient, hClient); 1419 1420 if (gpumgrGetBcEnabledStatus(pGpu) != bBcState) 1421 { 1422 // Corrupted broadcast state! 1423 NV_ASSERT(gpumgrGetBcEnabledStatus(pGpu) != bBcState); 1424 gpumgrSetBcEnabledStatus(pGpu, bBcState); 1425 } 1426 1427 portMemFree(pParams); 1428 1429 return status; 1430 } 1431 1432 /*! Return if GFX is supported for the given kernel graphics engine */ 1433 NvBool 1434 kgraphicsIsGFXSupported_IMPL 1435 ( 1436 OBJGPU *pGpu, 1437 KernelGraphics *pKernelGraphics 1438 ) 1439 { 1440 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 1441 NvU32 gfxCapabilites; 1442 1443 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_FALSE); 1444 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pGrInfo != NULL, NV_FALSE); 1445 1446 gfxCapabilites = pKernelGraphicsStaticInfo->pGrInfo->infoList[NV2080_CTRL_GR_INFO_INDEX_GFX_CAPABILITIES].data; 1447 1448 return (FLD_TEST_DRF(2080_CTRL_GR, _INFO_GFX_CAPABILITIES, _2D, _TRUE, gfxCapabilites) && 1449 FLD_TEST_DRF(2080_CTRL_GR, _INFO_GFX_CAPABILITIES, _3D, _TRUE, gfxCapabilites) && 1450 FLD_TEST_DRF(2080_CTRL_GR, _INFO_GFX_CAPABILITIES, _I2M, _TRUE, gfxCapabilites)); 1451 } 1452 1453 /*! Retrieve ctxbufpool parameters for given local ctx buffer */ 1454 const CTX_BUF_INFO * 1455 kgraphicsGetCtxBufferInfo_IMPL 1456 ( 1457 OBJGPU *pGpu, 1458 KernelGraphics *pKernelGraphics, 1459 GR_CTX_BUFFER buf 1460 ) 1461 { 1462 NV_ASSERT_OR_RETURN(NV_ENUM_IS(GR_CTX_BUFFER, buf), NULL); 1463 return &pKernelGraphics->maxCtxBufSize[buf]; 1464 } 1465 1466 /*! Set ctxbufpool parameters for given local ctx buffer */ 1467 void 1468 kgraphicsSetCtxBufferInfo_IMPL 1469 ( 1470 OBJGPU *pGpu, 1471 KernelGraphics *pKernelGraphics, 1472 GR_CTX_BUFFER buf, 1473 NvU64 size, 1474 NvU64 align, 1475 RM_ATTR_PAGE_SIZE attr, 1476 NvBool bContiguous 1477 ) 1478 { 1479 CTX_BUF_INFO *pInfo; 1480 NV_ASSERT_OR_RETURN_VOID(NV_ENUM_IS(GR_CTX_BUFFER, buf)); 1481 1482 pInfo = &pKernelGraphics->maxCtxBufSize[buf]; 1483 pInfo->size = size; 1484 pInfo->align = align; 1485 pInfo->attr = attr; 1486 pInfo->bContig = bContiguous; 1487 } 1488 1489 /*! Clear ctxbufpool parameters for all local ctx buffers */ 1490 void 1491 kgraphicsClearCtxBufferInfo_IMPL 1492 ( 1493 OBJGPU *pGpu, 1494 KernelGraphics *pKernelGraphics 1495 ) 1496 { 1497 portMemSet(pKernelGraphics->maxCtxBufSize, 0, sizeof(pKernelGraphics->maxCtxBufSize)); 1498 } 1499 1500 /*! Initialize ctxbufpool for this engine */ 1501 NV_STATUS 1502 kgraphicsInitCtxBufPool_IMPL 1503 ( 1504 OBJGPU *pGpu, 1505 KernelGraphics *pKernelGraphics, 1506 Heap *pHeap 1507 ) 1508 { 1509 return ctxBufPoolInit(pGpu, pHeap, &pKernelGraphics->pCtxBufPool); 1510 } 1511 1512 /*! Retrieve ctxbufpool for this engine */ 1513 CTX_BUF_POOL_INFO * 1514 kgraphicsGetCtxBufPool_IMPL 1515 ( 1516 OBJGPU *pGpu, 1517 KernelGraphics *pKernelGraphics 1518 ) 1519 { 1520 return pKernelGraphics->pCtxBufPool; 1521 } 1522 1523 /*! destroy ctxbufpool for this engine */ 1524 void 1525 kgraphicsDestroyCtxBufPool_IMPL 1526 ( 1527 OBJGPU *pGpu, 1528 KernelGraphics *pKernelGraphics 1529 ) 1530 { 1531 if (pKernelGraphics->pCtxBufPool == NULL) 1532 return; 1533 1534 ctxBufPoolRelease(pKernelGraphics->pCtxBufPool); 1535 ctxBufPoolDestroy(&pKernelGraphics->pCtxBufPool); 1536 pKernelGraphics->pCtxBufPool = NULL; 1537 } 1538 1539 /*! Get the global ctx buffers for the given GFID */ 1540 GR_GLOBALCTX_BUFFERS * 1541 kgraphicsGetGlobalCtxBuffers_IMPL 1542 ( 1543 OBJGPU *pGpu, 1544 KernelGraphics *pKernelGraphics, 1545 NvU32 gfid 1546 ) 1547 { 1548 if (pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers == NULL) 1549 return NULL; 1550 return &pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers[gfid]; 1551 } 1552 1553 /*! Should this global ctx buffer be mapped as size aligned? */ 1554 NvBool 1555 kgraphicsIsGlobalCtxBufferSizeAligned_IMPL 1556 ( 1557 OBJGPU *pGpu, 1558 KernelGraphics *pKernelGraphics, 1559 GR_GLOBALCTX_BUFFER buf 1560 ) 1561 { 1562 NV_ASSERT_OR_RETURN(NV_ENUM_IS(GR_GLOBALCTX_BUFFER, buf), NV_FALSE); 1563 return pKernelGraphics->globalCtxBuffersInfo.bSizeAligned[buf]; 1564 } 1565 1566 /*! Get ctx buf attr for global priv access map */ 1567 const GR_BUFFER_ATTR * 1568 kgraphicsGetGlobalPrivAccessMapAttr_IMPL 1569 ( 1570 OBJGPU *pGpu, 1571 KernelGraphics *pKernelGraphics 1572 ) 1573 { 1574 return &pKernelGraphics->globalCtxBuffersInfo.globalCtxAttr[GR_GLOBAL_BUFFER_GLOBAL_PRIV_ACCESS_MAP]; 1575 } 1576 1577 /* 1578 * @brief Get Main context buffer size 1579 * 1580 * @param[in] pGpu 1581 * @param[in] pKernelGraphics 1582 * @param[in] bIncludeSubctxHdrs If subctx headers should be included in size calculation 1583 * @param[out] pSize Main Context buffer size 1584 */ 1585 NV_STATUS 1586 kgraphicsGetMainCtxBufferSize_IMPL 1587 ( 1588 OBJGPU *pGpu, 1589 KernelGraphics *pKernelGraphics, 1590 NvBool bIncludeSubctxHdrs, 1591 NvU32 *pSize 1592 ) 1593 { 1594 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 1595 NvU32 size; 1596 1597 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 1598 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pGrInfo != NULL, NV_ERR_INVALID_STATE); 1599 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pContextBuffersInfo != NULL, NV_ERR_INVALID_STATE); 1600 1601 size = pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS].size; 1602 1603 // Allocate space for per VEID headers in the golden context buffer. 1604 if (bIncludeSubctxHdrs && 1605 kgraphicsIsPerSubcontextContextHeaderSupported(pGpu, pKernelGraphics)) 1606 { 1607 // TODO size this down to max per-engine subcontexts 1608 NvU32 maxSubctx = pKernelGraphicsStaticInfo->pGrInfo->infoList[NV2080_CTRL_GR_INFO_INDEX_MAX_SUBCONTEXT_COUNT].data; 1609 1610 // The header needs to start at a 4 KB aligned address 1611 size = RM_ALIGN_UP(size, RM_PAGE_SIZE); 1612 1613 // The header is only 256 bytes; but needs to be 4KB aligned. 1614 size += (RM_PAGE_SIZE * maxSubctx); 1615 } 1616 1617 *pSize = size; 1618 return NV_OK; 1619 } 1620 1621 NV_STATUS 1622 kgraphicsAllocKgraphicsBuffers_KERNEL 1623 ( 1624 OBJGPU *pGpu, 1625 KernelGraphics *pKernelGraphics, 1626 KernelGraphicsContext *pKernelGraphicsContext, 1627 KernelChannel *pKernelChannel 1628 ) 1629 { 1630 NvU32 gfid; 1631 OBJGVASPACE *pGVAS; 1632 1633 NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_CHANNEL); 1634 pGVAS = dynamicCast(pKernelChannel->pVAS, OBJGVASPACE); 1635 gfid = kchannelGetGfid(pKernelChannel); 1636 1637 // Deferred static info is necessary at this point for FECS buffer allocation. Skip for guest RM 1638 if (!IS_VIRTUAL(pGpu)) 1639 { 1640 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 1641 kgraphicsInitializeDeferredStaticData(pGpu, pKernelGraphics, NV01_NULL_OBJECT, NV01_NULL_OBJECT)); 1642 } 1643 1644 // 1645 // Allocate global context buffers for this gfid, if they haven't been 1646 // already 1647 // 1648 if (kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid)->memDesc[GR_GLOBALCTX_BUFFER_FECS_EVENT] == NULL) 1649 { 1650 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 1651 kgraphicsAllocGlobalCtxBuffers_HAL(pGpu, pKernelGraphics, gfid)); 1652 } 1653 1654 if (kgraphicsIsCtxswLoggingSupported(pGpu, pKernelGraphics) && 1655 (kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid)->memDesc[GR_GLOBALCTX_BUFFER_FECS_EVENT] != NULL)) 1656 { 1657 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 1658 1659 if (!gvaspaceIsExternallyOwned(pGVAS) && !IS_VIRTUAL_WITHOUT_SRIOV(pGpu)) 1660 { 1661 // 1662 // We map CTXSW buffer on each object allocation including compute object 1663 // Other global context buffers are not mapped during compute object alloc with subcontexts since 1664 // those are GR-only buffers and not needed for compute-only contexts 1665 // 1666 _kgraphicsMapGlobalCtxBuffer(pGpu, pKernelGraphics, gfid, pKernelChannel->pVAS, pKernelGraphicsContext, 1667 GR_GLOBALCTX_BUFFER_FECS_EVENT, NV_FALSE); 1668 } 1669 1670 if (!fecsBufferIsMapped(pGpu, pKernelGraphics)) 1671 { 1672 fecsBufferMap(pGpu, pKernelGraphics); 1673 } 1674 1675 if (fecsGetCtxswLogConsumerCount(pGpu, pKernelGraphicsManager) > 0) 1676 fecsBufferReset(pGpu, pKernelGraphics); 1677 } 1678 1679 return NV_OK; 1680 } 1681 1682 static NV_STATUS 1683 _kgraphicsMapGlobalCtxBuffer 1684 ( 1685 OBJGPU *pGpu, 1686 KernelGraphics *pKernelGraphics, 1687 NvU32 gfid, 1688 OBJVASPACE *pVAS, 1689 KernelGraphicsContext *pKernelGraphicsContext, 1690 GR_GLOBALCTX_BUFFER buffId, 1691 NvBool bIsReadOnly 1692 ) 1693 { 1694 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 1695 NV_STATUS status = NV_OK; 1696 NvU64 vaddr = 0; 1697 MEMORY_DESCRIPTOR *pMemDesc; 1698 NvBool bSizeAligned; 1699 1700 NV_ASSERT_OK_OR_RETURN( 1701 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast)); 1702 1703 bSizeAligned = pKernelGraphics->globalCtxBuffersInfo.bSizeAligned[buffId]; 1704 pMemDesc = pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers[gfid].memDesc[buffId]; 1705 1706 if (pMemDesc == NULL) 1707 { 1708 NvU32 buffSize; 1709 NvU32 fifoEngineId; 1710 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 1711 1712 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 1713 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 1714 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pContextBuffersInfo != NULL, NV_ERR_INVALID_STATE); 1715 1716 NV_ASSERT_OK_OR_RETURN(kgrctxGlobalCtxBufferToFifoEngineId(buffId, &fifoEngineId)); 1717 buffSize = pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[fifoEngineId].size; 1718 if (buffSize == 0) 1719 { 1720 NV_PRINTF(LEVEL_INFO, 1721 "Could not map %s Buffer as buffer is not supported\n", 1722 NV_ENUM_TO_STRING(GR_GLOBALCTX_BUFFER, buffId)); 1723 return NV_OK; 1724 } 1725 else 1726 { 1727 NV_PRINTF(LEVEL_ERROR, 1728 "Could not map %s Buffer, no memory allocated for it!\n", 1729 NV_ENUM_TO_STRING(GR_GLOBALCTX_BUFFER, buffId)); 1730 return NV_ERR_INVALID_ARGUMENT; 1731 } 1732 } 1733 1734 // Unconditionally call map for refcounting 1735 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 1736 kgraphicsMapCtxBuffer(pGpu, pKernelGraphics, pMemDesc, pVAS, 1737 &pKernelGraphicsContextUnicast->globalCtxBufferVaList[buffId], 1738 bSizeAligned, 1739 bIsReadOnly)); 1740 1741 NV_ASSERT_OK(vaListFindVa(&pKernelGraphicsContextUnicast->globalCtxBufferVaList[buffId], pVAS, &vaddr)); 1742 1743 NV_PRINTF(LEVEL_INFO, 1744 "GPU:%d %s Buffer PA @ 0x%llx VA @ 0x%llx of Size 0x%llx\n", 1745 pGpu->gpuInstance, NV_ENUM_TO_STRING(GR_GLOBALCTX_BUFFER, buffId), 1746 memdescGetPhysAddr(memdescGetMemDescFromGpu(pMemDesc, pGpu), AT_GPU, 0), 1747 vaddr, pMemDesc->Size); 1748 1749 return status; 1750 } 1751 1752 /*! 1753 * @brief Map a GR ctx buffer 1754 */ 1755 NV_STATUS 1756 kgraphicsMapCtxBuffer_IMPL 1757 ( 1758 OBJGPU *pGpu, 1759 KernelGraphics *pKernelGraphics, 1760 MEMORY_DESCRIPTOR *pMemDesc, 1761 OBJVASPACE *pVAS, 1762 VA_LIST *pVaList, 1763 NvBool bAlignSize, 1764 NvBool bIsReadOnly 1765 ) 1766 { 1767 NV_STATUS status = NV_OK; 1768 NvU64 vaddr = 0; 1769 OBJGVASPACE *pGVAS; 1770 1771 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 1772 1773 NV_PRINTF(LEVEL_INFO, "gpu:%d isBC=%d\n", pGpu->gpuInstance, 1774 gpumgrGetBcEnabledStatus(pGpu)); 1775 1776 pGVAS = dynamicCast(pVAS, OBJGVASPACE); 1777 NV_ASSERT_OR_RETURN(!gvaspaceIsExternallyOwned(pGVAS), NV_ERR_INVALID_OPERATION); 1778 1779 status = vaListFindVa(pVaList, pVAS, &vaddr); 1780 if (status == NV_ERR_OBJECT_NOT_FOUND) 1781 { 1782 // create a new subcontext mapping 1783 NvU32 allocFlags = bAlignSize ? DMA_ALLOC_VASPACE_SIZE_ALIGNED : DMA_ALLOC_VASPACE_NONE; 1784 NvU32 updateFlags = bIsReadOnly ? (DMA_UPDATE_VASPACE_FLAGS_READ_ONLY | 1785 DMA_UPDATE_VASPACE_FLAGS_SHADER_READ_ONLY) : DMA_UPDATE_VASPACE_FLAGS_NONE; 1786 1787 if (pGVAS->flags & VASPACE_FLAGS_RESTRICTED_RM_INTERNAL_VALIMITS) 1788 { 1789 allocFlags |= DMA_ALLOC_VASPACE_USE_RM_INTERNAL_VALIMITS; 1790 } 1791 1792 if (kgraphicsIsPerSubcontextContextHeaderSupported(pGpu, pKernelGraphics)) 1793 { 1794 status = dmaMapBuffer_HAL(pGpu, GPU_GET_DMA(pGpu), pVAS, pMemDesc, &vaddr, 1795 allocFlags, updateFlags); 1796 } 1797 else 1798 { 1799 // 1800 // Per subcontext headers not enabled. 1801 // If subcontext is not supported, create a new mapping. 1802 // If subcontext is supported, create an identity mapping to the existing one. 1803 // 1804 1805 if (vaListMapCount(pVaList) == 0) 1806 { 1807 status = dmaMapBuffer_HAL(pGpu, GPU_GET_DMA(pGpu), pVAS, 1808 pMemDesc, 1809 &vaddr, 1810 allocFlags, 1811 updateFlags); 1812 } 1813 else 1814 { 1815 OBJVASPACE *pVas; 1816 NvU32 mapFlags = 0x0; 1817 NvU64 vaddrCached; 1818 1819 FOR_EACH_IN_VADDR_LIST(pVaList, pVas, vaddr) 1820 { 1821 // Find the first virtual address in any VAS 1822 break; 1823 } 1824 FOR_EACH_IN_VADDR_LIST_END(pVaList, pVas, vaddr); 1825 1826 if (bIsReadOnly) 1827 { 1828 mapFlags = FLD_SET_DRF(OS46, _FLAGS, _ACCESS, _READ_ONLY, mapFlags); 1829 mapFlags = FLD_SET_DRF(OS46, _FLAGS, _SHADER_ACCESS, _READ_ONLY, mapFlags); 1830 } 1831 mapFlags = FLD_SET_DRF(OS46, _FLAGS, _DMA_UNICAST_REUSE_ALLOC, _FALSE, mapFlags); 1832 mapFlags = FLD_SET_DRF(OS46, _FLAGS, _DMA_OFFSET_FIXED, _TRUE, mapFlags); 1833 1834 NV_ASSERT(!bAlignSize); // TODO: Add support for size align 1835 vaddrCached = vaddr; 1836 NV_ASSERT_OK_OR_ELSE(status, 1837 dmaAllocMapping_HAL(pGpu, GPU_GET_DMA(pGpu), pVAS, pMemDesc, 1838 &vaddr, 1839 mapFlags, 1840 NULL, 1841 KMIGMGR_SWIZZID_INVALID), 1842 /* do nothing on error, but make sure we overwrite status */;); 1843 NV_ASSERT(vaddr == vaddrCached); 1844 } 1845 } 1846 1847 NV_PRINTF(LEVEL_INFO, "New ctx buffer mapping at VA 0x%llx\n", vaddr); 1848 } 1849 1850 if (status == NV_OK) 1851 NV_ASSERT_OK_OR_RETURN(vaListAddVa(pVaList, pVAS, vaddr)); 1852 1853 return status; 1854 } 1855 1856 /*! 1857 * @brief Unmap a GR ctx buffer 1858 */ 1859 void 1860 kgraphicsUnmapCtxBuffer_IMPL 1861 ( 1862 OBJGPU *pGpu, 1863 KernelGraphics *pKernelGraphics, 1864 OBJVASPACE *pVAS, 1865 VA_LIST *pVaList 1866 ) 1867 { 1868 NV_STATUS status = NV_OK; 1869 NvU64 vaddr = 0; 1870 1871 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 1872 1873 NV_PRINTF(LEVEL_INFO, "gpu:%d isBC=%d\n", pGpu->gpuInstance, 1874 gpumgrGetBcEnabledStatus(pGpu)); 1875 1876 status = vaListFindVa(pVaList, pVAS, &vaddr); 1877 if (status == NV_OK) 1878 { 1879 NV_ASSERT_OK(vaListRemoveVa(pVaList, pVAS)); 1880 1881 status = vaListFindVa(pVaList, pVAS, &vaddr); 1882 1883 NV_ASSERT((NV_OK == status) || (NV_ERR_OBJECT_NOT_FOUND == status)); 1884 if (NV_ERR_OBJECT_NOT_FOUND == status) 1885 { 1886 if (vaListGetManaged(pVaList)) 1887 { 1888 dmaUnmapBuffer_HAL(pGpu, GPU_GET_DMA(pGpu), pVAS, vaddr); 1889 } 1890 1891 NV_PRINTF(LEVEL_INFO, "Freed ctx buffer mapping at VA 0x%llx\n", 1892 vaddr); 1893 } 1894 } 1895 } 1896 1897 /*! 1898 * @brief Get the Class number for a given gr object type 1899 * 1900 * @param[in] pGpu 1901 * @param[in] pKernelGraphics 1902 * @param[in] wantObjectType GR object type to lookup 1903 * @param[out] pClass class number 1904 */ 1905 NV_STATUS 1906 kgraphicsGetClassByType_IMPL 1907 ( 1908 OBJGPU *pGpu, 1909 KernelGraphics *pKernelGraphics, 1910 NvU32 wantObjectType, 1911 NvU32 *pClass 1912 ) 1913 { 1914 NV_STATUS status = NV_OK; 1915 NvU32 objectType; 1916 NvU32 i; 1917 NvU32 numClasses; 1918 NvU32 *pClassesSupported; 1919 1920 *pClass = 0; 1921 1922 if (wantObjectType >= GR_OBJECT_TYPE_INVALID) 1923 { 1924 NV_PRINTF(LEVEL_ERROR, "bad requested object type : %d\n", 1925 wantObjectType); 1926 return NV_ERR_INVALID_ARGUMENT; 1927 } 1928 1929 // find out how many classes of type ENG_GR(0) we have 1930 NV_ASSERT_OK_OR_RETURN( 1931 gpuGetClassList(pGpu, &numClasses, NULL, ENG_GR(pKernelGraphics->instance))); 1932 1933 pClassesSupported = portMemAllocNonPaged(sizeof(NvU32) * numClasses); 1934 NV_CHECK_OR_RETURN(LEVEL_ERROR, pClassesSupported != NULL, NV_ERR_NO_MEMORY); 1935 1936 status = gpuGetClassList(pGpu, &numClasses, pClassesSupported, ENG_GR(pKernelGraphics->instance)); 1937 1938 if (status == NV_OK) 1939 { 1940 for (i = 0; i < numClasses; i++) 1941 { 1942 kgrmgrGetGrObjectType(pClassesSupported[i], &objectType); 1943 1944 NV_PRINTF(LEVEL_INFO, "classNum=0x%08x, type=%d\n", 1945 pClassesSupported[i], objectType); 1946 1947 if (objectType == wantObjectType) 1948 *pClass = pClassesSupported[i]; 1949 } 1950 } 1951 1952 portMemFree(pClassesSupported); 1953 1954 return (*pClass != 0) ? NV_OK : NV_ERR_INVALID_CLASS; 1955 } 1956 1957 /*! 1958 * @brief retrieve the ctx attributes for the given buffer 1959 */ 1960 const GR_BUFFER_ATTR * 1961 kgraphicsGetContextBufferAttr_IMPL 1962 ( 1963 OBJGPU *pGpu, 1964 KernelGraphics *pKernelGraphics, 1965 GR_CTX_BUFFER buf 1966 ) 1967 { 1968 NV_ASSERT_OR_RETURN(NV_ENUM_IS(GR_CTX_BUFFER, buf), NULL); 1969 return &pKernelGraphics->ctxAttr[buf]; 1970 } 1971 1972 /*! 1973 * @brief Creates a VEID0 channel for Golden Image creation 1974 * 1975 * @return NV_OK if channel and golden image created successfully 1976 */ 1977 NV_STATUS 1978 kgraphicsCreateGoldenImageChannel_IMPL 1979 ( 1980 OBJGPU *pGpu, 1981 KernelGraphics *pKernelGraphics 1982 ) 1983 { 1984 NV_STATUS status = NV_OK; 1985 NvHandle hClientId = NV01_NULL_OBJECT; 1986 NvHandle hDeviceId; 1987 NvHandle hSubdeviceId; 1988 NvHandle hVASpace = 0xbaba0042; 1989 NvHandle hPBVirtMemId = 0xbaba0043; 1990 NvHandle hPBPhysMemId = 0xbaba0044; 1991 NvHandle hChannelId = 0xbaba0045; 1992 NvHandle hObj3D = 0xbaba0046; 1993 NvHandle hUserdId = 0xbaba0049; 1994 NvU32 gpFifoEntries = 32; // power-of-2 random choice 1995 NvU64 gpFifoSize = NVA06F_GP_ENTRY__SIZE * gpFifoEntries; 1996 NvU64 chSize = gpFifoSize; 1997 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 1998 RsClient *pClientId; 1999 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 2000 NvBool bNeedMIGWar; 2001 NvBool bBcStatus; 2002 NvBool bClientUserd = IsVOLTAorBetter(pGpu); 2003 NvBool bAcquireLock = NV_FALSE; 2004 NvU32 sliLoopReentrancy; 2005 NV_VASPACE_ALLOCATION_PARAMETERS vaParams; 2006 NV_MEMORY_ALLOCATION_PARAMS memAllocParams; 2007 NV_CHANNEL_ALLOC_PARAMS channelGPFIFOAllocParams; 2008 NvU32 classNum; 2009 MIG_INSTANCE_REF ref; 2010 NvU32 objectType; 2011 NvU32 primarySliSubDeviceInstance; 2012 2013 // XXX This should be removed when broadcast SLI support is deprecated 2014 if (!gpumgrIsParentGPU(pGpu)) 2015 { 2016 return NV_OK; 2017 } 2018 2019 bBcStatus = gpumgrGetBcEnabledStatus(pGpu); 2020 2021 // FIXME these allocations corrupt BC state 2022 NV_ASSERT_OK_OR_RETURN( 2023 rmapiutilAllocClientAndDeviceHandles(pRmApi, pGpu, &hClientId, &hDeviceId, &hSubdeviceId)); 2024 // rmapiutilAllocClientAndDeviceHandles allocates a subdevice object for this subDeviceInstance 2025 primarySliSubDeviceInstance = gpumgrGetSubDeviceInstanceFromGpu(pGpu); 2026 2027 NV_ASSERT_OK_OR_RETURN(serverGetClientUnderLock(&g_resServ, hClientId, &pClientId)); 2028 2029 gpumgrSetBcEnabledStatus(pGpu, NV_TRUE); 2030 2031 // As we have forced here SLI broadcast mode, temporarily reset the reentrancy count 2032 sliLoopReentrancy = gpumgrSLILoopReentrancyPop(pGpu); 2033 2034 bNeedMIGWar = IS_MIG_IN_USE(pGpu); 2035 2036 // Allocate subdevices for secondary GPUs 2037 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY) 2038 { 2039 NvHandle hSecondary; 2040 NV2080_ALLOC_PARAMETERS nv2080AllocParams; 2041 NvU32 thisSubDeviceInstance = gpumgrGetSubDeviceInstanceFromGpu(pGpu); 2042 2043 // Skip if already allocated by rmapiutilAllocClientAndDeviceHandles() 2044 if (thisSubDeviceInstance == primarySliSubDeviceInstance) 2045 SLI_LOOP_CONTINUE; 2046 2047 // Allocate a subDevice 2048 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, 2049 clientGenResourceHandle(pClientId, &hSecondary), 2050 cleanup); 2051 2052 portMemSet(&nv2080AllocParams, 0, sizeof(nv2080AllocParams)); 2053 nv2080AllocParams.subDeviceId = thisSubDeviceInstance; 2054 2055 NV_CHECK_OK(status, LEVEL_SILENT, 2056 pRmApi->AllocWithHandle(pRmApi, 2057 hClientId, 2058 hDeviceId, 2059 hSecondary, 2060 NV20_SUBDEVICE_0, 2061 &nv2080AllocParams, 2062 sizeof(nv2080AllocParams))); 2063 } 2064 SLI_LOOP_END; 2065 2066 if (bNeedMIGWar) 2067 { 2068 NvHandle hPartitionRef = 0xbaba0048; 2069 NvHandle hExecPartitionRef = 0xbaba004a; 2070 NVC637_ALLOCATION_PARAMETERS nvC637AllocParams = {0}; 2071 2072 // Get swizzId for this GR 2073 NV_ASSERT_OK_OR_GOTO(status, 2074 kmigmgrGetMIGReferenceFromEngineType(pGpu, pKernelMIGManager, 2075 RM_ENGINE_TYPE_GR(pKernelGraphics->instance), &ref), 2076 cleanup); 2077 2078 portMemSet(&nvC637AllocParams, 0, sizeof(NVC637_ALLOCATION_PARAMETERS)); 2079 nvC637AllocParams.swizzId = ref.pKernelMIGGpuInstance->swizzId; 2080 2081 // allocate partition reference 2082 NV_ASSERT_OK_OR_GOTO(status, 2083 pRmApi->AllocWithHandle(pRmApi, 2084 hClientId, 2085 hSubdeviceId, 2086 hPartitionRef, 2087 AMPERE_SMC_PARTITION_REF, 2088 &nvC637AllocParams, 2089 sizeof(nvC637AllocParams)), 2090 cleanup); 2091 2092 if (ref.pMIGComputeInstance != NULL) 2093 { 2094 NVC638_ALLOCATION_PARAMETERS nvC638AllocParams = {0}; 2095 nvC638AllocParams.execPartitionId = ref.pMIGComputeInstance->id; 2096 NV_ASSERT_OK_OR_GOTO(status, 2097 pRmApi->AllocWithHandle(pRmApi, 2098 hClientId, 2099 hPartitionRef, 2100 hExecPartitionRef, 2101 AMPERE_SMC_EXEC_PARTITION_REF, 2102 &nvC638AllocParams, 2103 sizeof(nvC638AllocParams)), 2104 cleanup); 2105 } 2106 } 2107 2108 // 2109 // VidHeapControl and vaspace creation calls should happen outside GPU locks 2110 // UVM/CUDA may be holding the GPU locks here and the allocation may subsequently fail 2111 // So explicitly release GPU locks before RmVidHeapControl 2112 // See Bug 1735851-#24 2113 // 2114 rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); 2115 bAcquireLock = NV_TRUE; 2116 pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL); 2117 2118 // Create a new VAspace for channel 2119 portMemSet(&vaParams, 0, sizeof(NV_VASPACE_ALLOCATION_PARAMETERS)); 2120 NV_ASSERT_OK_OR_GOTO(status, 2121 pRmApi->AllocWithHandle(pRmApi, hClientId, hDeviceId, hVASpace, FERMI_VASPACE_A, &vaParams, sizeof(vaParams)), 2122 cleanup); 2123 2124 // Allocate gpfifo entries 2125 portMemSet(&memAllocParams, 0, sizeof(NV_MEMORY_ALLOCATION_PARAMS)); 2126 memAllocParams.owner = HEAP_OWNER_RM_CLIENT_GENERIC; 2127 memAllocParams.type = NVOS32_TYPE_IMAGE; 2128 memAllocParams.size = chSize; 2129 memAllocParams.attr = DRF_DEF(OS32, _ATTR, _LOCATION, _PCI); 2130 memAllocParams.hVASpace = 0; // Physical allocations don't expect vaSpace handles 2131 2132 // 2133 // When APM feature is enabled all RM internal sysmem allocations must 2134 // be in unprotected memory 2135 // When Hopper CC is enabled all RM internal sysmem allocations that 2136 // are required to be accessed from GPU should be in unprotected memory 2137 // Other sysmem allocations that are not required to be accessed from GPU 2138 // must be in protected memory 2139 // 2140 memAllocParams.attr2 |= DRF_DEF(OS32, _ATTR2, _MEMORY_PROTECTION, _UNPROTECTED); 2141 2142 NV_ASSERT_OK_OR_GOTO(status, 2143 pRmApi->AllocWithHandle(pRmApi, hClientId, hDeviceId, hPBPhysMemId, NV01_MEMORY_SYSTEM, &memAllocParams, sizeof(memAllocParams)), 2144 cleanup); 2145 2146 portMemSet(&memAllocParams, 0, sizeof(NV_MEMORY_ALLOCATION_PARAMS)); 2147 memAllocParams.owner = HEAP_OWNER_RM_CLIENT_GENERIC; 2148 memAllocParams.type = NVOS32_TYPE_IMAGE; 2149 memAllocParams.size = chSize; 2150 memAllocParams.attr = DRF_DEF(OS32, _ATTR, _LOCATION, _PCI); 2151 memAllocParams.flags = NVOS32_ALLOC_FLAGS_VIRTUAL; 2152 memAllocParams.hVASpace = hVASpace; // Virtual allocation expect vaSpace handles 2153 // 0 handle = allocations on gpu default vaSpace 2154 2155 NV_ASSERT_OK_OR_GOTO(status, 2156 pRmApi->AllocWithHandle(pRmApi, hClientId, hDeviceId, hPBVirtMemId, NV50_MEMORY_VIRTUAL, &memAllocParams, sizeof(memAllocParams)), 2157 cleanup); 2158 2159 // Allocate Userd 2160 if (bClientUserd) 2161 { 2162 NvU32 userdMemClass = NV01_MEMORY_LOCAL_USER; 2163 NvU32 ctrlSize; 2164 2165 if (gpuIsClassSupported(pGpu, VOLTA_CHANNEL_GPFIFO_A)) 2166 { 2167 ctrlSize = sizeof(Nvc36fControl); 2168 } 2169 else if (gpuIsClassSupported(pGpu, TURING_CHANNEL_GPFIFO_A)) 2170 { 2171 ctrlSize = sizeof(Nvc46fControl); 2172 } 2173 else if (gpuIsClassSupported(pGpu, AMPERE_CHANNEL_GPFIFO_A)) 2174 { 2175 ctrlSize = sizeof(Nvc56fControl); 2176 } 2177 else if (gpuIsClassSupported(pGpu, HOPPER_CHANNEL_GPFIFO_A)) 2178 { 2179 ctrlSize = sizeof(Nvc86fControl); 2180 } 2181 else 2182 { 2183 status = NV_ERR_NOT_SUPPORTED; 2184 goto cleanup; 2185 } 2186 2187 portMemSet(&memAllocParams, 0, sizeof(NV_MEMORY_ALLOCATION_PARAMS)); 2188 memAllocParams.owner = HEAP_OWNER_RM_CLIENT_GENERIC; 2189 memAllocParams.size = ctrlSize; 2190 memAllocParams.type = NVOS32_TYPE_IMAGE; 2191 2192 // Apply registry overrides to USERD. 2193 switch (DRF_VAL(_REG_STR_RM, _INST_LOC, _USERD, pGpu->instLocOverrides)) 2194 { 2195 case NV_REG_STR_RM_INST_LOC_USERD_NCOH: 2196 case NV_REG_STR_RM_INST_LOC_USERD_COH: 2197 userdMemClass = NV01_MEMORY_SYSTEM; 2198 memAllocParams.attr = DRF_DEF(OS32, _ATTR, _LOCATION, _PCI); 2199 break; 2200 2201 case NV_REG_STR_RM_INST_LOC_USERD_VID: 2202 case NV_REG_STR_RM_INST_LOC_USERD_DEFAULT: 2203 memAllocParams.attr = DRF_DEF(OS32, _ATTR, _LOCATION, _VIDMEM); 2204 break; 2205 } 2206 2207 // 2208 // When APM is enabled all RM internal allocations must to go to 2209 // unprotected memory irrespective of vidmem or sysmem 2210 // When Hopper CC is enabled all RM internal sysmem allocations that 2211 // are required to be accessed from GPU should be in unprotected memory 2212 // and all vidmem allocations must go to protected memory 2213 // 2214 if (gpuIsApmFeatureEnabled(pGpu) || 2215 FLD_TEST_DRF(OS32, _ATTR, _LOCATION, _PCI, memAllocParams.attr)) 2216 { 2217 memAllocParams.attr2 |= DRF_DEF(OS32, _ATTR2, _MEMORY_PROTECTION, 2218 _UNPROTECTED); 2219 } 2220 2221 NV_ASSERT_OK_OR_GOTO(status, 2222 pRmApi->AllocWithHandle(pRmApi, hClientId, hDeviceId, hUserdId, 2223 userdMemClass, &memAllocParams, sizeof(memAllocParams)), 2224 cleanup); 2225 } 2226 2227 // Get fifo channel class Id 2228 classNum = kfifoGetChannelClassId(pGpu, GPU_GET_KERNEL_FIFO(pGpu)); 2229 NV_ASSERT_OR_GOTO(classNum != 0, cleanup); 2230 2231 // Allocate a bare channel 2232 portMemSet(&channelGPFIFOAllocParams, 0, sizeof(NV_CHANNEL_ALLOC_PARAMS)); 2233 channelGPFIFOAllocParams.hVASpace = hVASpace; 2234 channelGPFIFOAllocParams.hObjectBuffer = hPBVirtMemId; 2235 channelGPFIFOAllocParams.gpFifoEntries = gpFifoEntries; 2236 // 2237 // Set the gpFifoOffset to zero intentionally since we only need this channel 2238 // to be created, but will not submit any work to it. So it's fine not to 2239 // provide a valid offset here. 2240 // 2241 channelGPFIFOAllocParams.gpFifoOffset = 0; 2242 if (bClientUserd) 2243 { 2244 channelGPFIFOAllocParams.hUserdMemory[0] = hUserdId; 2245 } 2246 2247 if (bNeedMIGWar) 2248 { 2249 RM_ENGINE_TYPE localRmEngineType; 2250 Device *pDevice; 2251 2252 NV_ASSERT_OK_OR_GOTO(status, 2253 deviceGetByHandle(pClientId, hDeviceId, &pDevice), 2254 cleanup); 2255 2256 NV_ASSERT_OK_OR_GOTO(status, 2257 kmigmgrGetInstanceRefFromDevice(pGpu, pKernelMIGManager, pDevice, &ref), 2258 cleanup); 2259 2260 NV_ASSERT_OK_OR_GOTO(status, 2261 kmigmgrGetGlobalToLocalEngineType(pGpu, pKernelMIGManager, ref, RM_ENGINE_TYPE_GR(pKernelGraphics->instance), &localRmEngineType), 2262 cleanup); 2263 2264 channelGPFIFOAllocParams.engineType = gpuGetNv2080EngineType(localRmEngineType); 2265 } 2266 else 2267 { 2268 channelGPFIFOAllocParams.engineType = gpuGetNv2080EngineType(RM_ENGINE_TYPE_GR0); 2269 } 2270 2271 NV_ASSERT_OK_OR_GOTO(status, 2272 pRmApi->AllocWithHandle(pRmApi, hClientId, hDeviceId, hChannelId, 2273 classNum, &channelGPFIFOAllocParams, sizeof(channelGPFIFOAllocParams)), 2274 cleanup); 2275 2276 // 2277 // When using split VAS, we need to reserve enough pagepool memory to 2278 // sustain large context buffer mappings. For GSPCLIENT where the golden 2279 // context buffer channel is initialized on boot, the pagepool does not have 2280 // enough reserved memory to accommodate these buffers, so we need to 2281 // reserve extra here. 2282 // 2283 if (IS_GSP_CLIENT(pGpu)) 2284 { 2285 KernelChannel *pKernelChannel; 2286 NvU64 reserveSize; 2287 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 2288 NvU32 i; 2289 2290 NV_ASSERT_OK(CliGetKernelChannel(pClientId, hChannelId, &pKernelChannel)); 2291 2292 NV_ASSERT_OR_ELSE(pKernelGraphicsStaticInfo != NULL, 2293 status = NV_ERR_INVALID_STATE; 2294 goto cleanup;); 2295 NV_ASSERT_OR_ELSE(pKernelGraphicsStaticInfo->pContextBuffersInfo != NULL, 2296 status = NV_ERR_INVALID_STATE; 2297 goto cleanup;); 2298 2299 reserveSize = 0; 2300 for (i = 0; i < NV_ARRAY_ELEMENTS(pKernelGraphicsStaticInfo->pContextBuffersInfo->engine); ++i) 2301 { 2302 if (pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[i].size != NV_U32_MAX) 2303 reserveSize += pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[i].size; 2304 } 2305 2306 NV_ASSERT_OK( 2307 vaspaceReserveMempool(pKernelChannel->pVAS, pGpu, 2308 GPU_RES_GET_DEVICE(pKernelChannel), 2309 reserveSize, RM_PAGE_SIZE, 2310 VASPACE_RESERVE_FLAGS_NONE)); 2311 } 2312 2313 // Reaquire the GPU locks 2314 NV_ASSERT_OK_OR_GOTO(status, 2315 rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_GR), 2316 cleanup); 2317 bAcquireLock = NV_FALSE; 2318 pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 2319 2320 if (!bNeedMIGWar) 2321 { 2322 objectType = GR_OBJECT_TYPE_3D; 2323 } 2324 else 2325 { 2326 objectType = GR_OBJECT_TYPE_COMPUTE; 2327 2328 } 2329 2330 // Get KernelGraphicsObject class Id 2331 NV_ASSERT_OK_OR_GOTO(status, 2332 kgraphicsGetClassByType(pGpu, pKernelGraphics, objectType, &classNum), 2333 cleanup); 2334 NV_ASSERT_OR_GOTO(classNum != 0, cleanup); 2335 2336 // Allocate a GR object on the channel 2337 NV_ASSERT_OK_OR_GOTO(status, 2338 pRmApi->AllocWithHandle(pRmApi, hClientId, hChannelId, hObj3D, classNum, NULL, 0), 2339 cleanup); 2340 2341 cleanup: 2342 2343 if (bAcquireLock) 2344 { 2345 NV_ASSERT_OK_OR_CAPTURE_FIRST_ERROR(status, 2346 rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_GR)); 2347 } 2348 2349 // Free all handles 2350 NV_ASSERT_OK_OR_CAPTURE_FIRST_ERROR(status, 2351 pRmApi->Free(pRmApi, hClientId, hClientId)); 2352 2353 // Restore the reentrancy count 2354 gpumgrSLILoopReentrancyPush(pGpu, sliLoopReentrancy); 2355 2356 gpumgrSetBcEnabledStatus(pGpu, bBcStatus); 2357 2358 return status; 2359 } 2360 2361 /*! 2362 * @brief Free context buffers shared by all/most graphics contexts 2363 */ 2364 void kgraphicsFreeGlobalCtxBuffers_IMPL 2365 ( 2366 OBJGPU *pGpu, 2367 KernelGraphics *pKernelGraphics, 2368 NvU32 gfid 2369 ) 2370 { 2371 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 2372 GR_GLOBALCTX_BUFFERS *pCtxBuffers; 2373 GR_GLOBALCTX_BUFFER buff; 2374 NvBool bEvict = NV_FALSE; 2375 2376 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2377 2378 if (pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers == NULL) 2379 return; 2380 2381 pCtxBuffers = &pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers[gfid]; 2382 2383 if (!pCtxBuffers->bAllocated) 2384 return; 2385 2386 FOR_EACH_IN_ENUM(GR_GLOBALCTX_BUFFER, buff) 2387 { 2388 if (pCtxBuffers->memDesc[buff] != NULL) 2389 { 2390 bEvict = NV_TRUE; 2391 2392 memdescFree(pCtxBuffers->memDesc[buff]); 2393 memdescDestroy(pCtxBuffers->memDesc[buff]); 2394 pCtxBuffers->memDesc[buff] = NULL; 2395 pCtxBuffers->bInitialized[buff] = NV_FALSE; 2396 } 2397 } 2398 FOR_EACH_IN_ENUM_END; 2399 2400 pCtxBuffers->bAllocated = NV_FALSE; 2401 2402 // make sure all L2 cache lines using CB buffers are clear after we free them 2403 if (bEvict) 2404 NV_ASSERT_OK(kmemsysCacheOp_HAL(pGpu, pKernelMemorySystem, NULL, FB_CACHE_VIDEO_MEMORY, FB_CACHE_EVICT)); 2405 } 2406 2407 NV_STATUS 2408 kgraphicsGetCaps_IMPL 2409 ( 2410 OBJGPU *pGpu, 2411 KernelGraphics *pKernelGraphics, 2412 NvU8 *pGrCaps 2413 ) 2414 { 2415 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 2416 2417 NV_ASSERT_OR_RETURN(pGrCaps != NULL, NV_ERR_INVALID_ARGUMENT); 2418 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 2419 2420 portMemCopy(pGrCaps, 2421 NV0080_CTRL_GR_CAPS_TBL_SIZE * sizeof(*pGrCaps), 2422 pKernelGraphicsStaticInfo->grCaps.capsTbl, 2423 NV0080_CTRL_GR_CAPS_TBL_SIZE * sizeof(*pGrCaps)); 2424 2425 return NV_OK; 2426 } 2427 2428 /*! 2429 * @brief Return whether unrestricted register access bufffer is supported or not. 2430 */ 2431 NvBool 2432 kgraphicsIsUnrestrictedAccessMapSupported_PF 2433 ( 2434 OBJGPU *pGpu, 2435 KernelGraphics *pKernelGraphics 2436 ) 2437 { 2438 return !hypervisorIsVgxHyper(); 2439 } 2440 2441 /*! 2442 * @brief Provides an opportunity to register some IntrService during intrStateInit. 2443 */ 2444 void 2445 kgraphicsRegisterIntrService_IMPL 2446 ( 2447 OBJGPU *pGpu, 2448 KernelGraphics *pKernelGraphics, 2449 IntrServiceRecord pRecords[MC_ENGINE_IDX_MAX] 2450 ) 2451 { 2452 NvU32 engineIdx = MC_ENGINE_IDX_GRn_FECS_LOG(pKernelGraphics->instance); 2453 2454 NV_ASSERT(pRecords[engineIdx].pInterruptService == NULL); 2455 pRecords[engineIdx].pInterruptService = staticCast(pKernelGraphics, IntrService); 2456 2457 engineIdx = MC_ENGINE_IDX_GRn(pKernelGraphics->instance); 2458 2459 NV_ASSERT(pRecords[engineIdx].pNotificationService == NULL); 2460 pRecords[engineIdx].bFifoWaiveNotify = NV_FALSE; 2461 pRecords[engineIdx].pNotificationService = staticCast(pKernelGraphics, IntrService); 2462 } 2463 2464 /*! 2465 * @brief Services the nonstall interrupt. 2466 */ 2467 NvU32 2468 kgraphicsServiceNotificationInterrupt_IMPL 2469 ( 2470 OBJGPU *pGpu, 2471 KernelGraphics *pKernelGraphics, 2472 IntrServiceServiceNotificationInterruptArguments *pParams 2473 ) 2474 { 2475 NvU32 grIdx = pKernelGraphics->instance; 2476 2477 NV_ASSERT_OR_RETURN(pParams != NULL, 0); 2478 NV_ASSERT_OR_RETURN(pParams->engineIdx == MC_ENGINE_IDX_GRn(grIdx), 0); 2479 2480 MODS_ARCH_REPORT(NV_ARCH_EVENT_NONSTALL_GR, "%s", "processing GR nonstall interrupt\n"); 2481 2482 kgraphicsNonstallIntrCheckAndClear_HAL(pGpu, pKernelGraphics, pParams->pThreadState); 2483 engineNonStallIntrNotify(pGpu, RM_ENGINE_TYPE_GR(pKernelGraphics->instance)); 2484 return NV_OK; 2485 } 2486 2487 /*! 2488 * KernelGraphics RM Device Controls 2489 */ 2490 2491 /*! 2492 * deviceCtrlCmdKGrGetCaps_IMPL 2493 * 2494 * Lock Requirements: 2495 * Assert that API lock held on entry 2496 * 2497 * TODO: remove once all uses have been migrated to V2 2498 */ 2499 NV_STATUS 2500 deviceCtrlCmdKGrGetCaps_IMPL 2501 ( 2502 Device *pDevice, 2503 NV0080_CTRL_GR_GET_CAPS_PARAMS *pParams 2504 ) 2505 { 2506 OBJGPU *pGpu = GPU_RES_GET_GPU(pDevice); 2507 NvU8 *pGrCaps = NvP64_VALUE(pParams->capsTbl); 2508 NvBool bCapsPopulated = NV_FALSE; 2509 2510 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner()); 2511 2512 if (IsDFPGA(pGpu)) 2513 { 2514 return NV_ERR_NOT_SUPPORTED; 2515 } 2516 2517 NV_CHECK_OR_RETURN(LEVEL_ERROR, pGrCaps != NULL, NV_ERR_INVALID_ARGUMENT); 2518 NV_CHECK_OR_RETURN(LEVEL_ERROR, pParams->capsTblSize == NV0080_CTRL_GR_CAPS_TBL_SIZE, NV_ERR_INVALID_ARGUMENT); 2519 2520 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY) 2521 { 2522 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 2523 KernelGraphics *pKernelGraphics; 2524 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo; 2525 NV_STATUS status; 2526 2527 portMemSet(&grRouteInfo, 0, sizeof(grRouteInfo)); 2528 kgrmgrCtrlSetEngineID(0, &grRouteInfo); 2529 NV_CHECK_OK_OR_ELSE(status, LEVEL_ERROR, 2530 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &grRouteInfo, &pKernelGraphics), 2531 SLI_LOOP_RETURN(status);); 2532 2533 if (!bCapsPopulated) 2534 { 2535 NV_CHECK_OK_OR_ELSE(status, LEVEL_ERROR, 2536 kgraphicsGetCaps(pGpu, pKernelGraphics, pGrCaps), 2537 SLI_LOOP_RETURN(status);); 2538 2539 bCapsPopulated = NV_TRUE; 2540 } 2541 } 2542 SLI_LOOP_END 2543 2544 return NV_OK; 2545 } 2546 2547 /*! 2548 * deviceCtrlCmdKGrGetCapsV2_IMPL 2549 * 2550 * Lock Requirements: 2551 * Assert that API lock held on entry 2552 */ 2553 NV_STATUS 2554 deviceCtrlCmdKGrGetCapsV2_IMPL 2555 ( 2556 Device *pDevice, 2557 NV0080_CTRL_GR_GET_CAPS_V2_PARAMS *pParams 2558 ) 2559 { 2560 OBJGPU *pGpu = GPU_RES_GET_GPU(pDevice); 2561 2562 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner()); 2563 2564 if (IsDFPGA(pGpu)) 2565 { 2566 return NV_ERR_NOT_SUPPORTED; 2567 } 2568 2569 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY) 2570 { 2571 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 2572 KernelGraphics *pKernelGraphics; 2573 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo = pParams->grRouteInfo; 2574 NV_STATUS status; 2575 2576 kgrmgrCtrlSetEngineID(0, &grRouteInfo); 2577 NV_CHECK_OK_OR_ELSE(status, LEVEL_ERROR, 2578 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &grRouteInfo, &pKernelGraphics), 2579 SLI_LOOP_RETURN(status);); 2580 2581 if (!pParams->bCapsPopulated) 2582 { 2583 NV_CHECK_OK_OR_ELSE(status, LEVEL_ERROR, 2584 kgraphicsGetCaps(pGpu, pKernelGraphics, pParams->capsTbl), 2585 SLI_LOOP_RETURN(status);); 2586 2587 pParams->bCapsPopulated = NV_TRUE; 2588 } 2589 } 2590 SLI_LOOP_END 2591 2592 return NV_OK; 2593 } 2594 2595 static NV_STATUS 2596 _kgraphicsCtrlCmdGrGetInfoV2 2597 ( 2598 OBJGPU *pGpu, 2599 Device *pDevice, 2600 NV2080_CTRL_GR_GET_INFO_V2_PARAMS *pParams 2601 ) 2602 { 2603 NV_STATUS status = NV_OK; 2604 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 2605 NvU32 grInfoListSize = pParams->grInfoListSize; 2606 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 2607 NV2080_CTRL_INTERNAL_STATIC_GR_INFO *pGrInfo; 2608 NvU32 i; 2609 2610 if (pKernelGraphicsManager == NULL) 2611 { 2612 return NV_ERR_NOT_SUPPORTED; 2613 } 2614 2615 if ((0 == grInfoListSize) || 2616 (grInfoListSize > NV2080_CTRL_GR_INFO_MAX_SIZE)) 2617 { 2618 NV_PRINTF(LEVEL_ERROR, "Invalid grInfoList size: 0x%x\n", grInfoListSize); 2619 return NV_ERR_INVALID_ARGUMENT; 2620 } 2621 2622 if (kmigmgrIsDeviceUsingDeviceProfiling(pGpu, pKernelMIGManager, pDevice)) 2623 { 2624 NV_ASSERT_OR_RETURN(kgrmgrGetLegacyKGraphicsStaticInfo(pGpu, pKernelGraphicsManager)->bInitialized, NV_ERR_INVALID_STATE); 2625 NV_ASSERT_OR_RETURN(kgrmgrGetLegacyKGraphicsStaticInfo(pGpu, pKernelGraphicsManager)->pGrInfo != NULL, NV_ERR_NOT_SUPPORTED); 2626 2627 pGrInfo = kgrmgrGetLegacyKGraphicsStaticInfo(pGpu, pKernelGraphicsManager)->pGrInfo; 2628 } 2629 else 2630 { 2631 KernelGraphics *pKernelGraphics; 2632 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 2633 2634 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2635 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &pParams->grRouteInfo, &pKernelGraphics)); 2636 2637 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 2638 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 2639 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pGrInfo != NULL, NV_ERR_NOT_SUPPORTED); 2640 2641 pGrInfo = pKernelGraphicsStaticInfo->pGrInfo; 2642 } 2643 2644 for (i = 0; i < grInfoListSize; i++) 2645 { 2646 NV_CHECK_OR_RETURN(LEVEL_ERROR, pParams->grInfoList[i].index < NV2080_CTRL_GR_INFO_MAX_SIZE, NV_ERR_INVALID_ARGUMENT); 2647 pParams->grInfoList[i].data = pGrInfo->infoList[pParams->grInfoList[i].index].data; 2648 } 2649 2650 return status; 2651 } 2652 2653 /*! 2654 * deviceCtrlCmdKGrGetInfo 2655 * 2656 * Lock Requirements: 2657 * Assert that API lock and GPUs lock held on entry 2658 * 2659 * TODO: remove once all uses have been migrated to V2 2660 */ 2661 NV_STATUS 2662 deviceCtrlCmdKGrGetInfo_IMPL 2663 ( 2664 Device *pDevice, 2665 NV0080_CTRL_GR_GET_INFO_PARAMS *pParams 2666 ) 2667 { 2668 OBJGPU *pGpu = GPU_RES_GET_GPU(pDevice); 2669 NV0080_CTRL_GR_GET_INFO_V2_PARAMS grInfoParamsV2; 2670 NV0080_CTRL_GR_INFO *pGrInfos = NvP64_VALUE(pParams->grInfoList); 2671 NvU32 grInfoListSize = NV_MIN(pParams->grInfoListSize, 2672 NV0080_CTRL_GR_INFO_MAX_SIZE); 2673 2674 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner()); 2675 2676 NV_CHECK_OR_RETURN(LEVEL_ERROR, pGrInfos != NULL, NV_ERR_INVALID_ARGUMENT); 2677 2678 portMemSet(&grInfoParamsV2, 0, sizeof(grInfoParamsV2)); 2679 portMemCopy(grInfoParamsV2.grInfoList, grInfoListSize * sizeof(*pGrInfos), 2680 pGrInfos, grInfoListSize * sizeof(*pGrInfos)); 2681 grInfoParamsV2.grInfoListSize = grInfoListSize; 2682 2683 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2684 _kgraphicsCtrlCmdGrGetInfoV2(pGpu, pDevice, &grInfoParamsV2)); 2685 2686 portMemCopy(pGrInfos, grInfoListSize * sizeof(*pGrInfos), 2687 grInfoParamsV2.grInfoList, grInfoListSize * sizeof(*pGrInfos)); 2688 return NV_OK; 2689 } 2690 2691 /*! 2692 * deviceCtrlCmdKGrGetInfoV2 2693 * 2694 * Lock Requirements: 2695 * Assert that API lock and GPUs lock held on entry 2696 */ 2697 NV_STATUS 2698 deviceCtrlCmdKGrGetInfoV2_IMPL 2699 ( 2700 Device *pDevice, 2701 NV0080_CTRL_GR_GET_INFO_V2_PARAMS *pParams 2702 ) 2703 { 2704 OBJGPU *pGpu = GPU_RES_GET_GPU(pDevice); 2705 2706 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner()); 2707 2708 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2709 _kgraphicsCtrlCmdGrGetInfoV2(pGpu, pDevice, pParams)); 2710 2711 return NV_OK; 2712 } 2713 2714 NV_STATUS 2715 kgraphicsDiscoverMaxLocalCtxBufferSize_IMPL 2716 ( 2717 OBJGPU *pGpu, 2718 KernelGraphics *pKernelGraphics 2719 ) 2720 { 2721 NvU32 bufId = 0; 2722 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 2723 2724 if (IS_MODS_AMODEL(pGpu)) 2725 return NV_OK; 2726 2727 NV_ASSERT_OK_OR_RETURN( 2728 kgraphicsInitializeDeferredStaticData(pGpu, pKernelGraphics, NV01_NULL_OBJECT, NV01_NULL_OBJECT)); 2729 2730 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pContextBuffersInfo != NULL, NV_ERR_INVALID_STATE); 2731 2732 FOR_EACH_IN_ENUM(GR_CTX_BUFFER, bufId) 2733 { 2734 if (bufId == GR_CTX_BUFFER_MAIN) 2735 { 2736 NvU32 size; 2737 2738 NV_ASSERT_OK_OR_RETURN(kgraphicsGetMainCtxBufferSize(pGpu, pKernelGraphics, NV_TRUE, &size)); 2739 kgraphicsSetCtxBufferInfo(pGpu, pKernelGraphics, bufId, 2740 size, 2741 RM_PAGE_SIZE, 2742 RM_ATTR_PAGE_SIZE_4KB, 2743 kgraphicsShouldForceMainCtxContiguity_HAL(pGpu, pKernelGraphics)); 2744 } 2745 else 2746 { 2747 NvU32 fifoEngineId; 2748 2749 NV_ASSERT_OK_OR_RETURN( 2750 kgrctxCtxBufferToFifoEngineId(bufId, &fifoEngineId)); 2751 2752 kgraphicsSetCtxBufferInfo(pGpu, pKernelGraphics, bufId, 2753 pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[fifoEngineId].size, 2754 RM_PAGE_SIZE, 2755 RM_ATTR_PAGE_SIZE_4KB, 2756 ((bufId == GR_CTX_BUFFER_PATCH) || (bufId == GR_CTX_BUFFER_PM))); 2757 } 2758 } 2759 FOR_EACH_IN_ENUM_END; 2760 return NV_OK; 2761 } 2762 2763 /*! 2764 * KernelGraphics RM SubDevice Controls 2765 */ 2766 2767 /*! 2768 * subdeviceCtrlCmdKGrGetCapsV2 2769 * 2770 * Lock Requirements: 2771 * Assert that API lock held on entry 2772 */ 2773 NV_STATUS 2774 subdeviceCtrlCmdKGrGetCapsV2_IMPL 2775 ( 2776 Subdevice *pSubdevice, 2777 NV2080_CTRL_GR_GET_CAPS_V2_PARAMS *pParams 2778 ) 2779 { 2780 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 2781 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 2782 KernelGraphics *pKernelGraphics; 2783 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo = pParams->grRouteInfo; 2784 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 2785 2786 kgrmgrCtrlSetEngineID(0, &grRouteInfo); 2787 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2788 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &grRouteInfo, &pKernelGraphics)); 2789 2790 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner()); 2791 2792 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 2793 if (pKernelGraphicsStaticInfo == NULL) 2794 { 2795 return NV_ERR_INVALID_STATE; 2796 } 2797 2798 if (!pParams->bCapsPopulated) 2799 { 2800 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2801 kgraphicsGetCaps(pGpu, pKernelGraphics, pParams->capsTbl)); 2802 2803 pParams->bCapsPopulated = NV_TRUE; 2804 } 2805 2806 return NV_OK; 2807 } 2808 2809 /*! 2810 * subdeviceCtrlCmdKGrGetInfo 2811 * 2812 * Lock Requirements: 2813 * Assert that API lock and GPUs lock held on entry 2814 * 2815 * TODO: remove once all uses have been migrated to V2 2816 */ 2817 NV_STATUS 2818 subdeviceCtrlCmdKGrGetInfo_IMPL 2819 ( 2820 Subdevice *pSubdevice, 2821 NV2080_CTRL_GR_GET_INFO_PARAMS *pParams 2822 ) 2823 { 2824 NV2080_CTRL_GR_GET_INFO_V2_PARAMS grInfoParamsV2; 2825 NV2080_CTRL_GR_INFO *pGrInfos = NvP64_VALUE(pParams->grInfoList); 2826 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo = pParams->grRouteInfo; 2827 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 2828 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 2829 NvU32 grInfoListSize = NV_MIN(pParams->grInfoListSize, 2830 NV2080_CTRL_GR_INFO_MAX_SIZE); 2831 2832 // 2833 // Adding the null check as engine GRMGR is missing for DFPGA. 2834 // 2835 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 2836 NV_CHECK_OR_RETURN(LEVEL_ERROR, pKernelGraphicsManager != NULL, NV_ERR_NOT_SUPPORTED); 2837 2838 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 2839 2840 NV_CHECK_OR_RETURN(LEVEL_ERROR, pGrInfos != NULL, NV_ERR_INVALID_ARGUMENT); 2841 2842 portMemSet(&grInfoParamsV2, 0, sizeof(grInfoParamsV2)); 2843 grInfoParamsV2.grInfoListSize = grInfoListSize; 2844 portMemCopy(grInfoParamsV2.grInfoList, grInfoListSize * sizeof(*pGrInfos), 2845 pGrInfos, grInfoListSize * sizeof(*pGrInfos)); 2846 grInfoParamsV2.grRouteInfo = grRouteInfo; 2847 2848 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2849 _kgraphicsCtrlCmdGrGetInfoV2(pGpu, pDevice, &grInfoParamsV2)); 2850 2851 portMemCopy(pGrInfos, grInfoListSize * sizeof(*pGrInfos), 2852 grInfoParamsV2.grInfoList, grInfoListSize * sizeof(*pGrInfos)); 2853 return NV_OK; 2854 } 2855 2856 /*! 2857 * subdeviceCtrlCmdKGrGetInfoV2 2858 * 2859 * Lock Requirements: 2860 * Assert that API lock and GPUs lock held on entry 2861 */ 2862 NV_STATUS 2863 subdeviceCtrlCmdKGrGetInfoV2_IMPL 2864 ( 2865 Subdevice *pSubdevice, 2866 NV2080_CTRL_GR_GET_INFO_V2_PARAMS *pParams 2867 ) 2868 { 2869 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 2870 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 2871 2872 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 2873 2874 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2875 _kgraphicsCtrlCmdGrGetInfoV2(pGpu, pDevice, pParams)); 2876 2877 return NV_OK; 2878 } 2879 2880 /*! 2881 * subdeviceCtrlCmdKGrGetSmToGpcTpcMappings 2882 * 2883 * Lock Requirements: 2884 * Assert that API lock and GPUs lock held on entry 2885 */ 2886 NV_STATUS 2887 subdeviceCtrlCmdKGrGetSmToGpcTpcMappings_IMPL 2888 ( 2889 Subdevice *pSubdevice, 2890 NV2080_CTRL_GR_GET_SM_TO_GPC_TPC_MAPPINGS_PARAMS *pParams 2891 ) 2892 { 2893 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 2894 KernelGraphics *pKernelGraphics; 2895 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 2896 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 2897 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 2898 const KGRAPHICS_STATIC_INFO *pStaticInfo; 2899 NvU32 i; 2900 2901 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 2902 2903 if (kmigmgrIsDeviceUsingDeviceProfiling(pGpu, pKernelMIGManager, pDevice)) 2904 { 2905 return NV_ERR_NOT_SUPPORTED; 2906 } 2907 else 2908 { 2909 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2910 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &pParams->grRouteInfo, &pKernelGraphics)); 2911 } 2912 2913 // Verify static info is available 2914 pStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 2915 NV_ASSERT_OR_RETURN(pStaticInfo != NULL, NV_ERR_INVALID_STATE); 2916 2917 // Verify limits are within bounds 2918 NV_ASSERT_OR_RETURN(pStaticInfo->globalSmOrder.numSm <= NV2080_CTRL_GR_GET_SM_TO_GPC_TPC_MAPPINGS_MAX_SM_COUNT, 2919 NV_ERR_INVALID_LIMIT); 2920 2921 // Populate output data 2922 pParams->smCount = pStaticInfo->globalSmOrder.numSm; 2923 for (i = 0; i < pStaticInfo->globalSmOrder.numSm; ++i) 2924 { 2925 pParams->smId[i].gpcId = pStaticInfo->globalSmOrder.globalSmId[i].gpcId; 2926 pParams->smId[i].tpcId = pStaticInfo->globalSmOrder.globalSmId[i].localTpcId; 2927 } 2928 2929 return NV_OK; 2930 } 2931 2932 NV_STATUS 2933 subdeviceCtrlCmdKGrGetGlobalSmOrder_IMPL 2934 ( 2935 Subdevice *pSubdevice, 2936 NV2080_CTRL_GR_GET_GLOBAL_SM_ORDER_PARAMS *pParams 2937 ) 2938 { 2939 NV_STATUS status = NV_OK; 2940 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 2941 KernelGraphics *pKernelGraphics; 2942 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 2943 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 2944 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 2945 const KGRAPHICS_STATIC_INFO *pStaticInfo; 2946 NvU32 i; 2947 2948 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 2949 2950 if (kmigmgrIsDeviceUsingDeviceProfiling(pGpu, pKernelMIGManager, pDevice)) 2951 { 2952 return NV_ERR_NOT_SUPPORTED; 2953 } 2954 else 2955 { 2956 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2957 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &pParams->grRouteInfo, &pKernelGraphics)); 2958 } 2959 2960 // Verify static info is available 2961 pStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 2962 NV_ASSERT_OR_RETURN(pStaticInfo != NULL, NV_ERR_INVALID_STATE); 2963 2964 // Verify limits are within bounds 2965 NV_ASSERT_OR_RETURN(pStaticInfo->globalSmOrder.numSm <= NV2080_CTRL_CMD_GR_GET_GLOBAL_SM_ORDER_MAX_SM_COUNT, 2966 NV_ERR_INVALID_LIMIT); 2967 2968 // Populate output data 2969 pParams->numSm = pStaticInfo->globalSmOrder.numSm; 2970 pParams->numTpc = pStaticInfo->globalSmOrder.numTpc; 2971 for (i = 0; i < pStaticInfo->globalSmOrder.numSm; ++i) 2972 { 2973 pParams->globalSmId[i].gpcId = pStaticInfo->globalSmOrder.globalSmId[i].gpcId; 2974 pParams->globalSmId[i].localTpcId = pStaticInfo->globalSmOrder.globalSmId[i].localTpcId; 2975 pParams->globalSmId[i].localSmId = pStaticInfo->globalSmOrder.globalSmId[i].localSmId; 2976 pParams->globalSmId[i].globalTpcId = pStaticInfo->globalSmOrder.globalSmId[i].globalTpcId; 2977 pParams->globalSmId[i].virtualGpcId = pStaticInfo->globalSmOrder.globalSmId[i].virtualGpcId; 2978 pParams->globalSmId[i].migratableTpcId = pStaticInfo->globalSmOrder.globalSmId[i].migratableTpcId; 2979 } 2980 2981 return status; 2982 } 2983 2984 /*! 2985 * subdeviceCtrlCmdKGrGetSmIssueRateModifier 2986 * 2987 * Lock Requirements: 2988 * Assert that API lock and GPUs lock held on entry 2989 */ 2990 NV_STATUS 2991 subdeviceCtrlCmdKGrGetSmIssueRateModifier_IMPL 2992 ( 2993 Subdevice *pSubdevice, 2994 NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_PARAMS *pParams 2995 ) 2996 { 2997 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 2998 KernelGraphics *pKernelGraphics; 2999 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 3000 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3001 const KGRAPHICS_STATIC_INFO *pStaticInfo; 3002 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 3003 3004 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner()); 3005 3006 if (kmigmgrIsDeviceUsingDeviceProfiling(pGpu, pKernelMIGManager, pDevice)) 3007 { 3008 NvU32 grIdx; 3009 for (grIdx = 0; grIdx < GPU_MAX_GRS; grIdx++) 3010 { 3011 pKernelGraphics = GPU_GET_KERNEL_GRAPHICS(pGpu, grIdx); 3012 if (pKernelGraphics != NULL) 3013 break; 3014 } 3015 if (pKernelGraphics == NULL) 3016 return NV_ERR_INVALID_STATE; 3017 } 3018 else 3019 { 3020 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3021 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &pParams->grRouteInfo, &pKernelGraphics)); 3022 } 3023 3024 // Verify static info is available 3025 pStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 3026 NV_ASSERT_OR_RETURN(pStaticInfo != NULL, NV_ERR_INVALID_STATE); 3027 NV_ASSERT_OR_RETURN(pStaticInfo->pSmIssueRateModifier != NULL, NV_ERR_NOT_SUPPORTED); 3028 3029 pParams->imla0 = pStaticInfo->pSmIssueRateModifier->imla0; 3030 pParams->fmla16 = pStaticInfo->pSmIssueRateModifier->fmla16; 3031 pParams->dp = pStaticInfo->pSmIssueRateModifier->dp; 3032 pParams->fmla32 = pStaticInfo->pSmIssueRateModifier->fmla32; 3033 pParams->ffma = pStaticInfo->pSmIssueRateModifier->ffma; 3034 pParams->imla1 = pStaticInfo->pSmIssueRateModifier->imla1; 3035 pParams->imla2 = pStaticInfo->pSmIssueRateModifier->imla2; 3036 pParams->imla3 = pStaticInfo->pSmIssueRateModifier->imla3; 3037 pParams->imla4 = pStaticInfo->pSmIssueRateModifier->imla4; 3038 3039 return NV_OK; 3040 } 3041 3042 /*! 3043 * subdeviceCtrlCmdKGrGetGpcMask 3044 * 3045 * Lock Requirements: 3046 * Assert that API lock and GPUs lock held on entry 3047 */ 3048 NV_STATUS 3049 subdeviceCtrlCmdKGrGetGpcMask_IMPL 3050 ( 3051 Subdevice *pSubdevice, 3052 NV2080_CTRL_GR_GET_GPC_MASK_PARAMS *pParams 3053 ) 3054 { 3055 NV_STATUS status = NV_OK; 3056 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3057 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 3058 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3059 KernelGraphics *pKernelGraphics; 3060 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 3061 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 3062 3063 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 3064 3065 if (!IS_MIG_IN_USE(pGpu) || 3066 kmigmgrIsDeviceUsingDeviceProfiling(pGpu, pKernelMIGManager, pDevice)) 3067 { 3068 pParams->gpcMask = kgrmgrGetLegacyGpcMask(pGpu, pKernelGraphicsManager); 3069 } 3070 else 3071 { 3072 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3073 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &pParams->grRouteInfo, &pKernelGraphics)); 3074 3075 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 3076 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 3077 3078 pParams->gpcMask = pKernelGraphicsStaticInfo->floorsweepingMasks.gpcMask; 3079 } 3080 3081 return status; 3082 } 3083 3084 /*! 3085 * subdeviceCtrlCmdKGrGetTpcMask 3086 * 3087 * Note: 3088 * pParams->gpcId is physical GPC id for non-MIG case, but logical GPC id for 3089 * MIG case. 3090 * 3091 * Lock Requirements: 3092 * Assert that API lock and GPUs lock held on entry 3093 */ 3094 NV_STATUS 3095 subdeviceCtrlCmdKGrGetTpcMask_IMPL 3096 ( 3097 Subdevice *pSubdevice, 3098 NV2080_CTRL_GR_GET_TPC_MASK_PARAMS *pParams 3099 ) 3100 { 3101 NV_STATUS status = NV_OK; 3102 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3103 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 3104 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3105 KernelGraphics *pKernelGraphics; 3106 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 3107 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 3108 NvU32 gpcCount; 3109 3110 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 3111 3112 if (!IS_MIG_IN_USE(pGpu) || 3113 kmigmgrIsDeviceUsingDeviceProfiling(pGpu, pKernelMIGManager, pDevice)) 3114 { 3115 pParams->tpcMask = kgrmgrGetLegacyTpcMask(pGpu, pKernelGraphicsManager, pParams->gpcId); 3116 } 3117 else 3118 { 3119 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3120 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &pParams->grRouteInfo, &pKernelGraphics)); 3121 3122 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 3123 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 3124 3125 gpcCount = nvPopCount32(pKernelGraphicsStaticInfo->floorsweepingMasks.gpcMask); 3126 if (pParams->gpcId >= gpcCount) 3127 { 3128 NV_PRINTF(LEVEL_ERROR, "Incorrect GPC-Idx provided = %d\n", pParams->gpcId); 3129 return NV_ERR_INVALID_ARGUMENT; 3130 } 3131 3132 pParams->tpcMask = pKernelGraphicsStaticInfo->floorsweepingMasks.tpcMask[pParams->gpcId]; 3133 } 3134 3135 return status; 3136 } 3137 3138 NV_STATUS 3139 subdeviceCtrlCmdKGrGetNumTpcsForGpc_IMPL 3140 ( 3141 Subdevice *pSubdevice, 3142 NV2080_CTRL_GR_GET_NUM_TPCS_FOR_GPC_PARAMS *pParams 3143 ) 3144 { 3145 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3146 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 3147 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3148 KernelGraphics *pKernelGraphics; 3149 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 3150 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo; 3151 NvU32 gpcCount; 3152 3153 portMemSet(&grRouteInfo, 0, sizeof(grRouteInfo)); 3154 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3155 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &grRouteInfo, &pKernelGraphics)); 3156 3157 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 3158 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 3159 3160 gpcCount = nvPopCount32(pKernelGraphicsStaticInfo->floorsweepingMasks.gpcMask); 3161 if (pParams->gpcId >= gpcCount) 3162 { 3163 NV_PRINTF(LEVEL_ERROR, "Incorrect GPC-Idx provided = %d\n", pParams->gpcId); 3164 return NV_ERR_INVALID_ARGUMENT; 3165 } 3166 3167 pParams->numTpcs = pKernelGraphicsStaticInfo->floorsweepingMasks.tpcCount[pParams->gpcId]; 3168 3169 return NV_OK; 3170 } 3171 3172 /*! 3173 * subdeviceCtrlCmdKGrGetPpcMask 3174 * 3175 * Lock Requirements: 3176 * Assert that API lock and GPUs lock held on entry 3177 */ 3178 NV_STATUS 3179 subdeviceCtrlCmdKGrGetPpcMask_IMPL 3180 ( 3181 Subdevice *pSubdevice, 3182 NV2080_CTRL_GR_GET_PPC_MASK_PARAMS *pParams 3183 ) 3184 { 3185 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3186 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 3187 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3188 KernelGraphics *pKernelGraphics; 3189 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 3190 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 3191 3192 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 3193 3194 if (kmigmgrIsDeviceUsingDeviceProfiling(pGpu, pKernelMIGManager, pDevice)) 3195 { 3196 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, kgrmgrGetLegacyPpcMask(pGpu, pKernelGraphicsManager, pParams->gpcId, &pParams->ppcMask)); 3197 } 3198 else 3199 { 3200 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3201 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &pParams->grRouteInfo, &pKernelGraphics)); 3202 3203 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 3204 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 3205 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pPpcMasks != NULL, NV_ERR_NOT_SUPPORTED); 3206 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pGrInfo != NULL, NV_ERR_NOT_SUPPORTED); 3207 3208 if (pParams->gpcId >= 3209 pKernelGraphicsStaticInfo->pGrInfo->infoList[NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_GPCS].data) 3210 { 3211 NV_PRINTF(LEVEL_ERROR, "Incorrect GPC-Idx provided = %d\n", pParams->gpcId); 3212 return NV_ERR_INVALID_ARGUMENT; 3213 } 3214 3215 pParams->ppcMask = pKernelGraphicsStaticInfo->pPpcMasks->mask[pParams->gpcId]; 3216 } 3217 3218 return NV_OK; 3219 } 3220 3221 // 3222 // subdeviceCtrlCmdKGrFecsBindEvtbufForUid 3223 // 3224 // Lock Requirements: 3225 // Assert that API lock and GPUs lock held on entry 3226 // 3227 NV_STATUS 3228 subdeviceCtrlCmdKGrFecsBindEvtbufForUid_IMPL 3229 ( 3230 Subdevice *pSubdevice, 3231 NV2080_CTRL_GR_FECS_BIND_EVTBUF_FOR_UID_PARAMS *pParams 3232 ) 3233 { 3234 NV_STATUS status; 3235 RmClient *pClient; 3236 RsResourceRef *pEventBufferRef = NULL; 3237 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3238 NvHandle hClient = RES_GET_CLIENT_HANDLE(pSubdevice); 3239 NvBool bMIGInUse = IS_MIG_IN_USE(pGpu); 3240 3241 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 3242 3243 NV_ASSERT_OK_OR_RETURN( 3244 serverutilGetResourceRefWithType(hClient, pParams->hEventBuffer, classId(EventBuffer), &pEventBufferRef)); 3245 3246 pClient = serverutilGetClientUnderLock(hClient); 3247 NV_ASSERT_OR_RETURN(pClient != NULL, NV_ERR_INVALID_CLIENT); 3248 3249 if (bMIGInUse) 3250 return NV_ERR_NOT_SUPPORTED; 3251 3252 status = fecsAddBindpoint(pGpu, 3253 pClient, 3254 pEventBufferRef, 3255 pSubdevice, 3256 pParams->bAllUsers, 3257 pParams->levelOfDetail, 3258 pParams->eventFilter, 3259 1, 3260 NULL); 3261 3262 return status; 3263 } 3264 3265 // 3266 // subdeviceCtrlCmdKGrFecsBindEvtbufForUidV2 3267 // 3268 // Lock Requirements: 3269 // Assert that API lock and GPUs lock held on entry 3270 // 3271 NV_STATUS 3272 subdeviceCtrlCmdKGrFecsBindEvtbufForUidV2_IMPL 3273 ( 3274 Subdevice *pSubdevice, 3275 NV2080_CTRL_GR_FECS_BIND_EVTBUF_FOR_UID_V2_PARAMS *pParams 3276 ) 3277 { 3278 NV_STATUS status; 3279 RmClient *pClient; 3280 RsResourceRef *pEventBufferRef = NULL; 3281 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3282 NvHandle hClient = RES_GET_CLIENT_HANDLE(pSubdevice); 3283 pParams->reasonCode = NV2080_CTRL_GR_FECS_BIND_REASON_CODE_NONE; 3284 3285 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 3286 3287 NV_ASSERT_OK_OR_RETURN( 3288 serverutilGetResourceRefWithType(hClient, pParams->hEventBuffer, classId(EventBuffer), &pEventBufferRef)); 3289 3290 pClient = serverutilGetClientUnderLock(hClient); 3291 NV_ASSERT_OR_RETURN(pClient != NULL, NV_ERR_INVALID_CLIENT); 3292 3293 status = fecsAddBindpoint(pGpu, 3294 pClient, 3295 pEventBufferRef, 3296 pSubdevice, 3297 pParams->bAllUsers, 3298 pParams->levelOfDetail, 3299 pParams->eventFilter, 3300 2, 3301 &pParams->reasonCode); 3302 return status; 3303 } 3304 3305 /*! 3306 * subdeviceCtrlCmdKGrGetPhysGpcMask 3307 * 3308 * Lock Requirements: 3309 * Assert that API lock and GPUs lock held on entry 3310 */ 3311 NV_STATUS 3312 subdeviceCtrlCmdKGrGetPhysGpcMask_IMPL 3313 ( 3314 Subdevice *pSubdevice, 3315 NV2080_CTRL_GR_GET_PHYS_GPC_MASK_PARAMS *pParams 3316 ) 3317 { 3318 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3319 KernelGraphics *pKernelGraphics; 3320 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 3321 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 3322 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 3323 NvU32 grIdx = 0; 3324 3325 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 3326 3327 if (!IS_MIG_ENABLED(pGpu)) 3328 { 3329 grIdx = 0; 3330 } 3331 // 3332 // if MIG is enabled we follow below policies: 3333 // For device level monitoring with no subscription - Return GPC mask for 3334 // requested syspipe 3335 // For valid subscription - Return physical GPC mask after validating that 3336 // a physical syspipe exist in given GPU instance 3337 // 3338 else if (kmigmgrIsDeviceUsingDeviceProfiling(pGpu, pKernelMIGManager, pDevice)) 3339 { 3340 NV_ASSERT_OR_RETURN(pParams->physSyspipeId < GPU_MAX_GRS, NV_ERR_INVALID_ARGUMENT); 3341 grIdx = pParams->physSyspipeId; 3342 } 3343 else 3344 { 3345 MIG_INSTANCE_REF ref; 3346 RM_ENGINE_TYPE localRmEngineType; 3347 3348 // 3349 // Get the relevant subscription and see if provided physicalId is 3350 // valid in defined GPU instance 3351 // 3352 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3353 kmigmgrGetInstanceRefFromDevice(pGpu, pKernelMIGManager, 3354 pDevice, &ref)); 3355 3356 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3357 kmigmgrGetGlobalToLocalEngineType(pGpu, pKernelMIGManager, ref, 3358 RM_ENGINE_TYPE_GR(pParams->physSyspipeId), 3359 &localRmEngineType)); 3360 // Not failing above means physSyspipeId is valid in GPU instance 3361 grIdx = pParams->physSyspipeId; 3362 } 3363 3364 pKernelGraphics = GPU_GET_KERNEL_GRAPHICS(pGpu, grIdx); 3365 NV_ASSERT_OR_RETURN(pKernelGraphics != NULL, NV_ERR_INVALID_STATE); 3366 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 3367 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 3368 3369 pParams->gpcMask = pKernelGraphicsStaticInfo->floorsweepingMasks.physGpcMask; 3370 3371 return NV_OK; 3372 } 3373 3374 /*! 3375 * subdeviceCtrlCmdKGrGetZcullMask_IMPL 3376 * 3377 * Lock Requirements: 3378 * Assert that API lock and GPUs lock held on entry 3379 */ 3380 NV_STATUS 3381 subdeviceCtrlCmdKGrGetZcullMask_IMPL 3382 ( 3383 Subdevice *pSubdevice, 3384 NV2080_CTRL_GR_GET_ZCULL_MASK_PARAMS *pParams 3385 ) 3386 { 3387 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3388 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 3389 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3390 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo; 3391 KernelGraphics *pKernelGraphics; 3392 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 3393 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 3394 3395 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 3396 3397 if (kmigmgrIsDeviceUsingDeviceProfiling(pGpu, pKernelMIGManager, pDevice)) 3398 { 3399 pParams->zcullMask = kgrmgrGetLegacyZcullMask(pGpu, pKernelGraphicsManager, pParams->gpcId); 3400 } 3401 else 3402 { 3403 portMemSet(&grRouteInfo, 0, sizeof(grRouteInfo)); 3404 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3405 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &grRouteInfo, &pKernelGraphics)); 3406 3407 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 3408 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 3409 3410 if (pParams->gpcId >= 3411 pKernelGraphicsStaticInfo->pGrInfo->infoList[NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_GPCS].data) 3412 { 3413 NV_PRINTF(LEVEL_ERROR, "Incorrect GPC-Idx provided = %d\n", pParams->gpcId); 3414 return NV_ERR_INVALID_ARGUMENT; 3415 } 3416 3417 if (pKernelGraphicsStaticInfo->floorsweepingMasks.zcullMask[pParams->gpcId] == NV_U32_MAX) 3418 { 3419 return NV_ERR_NOT_SUPPORTED; 3420 } 3421 else 3422 { 3423 pParams->zcullMask = pKernelGraphicsStaticInfo->floorsweepingMasks.zcullMask[pParams->gpcId]; 3424 } 3425 } 3426 3427 return NV_OK; 3428 } 3429 3430 /*! 3431 * subdeviceCtrlCmdKGrGetZcullInfo 3432 * 3433 * Lock Requirements: 3434 * Assert that API lock held on entry 3435 */ 3436 NV_STATUS 3437 subdeviceCtrlCmdKGrGetZcullInfo_IMPL 3438 ( 3439 Subdevice *pSubdevice, 3440 NV2080_CTRL_GR_GET_ZCULL_INFO_PARAMS *pParams 3441 ) 3442 { 3443 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3444 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3445 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 3446 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo; 3447 KernelGraphics *pKernelGraphics; 3448 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 3449 3450 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner()); 3451 3452 if (pKernelGraphicsManager == NULL) 3453 { 3454 return NV_ERR_NOT_SUPPORTED; 3455 } 3456 3457 portMemSet(&grRouteInfo, 0, sizeof(grRouteInfo)); 3458 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3459 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &grRouteInfo, &pKernelGraphics)); 3460 3461 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 3462 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 3463 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pZcullInfo != NULL, NV_ERR_NOT_SUPPORTED); 3464 3465 portMemCopy(pParams, 3466 sizeof(*pParams), 3467 pKernelGraphicsStaticInfo->pZcullInfo, 3468 sizeof(*pKernelGraphicsStaticInfo->pZcullInfo)); 3469 3470 return NV_OK; 3471 } 3472 3473 NV_STATUS 3474 subdeviceCtrlCmdKGrCtxswPmMode_IMPL 3475 ( 3476 Subdevice *pSubdevice, 3477 NV2080_CTRL_GR_CTXSW_PM_MODE_PARAMS *pParams 3478 ) 3479 { 3480 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3481 NV_STATUS status = NV_OK; 3482 3483 if (IS_GSP_CLIENT(pGpu)) 3484 { 3485 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo = pParams->grRouteInfo; 3486 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3487 KernelGraphics *pKernelGraphics; 3488 KernelChannel *pKernelChannel; 3489 KernelGraphicsContext *pKernelGraphicsContext; 3490 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); 3491 3492 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 3493 3494 if (pParams->pmMode != NV2080_CTRL_CTXSW_PM_MODE_NO_CTXSW) 3495 { 3496 kgrmgrCtrlSetChannelHandle(pParams->hChannel, &grRouteInfo); 3497 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3498 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, 3499 GPU_RES_GET_DEVICE(pSubdevice), 3500 &grRouteInfo, 3501 &pKernelGraphics)); 3502 3503 // Retrieve channel from either bare channel or TSG handle 3504 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3505 kchannelGetFromDualHandleRestricted(RES_GET_CLIENT(pSubdevice), 3506 pParams->hChannel, &pKernelChannel)); 3507 3508 NV_ASSERT_OK_OR_RETURN( 3509 kgrctxFromKernelChannel(pKernelChannel, &pKernelGraphicsContext)); 3510 3511 // Setup / promote the PM ctx buffer if required 3512 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3513 kgrctxSetupDeferredPmBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel)); 3514 } 3515 3516 status = pRmApi->Control(pRmApi, 3517 RES_GET_CLIENT_HANDLE(pSubdevice), 3518 RES_GET_HANDLE(pSubdevice), 3519 NV2080_CTRL_CMD_GR_CTXSW_PM_MODE, 3520 pParams, 3521 sizeof(*pParams)); 3522 } 3523 3524 return status; 3525 } 3526 3527 /*! 3528 * @brief Gets information about ROPs. 3529 * 3530 * Lock Requirements: 3531 * Assert that API and Gpus lock held on entry 3532 * 3533 * @return NV_OK if success. Error otherwise. 3534 */ 3535 NV_STATUS 3536 subdeviceCtrlCmdKGrGetROPInfo_IMPL 3537 ( 3538 Subdevice *pSubdevice, 3539 NV2080_CTRL_GR_GET_ROP_INFO_PARAMS *pRopInfoParams 3540 ) 3541 { 3542 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3543 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3544 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 3545 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo; 3546 KernelGraphics *pKernelGraphics; 3547 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 3548 3549 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner()); 3550 3551 portMemSet(&grRouteInfo, 0, sizeof(grRouteInfo)); 3552 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3553 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &grRouteInfo, &pKernelGraphics)); 3554 3555 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 3556 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 3557 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pRopInfo != NULL, NV_ERR_NOT_SUPPORTED); 3558 3559 portMemCopy(pRopInfoParams, 3560 sizeof(*pRopInfoParams), 3561 pKernelGraphicsStaticInfo->pRopInfo, 3562 sizeof(*pKernelGraphicsStaticInfo->pRopInfo)); 3563 3564 return NV_OK; 3565 } 3566 3567 /*! 3568 * @brief Gets the current attribute buffer size. 3569 * 3570 * Lock Requirements: 3571 * Assert that API lock held on entry 3572 * 3573 * @return NV_OK if success. Error otherwise. 3574 */ 3575 NV_STATUS 3576 subdeviceCtrlCmdKGrGetAttributeBufferSize_IMPL 3577 ( 3578 Subdevice *pSubdevice, 3579 NV2080_CTRL_GR_GET_ATTRIBUTE_BUFFER_SIZE_PARAMS *pAttribBufferSizeParams 3580 ) 3581 { 3582 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3583 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3584 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo; 3585 KernelGraphics *pKernelGraphics; 3586 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 3587 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 3588 NvU32 engineId; 3589 3590 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner()); 3591 3592 portMemSet(&grRouteInfo, 0, sizeof(grRouteInfo)); 3593 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3594 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &grRouteInfo, &pKernelGraphics)); 3595 3596 // Verify static info is available 3597 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 3598 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 3599 3600 if (pKernelGraphicsStaticInfo->pContextBuffersInfo == NULL) 3601 { 3602 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3603 kgraphicsInitializeDeferredStaticData(pGpu, pKernelGraphics, NV01_NULL_OBJECT, NV01_NULL_OBJECT)); 3604 3605 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pContextBuffersInfo != NULL, NV_ERR_INVALID_STATE); 3606 } 3607 3608 engineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_ATTRIBUTE_CB; 3609 pAttribBufferSizeParams->attribBufferSize = pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[engineId].size; 3610 3611 return NV_OK; 3612 } 3613 3614 /*! 3615 * subdeviceCtrlCmdKGrGetEngineContextProperties 3616 * 3617 * Lock Requirements: 3618 * Assert that API lock and GPUs lock held on entry 3619 */ 3620 NV_STATUS 3621 subdeviceCtrlCmdKGrGetEngineContextProperties_IMPL 3622 ( 3623 Subdevice *pSubdevice, 3624 NV2080_CTRL_GR_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS *pParams 3625 ) 3626 { 3627 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3628 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3629 KernelGraphics *pKernelGraphics; 3630 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 3631 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 3632 NvU32 size = 0; 3633 NvU32 alignment = RM_PAGE_SIZE; 3634 NvU32 engineId; 3635 3636 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner()); 3637 3638 engineId = DRF_VAL(0080_CTRL_FIFO, _GET_ENGINE_CONTEXT_PROPERTIES, _ENGINE_ID, pParams->engineId); 3639 3640 if (engineId >= NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_COUNT) 3641 { 3642 return NV_ERR_INVALID_ARGUMENT; 3643 } 3644 3645 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3646 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, 3647 &pParams->grRouteInfo, &pKernelGraphics)); 3648 3649 // Verify static info is available 3650 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 3651 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 3652 3653 if (pKernelGraphicsStaticInfo->pContextBuffersInfo == NULL) 3654 { 3655 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3656 kgraphicsInitializeDeferredStaticData(pGpu, pKernelGraphics, NV01_NULL_OBJECT, NV01_NULL_OBJECT)); 3657 3658 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pContextBuffersInfo != NULL, NV_ERR_INVALID_STATE); 3659 } 3660 3661 size = pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[engineId].size; 3662 alignment = pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[engineId].alignment; 3663 3664 if (size == NV_U32_MAX) 3665 { 3666 return NV_ERR_NOT_SUPPORTED; 3667 } 3668 3669 if (pParams->bInfoPopulated) 3670 { 3671 size = NV_MAX(size, pParams->size); 3672 alignment = NV_MAX(alignment, pParams->alignment); 3673 } 3674 3675 pParams->size = size; 3676 pParams->alignment = alignment; 3677 pParams->bInfoPopulated = NV_TRUE; 3678 3679 return NV_OK; 3680 } 3681 3682 /*! 3683 * @brief Gets the Graphics Context buffer size and alignment 3684 * 3685 * Lock Requirements: 3686 * Assert that API and Gpus lock held on entry 3687 */ 3688 NV_STATUS 3689 subdeviceCtrlCmdKGrGetCtxBufferSize_IMPL 3690 ( 3691 Subdevice *pSubdevice, 3692 NV2080_CTRL_GR_GET_CTX_BUFFER_SIZE_PARAMS *pParams 3693 ) 3694 { 3695 NV_STATUS status = NV_OK; 3696 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3697 KernelGraphics *pKernelGraphics; 3698 RsClient *pClient = RES_GET_CLIENT(pSubdevice); 3699 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 3700 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3701 NV2080_CTRL_GR_CTX_BUFFER_INFO *pCtxBufferInfo; 3702 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo; 3703 KernelChannel *pKernelChannel; 3704 KernelGraphicsContext *pKernelGraphicsContext; 3705 NvU32 bufferCount; 3706 NvU64 totalBufferSize; 3707 NvU64 prevAlignment; 3708 NvU32 i; 3709 3710 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 3711 3712 // 3713 // vGPU: 3714 // 3715 // Since vGPU does all real hardware management in the 3716 // host, if we are in guest OS (where IS_VIRTUAL(pGpu) is true), 3717 // do an RPC to the host to fetch the total GR Context Buffer Size. 3718 // 3719 if (IS_VIRTUAL_WITHOUT_SRIOV(pGpu) || 3720 (IS_VIRTUAL_WITH_SRIOV(pGpu) && gpuIsWarBug200577889SriovHeavyEnabled(pGpu))) 3721 { 3722 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 3723 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams; 3724 3725 NV_RM_RPC_CONTROL(pGpu, 3726 pRmCtrlParams->hClient, 3727 pRmCtrlParams->hObject, 3728 pRmCtrlParams->cmd, 3729 pRmCtrlParams->pParams, 3730 pRmCtrlParams->paramsSize, 3731 status); 3732 return status; 3733 } 3734 3735 portMemSet(&grRouteInfo, 0, sizeof(grRouteInfo)); 3736 kgrmgrCtrlSetChannelHandle(pParams->hChannel, &grRouteInfo); 3737 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3738 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, 3739 &grRouteInfo, &pKernelGraphics)); 3740 3741 // Get channel from provided handle and owner client 3742 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, 3743 CliGetKernelChannel(pClient, pParams->hChannel, &pKernelChannel)); 3744 3745 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3746 kgrctxFromKernelChannel(pKernelChannel, &pKernelGraphicsContext)); 3747 3748 // Get the total buffer count 3749 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3750 kgrctxGetBufferCount(pGpu, pKernelGraphicsContext, pKernelGraphics, &bufferCount)); 3751 3752 pCtxBufferInfo = portMemAllocNonPaged(bufferCount * sizeof(NV2080_CTRL_GR_CTX_BUFFER_INFO)); 3753 NV_CHECK_OR_ELSE(LEVEL_ERROR, 3754 pCtxBufferInfo != NULL, 3755 status = NV_ERR_NO_MEMORY; 3756 goto done;); 3757 portMemSet(pCtxBufferInfo, 0, bufferCount * sizeof(NV2080_CTRL_GR_CTX_BUFFER_INFO)); 3758 3759 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, 3760 kgrctxGetCtxBufferInfo(pGpu, 3761 pKernelGraphicsContext, 3762 pKernelGraphics, 3763 kchannelGetGfid(pKernelChannel), 3764 bufferCount, 3765 &bufferCount, 3766 pCtxBufferInfo), 3767 done); 3768 3769 // 3770 // Calculate total size by walking thru all buffers & alignments. Adjust the total size 3771 // by adding the respective alignment so that the mapping VA can be adjusted. 3772 // 3773 totalBufferSize = 0; 3774 prevAlignment = 0x0; 3775 for (i = 0; i < bufferCount; i++) 3776 { 3777 if (prevAlignment != pCtxBufferInfo[i].alignment) 3778 { 3779 totalBufferSize += pCtxBufferInfo[i].alignment; 3780 prevAlignment = pCtxBufferInfo[i].alignment; 3781 } 3782 3783 totalBufferSize += (pCtxBufferInfo[i].alignment != 0x0) ? 3784 NV_ALIGN_UP(pCtxBufferInfo[i].size, pCtxBufferInfo[i].alignment) : pCtxBufferInfo[i].size; 3785 } 3786 3787 pParams->totalBufferSize = totalBufferSize; 3788 3789 done: 3790 portMemFree(pCtxBufferInfo); 3791 return status; 3792 } 3793 3794 /*! 3795 * @brief Gets the Graphics Context buffer info like opaque buffer pointer 3796 * size, alignment, aperture, allocation contiguity etc. 3797 * 3798 * Lock Requirements: 3799 * Assert that API and Gpus lock held on entry 3800 */ 3801 NV_STATUS 3802 subdeviceCtrlCmdKGrGetCtxBufferInfo_IMPL 3803 ( 3804 Subdevice *pSubdevice, 3805 NV2080_CTRL_GR_GET_CTX_BUFFER_INFO_PARAMS *pParams 3806 ) 3807 { 3808 NV_STATUS status = NV_OK; 3809 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3810 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3811 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo; 3812 RsClient *pUserClient; 3813 KernelGraphics *pKernelGraphics; 3814 KernelChannel *pKernelChannel; 3815 KernelGraphicsContext *pKernelGraphicsContext; 3816 3817 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner()); 3818 3819 // 3820 // vGPU: 3821 // 3822 // Since vGPU does all real hardware management in the 3823 // host, if we are in guest OS (where IS_VIRTUAL(pGpu) is true), 3824 // do an RPC to the host to get Graphics context buffers information. 3825 // 3826 if (IS_VIRTUAL_WITHOUT_SRIOV(pGpu) || 3827 (IS_VIRTUAL_WITH_SRIOV(pGpu) && gpuIsWarBug200577889SriovHeavyEnabled(pGpu))) 3828 { 3829 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 3830 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams; 3831 3832 NV_RM_RPC_CONTROL(pGpu, 3833 pRmCtrlParams->hClient, 3834 pRmCtrlParams->hObject, 3835 pRmCtrlParams->cmd, 3836 pRmCtrlParams->pParams, 3837 pRmCtrlParams->paramsSize, 3838 status); 3839 return status; 3840 } 3841 3842 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, 3843 serverGetClientUnderLock(&g_resServ, pParams->hUserClient, &pUserClient)); 3844 3845 // Get channel from provided handle and owner client 3846 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, 3847 CliGetKernelChannel(pUserClient, pParams->hChannel, &pKernelChannel)); 3848 3849 portMemSet(&grRouteInfo, 0, sizeof(grRouteInfo)); 3850 kgrmgrCtrlSetChannelHandle(pParams->hChannel, &grRouteInfo); 3851 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3852 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, 3853 GPU_RES_GET_DEVICE(pKernelChannel), 3854 &grRouteInfo, &pKernelGraphics)); 3855 3856 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3857 kgrctxFromKernelChannel(pKernelChannel, &pKernelGraphicsContext)); 3858 3859 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3860 kgrctxGetCtxBufferInfo(pGpu, 3861 pKernelGraphicsContext, 3862 pKernelGraphics, 3863 kchannelGetGfid(pKernelChannel), 3864 NV_ARRAY_ELEMENTS(pParams->ctxBufferInfo), 3865 &pParams->bufferCount, 3866 pParams->ctxBufferInfo)); 3867 3868 return status; 3869 } 3870 3871 /*! 3872 * subdeviceCtrlCmdKGrInternalGetCtxBufferPtes 3873 * 3874 * Lock Requirements: 3875 * Assert that API lock and GPUs lock held on entry 3876 */ 3877 NV_STATUS 3878 subdeviceCtrlCmdKGrGetCtxBufferPtes_IMPL 3879 ( 3880 Subdevice *pSubdevice, 3881 NV2080_CTRL_KGR_GET_CTX_BUFFER_PTES_PARAMS *pParams 3882 ) 3883 { 3884 NV_STATUS status = NV_OK; 3885 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3886 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3887 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo; 3888 RsClient *pUserClient; 3889 KernelGraphics *pKernelGraphics; 3890 KernelChannel *pKernelChannel; 3891 KernelGraphicsContext *pKernelGraphicsContext; 3892 3893 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner()); 3894 3895 // 3896 // Currently, ROUTE_TO_VGPU_HOST instructs resource server to call the RPC 3897 // on all vGPU configurations including SRIOV Standard which is not required. 3898 // Hence, manually dispatching the RPC for required vGPU configs. 3899 // 3900 // vGPU: 3901 // 3902 // Since vGPU does all real hardware management in the 3903 // host, if we are in guest OS (where IS_VIRTUAL(pGpu) is true), 3904 // do an RPC to the host to get Graphics context buffers PTEs information. 3905 // 3906 if (IS_VIRTUAL_WITHOUT_SRIOV(pGpu) || 3907 (IS_VIRTUAL_WITH_SRIOV(pGpu) && gpuIsWarBug200577889SriovHeavyEnabled(pGpu))) 3908 { 3909 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 3910 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams; 3911 NvHandle hClient = RES_GET_CLIENT_HANDLE(pSubdevice); 3912 NvHandle hObject = RES_GET_HANDLE(pSubdevice); 3913 3914 NV_RM_RPC_CONTROL(pGpu, 3915 hClient, 3916 hObject, 3917 pRmCtrlParams->cmd, 3918 pRmCtrlParams->pParams, 3919 pRmCtrlParams->paramsSize, 3920 status); 3921 return status; 3922 } 3923 3924 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, 3925 serverGetClientUnderLock(&g_resServ, pParams->hUserClient, &pUserClient)); 3926 3927 // Get channel from provided handle and owner client 3928 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, 3929 CliGetKernelChannel(pUserClient, pParams->hChannel, &pKernelChannel)); 3930 3931 portMemSet(&grRouteInfo, 0, sizeof(grRouteInfo)); 3932 kgrmgrCtrlSetChannelHandle(pParams->hChannel, &grRouteInfo); 3933 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, 3934 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, 3935 GPU_RES_GET_DEVICE(pKernelChannel), 3936 &grRouteInfo, &pKernelGraphics)); 3937 3938 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3939 kgrctxFromKernelChannel(pKernelChannel, &pKernelGraphicsContext)); 3940 3941 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3942 kgrctxGetCtxBufferPtes(pGpu, 3943 pKernelGraphicsContext, 3944 pKernelGraphics, 3945 kchannelGetGfid(pKernelChannel), 3946 pParams->bufferType, 3947 pParams->firstPage, 3948 pParams->physAddrs, 3949 NV_ARRAY_ELEMENTS(pParams->physAddrs), 3950 &pParams->numPages, 3951 &pParams->bNoMorePages)); 3952 3953 return status; 3954 } 3955 3956 /*! 3957 * subdeviceCtrlCmdKGrGetGfxGpcAndTpcInfo 3958 * 3959 * Lock Requirements: 3960 * Assert that API lock and GPUs lock held on entry 3961 */ 3962 NV_STATUS 3963 subdeviceCtrlCmdKGrGetGfxGpcAndTpcInfo_IMPL 3964 ( 3965 Subdevice *pSubdevice, 3966 NV2080_CTRL_GR_GET_GFX_GPC_AND_TPC_INFO_PARAMS *pParams 3967 ) 3968 { 3969 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3970 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3971 KernelGraphics *pKernelGraphics; 3972 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 3973 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 3974 3975 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner()); 3976 3977 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3978 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, 3979 &pParams->grRouteInfo, &pKernelGraphics)); 3980 3981 // Verify static info is available 3982 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 3983 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 3984 3985 pParams->physGfxGpcMask = pKernelGraphicsStaticInfo->floorsweepingMasks.physGfxGpcMask; 3986 pParams->numGfxTpc = pKernelGraphicsStaticInfo->floorsweepingMasks.numGfxTpc; 3987 3988 return NV_OK; 3989 } 3990 3991 #define KGR_DO_WITH_GR(pGpu, pKernelGraphics, body) do \ 3992 { \ 3993 (body); \ 3994 } while (0); 3995 3996 NV_STATUS 3997 subdeviceCtrlCmdGrInternalSetFecsTraceHwEnable_IMPL 3998 ( 3999 Subdevice *pSubdevice, 4000 NV2080_CTRL_INTERNAL_GR_SET_FECS_TRACE_HW_ENABLE_PARAMS *pParams 4001 ) 4002 { 4003 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 4004 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 4005 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 4006 NV_STATUS status = NV_OK; 4007 KernelGraphics *pKernelGraphics; 4008 4009 LOCK_ASSERT_AND_RETURN(rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 4010 4011 NV_CHECK_OK_OR_RETURN( 4012 LEVEL_ERROR, 4013 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, 4014 &pParams->grRouteInfo, &pKernelGraphics)); 4015 4016 KGR_DO_WITH_GR(pGpu, pKernelGraphics, 4017 kgraphicsSetFecsTraceHwEnable_HAL(pGpu, pKernelGraphics, pParams->bEnable)); 4018 pKernelGraphics->bCtxswLoggingEnabled = pParams->bEnable; 4019 4020 return status; 4021 } 4022 4023 NV_STATUS 4024 subdeviceCtrlCmdGrInternalGetFecsTraceHwEnable_IMPL 4025 ( 4026 Subdevice *pSubdevice, 4027 NV2080_CTRL_INTERNAL_GR_GET_FECS_TRACE_HW_ENABLE_PARAMS *pParams 4028 ) 4029 { 4030 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 4031 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 4032 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 4033 NV_STATUS status = NV_OK; 4034 KernelGraphics *pKernelGraphics; 4035 4036 LOCK_ASSERT_AND_RETURN(rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 4037 4038 NV_CHECK_OK_OR_RETURN( 4039 LEVEL_ERROR, 4040 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &pParams->grRouteInfo, 4041 &pKernelGraphics)); 4042 4043 KGR_DO_WITH_GR(pGpu, pKernelGraphics, 4044 pParams->bEnable = kgraphicsIsCtxswLoggingEnabled(pGpu, pKernelGraphics)); 4045 4046 return status; 4047 } 4048 4049 NV_STATUS 4050 subdeviceCtrlCmdGrInternalSetFecsTraceRdOffset_IMPL 4051 ( 4052 Subdevice *pSubdevice, 4053 NV2080_CTRL_INTERNAL_GR_SET_FECS_TRACE_RD_OFFSET_PARAMS *pParams 4054 ) 4055 { 4056 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 4057 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 4058 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 4059 NV_STATUS status = NV_OK; 4060 KernelGraphics *pKernelGraphics; 4061 4062 LOCK_ASSERT_AND_RETURN(rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 4063 4064 NV_CHECK_OK_OR_RETURN( 4065 LEVEL_ERROR, 4066 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &pParams->grRouteInfo, 4067 &pKernelGraphics)); 4068 4069 KGR_DO_WITH_GR(pGpu, pKernelGraphics, 4070 kgraphicsSetFecsTraceRdOffset_HAL(pGpu, pKernelGraphics, pParams->offset)); 4071 4072 return status; 4073 } 4074 4075 NV_STATUS 4076 subdeviceCtrlCmdGrInternalGetFecsTraceRdOffset_IMPL 4077 ( 4078 Subdevice *pSubdevice, 4079 NV2080_CTRL_INTERNAL_GR_GET_FECS_TRACE_RD_OFFSET_PARAMS *pParams 4080 ) 4081 { 4082 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 4083 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 4084 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 4085 NV_STATUS status = NV_OK; 4086 KernelGraphics *pKernelGraphics; 4087 4088 LOCK_ASSERT_AND_RETURN(rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 4089 4090 NV_CHECK_OK_OR_RETURN( 4091 LEVEL_ERROR, 4092 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &pParams->grRouteInfo, 4093 &pKernelGraphics)); 4094 4095 KGR_DO_WITH_GR(pGpu, pKernelGraphics, 4096 pParams->offset = kgraphicsGetFecsTraceRdOffset_HAL(pGpu, pKernelGraphics)); 4097 4098 return status; 4099 } 4100 4101 NV_STATUS 4102 subdeviceCtrlCmdGrInternalSetFecsTraceWrOffset_IMPL 4103 ( 4104 Subdevice *pSubdevice, 4105 NV2080_CTRL_INTERNAL_GR_SET_FECS_TRACE_WR_OFFSET_PARAMS *pParams 4106 ) 4107 { 4108 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 4109 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 4110 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 4111 NV_STATUS status = NV_OK; 4112 KernelGraphics *pKernelGraphics; 4113 4114 LOCK_ASSERT_AND_RETURN(rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 4115 4116 NV_CHECK_OK_OR_RETURN( 4117 LEVEL_ERROR, 4118 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &pParams->grRouteInfo, 4119 &pKernelGraphics)); 4120 4121 KGR_DO_WITH_GR(pGpu, pKernelGraphics, 4122 kgraphicsSetFecsTraceWrOffset_HAL(pGpu, pKernelGraphics, pParams->offset)); 4123 4124 return status; 4125 } 4126