1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #define NVOC_KERNEL_GRAPHICS_H_PRIVATE_ACCESS_ALLOWED 25 26 27 #include "rmconfig.h" 28 29 #include "kernel/gpu/gr/kernel_graphics_manager.h" 30 #include "kernel/gpu/gr/kernel_graphics.h" 31 #include "kernel/gpu/mig_mgr/kernel_mig_manager.h" 32 #include "kernel/gpu/device/device.h" 33 #include "kernel/gpu/subdevice/subdevice.h" 34 #include "kernel/rmapi/rmapi_utils.h" 35 #include "kernel/core/locks.h" 36 #include "kernel/gpu/mem_sys/kern_mem_sys.h" 37 #include "kernel/mem_mgr/gpu_vaspace.h" 38 #include "virtualization/hypervisor/hypervisor.h" 39 #include "nvRmReg.h" 40 #include "kernel/gpu/mem_mgr/mem_mgr.h" 41 #include "kernel/gpu/mem_mgr/heap.h" 42 #include "kernel/gpu/intr/engine_idx.h" 43 #include "gpu/mem_mgr/virt_mem_allocator.h" 44 #include "gpu/mmu/kern_gmmu.h" 45 #include "platform/sli/sli.h" 46 #include "rmapi/rs_utils.h" 47 #include "rmapi/client.h" 48 49 #include "vgpu/vgpu_events.h" 50 #include "vgpu/rpc.h" 51 52 #include "class/clb0c0.h" 53 #include "class/clb1c0.h" 54 #include "class/clc0c0.h" 55 #include "class/clc1c0.h" 56 #include "class/clc3c0.h" 57 #include "class/clc5c0.h" 58 #include "class/clc6c0.h" 59 #include "class/clc7c0.h" 60 #include "class/clcbc0.h" 61 62 #include "class/cl0080.h" 63 #include "class/cl2080.h" 64 #include "class/cla06f.h" 65 #include "class/cla06fsubch.h" 66 #include "class/cl90f1.h" // FERMI_VASPACE_A 67 #include "class/cl003e.h" // NV01_MEMORY_SYSTEM 68 #include "class/cl50a0.h" // NV50_MEMORY_VIRTUAL 69 #include "class/cl0040.h" // NV01_MEMORY_LOCAL_USER 70 #include "class/clc36f.h" // VOLTA_CHANNEL_GPFIFO_A 71 #include "class/clc46f.h" // TURING_CHANNEL_GPFIFO_A 72 #include "class/clc56f.h" // AMPERE_CHANNEL_GPFIFO_A 73 #include "class/clc86f.h" // HOPPER_CHANNEL_GPFIFO_A 74 #include "class/clc637.h" 75 #include "class/clc638.h" 76 77 // 78 // We use NV2080_CTRL_INTERNAL_GR_MAX_GPC to statically allocate certain 79 // GPC related array in ctrl call header file. We will need to adjust 80 // NV2080_CTRL_INTERNAL_GR_MAX_GPC if some day KGRMGR_MAX_GPC gets changed 81 // 82 ct_assert(NV2080_CTRL_INTERNAL_GR_MAX_GPC == KGRMGR_MAX_GPC); 83 84 // 85 // Ensure the external and internal ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_COUNT 86 // will always in sync 87 // 88 ct_assert(NV2080_CTRL_INTERNAL_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_COUNT == 89 NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_COUNT); 90 91 typedef struct KGRAPHICS_PRIVATE_DATA 92 { 93 NvBool bInitialized; 94 KGRAPHICS_STATIC_INFO staticInfo; 95 } KGRAPHICS_PRIVATE_DATA; 96 static NV_STATUS _kgraphicsMapGlobalCtxBuffer(OBJGPU *pGpu, KernelGraphics *pKernelGraphics, NvU32 gfid, OBJVASPACE *, 97 KernelGraphicsContext *, GR_GLOBALCTX_BUFFER, NvBool bIsReadOnly); 98 static NV_STATUS _kgraphicsPostSchedulingEnableHandler(OBJGPU *, void *); 99 100 NV_STATUS 101 kgraphicsConstructEngine_IMPL 102 ( 103 OBJGPU *pGpu, 104 KernelGraphics *pKernelGraphics, 105 ENGDESCRIPTOR engDesc 106 ) 107 { 108 KGRAPHICS_PRIVATE_DATA *pPrivate; 109 NvU32 idx; 110 GR_GLOBALCTX_BUFFER buf; 111 GR_CTX_BUFFER localBuf; 112 113 pKernelGraphics->instance = ENGDESC_FIELD(engDesc, _INST); 114 115 pPrivate = portMemAllocNonPaged(sizeof(*pPrivate)); 116 if (pPrivate == NULL) 117 return NV_ERR_NO_MEMORY; 118 portMemSet(pPrivate, 0, sizeof(*pPrivate)); 119 pKernelGraphics->pPrivate = pPrivate; 120 121 // All local ctx buffers default to uncached FB preferred 122 FOR_EACH_IN_ENUM(GR_CTX_BUFFER, localBuf) 123 { 124 pKernelGraphics->ctxAttr[localBuf].pAllocList = ADDRLIST_FBMEM_PREFERRED; 125 pKernelGraphics->ctxAttr[localBuf].cpuAttr = NV_MEMORY_UNCACHED; 126 } 127 FOR_EACH_IN_ENUM_END; 128 129 // Process instloc overrides 130 { 131 struct 132 { 133 GR_CTX_BUFFER buf; 134 NvU32 override; 135 } instlocOverrides[] = 136 { 137 { GR_CTX_BUFFER_MAIN, DRF_VAL(_REG_STR_RM, _INST_LOC, _GRCTX, pGpu->instLocOverrides) }, 138 { GR_CTX_BUFFER_PATCH, DRF_VAL(_REG_STR_RM, _INST_LOC_2, _CTX_PATCH, pGpu->instLocOverrides2) }, 139 { GR_CTX_BUFFER_ZCULL, DRF_VAL(_REG_STR_RM, _INST_LOC_2, _ZCULLCTX, pGpu->instLocOverrides2) }, 140 { GR_CTX_BUFFER_PM, DRF_VAL(_REG_STR_RM, _INST_LOC_2, _PMCTX, pGpu->instLocOverrides2) }, 141 { GR_CTX_BUFFER_PREEMPT, DRF_VAL(_REG_STR_RM, _INST_LOC_3, _PREEMPT_BUFFER, pGpu->instLocOverrides3) }, 142 { GR_CTX_BUFFER_BETA_CB, DRF_VAL(_REG_STR_RM, _INST_LOC_3, _GFXP_BETACB_BUFFER, pGpu->instLocOverrides3) }, 143 { GR_CTX_BUFFER_PAGEPOOL, DRF_VAL(_REG_STR_RM, _INST_LOC_3, _GFXP_PAGEPOOL_BUFFER, pGpu->instLocOverrides3) }, 144 { GR_CTX_BUFFER_SPILL, DRF_VAL(_REG_STR_RM, _INST_LOC_3, _GFXP_SPILL_BUFFER, pGpu->instLocOverrides3) }, 145 { GR_CTX_BUFFER_RTV_CB, DRF_VAL(_REG_STR_RM, _INST_LOC_3, _GFXP_RTVCB_BUFFER, pGpu->instLocOverrides3) }, 146 { GR_CTX_BUFFER_SETUP, DRF_VAL(_REG_STR_RM, _INST_LOC_4, _GFXP_SETUP_BUFFER, pGpu->instLocOverrides4) } 147 }; 148 149 for (idx = 0; idx < NV_ARRAY_ELEMENTS(instlocOverrides); ++idx) 150 { 151 memdescOverrideInstLocList(instlocOverrides[idx].override, 152 NV_ENUM_TO_STRING(GR_CTX_BUFFER, instlocOverrides[idx].buf), 153 &pKernelGraphics->ctxAttr[instlocOverrides[idx].buf].pAllocList, 154 &pKernelGraphics->ctxAttr[instlocOverrides[idx].buf].cpuAttr); 155 } 156 } 157 158 // Most global ctx buffers default to uncached FB preferred 159 FOR_EACH_IN_ENUM(GR_GLOBALCTX_BUFFER, buf) 160 { 161 pKernelGraphics->globalCtxBuffersInfo.globalCtxAttr[buf].pAllocList = ADDRLIST_FBMEM_PREFERRED; 162 pKernelGraphics->globalCtxBuffersInfo.globalCtxAttr[buf].cpuAttr = NV_MEMORY_UNCACHED; 163 } 164 FOR_EACH_IN_ENUM_END; 165 166 // FECS event buffer defaults to cached SYSMEM 167 pKernelGraphics->globalCtxBuffersInfo.globalCtxAttr[GR_GLOBALCTX_BUFFER_FECS_EVENT].pAllocList = ADDRLIST_SYSMEM_ONLY; 168 pKernelGraphics->globalCtxBuffersInfo.globalCtxAttr[GR_GLOBALCTX_BUFFER_FECS_EVENT].cpuAttr = NV_MEMORY_CACHED; 169 170 // Process instloc overrides 171 { 172 struct 173 { 174 GR_GLOBALCTX_BUFFER buf; 175 NvU32 override; 176 } instlocOverrides[] = 177 { 178 { GR_GLOBALCTX_BUFFER_FECS_EVENT, DRF_VAL(_REG_STR_RM, _INST_LOC_4, _FECS_EVENT_BUF, pGpu->instLocOverrides4) }, 179 { GR_GLOBALCTX_BUFFER_ATTRIBUTE_CB, DRF_VAL(_REG_STR_RM, _INST_LOC_2, _ATTR_CB, pGpu->instLocOverrides2) }, 180 { GR_GLOBALCTX_BUFFER_BUNDLE_CB, DRF_VAL(_REG_STR_RM, _INST_LOC_2, _BUNDLE_CB, pGpu->instLocOverrides2) }, 181 { GR_GLOBALCTX_BUFFER_PAGEPOOL, DRF_VAL(_REG_STR_RM, _INST_LOC_2, _PAGEPOOL, pGpu->instLocOverrides2) }, 182 { GR_GLOBALCTX_BUFFER_PRIV_ACCESS_MAP, DRF_VAL(_REG_STR_RM, _INST_LOC_3, _PRIV_ACCESS_MAP, pGpu->instLocOverrides3) }, 183 { GR_GLOBALCTX_BUFFER_UNRESTRICTED_PRIV_ACCESS_MAP, DRF_VAL(_REG_STR_RM, _INST_LOC_3, _PRIV_ACCESS_MAP, pGpu->instLocOverrides3) }, 184 { GR_GLOBAL_BUFFER_GLOBAL_PRIV_ACCESS_MAP, DRF_VAL(_REG_STR_RM, _INST_LOC_3, _PRIV_ACCESS_MAP, pGpu->instLocOverrides3) }, 185 { GR_GLOBALCTX_BUFFER_RTV_CB, DRF_VAL(_REG_STR_RM, _INST_LOC_3, _RTVCB_BUFFER, pGpu->instLocOverrides3) } 186 }; 187 188 for (idx = 0; idx < NV_ARRAY_ELEMENTS(instlocOverrides); ++idx) 189 { 190 memdescOverrideInstLocList(instlocOverrides[idx].override, 191 NV_ENUM_TO_STRING(GR_GLOBALCTX_BUFFER, instlocOverrides[idx].buf), 192 &pKernelGraphics->globalCtxBuffersInfo.globalCtxAttr[instlocOverrides[idx].buf].pAllocList, 193 &pKernelGraphics->globalCtxBuffersInfo.globalCtxAttr[instlocOverrides[idx].buf].cpuAttr); 194 } 195 } 196 197 // Copy final global buffer attributes for local versions 198 FOR_EACH_IN_ENUM(GR_GLOBALCTX_BUFFER, buf) 199 { 200 // Host RM cannot allocate system memory on behalf of the VF RM, so force FB. 201 pKernelGraphics->globalCtxBuffersInfo.vfGlobalCtxAttr[buf].pAllocList = ADDRLIST_FBMEM_ONLY; 202 pKernelGraphics->globalCtxBuffersInfo.vfGlobalCtxAttr[buf].cpuAttr = NV_MEMORY_UNCACHED; 203 204 // Local context allocation 205 pKernelGraphics->globalCtxBuffersInfo.localCtxAttr[buf] = pKernelGraphics->globalCtxBuffersInfo.globalCtxAttr[buf]; 206 } 207 FOR_EACH_IN_ENUM_END; 208 209 // 210 // Default context buffers to non size aligned. The attribute buffer is 211 // required to be mapped with an offset naturally aligned to the size. 212 // 213 for (idx = 0; idx < GR_GLOBALCTX_BUFFER_COUNT; idx++) 214 pKernelGraphics->globalCtxBuffersInfo.bSizeAligned[idx] = NV_FALSE; 215 pKernelGraphics->globalCtxBuffersInfo.bSizeAligned[GR_GLOBALCTX_BUFFER_ATTRIBUTE_CB] = NV_TRUE; 216 217 NV_ASSERT_OK_OR_RETURN(fecsCtxswLoggingInit(pGpu, pKernelGraphics, &pKernelGraphics->pFecsTraceInfo)); 218 219 return NV_OK; 220 } 221 222 void 223 kgraphicsDestruct_IMPL 224 ( 225 KernelGraphics *pKernelGraphics 226 ) 227 { 228 OBJGPU *pGpu = ENG_GET_GPU(pKernelGraphics); 229 230 fecsCtxswLoggingTeardown(pGpu, pKernelGraphics); 231 pKernelGraphics->pFecsTraceInfo = NULL; 232 kgraphicsInvalidateStaticInfo(pGpu, pKernelGraphics); 233 234 portMemFree(pKernelGraphics->pPrivate); 235 pKernelGraphics->pPrivate = NULL; 236 } 237 238 NV_STATUS 239 kgraphicsStateInitLocked_IMPL 240 ( 241 OBJGPU *pGpu, 242 KernelGraphics *pKernelGraphics 243 ) 244 { 245 NvU32 nGlobalCtx = 1; 246 NvU32 numClasses; 247 248 NV_ASSERT_OK_OR_RETURN(gpuGetClassList(pGpu, &numClasses, NULL, ENG_GR(pKernelGraphics->instance))); 249 250 // 251 // Number of supported class can be zero when Graphics engine is disabled, but we still 252 // need those classes in ClassDB for KernelGraphics engine operation, thus here we are adding 253 // the ENG_GR(X) supported classes back to ClassDB 254 // 255 if (numClasses == 0) 256 { 257 PGPU_ENGINE_ORDER pEngineOrder = &pGpu->engineOrder; 258 const CLASSDESCRIPTOR *pClassDesc = &pEngineOrder->pClassDescriptors[0]; 259 NvU32 i; 260 NvU32 classNum; 261 262 for (i = 0; i < pEngineOrder->numClassDescriptors; i++) 263 { 264 classNum = pClassDesc[i].externalClassId; 265 if (classNum == (NvU32)~0) 266 continue; 267 268 if (ENG_GR(pKernelGraphics->instance) == pClassDesc[i].engDesc) 269 { 270 NV_PRINTF(LEVEL_INFO, "Adding class ID 0x%x to ClassDB\n", classNum); 271 NV_ASSERT_OK_OR_RETURN( 272 gpuAddClassToClassDBByEngTagClassId(pGpu, ENG_GR(pKernelGraphics->instance), classNum)); 273 } 274 } 275 } 276 277 // 278 // Allocate guest context db array 279 // 280 if (gpuIsSriovEnabled(pGpu)) 281 { 282 nGlobalCtx = VMMU_MAX_GFID; 283 } 284 285 pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers = portMemAllocNonPaged(sizeof(*pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers) * nGlobalCtx); 286 if (pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers == NULL) 287 { 288 return NV_ERR_NO_MEMORY; 289 } 290 portMemSet(pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers, 0, 291 sizeof(*pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers) * nGlobalCtx); 292 293 if (pKernelGraphics->instance == 0) 294 { 295 // 296 // GSP_CLIENT creates the golden context channel GR post load. However, 297 // if PMA scrubber is enabled, a scrubber channel must be constructed 298 // first as a part of Fifo post load. Hence, add the golden channel 299 // creation as a fifo post-scheduling-enablement callback. 300 // 301 NV_ASSERT_OK_OR_RETURN( 302 kfifoAddSchedulingHandler(pGpu, GPU_GET_KERNEL_FIFO(pGpu), 303 _kgraphicsPostSchedulingEnableHandler, 304 (void *)((NvUPtr)(pKernelGraphics->instance)), 305 NULL, NULL)); 306 } 307 308 return NV_OK; 309 } 310 311 NV_STATUS 312 kgraphicsStateUnload_IMPL 313 ( 314 OBJGPU *pGpu, 315 KernelGraphics *pKernelGraphics, 316 NvU32 flags 317 318 ) 319 { 320 if (pKernelGraphics->instance != 0) 321 return NV_OK; 322 323 kfifoRemoveSchedulingHandler(pGpu, GPU_GET_KERNEL_FIFO(pGpu), 324 _kgraphicsPostSchedulingEnableHandler, 325 (void *)((NvUPtr)(pKernelGraphics->instance)), 326 NULL, NULL); 327 328 return NV_OK; 329 } 330 331 NV_STATUS 332 kgraphicsStateLoad_IMPL 333 ( 334 OBJGPU *pGpu, 335 KernelGraphics *pKernelGraphics, 336 NvU32 flags 337 ) 338 { 339 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 340 341 if (fecsGetCtxswLogConsumerCount(pGpu, pKernelGraphicsManager) > 0) 342 { 343 fecsBufferMap(pGpu, pKernelGraphics); 344 fecsBufferReset(pGpu, pKernelGraphics); 345 } 346 347 return NV_OK; 348 } 349 350 NV_STATUS 351 kgraphicsStatePreUnload_IMPL 352 ( 353 OBJGPU *pGpu, 354 KernelGraphics *pKernelGraphics, 355 NvU32 flags 356 ) 357 { 358 fecsBufferUnmap(pGpu, pKernelGraphics); 359 360 // Release global buffers used as part of the gr context, when not in S/R 361 if (!(flags & GPU_STATE_FLAGS_PRESERVING)) 362 kgraphicsFreeGlobalCtxBuffers(pGpu, pKernelGraphics, GPU_GFID_PF); 363 364 return NV_OK; 365 } 366 367 void 368 kgraphicsStateDestroy_IMPL 369 ( 370 OBJGPU *pGpu, 371 KernelGraphics *pKernelGraphics 372 ) 373 { 374 fecsBufferTeardown(pGpu, pKernelGraphics); 375 376 portMemFree(pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers); 377 pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers = NULL; 378 } 379 380 NvBool kgraphicsIsPresent_IMPL 381 ( 382 OBJGPU *pGpu, 383 KernelGraphics *pKernelGraphics 384 ) 385 { 386 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 387 NvU32 unused; 388 389 if (IsDFPGA(pGpu)) 390 return NV_FALSE; 391 392 if (IS_MODS_AMODEL(pGpu)) 393 return NV_TRUE; 394 395 return kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo, 396 ENGINE_INFO_TYPE_RM_ENGINE_TYPE, (NvU32)RM_ENGINE_TYPE_GR(pKernelGraphics->instance), 397 ENGINE_INFO_TYPE_INVALID, &unused) == NV_OK; 398 } 399 400 NV_STATUS 401 kgraphicsStatePostLoad_IMPL 402 ( 403 OBJGPU *pGpu, 404 KernelGraphics *pKernelGraphics, 405 NvU32 flags 406 ) 407 { 408 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 409 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, kgraphicsLoadStaticInfo(pGpu, pKernelGraphics, KMIGMGR_SWIZZID_INVALID)); 410 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 411 412 if ((!IS_VIRTUAL(pGpu)) && 413 (pKernelGraphicsStaticInfo != NULL) && 414 (pKernelGraphicsStaticInfo->pContextBuffersInfo != NULL) && 415 (kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, GPU_GFID_PF)->memDesc[GR_GLOBALCTX_BUFFER_FECS_EVENT] == NULL)) 416 { 417 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 418 kgraphicsAllocGlobalCtxBuffers_HAL(pGpu, pKernelGraphics, GPU_GFID_PF)); 419 } 420 421 return NV_OK; 422 } 423 424 /*! 425 * @brief Create a golden image channel after Fifo post load 426 * Instead of lazily waiting until first client request, we proactively create a 427 * golden channel here. 428 */ 429 static NV_STATUS 430 _kgraphicsPostSchedulingEnableHandler 431 ( 432 OBJGPU *pGpu, 433 void *pGrIndex 434 ) 435 { 436 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 437 KernelGraphics *pKernelGraphics = GPU_GET_KERNEL_GRAPHICS(pGpu, ((NvU32)(NvUPtr)pGrIndex)); 438 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 439 440 // Nothing to do for non-GSPCLIENT 441 if (!IS_GSP_CLIENT(pGpu)) 442 return NV_OK; 443 444 // Defer golden context channel creation to GPU instance configuration 445 if (IS_MIG_ENABLED(pGpu)) 446 return NV_OK; 447 448 // Skip for MIG engines with 0 GPCs 449 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 450 if (pKernelGraphicsStaticInfo->floorsweepingMasks.gpcMask == 0x0) 451 return NV_OK; 452 453 if (memmgrIsPmaInitialized(pMemoryManager)) 454 { 455 Heap *pHeap = GPU_GET_HEAP(pGpu); 456 NvU32 pmaConfig = PMA_QUERY_SCRUB_ENABLED | PMA_QUERY_SCRUB_VALID; 457 458 NV_ASSERT_OK_OR_RETURN(pmaQueryConfigs(&pHeap->pmaObject, &pmaConfig)); 459 460 // 461 // Scrubber is also constructed from the same Fifo post scheduling 462 // enable callback queue. This check enforces the dependency that 463 // scrubber must be initialized first 464 // 465 if ((pmaConfig & PMA_QUERY_SCRUB_ENABLED) && 466 !(pmaConfig & PMA_QUERY_SCRUB_VALID)) 467 { 468 return NV_WARN_MORE_PROCESSING_REQUIRED; 469 } 470 } 471 472 return kgraphicsCreateGoldenImageChannel(pGpu, pKernelGraphics); 473 } 474 475 void 476 kgraphicsInvalidateStaticInfo_IMPL 477 ( 478 OBJGPU *pGpu, 479 KernelGraphics *pKernelGraphics 480 ) 481 { 482 portMemFree(pKernelGraphics->pPrivate->staticInfo.pGrInfo); 483 pKernelGraphics->pPrivate->staticInfo.pGrInfo = NULL; 484 485 portMemFree(pKernelGraphics->pPrivate->staticInfo.pPpcMasks); 486 pKernelGraphics->pPrivate->staticInfo.pPpcMasks = NULL; 487 488 portMemFree(pKernelGraphics->pPrivate->staticInfo.pZcullInfo); 489 pKernelGraphics->pPrivate->staticInfo.pZcullInfo = NULL; 490 491 portMemFree(pKernelGraphics->pPrivate->staticInfo.pRopInfo); 492 pKernelGraphics->pPrivate->staticInfo.pRopInfo = NULL; 493 494 portMemFree(pKernelGraphics->pPrivate->staticInfo.pContextBuffersInfo); 495 pKernelGraphics->pPrivate->staticInfo.pContextBuffersInfo = NULL; 496 497 portMemFree(pKernelGraphics->pPrivate->staticInfo.pSmIssueRateModifier); 498 pKernelGraphics->pPrivate->staticInfo.pSmIssueRateModifier = NULL; 499 500 portMemFree(pKernelGraphics->pPrivate->staticInfo.pFecsTraceDefines); 501 pKernelGraphics->pPrivate->staticInfo.pFecsTraceDefines = NULL; 502 503 portMemSet(&pKernelGraphics->pPrivate->staticInfo, 0, sizeof(pKernelGraphics->pPrivate->staticInfo)); 504 pKernelGraphics->pPrivate->bInitialized = NV_FALSE; 505 } 506 507 const KGRAPHICS_STATIC_INFO * 508 kgraphicsGetStaticInfo_IMPL 509 ( 510 OBJGPU *pGpu, 511 KernelGraphics *pKernelGraphics 512 ) 513 { 514 KGRAPHICS_PRIVATE_DATA *pPrivate = pKernelGraphics->pPrivate; 515 return ((pPrivate != NULL) && pPrivate->bInitialized) ? &pPrivate->staticInfo : NULL; 516 } 517 518 static NV_STATUS 519 _kgraphicsInternalClientAlloc 520 ( 521 OBJGPU *pGpu, 522 KernelGraphics *pKernelGraphics, 523 NvU32 swizzId, 524 NvHandle *phClient, 525 NvHandle *phDevice, 526 NvHandle *phSubdevice 527 ) 528 { 529 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); //FIXME = GPU_GET_PHYSICAL_RMAPI(pGpu); 530 NvU32 grIdx = pKernelGraphics->instance; 531 532 NV_ASSERT_OR_RETURN(phClient != NULL, NV_ERR_INVALID_ARGUMENT); 533 NV_ASSERT_OR_RETURN(phDevice != NULL, NV_ERR_INVALID_ARGUMENT); 534 NV_ASSERT_OR_RETURN(phSubdevice != NULL, NV_ERR_INVALID_ARGUMENT); 535 536 if (IS_MIG_IN_USE(pGpu)) 537 { 538 NvHandle hSubscription; 539 540 // Delay initialization to GPU instance configuration 541 if (swizzId == KMIGMGR_SWIZZID_INVALID) 542 return NV_ERR_NOT_READY; 543 544 // With MIG enabled, we need to use a client subscribed to the correct GPU instance. 545 NV_ASSERT_OK_OR_RETURN( 546 rmapiutilAllocClientAndDeviceHandles(pRmApi, pGpu, phClient, phDevice, phSubdevice)); 547 548 { 549 NVC637_ALLOCATION_PARAMETERS params; 550 NV_ASSERT_OK( 551 serverutilGenResourceHandle(*phClient, &hSubscription)); 552 portMemSet(¶ms, 0, sizeof(params)); 553 params.swizzId = swizzId; 554 NV_ASSERT_OK( 555 pRmApi->AllocWithHandle(pRmApi, *phClient, *phSubdevice, hSubscription, AMPERE_SMC_PARTITION_REF, ¶ms, sizeof(params))); 556 } 557 558 } 559 else if (grIdx != 0) 560 { 561 // Static data is only defined for GR0 in legacy mode 562 return NV_ERR_NOT_READY; 563 } 564 else 565 { 566 NV_ASSERT_OK_OR_RETURN( 567 rmapiutilAllocClientAndDeviceHandles(pRmApi, pGpu, phClient, phDevice, phSubdevice)); 568 } 569 570 return NV_OK; 571 } 572 573 /*! 574 * @brief Initialize static data that isn't collected right away 575 */ 576 NV_STATUS 577 kgraphicsInitializeDeferredStaticData_IMPL 578 ( 579 OBJGPU *pGpu, 580 KernelGraphics *pKernelGraphics, 581 NvHandle hClient, 582 NvHandle hSubdevice 583 ) 584 { 585 NV2080_CTRL_INTERNAL_STATIC_GR_GET_CONTEXT_BUFFERS_INFO_PARAMS *pParams; 586 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 587 KGRAPHICS_PRIVATE_DATA *pPrivate = pKernelGraphics->pPrivate; 588 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 589 NvU32 grIdx = pKernelGraphics->instance; 590 NV_STATUS status = NV_OK; 591 NvBool bInternalClientAllocated = NV_FALSE; 592 NvU32 gfid; 593 NvBool bCallingContextPlugin; 594 595 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &gfid)); 596 NV_ASSERT_OK_OR_RETURN(vgpuIsCallingContextPlugin(pGpu, &bCallingContextPlugin)); 597 598 if (bCallingContextPlugin) 599 { 600 gfid = GPU_GFID_PF; 601 } 602 603 // 604 // Most of GR is stub'd so context related things are not needed in AMODEL. 605 // But this function can be called in some MODS test, so return OK directly 606 // to avoid failing the test. 607 // 608 if (IS_MODS_AMODEL(pGpu)) 609 return NV_OK; 610 611 // Not ready 612 if (!pPrivate->bInitialized) 613 return NV_OK; 614 615 // Already done 616 if (pPrivate->staticInfo.pContextBuffersInfo != NULL) 617 return NV_OK; 618 619 // In progress 620 if (pKernelGraphics->bCollectingDeferredStaticData) 621 return NV_OK; 622 623 if (hClient == NV01_NULL_OBJECT) 624 { 625 NvHandle hDevice = NV01_NULL_OBJECT; 626 NvU32 swizzId = KMIGMGR_SWIZZID_INVALID; 627 628 if (IS_MIG_IN_USE(pGpu)) 629 { 630 MIG_INSTANCE_REF ref; 631 632 NV_ASSERT_OK_OR_RETURN( 633 kmigmgrGetMIGReferenceFromEngineType(pGpu, pKernelMIGManager, 634 RM_ENGINE_TYPE_GR(pKernelGraphics->instance), &ref)); 635 636 swizzId = ref.pKernelMIGGpuInstance->swizzId; 637 } 638 639 status = _kgraphicsInternalClientAlloc(pGpu, pKernelGraphics, swizzId, &hClient, &hDevice, &hSubdevice); 640 if (status == NV_ERR_NOT_READY) 641 { 642 return NV_OK; 643 } 644 NV_ASSERT_OR_RETURN(status == NV_OK, status); 645 NV_ASSERT_OR_RETURN(hClient != NV01_NULL_OBJECT, NV_ERR_INVALID_STATE); 646 bInternalClientAllocated = NV_TRUE; 647 } 648 649 // Prevent recursion when deferred static data collection is ON 650 pKernelGraphics->bCollectingDeferredStaticData = NV_TRUE; 651 652 if (IS_MIG_IN_USE(pGpu)) 653 { 654 MIG_INSTANCE_REF ref; 655 RM_ENGINE_TYPE localRmEngineType; 656 Subdevice *pSubdevice; 657 RsClient *pClient; 658 659 NV_CHECK_OK_OR_GOTO( 660 status, 661 LEVEL_ERROR, 662 serverGetClientUnderLock(&g_resServ, hClient, &pClient), 663 cleanup); 664 665 NV_CHECK_OK_OR_GOTO( 666 status, 667 LEVEL_ERROR, 668 subdeviceGetByHandle(pClient, hSubdevice, &pSubdevice), 669 cleanup); 670 671 // Physical RM will fill with local indices, so localize the index 672 NV_CHECK_OK_OR_GOTO( 673 status, 674 LEVEL_ERROR, 675 kmigmgrGetInstanceRefFromDevice(pGpu, pKernelMIGManager, 676 GPU_RES_GET_DEVICE(pSubdevice), 677 &ref), 678 cleanup); 679 NV_CHECK_OK_OR_GOTO( 680 status, 681 LEVEL_ERROR, 682 kmigmgrGetGlobalToLocalEngineType(pGpu, pKernelMIGManager, ref, 683 RM_ENGINE_TYPE_GR(grIdx), 684 &localRmEngineType), 685 cleanup); 686 grIdx = RM_ENGINE_TYPE_GR_IDX(localRmEngineType); 687 } 688 689 pParams = portMemAllocNonPaged(sizeof(*pParams)); 690 if (pParams == NULL) 691 { 692 return NV_ERR_NO_MEMORY; 693 } 694 portMemSet(pParams, 0, sizeof(*pParams)); 695 NV_CHECK_OK_OR_GOTO( 696 status, 697 LEVEL_ERROR, 698 pRmApi->Control(pRmApi, 699 hClient, 700 hSubdevice, 701 NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_CONTEXT_BUFFERS_INFO, 702 pParams, 703 sizeof(*pParams)), 704 cleanup_context_buffers_info); 705 706 pPrivate->staticInfo.pContextBuffersInfo = 707 portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pContextBuffersInfo)); 708 709 if (pPrivate->staticInfo.pContextBuffersInfo == NULL) 710 { 711 status = NV_ERR_NO_MEMORY; 712 goto cleanup_context_buffers_info; 713 } 714 715 portMemCopy(pPrivate->staticInfo.pContextBuffersInfo, 716 sizeof(*pPrivate->staticInfo.pContextBuffersInfo), 717 &pParams->engineContextBuffersInfo[grIdx], 718 sizeof(pParams->engineContextBuffersInfo[grIdx])); 719 720 cleanup_context_buffers_info: 721 portMemFree(pParams); 722 723 // 724 // We are not cleaning pContextBuffersInfo here since it's used after this 725 // function so has to be cleaned after used. 726 // 727 728 cleanup: 729 if (bInternalClientAllocated) 730 { 731 pRmApi->Free(pRmApi, hClient, hClient); 732 } 733 734 pKernelGraphics->bCollectingDeferredStaticData = NV_FALSE; 735 736 if (status == NV_OK) 737 { 738 // 739 // Allocate Ctx Buffers that are global to all channels if they have yet 740 // to be allocated. We delay them until now to save memory when runs 741 // are done without using graphics contexts! 742 // 743 if (!pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers[gfid].bAllocated && 744 (!gpuIsClientRmAllocatedCtxBufferEnabled(pGpu) || 745 (gpuIsSriovEnabled(pGpu) && IS_GFID_PF(gfid)))) 746 { 747 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 748 kgraphicsAllocGrGlobalCtxBuffers_HAL(pGpu, pKernelGraphics, gfid, NULL)); 749 } 750 } 751 752 return status; 753 } 754 755 NV_STATUS 756 kgraphicsLoadStaticInfo_VF 757 ( 758 OBJGPU *pGpu, 759 KernelGraphics *pKernelGraphics, 760 NvU32 swizzId 761 ) 762 { 763 KGRAPHICS_PRIVATE_DATA *pPrivate = pKernelGraphics->pPrivate; 764 VGPU_STATIC_INFO *pVSI = GPU_GET_STATIC_INFO(pGpu); 765 NvU32 grIdx = pKernelGraphics->instance; 766 NVOS_STATUS status = NV_OK; 767 768 NV_ASSERT_OR_RETURN(pVSI != NULL, NV_ERR_INVALID_STATE); 769 NV_ASSERT_OR_RETURN(pPrivate != NULL, NV_ERR_INVALID_STATE); 770 771 if (pPrivate->bInitialized) 772 return status; 773 774 if (IS_MIG_IN_USE(pGpu)) 775 { 776 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 777 778 // 779 // Delay initialization to GPU instance configuration, unless MODS is using 780 // legacy VGPU mode, in which case the guest never receives a 781 // configuration call 782 // 783 if ((swizzId == KMIGMGR_SWIZZID_INVALID) && !kmigmgrUseLegacyVgpuPolicy(pGpu, pKernelMIGManager)) 784 return status; 785 786 pPrivate->staticInfo.pGrInfo = portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pGrInfo)); 787 if (pPrivate->staticInfo.pGrInfo == NULL) 788 { 789 status = NV_ERR_NO_MEMORY; 790 goto cleanup; 791 } 792 793 portMemCopy(pPrivate->staticInfo.pGrInfo->infoList, 794 NV0080_CTRL_GR_INFO_MAX_SIZE * sizeof(*pPrivate->staticInfo.pGrInfo->infoList), 795 pVSI->grInfoParams.engineInfo[grIdx].infoList, 796 NV0080_CTRL_GR_INFO_MAX_SIZE * sizeof(*pVSI->grInfoParams.engineInfo[grIdx].infoList)); 797 798 portMemCopy(&pPrivate->staticInfo.globalSmOrder, sizeof(pPrivate->staticInfo.globalSmOrder), 799 &pVSI->globalSmOrder.globalSmOrder[grIdx], sizeof(pVSI->globalSmOrder.globalSmOrder[grIdx])); 800 801 // grCaps are the same for all GR and can be copied from VGPU static info 802 portMemCopy(pPrivate->staticInfo.grCaps.capsTbl, sizeof(pPrivate->staticInfo.grCaps.capsTbl), 803 pVSI->grCapsBits, sizeof(pVSI->grCapsBits)); 804 805 // Initialize PDB properties synchronized with physical RM 806 pPrivate->staticInfo.pdbTable.bPerSubCtxheaderSupported = pVSI->bPerSubCtxheaderSupported; 807 kgraphicsSetPerSubcontextContextHeaderSupported(pGpu, pKernelGraphics, pPrivate->staticInfo.pdbTable.bPerSubCtxheaderSupported); 808 809 pPrivate->staticInfo.pSmIssueRateModifier = 810 portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pSmIssueRateModifier)); 811 if (pPrivate->staticInfo.pSmIssueRateModifier == NULL) 812 { 813 status = NV_ERR_NO_MEMORY; 814 goto cleanup; 815 } 816 817 portMemCopy(pPrivate->staticInfo.pSmIssueRateModifier, sizeof(*pPrivate->staticInfo.pSmIssueRateModifier), 818 &pVSI->smIssueRateModifier.smIssueRateModifier[grIdx], sizeof(pVSI->smIssueRateModifier.smIssueRateModifier[grIdx])); 819 820 pPrivate->staticInfo.pPpcMasks = portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pPpcMasks)); 821 if (pPrivate->staticInfo.pPpcMasks == NULL) 822 { 823 status = NV_ERR_NO_MEMORY; 824 goto cleanup; 825 } 826 827 portMemCopy(pPrivate->staticInfo.pPpcMasks, sizeof(*pPrivate->staticInfo.pPpcMasks), 828 &pVSI->ppcMaskParams.enginePpcMasks[grIdx], sizeof(pVSI->ppcMaskParams.enginePpcMasks[grIdx])); 829 830 portMemCopy(&pPrivate->staticInfo.floorsweepingMasks, sizeof(pPrivate->staticInfo.floorsweepingMasks), 831 &pVSI->floorsweepMaskParams.floorsweepingMasks[grIdx], sizeof(pVSI->floorsweepMaskParams.floorsweepingMasks[grIdx])); 832 833 pPrivate->staticInfo.pContextBuffersInfo = 834 portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pContextBuffersInfo)); 835 836 if (pPrivate->staticInfo.pContextBuffersInfo == NULL) 837 { 838 status = NV_ERR_NO_MEMORY; 839 goto cleanup; 840 } 841 842 portMemCopy(pPrivate->staticInfo.pContextBuffersInfo, 843 sizeof(*pPrivate->staticInfo.pContextBuffersInfo), 844 &pVSI->ctxBuffInfo.engineContextBuffersInfo[grIdx], 845 sizeof(pVSI->ctxBuffInfo.engineContextBuffersInfo[grIdx])); 846 847 pPrivate->staticInfo.fecsRecordSize.fecsRecordSize = pVSI->fecsRecordSize.fecsRecordSize[grIdx].fecsRecordSize; 848 849 pPrivate->staticInfo.pFecsTraceDefines = 850 portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pFecsTraceDefines)); 851 852 if (pPrivate->staticInfo.pFecsTraceDefines == NULL) 853 { 854 status = NV_ERR_NO_MEMORY; 855 goto cleanup; 856 } 857 858 portMemCopy(pPrivate->staticInfo.pFecsTraceDefines, 859 sizeof(*pPrivate->staticInfo.pFecsTraceDefines), 860 &pVSI->fecsTraceDefines.fecsTraceDefines[grIdx], 861 sizeof(pVSI->fecsTraceDefines.fecsTraceDefines[grIdx])); 862 863 portMemCopy(&pPrivate->staticInfo.pdbTable, sizeof(pPrivate->staticInfo.pdbTable), 864 &pVSI->pdbTableParams.pdbTable[grIdx], sizeof(pVSI->pdbTableParams.pdbTable[grIdx])); 865 } 866 else if (grIdx == 0) 867 { 868 portMemCopy(pPrivate->staticInfo.grCaps.capsTbl, sizeof(pPrivate->staticInfo.grCaps.capsTbl), 869 pVSI->grCapsBits, sizeof(pVSI->grCapsBits)); 870 871 pPrivate->staticInfo.pGrInfo = portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pGrInfo)); 872 if (pPrivate->staticInfo.pGrInfo == NULL) 873 { 874 status = NV_ERR_NO_MEMORY; 875 goto cleanup; 876 } 877 878 portMemCopy(pPrivate->staticInfo.pGrInfo->infoList, 879 NV0080_CTRL_GR_INFO_MAX_SIZE * sizeof(*pPrivate->staticInfo.pGrInfo->infoList), 880 pVSI->grInfoParams.engineInfo[grIdx].infoList, 881 NV0080_CTRL_GR_INFO_MAX_SIZE * sizeof(*pVSI->grInfoParams.engineInfo[grIdx].infoList)); 882 883 // Initialize PDB properties synchronized with physical RM 884 pPrivate->staticInfo.pdbTable.bPerSubCtxheaderSupported = pVSI->bPerSubCtxheaderSupported; 885 kgraphicsSetPerSubcontextContextHeaderSupported(pGpu, pKernelGraphics, pPrivate->staticInfo.pdbTable.bPerSubCtxheaderSupported); 886 887 portMemCopy(&pPrivate->staticInfo.globalSmOrder, sizeof(pPrivate->staticInfo.globalSmOrder), 888 &pVSI->globalSmOrder.globalSmOrder[grIdx], sizeof(pVSI->globalSmOrder.globalSmOrder[grIdx])); 889 890 pPrivate->staticInfo.pSmIssueRateModifier = 891 portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pSmIssueRateModifier)); 892 if (pPrivate->staticInfo.pSmIssueRateModifier == NULL) 893 { 894 status = NV_ERR_NO_MEMORY; 895 goto cleanup; 896 } 897 898 portMemCopy(pPrivate->staticInfo.pSmIssueRateModifier, sizeof(*pPrivate->staticInfo.pSmIssueRateModifier), 899 &pVSI->smIssueRateModifier.smIssueRateModifier[grIdx], sizeof(pVSI->smIssueRateModifier.smIssueRateModifier[grIdx])); 900 901 pPrivate->staticInfo.pPpcMasks = portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pPpcMasks)); 902 if (pPrivate->staticInfo.pPpcMasks == NULL) 903 { 904 status = NV_ERR_NO_MEMORY; 905 goto cleanup; 906 } 907 908 portMemCopy(pPrivate->staticInfo.pPpcMasks, sizeof(*pPrivate->staticInfo.pPpcMasks), 909 &pVSI->ppcMaskParams.enginePpcMasks[grIdx], sizeof(pVSI->ppcMaskParams.enginePpcMasks[grIdx])); 910 911 pPrivate->staticInfo.pContextBuffersInfo = 912 portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pContextBuffersInfo)); 913 914 if (pPrivate->staticInfo.pContextBuffersInfo == NULL) 915 { 916 status = NV_ERR_NO_MEMORY; 917 goto cleanup; 918 } 919 920 portMemCopy(pPrivate->staticInfo.pContextBuffersInfo, 921 sizeof(*pPrivate->staticInfo.pContextBuffersInfo), 922 &pVSI->ctxBuffInfo.engineContextBuffersInfo[grIdx], 923 sizeof(pVSI->ctxBuffInfo.engineContextBuffersInfo[grIdx])); 924 925 portMemCopy(&pPrivate->staticInfo.floorsweepingMasks, sizeof(pPrivate->staticInfo.floorsweepingMasks), 926 &pVSI->floorsweepMaskParams.floorsweepingMasks[grIdx], sizeof(pVSI->floorsweepMaskParams.floorsweepingMasks[grIdx])); 927 928 pPrivate->staticInfo.pRopInfo = portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pRopInfo)); 929 if (pPrivate->staticInfo.pRopInfo == NULL) 930 { 931 status = NV_ERR_NO_MEMORY; 932 goto cleanup; 933 } 934 935 portMemCopy(pPrivate->staticInfo.pRopInfo, sizeof(*pPrivate->staticInfo.pRopInfo), 936 &pVSI->ropInfoParams.engineRopInfo[grIdx], sizeof(pVSI->ropInfoParams.engineRopInfo[grIdx])); 937 938 pPrivate->staticInfo.pZcullInfo = portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pZcullInfo)); 939 if (pPrivate->staticInfo.pZcullInfo == NULL) 940 { 941 status = NV_ERR_NO_MEMORY; 942 goto cleanup; 943 } 944 945 portMemCopy(pPrivate->staticInfo.pZcullInfo, sizeof(*pPrivate->staticInfo.pZcullInfo), 946 &pVSI->zcullInfoParams.engineZcullInfo[grIdx], sizeof(pVSI->zcullInfoParams.engineZcullInfo[grIdx])); 947 948 pPrivate->staticInfo.fecsRecordSize.fecsRecordSize = pVSI->fecsRecordSize.fecsRecordSize[grIdx].fecsRecordSize; 949 950 pPrivate->staticInfo.pFecsTraceDefines = 951 portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pFecsTraceDefines)); 952 if (pPrivate->staticInfo.pFecsTraceDefines == NULL) 953 { 954 status = NV_ERR_NO_MEMORY; 955 goto cleanup; 956 } 957 958 portMemCopy(pPrivate->staticInfo.pFecsTraceDefines, 959 sizeof(*pPrivate->staticInfo.pFecsTraceDefines), 960 &pVSI->fecsTraceDefines.fecsTraceDefines[grIdx], 961 sizeof(pVSI->fecsTraceDefines.fecsTraceDefines[grIdx])); 962 963 portMemCopy(&pPrivate->staticInfo.pdbTable, sizeof(pPrivate->staticInfo.pdbTable), 964 &pVSI->pdbTableParams.pdbTable[grIdx], sizeof(pVSI->pdbTableParams.pdbTable[grIdx])); 965 } 966 else 967 { 968 // if MIG disabled, only GR0 static data needs to be published 969 return status; 970 } 971 972 if (status == NV_OK) 973 { 974 // Publish static configuration 975 pPrivate->bInitialized = NV_TRUE; 976 } 977 978 if (!IS_MIG_IN_USE(pGpu) && (grIdx == 0)) 979 { 980 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 981 982 // Cache legacy GR mask info (i.e. GR0 with MIG disabled) to pKernelGraphicsManager->legacyFsMaskState 983 kgrmgrSetLegacyKgraphicsStaticInfo(pGpu, pKernelGraphicsManager, pKernelGraphics); 984 } 985 986 cleanup : 987 988 if (status != NV_OK) 989 { 990 portMemFree(pPrivate->staticInfo.pGrInfo); 991 pPrivate->staticInfo.pGrInfo = NULL; 992 993 portMemFree(pPrivate->staticInfo.pPpcMasks); 994 pPrivate->staticInfo.pPpcMasks = NULL; 995 996 portMemFree(pPrivate->staticInfo.pZcullInfo); 997 pPrivate->staticInfo.pZcullInfo = NULL; 998 999 portMemFree(pPrivate->staticInfo.pRopInfo); 1000 pPrivate->staticInfo.pRopInfo = NULL; 1001 1002 portMemFree(pPrivate->staticInfo.pContextBuffersInfo); 1003 pPrivate->staticInfo.pContextBuffersInfo = NULL; 1004 1005 portMemFree(pPrivate->staticInfo.pSmIssueRateModifier); 1006 pPrivate->staticInfo.pSmIssueRateModifier = NULL; 1007 1008 portMemFree(pPrivate->staticInfo.pFecsTraceDefines); 1009 pPrivate->staticInfo.pFecsTraceDefines = NULL; 1010 } 1011 1012 return status; 1013 } 1014 1015 NV_STATUS 1016 kgraphicsLoadStaticInfo_KERNEL 1017 ( 1018 OBJGPU *pGpu, 1019 KernelGraphics *pKernelGraphics, 1020 NvU32 swizzId 1021 ) 1022 { 1023 KGRAPHICS_PRIVATE_DATA *pPrivate = pKernelGraphics->pPrivate; 1024 NvHandle hClient = NV01_NULL_OBJECT; 1025 NvHandle hDevice; 1026 NvHandle hSubdevice; 1027 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 1028 NvU32 grIdx = pKernelGraphics->instance; 1029 NV_STATUS status = NV_OK; 1030 NvBool bBcState = gpumgrGetBcEnabledStatus(pGpu); 1031 union 1032 { 1033 NV2080_CTRL_INTERNAL_STATIC_GR_GET_CAPS_PARAMS caps; 1034 NV2080_CTRL_INTERNAL_STATIC_GR_GET_INFO_PARAMS info; 1035 NV2080_CTRL_INTERNAL_STATIC_GR_GET_GLOBAL_SM_ORDER_PARAMS globalSmOrder; 1036 NV2080_CTRL_INTERNAL_STATIC_GR_GET_FLOORSWEEPING_MASKS_PARAMS floorsweepingMasks; 1037 NV2080_CTRL_INTERNAL_STATIC_GR_GET_PPC_MASKS_PARAMS ppcMasks; 1038 NV2080_CTRL_INTERNAL_STATIC_GR_GET_ZCULL_INFO_PARAMS zcullInfo; 1039 NV2080_CTRL_INTERNAL_STATIC_GR_GET_ROP_INFO_PARAMS ropInfo; 1040 NV2080_CTRL_INTERNAL_STATIC_GR_GET_SM_ISSUE_RATE_MODIFIER_PARAMS smIssueRateModifier; 1041 NV2080_CTRL_INTERNAL_STATIC_GR_GET_FECS_RECORD_SIZE_PARAMS fecsRecordSize; 1042 NV2080_CTRL_INTERNAL_STATIC_GR_GET_FECS_TRACE_DEFINES_PARAMS fecsTraceDefines; 1043 NV2080_CTRL_INTERNAL_STATIC_GR_GET_PDB_PROPERTIES_PARAMS pdbProperties; 1044 } *pParams = NULL; 1045 1046 NV_ASSERT_OR_RETURN(pPrivate != NULL, NV_ERR_INVALID_STATE); 1047 1048 if (pPrivate->bInitialized) 1049 return NV_OK; 1050 1051 status = _kgraphicsInternalClientAlloc(pGpu, pKernelGraphics, swizzId, &hClient, &hDevice, &hSubdevice); 1052 1053 if (status == NV_ERR_NOT_READY) 1054 { 1055 return NV_OK; 1056 } 1057 NV_ASSERT_OR_RETURN(status == NV_OK, status); 1058 NV_ASSERT_OR_RETURN(hClient != NV01_NULL_OBJECT, NV_ERR_INVALID_STATE); 1059 1060 pParams = portMemAllocNonPaged(sizeof(*pParams)); 1061 if (pParams == NULL) 1062 { 1063 status = NV_ERR_NO_MEMORY; 1064 goto cleanup; 1065 } 1066 1067 if (IS_MIG_IN_USE(pGpu)) 1068 { 1069 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 1070 MIG_INSTANCE_REF ref; 1071 RM_ENGINE_TYPE localRmEngineType; 1072 RsClient *pClient; 1073 Device *pDevice; 1074 1075 NV_CHECK_OK_OR_GOTO( 1076 status, 1077 LEVEL_ERROR, 1078 serverGetClientUnderLock(&g_resServ, hClient, &pClient), 1079 cleanup); 1080 1081 NV_CHECK_OK_OR_GOTO( 1082 status, 1083 LEVEL_ERROR, 1084 deviceGetByHandle(pClient, hDevice, &pDevice), 1085 cleanup); 1086 1087 // Physical RM will fill with local indices, so localize the index 1088 NV_CHECK_OK_OR_GOTO( 1089 status, 1090 LEVEL_ERROR, 1091 kmigmgrGetInstanceRefFromDevice(pGpu, pKernelMIGManager, pDevice, &ref), 1092 cleanup); 1093 NV_CHECK_OK_OR_GOTO( 1094 status, 1095 LEVEL_ERROR, 1096 kmigmgrGetGlobalToLocalEngineType(pGpu, pKernelMIGManager, ref, 1097 RM_ENGINE_TYPE_GR(grIdx), 1098 &localRmEngineType), 1099 cleanup); 1100 grIdx = RM_ENGINE_TYPE_GR_IDX(localRmEngineType); 1101 } 1102 1103 // GR Caps 1104 portMemSet(pParams, 0, sizeof(*pParams)); 1105 NV_CHECK_OK_OR_GOTO( 1106 status, 1107 LEVEL_ERROR, 1108 pRmApi->Control(pRmApi, 1109 hClient, 1110 hSubdevice, 1111 NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_CAPS, 1112 pParams, 1113 sizeof(pParams->caps)), 1114 cleanup); 1115 1116 portMemCopy(&pPrivate->staticInfo.grCaps, sizeof(pPrivate->staticInfo.grCaps), 1117 &pParams->caps.engineCaps[grIdx], sizeof(pParams->caps.engineCaps[grIdx])); 1118 1119 // GR Info 1120 portMemSet(pParams, 0, sizeof(*pParams)); 1121 status = pRmApi->Control(pRmApi, 1122 hClient, 1123 hSubdevice, 1124 NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_INFO, 1125 pParams, 1126 sizeof(pParams->info)); 1127 1128 if (status == NV_OK) 1129 { 1130 pPrivate->staticInfo.pGrInfo = portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pGrInfo)); 1131 if (pPrivate->staticInfo.pGrInfo == NULL) 1132 { 1133 status = NV_ERR_NO_MEMORY; 1134 goto cleanup; 1135 } 1136 1137 portMemCopy(pPrivate->staticInfo.pGrInfo->infoList, 1138 NV0080_CTRL_GR_INFO_MAX_SIZE * sizeof(*pPrivate->staticInfo.pGrInfo->infoList), 1139 pParams->info.engineInfo[grIdx].infoList, 1140 NV0080_CTRL_GR_INFO_MAX_SIZE * sizeof(*pParams->info.engineInfo[grIdx].infoList)); 1141 1142 } 1143 1144 // Floorsweeping masks 1145 portMemSet(pParams, 0, sizeof(*pParams)); 1146 NV_CHECK_OK_OR_GOTO( 1147 status, 1148 LEVEL_ERROR, 1149 pRmApi->Control(pRmApi, 1150 hClient, 1151 hSubdevice, 1152 NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_FLOORSWEEPING_MASKS, 1153 pParams, 1154 sizeof(pParams->floorsweepingMasks)), 1155 cleanup); 1156 1157 portMemCopy(&pPrivate->staticInfo.floorsweepingMasks, sizeof(pPrivate->staticInfo.floorsweepingMasks), 1158 &pParams->floorsweepingMasks.floorsweepingMasks[grIdx], sizeof(pParams->floorsweepingMasks.floorsweepingMasks[grIdx])); 1159 1160 // 1161 // Most of GR is stub'd in AMODEL. However, some tests still need the CAPS/INFO data, 1162 // so we still need to generate CAPS/INFO data for AMODEL 1163 // 1164 if (IS_MODS_AMODEL(pGpu)) 1165 { 1166 pPrivate->bInitialized = NV_TRUE; 1167 if (!IS_MIG_IN_USE(pGpu) && (grIdx == 0)) 1168 { 1169 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 1170 1171 // Cache legacy GR mask info (i.e. GR0 with MIG disabled) to pKernelGraphicsManager->legacyFsMaskState 1172 kgrmgrSetLegacyKgraphicsStaticInfo(pGpu, pKernelGraphicsManager, pKernelGraphics); 1173 } 1174 status = NV_OK; 1175 goto cleanup; 1176 } 1177 1178 // GR Global SM Order 1179 portMemSet(pParams, 0, sizeof(*pParams)); 1180 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, 1181 pRmApi->Control(pRmApi, 1182 hClient, 1183 hSubdevice, 1184 NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_GLOBAL_SM_ORDER, 1185 pParams, 1186 sizeof(pParams->globalSmOrder)), 1187 cleanup); 1188 1189 portMemCopy(&pPrivate->staticInfo.globalSmOrder, sizeof(pPrivate->staticInfo.globalSmOrder), 1190 &pParams->globalSmOrder.globalSmOrder[grIdx], sizeof(pParams->globalSmOrder.globalSmOrder[grIdx])); 1191 1192 // PPC Mask 1193 portMemSet(pParams, 0, sizeof(*pParams)); 1194 status = pRmApi->Control(pRmApi, 1195 hClient, 1196 hSubdevice, 1197 NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_PPC_MASKS, 1198 pParams, 1199 sizeof(pParams->ppcMasks)); 1200 1201 if (status == NV_OK) 1202 { 1203 pPrivate->staticInfo.pPpcMasks = portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pPpcMasks)); 1204 if (pPrivate->staticInfo.pPpcMasks == NULL) 1205 { 1206 status = NV_ERR_NO_MEMORY; 1207 goto cleanup; 1208 } 1209 1210 portMemCopy(pPrivate->staticInfo.pPpcMasks, sizeof(*pPrivate->staticInfo.pPpcMasks), 1211 &pParams->ppcMasks.enginePpcMasks[grIdx], sizeof(pParams->ppcMasks.enginePpcMasks[grIdx])); 1212 } 1213 else if (status == NV_ERR_NOT_SUPPORTED) 1214 { 1215 // 1216 // Some chips don't support this call, so just keep the pPpcMasks 1217 // pointer as NULL, but don't return error 1218 // 1219 status = NV_OK; 1220 } 1221 1222 portMemSet(pParams, 0, sizeof(*pParams)); 1223 status = pRmApi->Control(pRmApi, 1224 hClient, 1225 hSubdevice, 1226 NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_ZCULL_INFO, 1227 pParams, 1228 sizeof(pParams->zcullInfo)); 1229 1230 if (status == NV_OK) 1231 { 1232 pPrivate->staticInfo.pZcullInfo = portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pZcullInfo)); 1233 if (pPrivate->staticInfo.pZcullInfo == NULL) 1234 { 1235 status = NV_ERR_NO_MEMORY; 1236 goto cleanup; 1237 } 1238 1239 portMemCopy(pPrivate->staticInfo.pZcullInfo, sizeof(*pPrivate->staticInfo.pZcullInfo), 1240 &pParams->zcullInfo.engineZcullInfo[grIdx], sizeof(pParams->zcullInfo.engineZcullInfo[grIdx])); 1241 } 1242 else if (status == NV_ERR_NOT_SUPPORTED) 1243 { 1244 // It's expected to get this error when MIG is enabled, thus don't return error 1245 if (IS_MIG_ENABLED(pGpu)) 1246 { 1247 status = NV_OK; 1248 } 1249 } 1250 1251 // ROP Info 1252 portMemSet(pParams, 0, sizeof(*pParams)); 1253 status = pRmApi->Control(pRmApi, 1254 hClient, 1255 hSubdevice, 1256 NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_ROP_INFO, 1257 pParams, 1258 sizeof(pParams->ropInfo)); 1259 1260 if (status == NV_OK) 1261 { 1262 pPrivate->staticInfo.pRopInfo = portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pRopInfo)); 1263 if (pPrivate->staticInfo.pRopInfo == NULL) 1264 { 1265 status = NV_ERR_NO_MEMORY; 1266 goto cleanup; 1267 } 1268 1269 portMemCopy(pPrivate->staticInfo.pRopInfo, sizeof(*pPrivate->staticInfo.pRopInfo), 1270 &pParams->ropInfo.engineRopInfo[grIdx], sizeof(pParams->ropInfo.engineRopInfo[grIdx])); 1271 } 1272 else if (status == NV_ERR_NOT_SUPPORTED) 1273 { 1274 // It's expected to get this error when MIG is enabled, thus don't return error 1275 if (IS_MIG_ENABLED(pGpu)) 1276 { 1277 status = NV_OK; 1278 } 1279 } 1280 1281 // SM Issue Rate Modifier 1282 portMemSet(pParams, 0, sizeof(*pParams)); 1283 status = pRmApi->Control(pRmApi, 1284 hClient, 1285 hSubdevice, 1286 NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_SM_ISSUE_RATE_MODIFIER, 1287 pParams, 1288 sizeof(pParams->smIssueRateModifier)); 1289 1290 if (status == NV_OK) 1291 { 1292 pPrivate->staticInfo.pSmIssueRateModifier = portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pSmIssueRateModifier)); 1293 if (pPrivate->staticInfo.pSmIssueRateModifier == NULL) 1294 { 1295 status = NV_ERR_NO_MEMORY; 1296 goto cleanup; 1297 } 1298 1299 portMemCopy(pPrivate->staticInfo.pSmIssueRateModifier, sizeof(*pPrivate->staticInfo.pSmIssueRateModifier), 1300 &pParams->smIssueRateModifier.smIssueRateModifier[grIdx], sizeof(pParams->smIssueRateModifier.smIssueRateModifier[grIdx])); 1301 } 1302 else if (status == NV_ERR_NOT_SUPPORTED) 1303 { 1304 status = NV_OK; 1305 } 1306 1307 // FECS Record Size 1308 portMemSet(pParams, 0, sizeof(*pParams)); 1309 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, 1310 pRmApi->Control(pRmApi, 1311 hClient, 1312 hSubdevice, 1313 NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_FECS_RECORD_SIZE, 1314 pParams, 1315 sizeof(pParams->fecsRecordSize)), 1316 cleanup); 1317 1318 pPrivate->staticInfo.fecsRecordSize.fecsRecordSize = pParams->fecsRecordSize.fecsRecordSize[grIdx].fecsRecordSize; 1319 1320 // FECS Trace Defines 1321 portMemSet(pParams, 0, sizeof(*pParams)); 1322 status = pRmApi->Control(pRmApi, 1323 hClient, 1324 hSubdevice, 1325 NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_FECS_TRACE_DEFINES, 1326 pParams, 1327 sizeof(pParams->fecsTraceDefines)); 1328 if (status == NV_OK) 1329 { 1330 pPrivate->staticInfo.pFecsTraceDefines = portMemAllocNonPaged(sizeof(*pPrivate->staticInfo.pFecsTraceDefines)); 1331 if (pPrivate->staticInfo.pFecsTraceDefines == NULL) 1332 { 1333 status = NV_ERR_NO_MEMORY; 1334 goto cleanup; 1335 } 1336 portMemCopy(pPrivate->staticInfo.pFecsTraceDefines, sizeof(*pPrivate->staticInfo.pFecsTraceDefines), 1337 &pParams->fecsTraceDefines.fecsTraceDefines[grIdx], sizeof(pParams->fecsTraceDefines.fecsTraceDefines[grIdx])); 1338 } 1339 else if (status == NV_ERR_NOT_SUPPORTED) 1340 { 1341 status = NV_OK; 1342 } 1343 1344 // PDB Properties 1345 portMemSet(pParams, 0, sizeof(*pParams)); 1346 NV_CHECK_OK_OR_GOTO( 1347 status, 1348 LEVEL_ERROR, 1349 pRmApi->Control(pRmApi, 1350 hClient, 1351 hSubdevice, 1352 NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_PDB_PROPERTIES, 1353 pParams, 1354 sizeof(pParams->pdbProperties)), 1355 cleanup); 1356 1357 portMemCopy(&pPrivate->staticInfo.pdbTable, sizeof(pPrivate->staticInfo.pdbTable), 1358 &pParams->pdbProperties.pdbTable[grIdx], sizeof(pParams->pdbProperties.pdbTable[grIdx])); 1359 kgraphicsSetPerSubcontextContextHeaderSupported(pGpu, pKernelGraphics, pPrivate->staticInfo.pdbTable.bPerSubCtxheaderSupported); 1360 1361 // Publish static configuration 1362 pPrivate->bInitialized = NV_TRUE; 1363 1364 // The deferred data is ready after MIG is enabled, so no need to defer the initialization 1365 if (IS_MIG_IN_USE(pGpu) || 1366 !kgraphicsShouldDeferContextInit(pGpu, pKernelGraphics)) 1367 { 1368 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, 1369 kgraphicsInitializeDeferredStaticData(pGpu, pKernelGraphics, hClient, hSubdevice), cleanup); 1370 } 1371 1372 if (!IS_MIG_IN_USE(pGpu) && (grIdx == 0)) 1373 { 1374 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 1375 1376 // Cache legacy GR mask info (i.e. GR0 with MIG disabled) to pKernelGraphicsManager->legacyFsMaskState 1377 kgrmgrSetLegacyKgraphicsStaticInfo(pGpu, pKernelGraphicsManager, pKernelGraphics); 1378 } 1379 1380 cleanup: 1381 if (status != NV_OK) 1382 { 1383 // Redact static configuration 1384 pPrivate->bInitialized = NV_FALSE; 1385 1386 portMemFree(pPrivate->staticInfo.pGrInfo); 1387 pPrivate->staticInfo.pGrInfo = NULL; 1388 1389 portMemFree(pPrivate->staticInfo.pPpcMasks); 1390 pPrivate->staticInfo.pPpcMasks = NULL; 1391 1392 portMemFree(pPrivate->staticInfo.pZcullInfo); 1393 pPrivate->staticInfo.pZcullInfo = NULL; 1394 1395 portMemFree(pPrivate->staticInfo.pRopInfo); 1396 pPrivate->staticInfo.pRopInfo = NULL; 1397 1398 portMemFree(pPrivate->staticInfo.pContextBuffersInfo); 1399 pPrivate->staticInfo.pContextBuffersInfo = NULL; 1400 1401 portMemFree(pPrivate->staticInfo.pSmIssueRateModifier); 1402 pPrivate->staticInfo.pSmIssueRateModifier = NULL; 1403 1404 portMemFree(pPrivate->staticInfo.pFecsTraceDefines); 1405 pPrivate->staticInfo.pFecsTraceDefines = NULL; 1406 } 1407 1408 // If we had to subscribe specifically, free the hclient we allocated 1409 if (hClient != NV01_NULL_OBJECT) 1410 pRmApi->Free(pRmApi, hClient, hClient); 1411 1412 if (gpumgrGetBcEnabledStatus(pGpu) != bBcState) 1413 { 1414 // Corrupted broadcast state! 1415 NV_ASSERT(gpumgrGetBcEnabledStatus(pGpu) != bBcState); 1416 gpumgrSetBcEnabledStatus(pGpu, bBcState); 1417 } 1418 1419 portMemFree(pParams); 1420 1421 return status; 1422 } 1423 1424 /*! Return if GFX is supported for the given kernel graphics engine */ 1425 NvBool 1426 kgraphicsIsGFXSupported_IMPL 1427 ( 1428 OBJGPU *pGpu, 1429 KernelGraphics *pKernelGraphics 1430 ) 1431 { 1432 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 1433 NvU32 gfxCapabilites; 1434 1435 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_FALSE); 1436 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pGrInfo != NULL, NV_FALSE); 1437 1438 gfxCapabilites = pKernelGraphicsStaticInfo->pGrInfo->infoList[NV2080_CTRL_GR_INFO_INDEX_GFX_CAPABILITIES].data; 1439 1440 return (FLD_TEST_DRF(2080_CTRL_GR, _INFO_GFX_CAPABILITIES, _2D, _TRUE, gfxCapabilites) && 1441 FLD_TEST_DRF(2080_CTRL_GR, _INFO_GFX_CAPABILITIES, _3D, _TRUE, gfxCapabilites) && 1442 FLD_TEST_DRF(2080_CTRL_GR, _INFO_GFX_CAPABILITIES, _I2M, _TRUE, gfxCapabilites)); 1443 } 1444 1445 /*! Retrieve ctxbufpool parameters for given local ctx buffer */ 1446 const CTX_BUF_INFO * 1447 kgraphicsGetCtxBufferInfo_IMPL 1448 ( 1449 OBJGPU *pGpu, 1450 KernelGraphics *pKernelGraphics, 1451 GR_CTX_BUFFER buf 1452 ) 1453 { 1454 NV_ASSERT_OR_RETURN(NV_ENUM_IS(GR_CTX_BUFFER, buf), NULL); 1455 return &pKernelGraphics->maxCtxBufSize[buf]; 1456 } 1457 1458 /*! Set ctxbufpool parameters for given local ctx buffer */ 1459 void 1460 kgraphicsSetCtxBufferInfo_IMPL 1461 ( 1462 OBJGPU *pGpu, 1463 KernelGraphics *pKernelGraphics, 1464 GR_CTX_BUFFER buf, 1465 NvU64 size, 1466 NvU64 align, 1467 RM_ATTR_PAGE_SIZE attr, 1468 NvBool bContiguous 1469 ) 1470 { 1471 CTX_BUF_INFO *pInfo; 1472 NV_ASSERT_OR_RETURN_VOID(NV_ENUM_IS(GR_CTX_BUFFER, buf)); 1473 1474 pInfo = &pKernelGraphics->maxCtxBufSize[buf]; 1475 pInfo->size = size; 1476 pInfo->align = align; 1477 pInfo->attr = attr; 1478 pInfo->bContig = bContiguous; 1479 } 1480 1481 /*! Clear ctxbufpool parameters for all local ctx buffers */ 1482 void 1483 kgraphicsClearCtxBufferInfo_IMPL 1484 ( 1485 OBJGPU *pGpu, 1486 KernelGraphics *pKernelGraphics 1487 ) 1488 { 1489 portMemSet(pKernelGraphics->maxCtxBufSize, 0, sizeof(pKernelGraphics->maxCtxBufSize)); 1490 } 1491 1492 /*! Initialize ctxbufpool for this engine */ 1493 NV_STATUS 1494 kgraphicsInitCtxBufPool_IMPL 1495 ( 1496 OBJGPU *pGpu, 1497 KernelGraphics *pKernelGraphics, 1498 Heap *pHeap 1499 ) 1500 { 1501 return ctxBufPoolInit(pGpu, pHeap, &pKernelGraphics->pCtxBufPool); 1502 } 1503 1504 /*! Retrieve ctxbufpool for this engine */ 1505 CTX_BUF_POOL_INFO * 1506 kgraphicsGetCtxBufPool_IMPL 1507 ( 1508 OBJGPU *pGpu, 1509 KernelGraphics *pKernelGraphics 1510 ) 1511 { 1512 return pKernelGraphics->pCtxBufPool; 1513 } 1514 1515 /*! destroy ctxbufpool for this engine */ 1516 void 1517 kgraphicsDestroyCtxBufPool_IMPL 1518 ( 1519 OBJGPU *pGpu, 1520 KernelGraphics *pKernelGraphics 1521 ) 1522 { 1523 if (pKernelGraphics->pCtxBufPool == NULL) 1524 return; 1525 1526 ctxBufPoolRelease(pKernelGraphics->pCtxBufPool); 1527 ctxBufPoolDestroy(&pKernelGraphics->pCtxBufPool); 1528 pKernelGraphics->pCtxBufPool = NULL; 1529 } 1530 1531 /*! Get the global ctx buffers for the given GFID */ 1532 GR_GLOBALCTX_BUFFERS * 1533 kgraphicsGetGlobalCtxBuffers_IMPL 1534 ( 1535 OBJGPU *pGpu, 1536 KernelGraphics *pKernelGraphics, 1537 NvU32 gfid 1538 ) 1539 { 1540 if (pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers == NULL) 1541 return NULL; 1542 return &pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers[gfid]; 1543 } 1544 1545 /*! Should this global ctx buffer be mapped as size aligned? */ 1546 NvBool 1547 kgraphicsIsGlobalCtxBufferSizeAligned_IMPL 1548 ( 1549 OBJGPU *pGpu, 1550 KernelGraphics *pKernelGraphics, 1551 GR_GLOBALCTX_BUFFER buf 1552 ) 1553 { 1554 NV_ASSERT_OR_RETURN(NV_ENUM_IS(GR_GLOBALCTX_BUFFER, buf), NV_FALSE); 1555 return pKernelGraphics->globalCtxBuffersInfo.bSizeAligned[buf]; 1556 } 1557 1558 /*! Get ctx buf attr for global priv access map */ 1559 const GR_BUFFER_ATTR * 1560 kgraphicsGetGlobalPrivAccessMapAttr_IMPL 1561 ( 1562 OBJGPU *pGpu, 1563 KernelGraphics *pKernelGraphics 1564 ) 1565 { 1566 return &pKernelGraphics->globalCtxBuffersInfo.globalCtxAttr[GR_GLOBAL_BUFFER_GLOBAL_PRIV_ACCESS_MAP]; 1567 } 1568 1569 /* 1570 * @brief Get Main context buffer size 1571 * 1572 * @param[in] pGpu 1573 * @param[in] pKernelGraphics 1574 * @param[in] bIncludeSubctxHdrs If subctx headers should be included in size calculation 1575 * @param[out] pSize Main Context buffer size 1576 */ 1577 NV_STATUS 1578 kgraphicsGetMainCtxBufferSize_IMPL 1579 ( 1580 OBJGPU *pGpu, 1581 KernelGraphics *pKernelGraphics, 1582 NvBool bIncludeSubctxHdrs, 1583 NvU32 *pSize 1584 ) 1585 { 1586 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 1587 NvU32 size; 1588 1589 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 1590 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pGrInfo != NULL, NV_ERR_INVALID_STATE); 1591 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pContextBuffersInfo != NULL, NV_ERR_INVALID_STATE); 1592 1593 size = pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS].size; 1594 1595 // Allocate space for per VEID headers in the golden context buffer. 1596 if (bIncludeSubctxHdrs && 1597 kgraphicsIsPerSubcontextContextHeaderSupported(pGpu, pKernelGraphics)) 1598 { 1599 // TODO size this down to max per-engine subcontexts 1600 NvU32 maxSubctx = pKernelGraphicsStaticInfo->pGrInfo->infoList[NV2080_CTRL_GR_INFO_INDEX_MAX_SUBCONTEXT_COUNT].data; 1601 1602 // The header needs to start at a 4 KB aligned address 1603 size = RM_ALIGN_UP(size, RM_PAGE_SIZE); 1604 1605 // The header is only 256 bytes; but needs to be 4KB aligned. 1606 size += (RM_PAGE_SIZE * maxSubctx); 1607 } 1608 1609 *pSize = size; 1610 return NV_OK; 1611 } 1612 1613 NV_STATUS 1614 kgraphicsAllocKgraphicsBuffers_KERNEL 1615 ( 1616 OBJGPU *pGpu, 1617 KernelGraphics *pKernelGraphics, 1618 KernelGraphicsContext *pKernelGraphicsContext, 1619 KernelChannel *pKernelChannel 1620 ) 1621 { 1622 NvU32 gfid; 1623 OBJGVASPACE *pGVAS; 1624 1625 NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_CHANNEL); 1626 pGVAS = dynamicCast(pKernelChannel->pVAS, OBJGVASPACE); 1627 gfid = kchannelGetGfid(pKernelChannel); 1628 1629 // Deferred static info is necessary at this point for FECS buffer allocation. Skip for guest RM 1630 if (!IS_VIRTUAL(pGpu)) 1631 { 1632 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 1633 kgraphicsInitializeDeferredStaticData(pGpu, pKernelGraphics, NV01_NULL_OBJECT, NV01_NULL_OBJECT)); 1634 } 1635 1636 // 1637 // Allocate global context buffers for this gfid, if they haven't been 1638 // already 1639 // 1640 if (kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid)->memDesc[GR_GLOBALCTX_BUFFER_FECS_EVENT] == NULL) 1641 { 1642 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 1643 kgraphicsAllocGlobalCtxBuffers_HAL(pGpu, pKernelGraphics, gfid)); 1644 } 1645 1646 if (kgraphicsIsCtxswLoggingSupported(pGpu, pKernelGraphics) && 1647 (kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid)->memDesc[GR_GLOBALCTX_BUFFER_FECS_EVENT] != NULL)) 1648 { 1649 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 1650 1651 if (!gvaspaceIsExternallyOwned(pGVAS) && !IS_VIRTUAL_WITHOUT_SRIOV(pGpu)) 1652 { 1653 // 1654 // We map CTXSW buffer on each object allocation including compute object 1655 // Other global context buffers are not mapped during compute object alloc with subcontexts since 1656 // those are GR-only buffers and not needed for compute-only contexts 1657 // 1658 _kgraphicsMapGlobalCtxBuffer(pGpu, pKernelGraphics, gfid, pKernelChannel->pVAS, pKernelGraphicsContext, 1659 GR_GLOBALCTX_BUFFER_FECS_EVENT, NV_FALSE); 1660 } 1661 1662 if (!fecsBufferIsMapped(pGpu, pKernelGraphics)) 1663 { 1664 fecsBufferMap(pGpu, pKernelGraphics); 1665 } 1666 1667 if (fecsGetCtxswLogConsumerCount(pGpu, pKernelGraphicsManager) > 0) 1668 fecsBufferReset(pGpu, pKernelGraphics); 1669 } 1670 1671 return NV_OK; 1672 } 1673 1674 static NV_STATUS 1675 _kgraphicsMapGlobalCtxBuffer 1676 ( 1677 OBJGPU *pGpu, 1678 KernelGraphics *pKernelGraphics, 1679 NvU32 gfid, 1680 OBJVASPACE *pVAS, 1681 KernelGraphicsContext *pKernelGraphicsContext, 1682 GR_GLOBALCTX_BUFFER buffId, 1683 NvBool bIsReadOnly 1684 ) 1685 { 1686 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 1687 NV_STATUS status = NV_OK; 1688 NvU64 vaddr = 0; 1689 MEMORY_DESCRIPTOR *pMemDesc; 1690 NvBool bSizeAligned; 1691 1692 NV_ASSERT_OK_OR_RETURN( 1693 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast)); 1694 1695 bSizeAligned = pKernelGraphics->globalCtxBuffersInfo.bSizeAligned[buffId]; 1696 pMemDesc = pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers[gfid].memDesc[buffId]; 1697 1698 if (pMemDesc == NULL) 1699 { 1700 NvU32 buffSize; 1701 NvU32 fifoEngineId; 1702 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 1703 1704 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 1705 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 1706 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pContextBuffersInfo != NULL, NV_ERR_INVALID_STATE); 1707 1708 NV_ASSERT_OK_OR_RETURN(kgrctxGlobalCtxBufferToFifoEngineId(buffId, &fifoEngineId)); 1709 buffSize = pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[fifoEngineId].size; 1710 if (buffSize == 0) 1711 { 1712 NV_PRINTF(LEVEL_INFO, 1713 "Could not map %s Buffer as buffer is not supported\n", 1714 NV_ENUM_TO_STRING(GR_GLOBALCTX_BUFFER, buffId)); 1715 return NV_OK; 1716 } 1717 else 1718 { 1719 NV_PRINTF(LEVEL_ERROR, 1720 "Could not map %s Buffer, no memory allocated for it!\n", 1721 NV_ENUM_TO_STRING(GR_GLOBALCTX_BUFFER, buffId)); 1722 return NV_ERR_INVALID_ARGUMENT; 1723 } 1724 } 1725 1726 // Unconditionally call map for refcounting 1727 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 1728 kgraphicsMapCtxBuffer(pGpu, pKernelGraphics, pMemDesc, pVAS, 1729 &pKernelGraphicsContextUnicast->globalCtxBufferVaList[buffId], 1730 bSizeAligned, 1731 bIsReadOnly)); 1732 1733 NV_ASSERT_OK(vaListFindVa(&pKernelGraphicsContextUnicast->globalCtxBufferVaList[buffId], pVAS, &vaddr)); 1734 1735 NV_PRINTF(LEVEL_INFO, 1736 "GPU:%d %s Buffer PA @ 0x%llx VA @ 0x%llx of Size 0x%llx\n", 1737 pGpu->gpuInstance, NV_ENUM_TO_STRING(GR_GLOBALCTX_BUFFER, buffId), 1738 memdescGetPhysAddr(memdescGetMemDescFromGpu(pMemDesc, pGpu), AT_GPU, 0), 1739 vaddr, pMemDesc->Size); 1740 1741 return status; 1742 } 1743 1744 /*! 1745 * @brief Map a GR ctx buffer 1746 */ 1747 NV_STATUS 1748 kgraphicsMapCtxBuffer_IMPL 1749 ( 1750 OBJGPU *pGpu, 1751 KernelGraphics *pKernelGraphics, 1752 MEMORY_DESCRIPTOR *pMemDesc, 1753 OBJVASPACE *pVAS, 1754 VA_LIST *pVaList, 1755 NvBool bAlignSize, 1756 NvBool bIsReadOnly 1757 ) 1758 { 1759 NV_STATUS status = NV_OK; 1760 NvU64 vaddr = 0; 1761 OBJGVASPACE *pGVAS; 1762 1763 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 1764 1765 NV_PRINTF(LEVEL_INFO, "gpu:%d isBC=%d\n", pGpu->gpuInstance, 1766 gpumgrGetBcEnabledStatus(pGpu)); 1767 1768 pGVAS = dynamicCast(pVAS, OBJGVASPACE); 1769 NV_ASSERT_OR_RETURN(!gvaspaceIsExternallyOwned(pGVAS), NV_ERR_INVALID_OPERATION); 1770 1771 status = vaListFindVa(pVaList, pVAS, &vaddr); 1772 if (status == NV_ERR_OBJECT_NOT_FOUND) 1773 { 1774 // create a new subcontext mapping 1775 NvU32 allocFlags = bAlignSize ? DMA_ALLOC_VASPACE_SIZE_ALIGNED : DMA_ALLOC_VASPACE_NONE; 1776 NvU32 updateFlags = bIsReadOnly ? (DMA_UPDATE_VASPACE_FLAGS_READ_ONLY | 1777 DMA_UPDATE_VASPACE_FLAGS_SHADER_READ_ONLY) : DMA_UPDATE_VASPACE_FLAGS_NONE; 1778 1779 if (pGVAS->flags & VASPACE_FLAGS_RESTRICTED_RM_INTERNAL_VALIMITS) 1780 { 1781 allocFlags |= DMA_ALLOC_VASPACE_USE_RM_INTERNAL_VALIMITS; 1782 } 1783 1784 if (kgraphicsIsPerSubcontextContextHeaderSupported(pGpu, pKernelGraphics)) 1785 { 1786 status = dmaMapBuffer_HAL(pGpu, GPU_GET_DMA(pGpu), pVAS, pMemDesc, &vaddr, 1787 allocFlags, updateFlags); 1788 } 1789 else 1790 { 1791 // 1792 // Per subcontext headers not enabled. 1793 // If subcontext is not supported, create a new mapping. 1794 // If subcontext is supported, create an identity mapping to the existing one. 1795 // 1796 1797 if (vaListMapCount(pVaList) == 0) 1798 { 1799 status = dmaMapBuffer_HAL(pGpu, GPU_GET_DMA(pGpu), pVAS, 1800 pMemDesc, 1801 &vaddr, 1802 allocFlags, 1803 updateFlags); 1804 } 1805 else 1806 { 1807 OBJVASPACE *pVas; 1808 NvU32 mapFlags = 0x0; 1809 NvU64 vaddrCached; 1810 1811 FOR_EACH_IN_VADDR_LIST(pVaList, pVas, vaddr) 1812 { 1813 // Find the first virtual address in any VAS 1814 break; 1815 } 1816 FOR_EACH_IN_VADDR_LIST_END(pVaList, pVas, vaddr); 1817 1818 if (bIsReadOnly) 1819 { 1820 mapFlags = FLD_SET_DRF(OS46, _FLAGS, _ACCESS, _READ_ONLY, mapFlags); 1821 mapFlags = FLD_SET_DRF(OS46, _FLAGS, _SHADER_ACCESS, _READ_ONLY, mapFlags); 1822 } 1823 mapFlags = FLD_SET_DRF(OS46, _FLAGS, _DMA_UNICAST_REUSE_ALLOC, _FALSE, mapFlags); 1824 mapFlags = FLD_SET_DRF(OS46, _FLAGS, _DMA_OFFSET_FIXED, _TRUE, mapFlags); 1825 1826 NV_ASSERT(!bAlignSize); // TODO: Add support for size align 1827 vaddrCached = vaddr; 1828 NV_ASSERT_OK_OR_ELSE(status, 1829 dmaAllocMapping_HAL(pGpu, GPU_GET_DMA(pGpu), pVAS, pMemDesc, 1830 &vaddr, 1831 mapFlags, 1832 NULL, 1833 KMIGMGR_SWIZZID_INVALID), 1834 /* do nothing on error, but make sure we overwrite status */;); 1835 NV_ASSERT(vaddr == vaddrCached); 1836 } 1837 } 1838 1839 NV_PRINTF(LEVEL_INFO, "New ctx buffer mapping at VA 0x%llx\n", vaddr); 1840 } 1841 1842 if (status == NV_OK) 1843 NV_ASSERT_OK_OR_RETURN(vaListAddVa(pVaList, pVAS, vaddr)); 1844 1845 return status; 1846 } 1847 1848 /*! 1849 * @brief Unmap a GR ctx buffer 1850 */ 1851 void 1852 kgraphicsUnmapCtxBuffer_IMPL 1853 ( 1854 OBJGPU *pGpu, 1855 KernelGraphics *pKernelGraphics, 1856 OBJVASPACE *pVAS, 1857 VA_LIST *pVaList 1858 ) 1859 { 1860 NV_STATUS status = NV_OK; 1861 NvU64 vaddr = 0; 1862 1863 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 1864 1865 NV_PRINTF(LEVEL_INFO, "gpu:%d isBC=%d\n", pGpu->gpuInstance, 1866 gpumgrGetBcEnabledStatus(pGpu)); 1867 1868 status = vaListFindVa(pVaList, pVAS, &vaddr); 1869 if (status == NV_OK) 1870 { 1871 NV_ASSERT_OK(vaListRemoveVa(pVaList, pVAS)); 1872 1873 status = vaListFindVa(pVaList, pVAS, &vaddr); 1874 1875 NV_ASSERT((NV_OK == status) || (NV_ERR_OBJECT_NOT_FOUND == status)); 1876 if (NV_ERR_OBJECT_NOT_FOUND == status) 1877 { 1878 if (vaListGetManaged(pVaList)) 1879 { 1880 dmaUnmapBuffer_HAL(pGpu, GPU_GET_DMA(pGpu), pVAS, vaddr); 1881 } 1882 1883 NV_PRINTF(LEVEL_INFO, "Freed ctx buffer mapping at VA 0x%llx\n", 1884 vaddr); 1885 } 1886 } 1887 } 1888 1889 /*! 1890 * @brief Get the Class number for a given gr object type 1891 * 1892 * @param[in] pGpu 1893 * @param[in] pKernelGraphics 1894 * @param[in] wantObjectType GR object type to lookup 1895 * @param[out] pClass class number 1896 */ 1897 NV_STATUS 1898 kgraphicsGetClassByType_IMPL 1899 ( 1900 OBJGPU *pGpu, 1901 KernelGraphics *pKernelGraphics, 1902 NvU32 wantObjectType, 1903 NvU32 *pClass 1904 ) 1905 { 1906 NV_STATUS status = NV_OK; 1907 NvU32 objectType; 1908 NvU32 i; 1909 NvU32 numClasses; 1910 NvU32 *pClassesSupported; 1911 1912 *pClass = 0; 1913 1914 if (wantObjectType >= GR_OBJECT_TYPE_INVALID) 1915 { 1916 NV_PRINTF(LEVEL_ERROR, "bad requested object type : %d\n", 1917 wantObjectType); 1918 return NV_ERR_INVALID_ARGUMENT; 1919 } 1920 1921 // find out how many classes of type ENG_GR(0) we have 1922 NV_ASSERT_OK_OR_RETURN( 1923 gpuGetClassList(pGpu, &numClasses, NULL, ENG_GR(pKernelGraphics->instance))); 1924 1925 pClassesSupported = portMemAllocNonPaged(sizeof(NvU32) * numClasses); 1926 NV_CHECK_OR_RETURN(LEVEL_ERROR, pClassesSupported != NULL, NV_ERR_NO_MEMORY); 1927 1928 status = gpuGetClassList(pGpu, &numClasses, pClassesSupported, ENG_GR(pKernelGraphics->instance)); 1929 1930 if (status == NV_OK) 1931 { 1932 for (i = 0; i < numClasses; i++) 1933 { 1934 kgrmgrGetGrObjectType(pClassesSupported[i], &objectType); 1935 1936 NV_PRINTF(LEVEL_INFO, "classNum=0x%08x, type=%d\n", 1937 pClassesSupported[i], objectType); 1938 1939 if (objectType == wantObjectType) 1940 *pClass = pClassesSupported[i]; 1941 } 1942 } 1943 1944 portMemFree(pClassesSupported); 1945 1946 return (*pClass != 0) ? NV_OK : NV_ERR_INVALID_CLASS; 1947 } 1948 1949 /*! 1950 * @brief retrieve the ctx attributes for the given buffer 1951 */ 1952 const GR_BUFFER_ATTR * 1953 kgraphicsGetContextBufferAttr_IMPL 1954 ( 1955 OBJGPU *pGpu, 1956 KernelGraphics *pKernelGraphics, 1957 GR_CTX_BUFFER buf 1958 ) 1959 { 1960 NV_ASSERT_OR_RETURN(NV_ENUM_IS(GR_CTX_BUFFER, buf), NULL); 1961 return &pKernelGraphics->ctxAttr[buf]; 1962 } 1963 1964 /*! 1965 * @brief Creates a VEID0 channel for Golden Image creation 1966 * 1967 * @return NV_OK if channel and golden image created successfully 1968 */ 1969 NV_STATUS 1970 kgraphicsCreateGoldenImageChannel_IMPL 1971 ( 1972 OBJGPU *pGpu, 1973 KernelGraphics *pKernelGraphics 1974 ) 1975 { 1976 NV_STATUS status = NV_OK; 1977 NvHandle hClientId = NV01_NULL_OBJECT; 1978 NvHandle hDeviceId; 1979 NvHandle hSubdeviceId; 1980 NvHandle hVASpace = 0xbaba0042; 1981 NvHandle hPBVirtMemId = 0xbaba0043; 1982 NvHandle hPBPhysMemId = 0xbaba0044; 1983 NvHandle hChannelId = 0xbaba0045; 1984 NvHandle hObj3D = 0xbaba0046; 1985 NvHandle hUserdId = 0xbaba0049; 1986 NvU32 gpFifoEntries = 32; // power-of-2 random choice 1987 NvU64 gpFifoSize = NVA06F_GP_ENTRY__SIZE * gpFifoEntries; 1988 NvU64 chSize = gpFifoSize; 1989 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 1990 RsClient *pClientId; 1991 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 1992 NvBool bNeedMIGWar; 1993 NvBool bBcStatus; 1994 NvBool bClientUserd = IsVOLTAorBetter(pGpu); 1995 NvBool bAcquireLock = NV_FALSE; 1996 NvU32 sliLoopReentrancy; 1997 NV_VASPACE_ALLOCATION_PARAMETERS vaParams; 1998 NV_MEMORY_ALLOCATION_PARAMS memAllocParams; 1999 NV_CHANNEL_ALLOC_PARAMS channelGPFIFOAllocParams; 2000 NvU32 classNum; 2001 MIG_INSTANCE_REF ref; 2002 NvU32 objectType; 2003 NvU32 primarySliSubDeviceInstance; 2004 2005 // XXX This should be removed when broadcast SLI support is deprecated 2006 if (!gpumgrIsParentGPU(pGpu)) 2007 { 2008 return NV_OK; 2009 } 2010 2011 bBcStatus = gpumgrGetBcEnabledStatus(pGpu); 2012 2013 // FIXME these allocations corrupt BC state 2014 NV_ASSERT_OK_OR_RETURN( 2015 rmapiutilAllocClientAndDeviceHandles(pRmApi, pGpu, &hClientId, &hDeviceId, &hSubdeviceId)); 2016 // rmapiutilAllocClientAndDeviceHandles allocates a subdevice object for this subDeviceInstance 2017 primarySliSubDeviceInstance = gpumgrGetSubDeviceInstanceFromGpu(pGpu); 2018 2019 NV_ASSERT_OK_OR_RETURN(serverGetClientUnderLock(&g_resServ, hClientId, &pClientId)); 2020 2021 gpumgrSetBcEnabledStatus(pGpu, NV_TRUE); 2022 2023 // As we have forced here SLI broadcast mode, temporarily reset the reentrancy count 2024 sliLoopReentrancy = gpumgrSLILoopReentrancyPop(pGpu); 2025 2026 bNeedMIGWar = IS_MIG_IN_USE(pGpu); 2027 2028 // Allocate subdevices for secondary GPUs 2029 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY) 2030 { 2031 NvHandle hSecondary; 2032 NV2080_ALLOC_PARAMETERS nv2080AllocParams; 2033 NvU32 thisSubDeviceInstance = gpumgrGetSubDeviceInstanceFromGpu(pGpu); 2034 2035 // Skip if already allocated by rmapiutilAllocClientAndDeviceHandles() 2036 if (thisSubDeviceInstance == primarySliSubDeviceInstance) 2037 SLI_LOOP_CONTINUE; 2038 2039 // Allocate a subDevice 2040 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, 2041 clientGenResourceHandle(pClientId, &hSecondary), 2042 cleanup); 2043 2044 portMemSet(&nv2080AllocParams, 0, sizeof(nv2080AllocParams)); 2045 nv2080AllocParams.subDeviceId = thisSubDeviceInstance; 2046 2047 NV_CHECK_OK(status, LEVEL_SILENT, 2048 pRmApi->AllocWithHandle(pRmApi, 2049 hClientId, 2050 hDeviceId, 2051 hSecondary, 2052 NV20_SUBDEVICE_0, 2053 &nv2080AllocParams, 2054 sizeof(nv2080AllocParams))); 2055 } 2056 SLI_LOOP_END; 2057 2058 if (bNeedMIGWar) 2059 { 2060 NvHandle hPartitionRef = 0xbaba0048; 2061 NvHandle hExecPartitionRef = 0xbaba004a; 2062 NVC637_ALLOCATION_PARAMETERS nvC637AllocParams = {0}; 2063 2064 // Get swizzId for this GR 2065 NV_ASSERT_OK_OR_GOTO(status, 2066 kmigmgrGetMIGReferenceFromEngineType(pGpu, pKernelMIGManager, 2067 RM_ENGINE_TYPE_GR(pKernelGraphics->instance), &ref), 2068 cleanup); 2069 2070 portMemSet(&nvC637AllocParams, 0, sizeof(NVC637_ALLOCATION_PARAMETERS)); 2071 nvC637AllocParams.swizzId = ref.pKernelMIGGpuInstance->swizzId; 2072 2073 // allocate partition reference 2074 NV_ASSERT_OK_OR_GOTO(status, 2075 pRmApi->AllocWithHandle(pRmApi, 2076 hClientId, 2077 hSubdeviceId, 2078 hPartitionRef, 2079 AMPERE_SMC_PARTITION_REF, 2080 &nvC637AllocParams, 2081 sizeof(nvC637AllocParams)), 2082 cleanup); 2083 2084 if (ref.pMIGComputeInstance != NULL) 2085 { 2086 NVC638_ALLOCATION_PARAMETERS nvC638AllocParams = {0}; 2087 nvC638AllocParams.execPartitionId = ref.pMIGComputeInstance->id; 2088 NV_ASSERT_OK_OR_GOTO(status, 2089 pRmApi->AllocWithHandle(pRmApi, 2090 hClientId, 2091 hPartitionRef, 2092 hExecPartitionRef, 2093 AMPERE_SMC_EXEC_PARTITION_REF, 2094 &nvC638AllocParams, 2095 sizeof(nvC638AllocParams)), 2096 cleanup); 2097 } 2098 } 2099 2100 // 2101 // VidHeapControl and vaspace creation calls should happen outside GPU locks 2102 // UVM/CUDA may be holding the GPU locks here and the allocation may subsequently fail 2103 // So explicitly release GPU locks before RmVidHeapControl 2104 // See Bug 1735851-#24 2105 // 2106 rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); 2107 bAcquireLock = NV_TRUE; 2108 pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL); 2109 2110 // Create a new VAspace for channel 2111 portMemSet(&vaParams, 0, sizeof(NV_VASPACE_ALLOCATION_PARAMETERS)); 2112 NV_ASSERT_OK_OR_GOTO(status, 2113 pRmApi->AllocWithHandle(pRmApi, hClientId, hDeviceId, hVASpace, FERMI_VASPACE_A, &vaParams, sizeof(vaParams)), 2114 cleanup); 2115 2116 // Allocate gpfifo entries 2117 portMemSet(&memAllocParams, 0, sizeof(NV_MEMORY_ALLOCATION_PARAMS)); 2118 memAllocParams.owner = HEAP_OWNER_RM_CLIENT_GENERIC; 2119 memAllocParams.type = NVOS32_TYPE_IMAGE; 2120 memAllocParams.size = chSize; 2121 memAllocParams.attr = DRF_DEF(OS32, _ATTR, _LOCATION, _PCI); 2122 memAllocParams.hVASpace = 0; // Physical allocations don't expect vaSpace handles 2123 2124 // 2125 // When APM feature is enabled all RM internal sysmem allocations must 2126 // be in unprotected memory 2127 // When Hopper CC is enabled all RM internal sysmem allocations that 2128 // are required to be accessed from GPU should be in unprotected memory 2129 // Other sysmem allocations that are not required to be accessed from GPU 2130 // must be in protected memory 2131 // 2132 memAllocParams.attr2 |= DRF_DEF(OS32, _ATTR2, _MEMORY_PROTECTION, _UNPROTECTED); 2133 2134 NV_ASSERT_OK_OR_GOTO(status, 2135 pRmApi->AllocWithHandle(pRmApi, hClientId, hDeviceId, hPBPhysMemId, NV01_MEMORY_SYSTEM, &memAllocParams, sizeof(memAllocParams)), 2136 cleanup); 2137 2138 portMemSet(&memAllocParams, 0, sizeof(NV_MEMORY_ALLOCATION_PARAMS)); 2139 memAllocParams.owner = HEAP_OWNER_RM_CLIENT_GENERIC; 2140 memAllocParams.type = NVOS32_TYPE_IMAGE; 2141 memAllocParams.size = chSize; 2142 memAllocParams.attr = DRF_DEF(OS32, _ATTR, _LOCATION, _PCI); 2143 memAllocParams.flags = NVOS32_ALLOC_FLAGS_VIRTUAL; 2144 memAllocParams.hVASpace = hVASpace; // Virtual allocation expect vaSpace handles 2145 // 0 handle = allocations on gpu default vaSpace 2146 2147 NV_ASSERT_OK_OR_GOTO(status, 2148 pRmApi->AllocWithHandle(pRmApi, hClientId, hDeviceId, hPBVirtMemId, NV50_MEMORY_VIRTUAL, &memAllocParams, sizeof(memAllocParams)), 2149 cleanup); 2150 2151 // Allocate Userd 2152 if (bClientUserd) 2153 { 2154 NvU32 userdMemClass = NV01_MEMORY_LOCAL_USER; 2155 NvU32 ctrlSize; 2156 2157 if (gpuIsClassSupported(pGpu, VOLTA_CHANNEL_GPFIFO_A)) 2158 { 2159 ctrlSize = sizeof(Nvc36fControl); 2160 } 2161 else if (gpuIsClassSupported(pGpu, TURING_CHANNEL_GPFIFO_A)) 2162 { 2163 ctrlSize = sizeof(Nvc46fControl); 2164 } 2165 else if (gpuIsClassSupported(pGpu, AMPERE_CHANNEL_GPFIFO_A)) 2166 { 2167 ctrlSize = sizeof(Nvc56fControl); 2168 } 2169 else if (gpuIsClassSupported(pGpu, HOPPER_CHANNEL_GPFIFO_A)) 2170 { 2171 ctrlSize = sizeof(Nvc86fControl); 2172 } 2173 else 2174 { 2175 status = NV_ERR_NOT_SUPPORTED; 2176 goto cleanup; 2177 } 2178 2179 portMemSet(&memAllocParams, 0, sizeof(NV_MEMORY_ALLOCATION_PARAMS)); 2180 memAllocParams.owner = HEAP_OWNER_RM_CLIENT_GENERIC; 2181 memAllocParams.size = ctrlSize; 2182 memAllocParams.type = NVOS32_TYPE_IMAGE; 2183 2184 // Apply registry overrides to USERD. 2185 switch (DRF_VAL(_REG_STR_RM, _INST_LOC, _USERD, pGpu->instLocOverrides)) 2186 { 2187 case NV_REG_STR_RM_INST_LOC_USERD_NCOH: 2188 case NV_REG_STR_RM_INST_LOC_USERD_COH: 2189 userdMemClass = NV01_MEMORY_SYSTEM; 2190 memAllocParams.attr = DRF_DEF(OS32, _ATTR, _LOCATION, _PCI); 2191 break; 2192 2193 case NV_REG_STR_RM_INST_LOC_USERD_VID: 2194 case NV_REG_STR_RM_INST_LOC_USERD_DEFAULT: 2195 memAllocParams.attr = DRF_DEF(OS32, _ATTR, _LOCATION, _VIDMEM); 2196 break; 2197 } 2198 2199 // 2200 // When APM is enabled all RM internal allocations must to go to 2201 // unprotected memory irrespective of vidmem or sysmem 2202 // When Hopper CC is enabled all RM internal sysmem allocations that 2203 // are required to be accessed from GPU should be in unprotected memory 2204 // and all vidmem allocations must go to protected memory 2205 // 2206 if (gpuIsApmFeatureEnabled(pGpu) || 2207 FLD_TEST_DRF(OS32, _ATTR, _LOCATION, _PCI, memAllocParams.attr)) 2208 { 2209 memAllocParams.attr2 |= DRF_DEF(OS32, _ATTR2, _MEMORY_PROTECTION, 2210 _UNPROTECTED); 2211 } 2212 2213 NV_ASSERT_OK_OR_GOTO(status, 2214 pRmApi->AllocWithHandle(pRmApi, hClientId, hDeviceId, hUserdId, 2215 userdMemClass, &memAllocParams, sizeof(memAllocParams)), 2216 cleanup); 2217 } 2218 2219 // Get fifo channel class Id 2220 classNum = kfifoGetChannelClassId(pGpu, GPU_GET_KERNEL_FIFO(pGpu)); 2221 NV_ASSERT_OR_GOTO(classNum != 0, cleanup); 2222 2223 // Allocate a bare channel 2224 portMemSet(&channelGPFIFOAllocParams, 0, sizeof(NV_CHANNEL_ALLOC_PARAMS)); 2225 channelGPFIFOAllocParams.hVASpace = hVASpace; 2226 channelGPFIFOAllocParams.hObjectBuffer = hPBVirtMemId; 2227 channelGPFIFOAllocParams.gpFifoEntries = gpFifoEntries; 2228 // 2229 // Set the gpFifoOffset to zero intentionally since we only need this channel 2230 // to be created, but will not submit any work to it. So it's fine not to 2231 // provide a valid offset here. 2232 // 2233 channelGPFIFOAllocParams.gpFifoOffset = 0; 2234 if (bClientUserd) 2235 { 2236 channelGPFIFOAllocParams.hUserdMemory[0] = hUserdId; 2237 } 2238 2239 if (bNeedMIGWar) 2240 { 2241 RM_ENGINE_TYPE localRmEngineType; 2242 Device *pDevice; 2243 2244 NV_ASSERT_OK_OR_GOTO(status, 2245 deviceGetByHandle(pClientId, hDeviceId, &pDevice), 2246 cleanup); 2247 2248 NV_ASSERT_OK_OR_GOTO(status, 2249 kmigmgrGetInstanceRefFromDevice(pGpu, pKernelMIGManager, pDevice, &ref), 2250 cleanup); 2251 2252 NV_ASSERT_OK_OR_GOTO(status, 2253 kmigmgrGetGlobalToLocalEngineType(pGpu, pKernelMIGManager, ref, RM_ENGINE_TYPE_GR(pKernelGraphics->instance), &localRmEngineType), 2254 cleanup); 2255 2256 channelGPFIFOAllocParams.engineType = gpuGetNv2080EngineType(localRmEngineType); 2257 } 2258 else 2259 { 2260 channelGPFIFOAllocParams.engineType = gpuGetNv2080EngineType(RM_ENGINE_TYPE_GR0); 2261 } 2262 2263 NV_ASSERT_OK_OR_GOTO(status, 2264 pRmApi->AllocWithHandle(pRmApi, hClientId, hDeviceId, hChannelId, 2265 classNum, &channelGPFIFOAllocParams, sizeof(channelGPFIFOAllocParams)), 2266 cleanup); 2267 2268 // 2269 // When using split VAS, we need to reserve enough pagepool memory to 2270 // sustain large context buffer mappings. For GSPCLIENT where the golden 2271 // context buffer channel is initialized on boot, the pagepool does not have 2272 // enough reserved memory to accommodate these buffers, so we need to 2273 // reserve extra here. 2274 // 2275 if (IS_GSP_CLIENT(pGpu)) 2276 { 2277 KernelChannel *pKernelChannel; 2278 NvU64 reserveSize; 2279 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 2280 NvU32 i; 2281 2282 NV_ASSERT_OK(CliGetKernelChannel(pClientId, hChannelId, &pKernelChannel)); 2283 2284 NV_ASSERT_OR_ELSE(pKernelGraphicsStaticInfo != NULL, 2285 status = NV_ERR_INVALID_STATE; 2286 goto cleanup;); 2287 NV_ASSERT_OR_ELSE(pKernelGraphicsStaticInfo->pContextBuffersInfo != NULL, 2288 status = NV_ERR_INVALID_STATE; 2289 goto cleanup;); 2290 2291 reserveSize = 0; 2292 for (i = 0; i < NV_ARRAY_ELEMENTS(pKernelGraphicsStaticInfo->pContextBuffersInfo->engine); ++i) 2293 { 2294 if (pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[i].size != NV_U32_MAX) 2295 reserveSize += pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[i].size; 2296 } 2297 2298 NV_ASSERT_OK( 2299 vaspaceReserveMempool(pKernelChannel->pVAS, pGpu, 2300 GPU_RES_GET_DEVICE(pKernelChannel), 2301 reserveSize, RM_PAGE_SIZE, 2302 VASPACE_RESERVE_FLAGS_NONE)); 2303 } 2304 2305 // Reaquire the GPU locks 2306 NV_ASSERT_OK_OR_GOTO(status, 2307 rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_GR), 2308 cleanup); 2309 bAcquireLock = NV_FALSE; 2310 pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 2311 2312 if (!bNeedMIGWar) 2313 { 2314 objectType = GR_OBJECT_TYPE_3D; 2315 } 2316 else 2317 { 2318 objectType = GR_OBJECT_TYPE_COMPUTE; 2319 2320 } 2321 2322 // Get KernelGraphicsObject class Id 2323 NV_ASSERT_OK_OR_GOTO(status, 2324 kgraphicsGetClassByType(pGpu, pKernelGraphics, objectType, &classNum), 2325 cleanup); 2326 NV_ASSERT_OR_GOTO(classNum != 0, cleanup); 2327 2328 // Allocate a GR object on the channel 2329 NV_ASSERT_OK_OR_GOTO(status, 2330 pRmApi->AllocWithHandle(pRmApi, hClientId, hChannelId, hObj3D, classNum, NULL, 0), 2331 cleanup); 2332 2333 cleanup: 2334 2335 if (bAcquireLock) 2336 { 2337 NV_ASSERT_OK_OR_CAPTURE_FIRST_ERROR(status, 2338 rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_GR)); 2339 } 2340 2341 // Free all handles 2342 NV_ASSERT_OK_OR_CAPTURE_FIRST_ERROR(status, 2343 pRmApi->Free(pRmApi, hClientId, hClientId)); 2344 2345 // Restore the reentrancy count 2346 gpumgrSLILoopReentrancyPush(pGpu, sliLoopReentrancy); 2347 2348 gpumgrSetBcEnabledStatus(pGpu, bBcStatus); 2349 2350 return status; 2351 } 2352 2353 /*! 2354 * @brief Free context buffers shared by all/most graphics contexts 2355 */ 2356 void kgraphicsFreeGlobalCtxBuffers_IMPL 2357 ( 2358 OBJGPU *pGpu, 2359 KernelGraphics *pKernelGraphics, 2360 NvU32 gfid 2361 ) 2362 { 2363 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 2364 GR_GLOBALCTX_BUFFERS *pCtxBuffers; 2365 GR_GLOBALCTX_BUFFER buff; 2366 NvBool bEvict = NV_FALSE; 2367 2368 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2369 2370 if (pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers == NULL) 2371 return; 2372 2373 pCtxBuffers = &pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers[gfid]; 2374 2375 if (!pCtxBuffers->bAllocated) 2376 return; 2377 2378 FOR_EACH_IN_ENUM(GR_GLOBALCTX_BUFFER, buff) 2379 { 2380 if (pCtxBuffers->memDesc[buff] != NULL) 2381 { 2382 bEvict = NV_TRUE; 2383 2384 memdescFree(pCtxBuffers->memDesc[buff]); 2385 memdescDestroy(pCtxBuffers->memDesc[buff]); 2386 pCtxBuffers->memDesc[buff] = NULL; 2387 pCtxBuffers->bInitialized[buff] = NV_FALSE; 2388 } 2389 } 2390 FOR_EACH_IN_ENUM_END; 2391 2392 pCtxBuffers->bAllocated = NV_FALSE; 2393 2394 // make sure all L2 cache lines using CB buffers are clear after we free them 2395 if (bEvict) 2396 NV_ASSERT_OK(kmemsysCacheOp_HAL(pGpu, pKernelMemorySystem, NULL, FB_CACHE_VIDEO_MEMORY, FB_CACHE_EVICT)); 2397 } 2398 2399 NV_STATUS 2400 kgraphicsGetCaps_IMPL 2401 ( 2402 OBJGPU *pGpu, 2403 KernelGraphics *pKernelGraphics, 2404 NvU8 *pGrCaps 2405 ) 2406 { 2407 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 2408 2409 NV_ASSERT_OR_RETURN(pGrCaps != NULL, NV_ERR_INVALID_ARGUMENT); 2410 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 2411 2412 portMemCopy(pGrCaps, 2413 NV0080_CTRL_GR_CAPS_TBL_SIZE * sizeof(*pGrCaps), 2414 pKernelGraphicsStaticInfo->grCaps.capsTbl, 2415 NV0080_CTRL_GR_CAPS_TBL_SIZE * sizeof(*pGrCaps)); 2416 2417 return NV_OK; 2418 } 2419 2420 /*! 2421 * @brief Return whether unrestricted register access bufffer is supported or not. 2422 */ 2423 NvBool 2424 kgraphicsIsUnrestrictedAccessMapSupported_PF 2425 ( 2426 OBJGPU *pGpu, 2427 KernelGraphics *pKernelGraphics 2428 ) 2429 { 2430 return !hypervisorIsVgxHyper(); 2431 } 2432 2433 /*! 2434 * @brief Provides an opportunity to register some IntrService during intrStateInit. 2435 */ 2436 void 2437 kgraphicsRegisterIntrService_IMPL 2438 ( 2439 OBJGPU *pGpu, 2440 KernelGraphics *pKernelGraphics, 2441 IntrServiceRecord pRecords[MC_ENGINE_IDX_MAX] 2442 ) 2443 { 2444 NvU32 engineIdx = MC_ENGINE_IDX_GRn_FECS_LOG(pKernelGraphics->instance); 2445 2446 NV_ASSERT(pRecords[engineIdx].pInterruptService == NULL); 2447 pRecords[engineIdx].pInterruptService = staticCast(pKernelGraphics, IntrService); 2448 2449 engineIdx = MC_ENGINE_IDX_GRn(pKernelGraphics->instance); 2450 2451 NV_ASSERT(pRecords[engineIdx].pNotificationService == NULL); 2452 pRecords[engineIdx].bFifoWaiveNotify = NV_FALSE; 2453 pRecords[engineIdx].pNotificationService = staticCast(pKernelGraphics, IntrService); 2454 } 2455 2456 /*! 2457 * @brief Services the nonstall interrupt. 2458 */ 2459 NvU32 2460 kgraphicsServiceNotificationInterrupt_IMPL 2461 ( 2462 OBJGPU *pGpu, 2463 KernelGraphics *pKernelGraphics, 2464 IntrServiceServiceNotificationInterruptArguments *pParams 2465 ) 2466 { 2467 NvU32 grIdx = pKernelGraphics->instance; 2468 2469 NV_ASSERT_OR_RETURN(pParams != NULL, 0); 2470 NV_ASSERT_OR_RETURN(pParams->engineIdx == MC_ENGINE_IDX_GRn(grIdx), 0); 2471 2472 MODS_ARCH_REPORT(NV_ARCH_EVENT_NONSTALL_GR, "%s", "processing GR nonstall interrupt\n"); 2473 2474 kgraphicsNonstallIntrCheckAndClear_HAL(pGpu, pKernelGraphics, pParams->pThreadState); 2475 engineNonStallIntrNotify(pGpu, RM_ENGINE_TYPE_GR(pKernelGraphics->instance)); 2476 return NV_OK; 2477 } 2478 2479 /*! 2480 * KernelGraphics RM Device Controls 2481 */ 2482 2483 /*! 2484 * deviceCtrlCmdKGrGetCaps_IMPL 2485 * 2486 * Lock Requirements: 2487 * Assert that API lock held on entry 2488 * 2489 * TODO: remove once all uses have been migrated to V2 2490 */ 2491 NV_STATUS 2492 deviceCtrlCmdKGrGetCaps_IMPL 2493 ( 2494 Device *pDevice, 2495 NV0080_CTRL_GR_GET_CAPS_PARAMS *pParams 2496 ) 2497 { 2498 OBJGPU *pGpu = GPU_RES_GET_GPU(pDevice); 2499 NvU8 *pGrCaps = NvP64_VALUE(pParams->capsTbl); 2500 NvBool bCapsPopulated = NV_FALSE; 2501 2502 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner()); 2503 2504 if (IsDFPGA(pGpu)) 2505 { 2506 return NV_ERR_NOT_SUPPORTED; 2507 } 2508 2509 NV_CHECK_OR_RETURN(LEVEL_ERROR, pGrCaps != NULL, NV_ERR_INVALID_ARGUMENT); 2510 NV_CHECK_OR_RETURN(LEVEL_ERROR, pParams->capsTblSize == NV0080_CTRL_GR_CAPS_TBL_SIZE, NV_ERR_INVALID_ARGUMENT); 2511 2512 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY) 2513 { 2514 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 2515 KernelGraphics *pKernelGraphics; 2516 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo; 2517 NV_STATUS status; 2518 2519 portMemSet(&grRouteInfo, 0, sizeof(grRouteInfo)); 2520 kgrmgrCtrlSetEngineID(0, &grRouteInfo); 2521 NV_CHECK_OK_OR_ELSE(status, LEVEL_ERROR, 2522 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &grRouteInfo, &pKernelGraphics), 2523 SLI_LOOP_RETURN(status);); 2524 2525 if (!bCapsPopulated) 2526 { 2527 NV_CHECK_OK_OR_ELSE(status, LEVEL_ERROR, 2528 kgraphicsGetCaps(pGpu, pKernelGraphics, pGrCaps), 2529 SLI_LOOP_RETURN(status);); 2530 2531 bCapsPopulated = NV_TRUE; 2532 } 2533 } 2534 SLI_LOOP_END 2535 2536 return NV_OK; 2537 } 2538 2539 /*! 2540 * deviceCtrlCmdKGrGetCapsV2_IMPL 2541 * 2542 * Lock Requirements: 2543 * Assert that API lock held on entry 2544 */ 2545 NV_STATUS 2546 deviceCtrlCmdKGrGetCapsV2_IMPL 2547 ( 2548 Device *pDevice, 2549 NV0080_CTRL_GR_GET_CAPS_V2_PARAMS *pParams 2550 ) 2551 { 2552 OBJGPU *pGpu = GPU_RES_GET_GPU(pDevice); 2553 2554 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner()); 2555 2556 if (IsDFPGA(pGpu)) 2557 { 2558 return NV_ERR_NOT_SUPPORTED; 2559 } 2560 2561 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY) 2562 { 2563 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 2564 KernelGraphics *pKernelGraphics; 2565 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo = pParams->grRouteInfo; 2566 NV_STATUS status; 2567 2568 kgrmgrCtrlSetEngineID(0, &grRouteInfo); 2569 NV_CHECK_OK_OR_ELSE(status, LEVEL_ERROR, 2570 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &grRouteInfo, &pKernelGraphics), 2571 SLI_LOOP_RETURN(status);); 2572 2573 if (!pParams->bCapsPopulated) 2574 { 2575 NV_CHECK_OK_OR_ELSE(status, LEVEL_ERROR, 2576 kgraphicsGetCaps(pGpu, pKernelGraphics, pParams->capsTbl), 2577 SLI_LOOP_RETURN(status);); 2578 2579 pParams->bCapsPopulated = NV_TRUE; 2580 } 2581 } 2582 SLI_LOOP_END 2583 2584 return NV_OK; 2585 } 2586 2587 static NV_STATUS 2588 _kgraphicsCtrlCmdGrGetInfoV2 2589 ( 2590 OBJGPU *pGpu, 2591 Device *pDevice, 2592 NV2080_CTRL_GR_GET_INFO_V2_PARAMS *pParams 2593 ) 2594 { 2595 NV_STATUS status = NV_OK; 2596 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 2597 NvU32 grInfoListSize = pParams->grInfoListSize; 2598 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 2599 NV2080_CTRL_INTERNAL_STATIC_GR_INFO *pGrInfo; 2600 NvU32 i; 2601 2602 if (pKernelGraphicsManager == NULL) 2603 { 2604 return NV_ERR_NOT_SUPPORTED; 2605 } 2606 2607 if ((0 == grInfoListSize) || 2608 (grInfoListSize > NV2080_CTRL_GR_INFO_MAX_SIZE)) 2609 { 2610 NV_PRINTF(LEVEL_ERROR, "Invalid grInfoList size: 0x%x\n", grInfoListSize); 2611 return NV_ERR_INVALID_ARGUMENT; 2612 } 2613 2614 if (kmigmgrIsDeviceUsingDeviceProfiling(pGpu, pKernelMIGManager, pDevice)) 2615 { 2616 NV_ASSERT_OR_RETURN(kgrmgrGetLegacyKGraphicsStaticInfo(pGpu, pKernelGraphicsManager)->bInitialized, NV_ERR_INVALID_STATE); 2617 NV_ASSERT_OR_RETURN(kgrmgrGetLegacyKGraphicsStaticInfo(pGpu, pKernelGraphicsManager)->pGrInfo != NULL, NV_ERR_NOT_SUPPORTED); 2618 2619 pGrInfo = kgrmgrGetLegacyKGraphicsStaticInfo(pGpu, pKernelGraphicsManager)->pGrInfo; 2620 } 2621 else 2622 { 2623 KernelGraphics *pKernelGraphics; 2624 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 2625 2626 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2627 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &pParams->grRouteInfo, &pKernelGraphics)); 2628 2629 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 2630 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 2631 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pGrInfo != NULL, NV_ERR_NOT_SUPPORTED); 2632 2633 pGrInfo = pKernelGraphicsStaticInfo->pGrInfo; 2634 } 2635 2636 for (i = 0; i < grInfoListSize; i++) 2637 { 2638 NV_CHECK_OR_RETURN(LEVEL_ERROR, pParams->grInfoList[i].index < NV2080_CTRL_GR_INFO_MAX_SIZE, NV_ERR_INVALID_ARGUMENT); 2639 pParams->grInfoList[i].data = pGrInfo->infoList[pParams->grInfoList[i].index].data; 2640 } 2641 2642 return status; 2643 } 2644 2645 /*! 2646 * deviceCtrlCmdKGrGetInfo 2647 * 2648 * Lock Requirements: 2649 * Assert that API lock and GPUs lock held on entry 2650 * 2651 * TODO: remove once all uses have been migrated to V2 2652 */ 2653 NV_STATUS 2654 deviceCtrlCmdKGrGetInfo_IMPL 2655 ( 2656 Device *pDevice, 2657 NV0080_CTRL_GR_GET_INFO_PARAMS *pParams 2658 ) 2659 { 2660 OBJGPU *pGpu = GPU_RES_GET_GPU(pDevice); 2661 NV0080_CTRL_GR_GET_INFO_V2_PARAMS grInfoParamsV2; 2662 NV0080_CTRL_GR_INFO *pGrInfos = NvP64_VALUE(pParams->grInfoList); 2663 NvU32 grInfoListSize = NV_MIN(pParams->grInfoListSize, 2664 NV0080_CTRL_GR_INFO_MAX_SIZE); 2665 2666 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner()); 2667 2668 NV_CHECK_OR_RETURN(LEVEL_ERROR, pGrInfos != NULL, NV_ERR_INVALID_ARGUMENT); 2669 2670 portMemSet(&grInfoParamsV2, 0, sizeof(grInfoParamsV2)); 2671 portMemCopy(grInfoParamsV2.grInfoList, grInfoListSize * sizeof(*pGrInfos), 2672 pGrInfos, grInfoListSize * sizeof(*pGrInfos)); 2673 grInfoParamsV2.grInfoListSize = grInfoListSize; 2674 2675 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2676 _kgraphicsCtrlCmdGrGetInfoV2(pGpu, pDevice, &grInfoParamsV2)); 2677 2678 portMemCopy(pGrInfos, grInfoListSize * sizeof(*pGrInfos), 2679 grInfoParamsV2.grInfoList, grInfoListSize * sizeof(*pGrInfos)); 2680 return NV_OK; 2681 } 2682 2683 /*! 2684 * deviceCtrlCmdKGrGetInfoV2 2685 * 2686 * Lock Requirements: 2687 * Assert that API lock and GPUs lock held on entry 2688 */ 2689 NV_STATUS 2690 deviceCtrlCmdKGrGetInfoV2_IMPL 2691 ( 2692 Device *pDevice, 2693 NV0080_CTRL_GR_GET_INFO_V2_PARAMS *pParams 2694 ) 2695 { 2696 OBJGPU *pGpu = GPU_RES_GET_GPU(pDevice); 2697 2698 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner()); 2699 2700 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2701 _kgraphicsCtrlCmdGrGetInfoV2(pGpu, pDevice, pParams)); 2702 2703 return NV_OK; 2704 } 2705 2706 NV_STATUS 2707 kgraphicsDiscoverMaxLocalCtxBufferSize_IMPL 2708 ( 2709 OBJGPU *pGpu, 2710 KernelGraphics *pKernelGraphics 2711 ) 2712 { 2713 NvU32 bufId = 0; 2714 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 2715 2716 if (IS_MODS_AMODEL(pGpu)) 2717 return NV_OK; 2718 2719 NV_ASSERT_OK_OR_RETURN( 2720 kgraphicsInitializeDeferredStaticData(pGpu, pKernelGraphics, NV01_NULL_OBJECT, NV01_NULL_OBJECT)); 2721 2722 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pContextBuffersInfo != NULL, NV_ERR_INVALID_STATE); 2723 2724 FOR_EACH_IN_ENUM(GR_CTX_BUFFER, bufId) 2725 { 2726 if (bufId == GR_CTX_BUFFER_MAIN) 2727 { 2728 NvU32 size; 2729 2730 NV_ASSERT_OK_OR_RETURN(kgraphicsGetMainCtxBufferSize(pGpu, pKernelGraphics, NV_TRUE, &size)); 2731 kgraphicsSetCtxBufferInfo(pGpu, pKernelGraphics, bufId, 2732 size, 2733 RM_PAGE_SIZE, 2734 RM_ATTR_PAGE_SIZE_4KB, 2735 kgraphicsShouldForceMainCtxContiguity_HAL(pGpu, pKernelGraphics)); 2736 } 2737 else 2738 { 2739 NvU32 fifoEngineId; 2740 2741 NV_ASSERT_OK_OR_RETURN( 2742 kgrctxCtxBufferToFifoEngineId(bufId, &fifoEngineId)); 2743 2744 kgraphicsSetCtxBufferInfo(pGpu, pKernelGraphics, bufId, 2745 pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[fifoEngineId].size, 2746 RM_PAGE_SIZE, 2747 RM_ATTR_PAGE_SIZE_4KB, 2748 ((bufId == GR_CTX_BUFFER_PATCH) || (bufId == GR_CTX_BUFFER_PM))); 2749 } 2750 } 2751 FOR_EACH_IN_ENUM_END; 2752 return NV_OK; 2753 } 2754 2755 /*! 2756 * KernelGraphics RM SubDevice Controls 2757 */ 2758 2759 /*! 2760 * subdeviceCtrlCmdKGrGetCapsV2 2761 * 2762 * Lock Requirements: 2763 * Assert that API lock held on entry 2764 */ 2765 NV_STATUS 2766 subdeviceCtrlCmdKGrGetCapsV2_IMPL 2767 ( 2768 Subdevice *pSubdevice, 2769 NV2080_CTRL_GR_GET_CAPS_V2_PARAMS *pParams 2770 ) 2771 { 2772 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 2773 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 2774 KernelGraphics *pKernelGraphics; 2775 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo = pParams->grRouteInfo; 2776 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 2777 2778 kgrmgrCtrlSetEngineID(0, &grRouteInfo); 2779 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2780 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &grRouteInfo, &pKernelGraphics)); 2781 2782 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner()); 2783 2784 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 2785 if (pKernelGraphicsStaticInfo == NULL) 2786 { 2787 return NV_ERR_INVALID_STATE; 2788 } 2789 2790 if (!pParams->bCapsPopulated) 2791 { 2792 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2793 kgraphicsGetCaps(pGpu, pKernelGraphics, pParams->capsTbl)); 2794 2795 pParams->bCapsPopulated = NV_TRUE; 2796 } 2797 2798 return NV_OK; 2799 } 2800 2801 /*! 2802 * subdeviceCtrlCmdKGrGetInfo 2803 * 2804 * Lock Requirements: 2805 * Assert that API lock and GPUs lock held on entry 2806 * 2807 * TODO: remove once all uses have been migrated to V2 2808 */ 2809 NV_STATUS 2810 subdeviceCtrlCmdKGrGetInfo_IMPL 2811 ( 2812 Subdevice *pSubdevice, 2813 NV2080_CTRL_GR_GET_INFO_PARAMS *pParams 2814 ) 2815 { 2816 NV2080_CTRL_GR_GET_INFO_V2_PARAMS grInfoParamsV2; 2817 NV2080_CTRL_GR_INFO *pGrInfos = NvP64_VALUE(pParams->grInfoList); 2818 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo = pParams->grRouteInfo; 2819 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 2820 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 2821 NvU32 grInfoListSize = NV_MIN(pParams->grInfoListSize, 2822 NV2080_CTRL_GR_INFO_MAX_SIZE); 2823 2824 // 2825 // Adding the null check as engine GRMGR is missing for DFPGA. 2826 // 2827 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 2828 NV_CHECK_OR_RETURN(LEVEL_ERROR, pKernelGraphicsManager != NULL, NV_ERR_NOT_SUPPORTED); 2829 2830 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 2831 2832 NV_CHECK_OR_RETURN(LEVEL_ERROR, pGrInfos != NULL, NV_ERR_INVALID_ARGUMENT); 2833 2834 portMemSet(&grInfoParamsV2, 0, sizeof(grInfoParamsV2)); 2835 grInfoParamsV2.grInfoListSize = grInfoListSize; 2836 portMemCopy(grInfoParamsV2.grInfoList, grInfoListSize * sizeof(*pGrInfos), 2837 pGrInfos, grInfoListSize * sizeof(*pGrInfos)); 2838 grInfoParamsV2.grRouteInfo = grRouteInfo; 2839 2840 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2841 _kgraphicsCtrlCmdGrGetInfoV2(pGpu, pDevice, &grInfoParamsV2)); 2842 2843 portMemCopy(pGrInfos, grInfoListSize * sizeof(*pGrInfos), 2844 grInfoParamsV2.grInfoList, grInfoListSize * sizeof(*pGrInfos)); 2845 return NV_OK; 2846 } 2847 2848 /*! 2849 * subdeviceCtrlCmdKGrGetInfoV2 2850 * 2851 * Lock Requirements: 2852 * Assert that API lock and GPUs lock held on entry 2853 */ 2854 NV_STATUS 2855 subdeviceCtrlCmdKGrGetInfoV2_IMPL 2856 ( 2857 Subdevice *pSubdevice, 2858 NV2080_CTRL_GR_GET_INFO_V2_PARAMS *pParams 2859 ) 2860 { 2861 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 2862 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 2863 2864 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 2865 2866 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2867 _kgraphicsCtrlCmdGrGetInfoV2(pGpu, pDevice, pParams)); 2868 2869 return NV_OK; 2870 } 2871 2872 /*! 2873 * subdeviceCtrlCmdKGrGetSmToGpcTpcMappings 2874 * 2875 * Lock Requirements: 2876 * Assert that API lock and GPUs lock held on entry 2877 */ 2878 NV_STATUS 2879 subdeviceCtrlCmdKGrGetSmToGpcTpcMappings_IMPL 2880 ( 2881 Subdevice *pSubdevice, 2882 NV2080_CTRL_GR_GET_SM_TO_GPC_TPC_MAPPINGS_PARAMS *pParams 2883 ) 2884 { 2885 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 2886 KernelGraphics *pKernelGraphics; 2887 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 2888 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 2889 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 2890 const KGRAPHICS_STATIC_INFO *pStaticInfo; 2891 NvU32 i; 2892 2893 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 2894 2895 if (kmigmgrIsDeviceUsingDeviceProfiling(pGpu, pKernelMIGManager, pDevice)) 2896 { 2897 return NV_ERR_NOT_SUPPORTED; 2898 } 2899 else 2900 { 2901 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2902 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &pParams->grRouteInfo, &pKernelGraphics)); 2903 } 2904 2905 // Verify static info is available 2906 pStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 2907 NV_ASSERT_OR_RETURN(pStaticInfo != NULL, NV_ERR_INVALID_STATE); 2908 2909 // Verify limits are within bounds 2910 NV_ASSERT_OR_RETURN(pStaticInfo->globalSmOrder.numSm <= NV2080_CTRL_GR_GET_SM_TO_GPC_TPC_MAPPINGS_MAX_SM_COUNT, 2911 NV_ERR_INVALID_LIMIT); 2912 2913 // Populate output data 2914 pParams->smCount = pStaticInfo->globalSmOrder.numSm; 2915 for (i = 0; i < pStaticInfo->globalSmOrder.numSm; ++i) 2916 { 2917 pParams->smId[i].gpcId = pStaticInfo->globalSmOrder.globalSmId[i].gpcId; 2918 pParams->smId[i].tpcId = pStaticInfo->globalSmOrder.globalSmId[i].localTpcId; 2919 } 2920 2921 return NV_OK; 2922 } 2923 2924 NV_STATUS 2925 subdeviceCtrlCmdKGrGetGlobalSmOrder_IMPL 2926 ( 2927 Subdevice *pSubdevice, 2928 NV2080_CTRL_GR_GET_GLOBAL_SM_ORDER_PARAMS *pParams 2929 ) 2930 { 2931 NV_STATUS status = NV_OK; 2932 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 2933 KernelGraphics *pKernelGraphics; 2934 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 2935 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 2936 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 2937 const KGRAPHICS_STATIC_INFO *pStaticInfo; 2938 NvU32 i; 2939 2940 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 2941 2942 if (kmigmgrIsDeviceUsingDeviceProfiling(pGpu, pKernelMIGManager, pDevice)) 2943 { 2944 return NV_ERR_NOT_SUPPORTED; 2945 } 2946 else 2947 { 2948 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2949 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &pParams->grRouteInfo, &pKernelGraphics)); 2950 } 2951 2952 // Verify static info is available 2953 pStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 2954 NV_ASSERT_OR_RETURN(pStaticInfo != NULL, NV_ERR_INVALID_STATE); 2955 2956 // Verify limits are within bounds 2957 NV_ASSERT_OR_RETURN(pStaticInfo->globalSmOrder.numSm <= NV2080_CTRL_CMD_GR_GET_GLOBAL_SM_ORDER_MAX_SM_COUNT, 2958 NV_ERR_INVALID_LIMIT); 2959 2960 // Populate output data 2961 pParams->numSm = pStaticInfo->globalSmOrder.numSm; 2962 pParams->numTpc = pStaticInfo->globalSmOrder.numTpc; 2963 for (i = 0; i < pStaticInfo->globalSmOrder.numSm; ++i) 2964 { 2965 pParams->globalSmId[i].gpcId = pStaticInfo->globalSmOrder.globalSmId[i].gpcId; 2966 pParams->globalSmId[i].localTpcId = pStaticInfo->globalSmOrder.globalSmId[i].localTpcId; 2967 pParams->globalSmId[i].localSmId = pStaticInfo->globalSmOrder.globalSmId[i].localSmId; 2968 pParams->globalSmId[i].globalTpcId = pStaticInfo->globalSmOrder.globalSmId[i].globalTpcId; 2969 pParams->globalSmId[i].virtualGpcId = pStaticInfo->globalSmOrder.globalSmId[i].virtualGpcId; 2970 pParams->globalSmId[i].migratableTpcId = pStaticInfo->globalSmOrder.globalSmId[i].migratableTpcId; 2971 } 2972 2973 return status; 2974 } 2975 2976 /*! 2977 * subdeviceCtrlCmdKGrGetSmIssueRateModifier 2978 * 2979 * Lock Requirements: 2980 * Assert that API lock and GPUs lock held on entry 2981 */ 2982 NV_STATUS 2983 subdeviceCtrlCmdKGrGetSmIssueRateModifier_IMPL 2984 ( 2985 Subdevice *pSubdevice, 2986 NV2080_CTRL_GR_GET_SM_ISSUE_RATE_MODIFIER_PARAMS *pParams 2987 ) 2988 { 2989 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 2990 KernelGraphics *pKernelGraphics; 2991 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 2992 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 2993 const KGRAPHICS_STATIC_INFO *pStaticInfo; 2994 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 2995 2996 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner()); 2997 2998 if (kmigmgrIsDeviceUsingDeviceProfiling(pGpu, pKernelMIGManager, pDevice)) 2999 { 3000 NvU32 grIdx; 3001 for (grIdx = 0; grIdx < GPU_MAX_GRS; grIdx++) 3002 { 3003 pKernelGraphics = GPU_GET_KERNEL_GRAPHICS(pGpu, grIdx); 3004 if (pKernelGraphics != NULL) 3005 break; 3006 } 3007 if (pKernelGraphics == NULL) 3008 return NV_ERR_INVALID_STATE; 3009 } 3010 else 3011 { 3012 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3013 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &pParams->grRouteInfo, &pKernelGraphics)); 3014 } 3015 3016 // Verify static info is available 3017 pStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 3018 NV_ASSERT_OR_RETURN(pStaticInfo != NULL, NV_ERR_INVALID_STATE); 3019 NV_ASSERT_OR_RETURN(pStaticInfo->pSmIssueRateModifier != NULL, NV_ERR_NOT_SUPPORTED); 3020 3021 pParams->imla0 = pStaticInfo->pSmIssueRateModifier->imla0; 3022 pParams->fmla16 = pStaticInfo->pSmIssueRateModifier->fmla16; 3023 pParams->dp = pStaticInfo->pSmIssueRateModifier->dp; 3024 pParams->fmla32 = pStaticInfo->pSmIssueRateModifier->fmla32; 3025 pParams->ffma = pStaticInfo->pSmIssueRateModifier->ffma; 3026 pParams->imla1 = pStaticInfo->pSmIssueRateModifier->imla1; 3027 pParams->imla2 = pStaticInfo->pSmIssueRateModifier->imla2; 3028 pParams->imla3 = pStaticInfo->pSmIssueRateModifier->imla3; 3029 pParams->imla4 = pStaticInfo->pSmIssueRateModifier->imla4; 3030 3031 return NV_OK; 3032 } 3033 3034 /*! 3035 * subdeviceCtrlCmdKGrGetGpcMask 3036 * 3037 * Lock Requirements: 3038 * Assert that API lock and GPUs lock held on entry 3039 */ 3040 NV_STATUS 3041 subdeviceCtrlCmdKGrGetGpcMask_IMPL 3042 ( 3043 Subdevice *pSubdevice, 3044 NV2080_CTRL_GR_GET_GPC_MASK_PARAMS *pParams 3045 ) 3046 { 3047 NV_STATUS status = NV_OK; 3048 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3049 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 3050 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3051 KernelGraphics *pKernelGraphics; 3052 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 3053 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 3054 3055 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 3056 3057 if (!IS_MIG_IN_USE(pGpu) || 3058 kmigmgrIsDeviceUsingDeviceProfiling(pGpu, pKernelMIGManager, pDevice)) 3059 { 3060 pParams->gpcMask = kgrmgrGetLegacyGpcMask(pGpu, pKernelGraphicsManager); 3061 } 3062 else 3063 { 3064 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3065 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &pParams->grRouteInfo, &pKernelGraphics)); 3066 3067 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 3068 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 3069 3070 pParams->gpcMask = pKernelGraphicsStaticInfo->floorsweepingMasks.gpcMask; 3071 } 3072 3073 return status; 3074 } 3075 3076 /*! 3077 * subdeviceCtrlCmdKGrGetTpcMask 3078 * 3079 * Note: 3080 * pParams->gpcId is physical GPC id for non-MIG case, but logical GPC id for 3081 * MIG case. 3082 * 3083 * Lock Requirements: 3084 * Assert that API lock and GPUs lock held on entry 3085 */ 3086 NV_STATUS 3087 subdeviceCtrlCmdKGrGetTpcMask_IMPL 3088 ( 3089 Subdevice *pSubdevice, 3090 NV2080_CTRL_GR_GET_TPC_MASK_PARAMS *pParams 3091 ) 3092 { 3093 NV_STATUS status = NV_OK; 3094 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3095 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 3096 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3097 KernelGraphics *pKernelGraphics; 3098 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 3099 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 3100 NvU32 gpcCount; 3101 3102 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 3103 3104 if (!IS_MIG_IN_USE(pGpu) || 3105 kmigmgrIsDeviceUsingDeviceProfiling(pGpu, pKernelMIGManager, pDevice)) 3106 { 3107 pParams->tpcMask = kgrmgrGetLegacyTpcMask(pGpu, pKernelGraphicsManager, pParams->gpcId); 3108 } 3109 else 3110 { 3111 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3112 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &pParams->grRouteInfo, &pKernelGraphics)); 3113 3114 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 3115 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 3116 3117 gpcCount = nvPopCount32(pKernelGraphicsStaticInfo->floorsweepingMasks.gpcMask); 3118 if (pParams->gpcId >= gpcCount) 3119 { 3120 NV_PRINTF(LEVEL_ERROR, "Incorrect GPC-Idx provided = %d\n", pParams->gpcId); 3121 return NV_ERR_INVALID_ARGUMENT; 3122 } 3123 3124 pParams->tpcMask = pKernelGraphicsStaticInfo->floorsweepingMasks.tpcMask[pParams->gpcId]; 3125 } 3126 3127 return status; 3128 } 3129 3130 NV_STATUS 3131 subdeviceCtrlCmdKGrGetNumTpcsForGpc_IMPL 3132 ( 3133 Subdevice *pSubdevice, 3134 NV2080_CTRL_GR_GET_NUM_TPCS_FOR_GPC_PARAMS *pParams 3135 ) 3136 { 3137 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3138 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 3139 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3140 KernelGraphics *pKernelGraphics; 3141 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 3142 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo; 3143 NvU32 gpcCount; 3144 3145 portMemSet(&grRouteInfo, 0, sizeof(grRouteInfo)); 3146 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3147 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &grRouteInfo, &pKernelGraphics)); 3148 3149 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 3150 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 3151 3152 gpcCount = nvPopCount32(pKernelGraphicsStaticInfo->floorsweepingMasks.gpcMask); 3153 if (pParams->gpcId >= gpcCount) 3154 { 3155 NV_PRINTF(LEVEL_ERROR, "Incorrect GPC-Idx provided = %d\n", pParams->gpcId); 3156 return NV_ERR_INVALID_ARGUMENT; 3157 } 3158 3159 pParams->numTpcs = pKernelGraphicsStaticInfo->floorsweepingMasks.tpcCount[pParams->gpcId]; 3160 3161 return NV_OK; 3162 } 3163 3164 /*! 3165 * subdeviceCtrlCmdKGrGetPpcMask 3166 * 3167 * Lock Requirements: 3168 * Assert that API lock and GPUs lock held on entry 3169 */ 3170 NV_STATUS 3171 subdeviceCtrlCmdKGrGetPpcMask_IMPL 3172 ( 3173 Subdevice *pSubdevice, 3174 NV2080_CTRL_GR_GET_PPC_MASK_PARAMS *pParams 3175 ) 3176 { 3177 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3178 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 3179 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3180 KernelGraphics *pKernelGraphics; 3181 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 3182 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 3183 3184 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 3185 3186 if (kmigmgrIsDeviceUsingDeviceProfiling(pGpu, pKernelMIGManager, pDevice)) 3187 { 3188 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, kgrmgrGetLegacyPpcMask(pGpu, pKernelGraphicsManager, pParams->gpcId, &pParams->ppcMask)); 3189 } 3190 else 3191 { 3192 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3193 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &pParams->grRouteInfo, &pKernelGraphics)); 3194 3195 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 3196 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 3197 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pPpcMasks != NULL, NV_ERR_NOT_SUPPORTED); 3198 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pGrInfo != NULL, NV_ERR_NOT_SUPPORTED); 3199 3200 if (pParams->gpcId >= 3201 pKernelGraphicsStaticInfo->pGrInfo->infoList[NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_GPCS].data) 3202 { 3203 NV_PRINTF(LEVEL_ERROR, "Incorrect GPC-Idx provided = %d\n", pParams->gpcId); 3204 return NV_ERR_INVALID_ARGUMENT; 3205 } 3206 3207 pParams->ppcMask = pKernelGraphicsStaticInfo->pPpcMasks->mask[pParams->gpcId]; 3208 } 3209 3210 return NV_OK; 3211 } 3212 3213 // 3214 // subdeviceCtrlCmdKGrFecsBindEvtbufForUid 3215 // 3216 // Lock Requirements: 3217 // Assert that API lock and GPUs lock held on entry 3218 // 3219 NV_STATUS 3220 subdeviceCtrlCmdKGrFecsBindEvtbufForUid_IMPL 3221 ( 3222 Subdevice *pSubdevice, 3223 NV2080_CTRL_GR_FECS_BIND_EVTBUF_FOR_UID_PARAMS *pParams 3224 ) 3225 { 3226 NV_STATUS status; 3227 RmClient *pClient; 3228 RsResourceRef *pEventBufferRef = NULL; 3229 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3230 NvHandle hClient = RES_GET_CLIENT_HANDLE(pSubdevice); 3231 NvBool bMIGInUse = IS_MIG_IN_USE(pGpu); 3232 3233 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 3234 3235 NV_ASSERT_OK_OR_RETURN( 3236 serverutilGetResourceRefWithType(hClient, pParams->hEventBuffer, classId(EventBuffer), &pEventBufferRef)); 3237 3238 pClient = serverutilGetClientUnderLock(hClient); 3239 NV_ASSERT_OR_RETURN(pClient != NULL, NV_ERR_INVALID_CLIENT); 3240 3241 if (bMIGInUse) 3242 return NV_ERR_NOT_SUPPORTED; 3243 3244 status = fecsAddBindpoint(pGpu, 3245 pClient, 3246 pEventBufferRef, 3247 pSubdevice, 3248 pParams->bAllUsers, 3249 pParams->levelOfDetail, 3250 pParams->eventFilter, 3251 1, 3252 NULL); 3253 3254 return status; 3255 } 3256 3257 // 3258 // subdeviceCtrlCmdKGrFecsBindEvtbufForUidV2 3259 // 3260 // Lock Requirements: 3261 // Assert that API lock and GPUs lock held on entry 3262 // 3263 NV_STATUS 3264 subdeviceCtrlCmdKGrFecsBindEvtbufForUidV2_IMPL 3265 ( 3266 Subdevice *pSubdevice, 3267 NV2080_CTRL_GR_FECS_BIND_EVTBUF_FOR_UID_V2_PARAMS *pParams 3268 ) 3269 { 3270 NV_STATUS status; 3271 RmClient *pClient; 3272 RsResourceRef *pEventBufferRef = NULL; 3273 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3274 NvHandle hClient = RES_GET_CLIENT_HANDLE(pSubdevice); 3275 pParams->reasonCode = NV2080_CTRL_GR_FECS_BIND_REASON_CODE_NONE; 3276 3277 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 3278 3279 NV_ASSERT_OK_OR_RETURN( 3280 serverutilGetResourceRefWithType(hClient, pParams->hEventBuffer, classId(EventBuffer), &pEventBufferRef)); 3281 3282 pClient = serverutilGetClientUnderLock(hClient); 3283 NV_ASSERT_OR_RETURN(pClient != NULL, NV_ERR_INVALID_CLIENT); 3284 3285 status = fecsAddBindpoint(pGpu, 3286 pClient, 3287 pEventBufferRef, 3288 pSubdevice, 3289 pParams->bAllUsers, 3290 pParams->levelOfDetail, 3291 pParams->eventFilter, 3292 2, 3293 &pParams->reasonCode); 3294 return status; 3295 } 3296 3297 /*! 3298 * subdeviceCtrlCmdKGrGetPhysGpcMask 3299 * 3300 * Lock Requirements: 3301 * Assert that API lock and GPUs lock held on entry 3302 */ 3303 NV_STATUS 3304 subdeviceCtrlCmdKGrGetPhysGpcMask_IMPL 3305 ( 3306 Subdevice *pSubdevice, 3307 NV2080_CTRL_GR_GET_PHYS_GPC_MASK_PARAMS *pParams 3308 ) 3309 { 3310 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3311 KernelGraphics *pKernelGraphics; 3312 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 3313 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 3314 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 3315 NvU32 grIdx = 0; 3316 3317 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 3318 3319 if (!IS_MIG_ENABLED(pGpu)) 3320 { 3321 grIdx = 0; 3322 } 3323 // 3324 // if MIG is enabled we follow below policies: 3325 // For device level monitoring with no subscription - Return GPC mask for 3326 // requested syspipe 3327 // For valid subscription - Return physical GPC mask after validating that 3328 // a physical syspipe exist in given GPU instance 3329 // 3330 else if (kmigmgrIsDeviceUsingDeviceProfiling(pGpu, pKernelMIGManager, pDevice)) 3331 { 3332 NV_ASSERT_OR_RETURN(pParams->physSyspipeId < GPU_MAX_GRS, NV_ERR_INVALID_ARGUMENT); 3333 grIdx = pParams->physSyspipeId; 3334 } 3335 else 3336 { 3337 MIG_INSTANCE_REF ref; 3338 RM_ENGINE_TYPE localRmEngineType; 3339 3340 // 3341 // Get the relevant subscription and see if provided physicalId is 3342 // valid in defined GPU instance 3343 // 3344 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3345 kmigmgrGetInstanceRefFromDevice(pGpu, pKernelMIGManager, 3346 pDevice, &ref)); 3347 3348 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3349 kmigmgrGetGlobalToLocalEngineType(pGpu, pKernelMIGManager, ref, 3350 RM_ENGINE_TYPE_GR(pParams->physSyspipeId), 3351 &localRmEngineType)); 3352 // Not failing above means physSyspipeId is valid in GPU instance 3353 grIdx = pParams->physSyspipeId; 3354 } 3355 3356 pKernelGraphics = GPU_GET_KERNEL_GRAPHICS(pGpu, grIdx); 3357 NV_ASSERT_OR_RETURN(pKernelGraphics != NULL, NV_ERR_INVALID_STATE); 3358 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 3359 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 3360 3361 pParams->gpcMask = pKernelGraphicsStaticInfo->floorsweepingMasks.physGpcMask; 3362 3363 return NV_OK; 3364 } 3365 3366 /*! 3367 * subdeviceCtrlCmdKGrGetZcullMask_IMPL 3368 * 3369 * Lock Requirements: 3370 * Assert that API lock and GPUs lock held on entry 3371 */ 3372 NV_STATUS 3373 subdeviceCtrlCmdKGrGetZcullMask_IMPL 3374 ( 3375 Subdevice *pSubdevice, 3376 NV2080_CTRL_GR_GET_ZCULL_MASK_PARAMS *pParams 3377 ) 3378 { 3379 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3380 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 3381 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3382 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo; 3383 KernelGraphics *pKernelGraphics; 3384 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 3385 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 3386 3387 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 3388 3389 if (kmigmgrIsDeviceUsingDeviceProfiling(pGpu, pKernelMIGManager, pDevice)) 3390 { 3391 pParams->zcullMask = kgrmgrGetLegacyZcullMask(pGpu, pKernelGraphicsManager, pParams->gpcId); 3392 } 3393 else 3394 { 3395 portMemSet(&grRouteInfo, 0, sizeof(grRouteInfo)); 3396 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3397 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &grRouteInfo, &pKernelGraphics)); 3398 3399 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 3400 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 3401 3402 if (pParams->gpcId >= 3403 pKernelGraphicsStaticInfo->pGrInfo->infoList[NV2080_CTRL_GR_INFO_INDEX_LITTER_NUM_GPCS].data) 3404 { 3405 NV_PRINTF(LEVEL_ERROR, "Incorrect GPC-Idx provided = %d\n", pParams->gpcId); 3406 return NV_ERR_INVALID_ARGUMENT; 3407 } 3408 3409 if (pKernelGraphicsStaticInfo->floorsweepingMasks.zcullMask[pParams->gpcId] == NV_U32_MAX) 3410 { 3411 return NV_ERR_NOT_SUPPORTED; 3412 } 3413 else 3414 { 3415 pParams->zcullMask = pKernelGraphicsStaticInfo->floorsweepingMasks.zcullMask[pParams->gpcId]; 3416 } 3417 } 3418 3419 return NV_OK; 3420 } 3421 3422 /*! 3423 * subdeviceCtrlCmdKGrGetZcullInfo 3424 * 3425 * Lock Requirements: 3426 * Assert that API lock held on entry 3427 */ 3428 NV_STATUS 3429 subdeviceCtrlCmdKGrGetZcullInfo_IMPL 3430 ( 3431 Subdevice *pSubdevice, 3432 NV2080_CTRL_GR_GET_ZCULL_INFO_PARAMS *pParams 3433 ) 3434 { 3435 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3436 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3437 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 3438 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo; 3439 KernelGraphics *pKernelGraphics; 3440 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 3441 3442 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner()); 3443 3444 if (pKernelGraphicsManager == NULL) 3445 { 3446 return NV_ERR_NOT_SUPPORTED; 3447 } 3448 3449 portMemSet(&grRouteInfo, 0, sizeof(grRouteInfo)); 3450 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3451 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &grRouteInfo, &pKernelGraphics)); 3452 3453 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 3454 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 3455 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pZcullInfo != NULL, NV_ERR_NOT_SUPPORTED); 3456 3457 portMemCopy(pParams, 3458 sizeof(*pParams), 3459 pKernelGraphicsStaticInfo->pZcullInfo, 3460 sizeof(*pKernelGraphicsStaticInfo->pZcullInfo)); 3461 3462 return NV_OK; 3463 } 3464 3465 NV_STATUS 3466 subdeviceCtrlCmdKGrCtxswPmMode_IMPL 3467 ( 3468 Subdevice *pSubdevice, 3469 NV2080_CTRL_GR_CTXSW_PM_MODE_PARAMS *pParams 3470 ) 3471 { 3472 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3473 NV_STATUS status = NV_OK; 3474 3475 if (IS_GSP_CLIENT(pGpu)) 3476 { 3477 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo = pParams->grRouteInfo; 3478 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3479 KernelGraphics *pKernelGraphics; 3480 KernelChannel *pKernelChannel; 3481 KernelGraphicsContext *pKernelGraphicsContext; 3482 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); 3483 3484 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 3485 3486 if (pParams->pmMode != NV2080_CTRL_CTXSW_PM_MODE_NO_CTXSW) 3487 { 3488 kgrmgrCtrlSetChannelHandle(pParams->hChannel, &grRouteInfo); 3489 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3490 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, 3491 GPU_RES_GET_DEVICE(pSubdevice), 3492 &grRouteInfo, 3493 &pKernelGraphics)); 3494 3495 // Retrieve channel from either bare channel or TSG handle 3496 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3497 kchannelGetFromDualHandleRestricted(RES_GET_CLIENT(pSubdevice), 3498 pParams->hChannel, &pKernelChannel)); 3499 3500 NV_ASSERT_OK_OR_RETURN( 3501 kgrctxFromKernelChannel(pKernelChannel, &pKernelGraphicsContext)); 3502 3503 // Setup / promote the PM ctx buffer if required 3504 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3505 kgrctxSetupDeferredPmBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel)); 3506 } 3507 3508 status = pRmApi->Control(pRmApi, 3509 RES_GET_CLIENT_HANDLE(pSubdevice), 3510 RES_GET_HANDLE(pSubdevice), 3511 NV2080_CTRL_CMD_GR_CTXSW_PM_MODE, 3512 pParams, 3513 sizeof(*pParams)); 3514 } 3515 3516 return status; 3517 } 3518 3519 /*! 3520 * @brief Gets information about ROPs. 3521 * 3522 * Lock Requirements: 3523 * Assert that API and Gpus lock held on entry 3524 * 3525 * @return NV_OK if success. Error otherwise. 3526 */ 3527 NV_STATUS 3528 subdeviceCtrlCmdKGrGetROPInfo_IMPL 3529 ( 3530 Subdevice *pSubdevice, 3531 NV2080_CTRL_GR_GET_ROP_INFO_PARAMS *pRopInfoParams 3532 ) 3533 { 3534 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3535 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3536 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 3537 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo; 3538 KernelGraphics *pKernelGraphics; 3539 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 3540 3541 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner()); 3542 3543 portMemSet(&grRouteInfo, 0, sizeof(grRouteInfo)); 3544 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3545 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &grRouteInfo, &pKernelGraphics)); 3546 3547 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 3548 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 3549 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pRopInfo != NULL, NV_ERR_NOT_SUPPORTED); 3550 3551 portMemCopy(pRopInfoParams, 3552 sizeof(*pRopInfoParams), 3553 pKernelGraphicsStaticInfo->pRopInfo, 3554 sizeof(*pKernelGraphicsStaticInfo->pRopInfo)); 3555 3556 return NV_OK; 3557 } 3558 3559 /*! 3560 * @brief Gets the current attribute buffer size. 3561 * 3562 * Lock Requirements: 3563 * Assert that API lock held on entry 3564 * 3565 * @return NV_OK if success. Error otherwise. 3566 */ 3567 NV_STATUS 3568 subdeviceCtrlCmdKGrGetAttributeBufferSize_IMPL 3569 ( 3570 Subdevice *pSubdevice, 3571 NV2080_CTRL_GR_GET_ATTRIBUTE_BUFFER_SIZE_PARAMS *pAttribBufferSizeParams 3572 ) 3573 { 3574 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3575 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3576 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo; 3577 KernelGraphics *pKernelGraphics; 3578 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 3579 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 3580 NvU32 engineId; 3581 3582 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner()); 3583 3584 portMemSet(&grRouteInfo, 0, sizeof(grRouteInfo)); 3585 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3586 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &grRouteInfo, &pKernelGraphics)); 3587 3588 // Verify static info is available 3589 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 3590 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 3591 3592 if (pKernelGraphicsStaticInfo->pContextBuffersInfo == NULL) 3593 { 3594 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3595 kgraphicsInitializeDeferredStaticData(pGpu, pKernelGraphics, NV01_NULL_OBJECT, NV01_NULL_OBJECT)); 3596 3597 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pContextBuffersInfo != NULL, NV_ERR_INVALID_STATE); 3598 } 3599 3600 engineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_ATTRIBUTE_CB; 3601 pAttribBufferSizeParams->attribBufferSize = pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[engineId].size; 3602 3603 return NV_OK; 3604 } 3605 3606 /*! 3607 * subdeviceCtrlCmdKGrGetEngineContextProperties 3608 * 3609 * Lock Requirements: 3610 * Assert that API lock and GPUs lock held on entry 3611 */ 3612 NV_STATUS 3613 subdeviceCtrlCmdKGrGetEngineContextProperties_IMPL 3614 ( 3615 Subdevice *pSubdevice, 3616 NV2080_CTRL_GR_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS *pParams 3617 ) 3618 { 3619 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3620 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3621 KernelGraphics *pKernelGraphics; 3622 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 3623 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 3624 NvU32 size = 0; 3625 NvU32 alignment = RM_PAGE_SIZE; 3626 NvU32 engineId; 3627 3628 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner()); 3629 3630 engineId = DRF_VAL(0080_CTRL_FIFO, _GET_ENGINE_CONTEXT_PROPERTIES, _ENGINE_ID, pParams->engineId); 3631 3632 if (engineId >= NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_COUNT) 3633 { 3634 return NV_ERR_INVALID_ARGUMENT; 3635 } 3636 3637 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3638 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, 3639 &pParams->grRouteInfo, &pKernelGraphics)); 3640 3641 // Verify static info is available 3642 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 3643 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 3644 3645 if (pKernelGraphicsStaticInfo->pContextBuffersInfo == NULL) 3646 { 3647 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3648 kgraphicsInitializeDeferredStaticData(pGpu, pKernelGraphics, NV01_NULL_OBJECT, NV01_NULL_OBJECT)); 3649 3650 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pContextBuffersInfo != NULL, NV_ERR_INVALID_STATE); 3651 } 3652 3653 size = pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[engineId].size; 3654 alignment = pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[engineId].alignment; 3655 3656 if (size == NV_U32_MAX) 3657 { 3658 return NV_ERR_NOT_SUPPORTED; 3659 } 3660 3661 if (pParams->bInfoPopulated) 3662 { 3663 size = NV_MAX(size, pParams->size); 3664 alignment = NV_MAX(alignment, pParams->alignment); 3665 } 3666 3667 pParams->size = size; 3668 pParams->alignment = alignment; 3669 pParams->bInfoPopulated = NV_TRUE; 3670 3671 return NV_OK; 3672 } 3673 3674 /*! 3675 * @brief Gets the Graphics Context buffer size and alignment 3676 * 3677 * Lock Requirements: 3678 * Assert that API and Gpus lock held on entry 3679 */ 3680 NV_STATUS 3681 subdeviceCtrlCmdKGrGetCtxBufferSize_IMPL 3682 ( 3683 Subdevice *pSubdevice, 3684 NV2080_CTRL_GR_GET_CTX_BUFFER_SIZE_PARAMS *pParams 3685 ) 3686 { 3687 NV_STATUS status = NV_OK; 3688 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3689 KernelGraphics *pKernelGraphics; 3690 RsClient *pClient = RES_GET_CLIENT(pSubdevice); 3691 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 3692 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3693 NV2080_CTRL_GR_CTX_BUFFER_INFO *pCtxBufferInfo; 3694 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo; 3695 KernelChannel *pKernelChannel; 3696 KernelGraphicsContext *pKernelGraphicsContext; 3697 NvU32 bufferCount; 3698 NvU64 totalBufferSize; 3699 NvU64 prevAlignment; 3700 NvU32 i; 3701 3702 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 3703 3704 // 3705 // vGPU: 3706 // 3707 // Since vGPU does all real hardware management in the 3708 // host, if we are in guest OS (where IS_VIRTUAL(pGpu) is true), 3709 // do an RPC to the host to fetch the total GR Context Buffer Size. 3710 // 3711 if (IS_VIRTUAL_WITHOUT_SRIOV(pGpu) || 3712 (IS_VIRTUAL_WITH_SRIOV(pGpu) && gpuIsWarBug200577889SriovHeavyEnabled(pGpu))) 3713 { 3714 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 3715 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams; 3716 3717 NV_RM_RPC_CONTROL(pGpu, 3718 pRmCtrlParams->hClient, 3719 pRmCtrlParams->hObject, 3720 pRmCtrlParams->cmd, 3721 pRmCtrlParams->pParams, 3722 pRmCtrlParams->paramsSize, 3723 status); 3724 return status; 3725 } 3726 3727 portMemSet(&grRouteInfo, 0, sizeof(grRouteInfo)); 3728 kgrmgrCtrlSetChannelHandle(pParams->hChannel, &grRouteInfo); 3729 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3730 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, 3731 &grRouteInfo, &pKernelGraphics)); 3732 3733 // Get channel from provided handle and owner client 3734 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, 3735 CliGetKernelChannel(pClient, pParams->hChannel, &pKernelChannel)); 3736 3737 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3738 kgrctxFromKernelChannel(pKernelChannel, &pKernelGraphicsContext)); 3739 3740 // Get the total buffer count 3741 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3742 kgrctxGetBufferCount(pGpu, pKernelGraphicsContext, pKernelGraphics, &bufferCount)); 3743 3744 pCtxBufferInfo = portMemAllocNonPaged(bufferCount * sizeof(NV2080_CTRL_GR_CTX_BUFFER_INFO)); 3745 NV_CHECK_OR_ELSE(LEVEL_ERROR, 3746 pCtxBufferInfo != NULL, 3747 status = NV_ERR_NO_MEMORY; 3748 goto done;); 3749 portMemSet(pCtxBufferInfo, 0, bufferCount * sizeof(NV2080_CTRL_GR_CTX_BUFFER_INFO)); 3750 3751 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, 3752 kgrctxGetCtxBufferInfo(pGpu, 3753 pKernelGraphicsContext, 3754 pKernelGraphics, 3755 kchannelGetGfid(pKernelChannel), 3756 bufferCount, 3757 &bufferCount, 3758 pCtxBufferInfo), 3759 done); 3760 3761 // 3762 // Calculate total size by walking thru all buffers & alignments. Adjust the total size 3763 // by adding the respective alignment so that the mapping VA can be adjusted. 3764 // 3765 totalBufferSize = 0; 3766 prevAlignment = 0x0; 3767 for (i = 0; i < bufferCount; i++) 3768 { 3769 if (prevAlignment != pCtxBufferInfo[i].alignment) 3770 { 3771 totalBufferSize += pCtxBufferInfo[i].alignment; 3772 prevAlignment = pCtxBufferInfo[i].alignment; 3773 } 3774 3775 totalBufferSize += (pCtxBufferInfo[i].alignment != 0x0) ? 3776 NV_ALIGN_UP(pCtxBufferInfo[i].size, pCtxBufferInfo[i].alignment) : pCtxBufferInfo[i].size; 3777 } 3778 3779 pParams->totalBufferSize = totalBufferSize; 3780 3781 done: 3782 portMemFree(pCtxBufferInfo); 3783 return status; 3784 } 3785 3786 /*! 3787 * @brief Gets the Graphics Context buffer info like opaque buffer pointer 3788 * size, alignment, aperture, allocation contiguity etc. 3789 * 3790 * Lock Requirements: 3791 * Assert that API and Gpus lock held on entry 3792 */ 3793 NV_STATUS 3794 subdeviceCtrlCmdKGrGetCtxBufferInfo_IMPL 3795 ( 3796 Subdevice *pSubdevice, 3797 NV2080_CTRL_GR_GET_CTX_BUFFER_INFO_PARAMS *pParams 3798 ) 3799 { 3800 NV_STATUS status = NV_OK; 3801 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3802 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3803 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo; 3804 RsClient *pUserClient; 3805 KernelGraphics *pKernelGraphics; 3806 KernelChannel *pKernelChannel; 3807 KernelGraphicsContext *pKernelGraphicsContext; 3808 3809 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner()); 3810 3811 // 3812 // vGPU: 3813 // 3814 // Since vGPU does all real hardware management in the 3815 // host, if we are in guest OS (where IS_VIRTUAL(pGpu) is true), 3816 // do an RPC to the host to get Graphics context buffers information. 3817 // 3818 if (IS_VIRTUAL_WITHOUT_SRIOV(pGpu) || 3819 (IS_VIRTUAL_WITH_SRIOV(pGpu) && gpuIsWarBug200577889SriovHeavyEnabled(pGpu))) 3820 { 3821 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 3822 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams; 3823 3824 NV_RM_RPC_CONTROL(pGpu, 3825 pRmCtrlParams->hClient, 3826 pRmCtrlParams->hObject, 3827 pRmCtrlParams->cmd, 3828 pRmCtrlParams->pParams, 3829 pRmCtrlParams->paramsSize, 3830 status); 3831 return status; 3832 } 3833 3834 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, 3835 serverGetClientUnderLock(&g_resServ, pParams->hUserClient, &pUserClient)); 3836 3837 // Get channel from provided handle and owner client 3838 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, 3839 CliGetKernelChannel(pUserClient, pParams->hChannel, &pKernelChannel)); 3840 3841 portMemSet(&grRouteInfo, 0, sizeof(grRouteInfo)); 3842 kgrmgrCtrlSetChannelHandle(pParams->hChannel, &grRouteInfo); 3843 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3844 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, 3845 GPU_RES_GET_DEVICE(pKernelChannel), 3846 &grRouteInfo, &pKernelGraphics)); 3847 3848 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3849 kgrctxFromKernelChannel(pKernelChannel, &pKernelGraphicsContext)); 3850 3851 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3852 kgrctxGetCtxBufferInfo(pGpu, 3853 pKernelGraphicsContext, 3854 pKernelGraphics, 3855 kchannelGetGfid(pKernelChannel), 3856 NV_ARRAY_ELEMENTS(pParams->ctxBufferInfo), 3857 &pParams->bufferCount, 3858 pParams->ctxBufferInfo)); 3859 3860 return status; 3861 } 3862 3863 /*! 3864 * subdeviceCtrlCmdKGrInternalGetCtxBufferPtes 3865 * 3866 * Lock Requirements: 3867 * Assert that API lock and GPUs lock held on entry 3868 */ 3869 NV_STATUS 3870 subdeviceCtrlCmdKGrGetCtxBufferPtes_IMPL 3871 ( 3872 Subdevice *pSubdevice, 3873 NV2080_CTRL_KGR_GET_CTX_BUFFER_PTES_PARAMS *pParams 3874 ) 3875 { 3876 NV_STATUS status = NV_OK; 3877 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3878 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3879 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo; 3880 RsClient *pUserClient; 3881 KernelGraphics *pKernelGraphics; 3882 KernelChannel *pKernelChannel; 3883 KernelGraphicsContext *pKernelGraphicsContext; 3884 3885 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner()); 3886 3887 // 3888 // Currently, ROUTE_TO_VGPU_HOST instructs resource server to call the RPC 3889 // on all vGPU configurations including SRIOV Standard which is not required. 3890 // Hence, manually dispatching the RPC for required vGPU configs. 3891 // 3892 // vGPU: 3893 // 3894 // Since vGPU does all real hardware management in the 3895 // host, if we are in guest OS (where IS_VIRTUAL(pGpu) is true), 3896 // do an RPC to the host to get Graphics context buffers PTEs information. 3897 // 3898 if (IS_VIRTUAL_WITHOUT_SRIOV(pGpu) || 3899 (IS_VIRTUAL_WITH_SRIOV(pGpu) && gpuIsWarBug200577889SriovHeavyEnabled(pGpu))) 3900 { 3901 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 3902 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams; 3903 NvHandle hClient = RES_GET_CLIENT_HANDLE(pSubdevice); 3904 NvHandle hObject = RES_GET_HANDLE(pSubdevice); 3905 3906 NV_RM_RPC_CONTROL(pGpu, 3907 hClient, 3908 hObject, 3909 pRmCtrlParams->cmd, 3910 pRmCtrlParams->pParams, 3911 pRmCtrlParams->paramsSize, 3912 status); 3913 return status; 3914 } 3915 3916 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, 3917 serverGetClientUnderLock(&g_resServ, pParams->hUserClient, &pUserClient)); 3918 3919 // Get channel from provided handle and owner client 3920 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, 3921 CliGetKernelChannel(pUserClient, pParams->hChannel, &pKernelChannel)); 3922 3923 portMemSet(&grRouteInfo, 0, sizeof(grRouteInfo)); 3924 kgrmgrCtrlSetChannelHandle(pParams->hChannel, &grRouteInfo); 3925 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, 3926 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, 3927 GPU_RES_GET_DEVICE(pKernelChannel), 3928 &grRouteInfo, &pKernelGraphics)); 3929 3930 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3931 kgrctxFromKernelChannel(pKernelChannel, &pKernelGraphicsContext)); 3932 3933 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3934 kgrctxGetCtxBufferPtes(pGpu, 3935 pKernelGraphicsContext, 3936 pKernelGraphics, 3937 kchannelGetGfid(pKernelChannel), 3938 pParams->bufferType, 3939 pParams->firstPage, 3940 pParams->physAddrs, 3941 NV_ARRAY_ELEMENTS(pParams->physAddrs), 3942 &pParams->numPages, 3943 &pParams->bNoMorePages)); 3944 3945 return status; 3946 } 3947 3948 /*! 3949 * subdeviceCtrlCmdKGrGetGfxGpcAndTpcInfo 3950 * 3951 * Lock Requirements: 3952 * Assert that API lock and GPUs lock held on entry 3953 */ 3954 NV_STATUS 3955 subdeviceCtrlCmdKGrGetGfxGpcAndTpcInfo_IMPL 3956 ( 3957 Subdevice *pSubdevice, 3958 NV2080_CTRL_GR_GET_GFX_GPC_AND_TPC_INFO_PARAMS *pParams 3959 ) 3960 { 3961 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3962 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3963 KernelGraphics *pKernelGraphics; 3964 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 3965 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 3966 3967 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner()); 3968 3969 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 3970 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, 3971 &pParams->grRouteInfo, &pKernelGraphics)); 3972 3973 // Verify static info is available 3974 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 3975 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 3976 3977 pParams->physGfxGpcMask = pKernelGraphicsStaticInfo->floorsweepingMasks.physGfxGpcMask; 3978 pParams->numGfxTpc = pKernelGraphicsStaticInfo->floorsweepingMasks.numGfxTpc; 3979 3980 return NV_OK; 3981 } 3982 3983 #define KGR_DO_WITH_GR(pGpu, pKernelGraphics, body) do \ 3984 { \ 3985 (body); \ 3986 } while (0); 3987 3988 NV_STATUS 3989 subdeviceCtrlCmdGrInternalSetFecsTraceHwEnable_IMPL 3990 ( 3991 Subdevice *pSubdevice, 3992 NV2080_CTRL_INTERNAL_GR_SET_FECS_TRACE_HW_ENABLE_PARAMS *pParams 3993 ) 3994 { 3995 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 3996 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 3997 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 3998 NV_STATUS status = NV_OK; 3999 KernelGraphics *pKernelGraphics; 4000 4001 LOCK_ASSERT_AND_RETURN(rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 4002 4003 NV_CHECK_OK_OR_RETURN( 4004 LEVEL_ERROR, 4005 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, 4006 &pParams->grRouteInfo, &pKernelGraphics)); 4007 4008 KGR_DO_WITH_GR(pGpu, pKernelGraphics, 4009 kgraphicsSetFecsTraceHwEnable_HAL(pGpu, pKernelGraphics, pParams->bEnable)); 4010 pKernelGraphics->bCtxswLoggingEnabled = pParams->bEnable; 4011 4012 return status; 4013 } 4014 4015 NV_STATUS 4016 subdeviceCtrlCmdGrInternalGetFecsTraceHwEnable_IMPL 4017 ( 4018 Subdevice *pSubdevice, 4019 NV2080_CTRL_INTERNAL_GR_GET_FECS_TRACE_HW_ENABLE_PARAMS *pParams 4020 ) 4021 { 4022 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 4023 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 4024 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 4025 NV_STATUS status = NV_OK; 4026 KernelGraphics *pKernelGraphics; 4027 4028 LOCK_ASSERT_AND_RETURN(rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 4029 4030 NV_CHECK_OK_OR_RETURN( 4031 LEVEL_ERROR, 4032 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &pParams->grRouteInfo, 4033 &pKernelGraphics)); 4034 4035 KGR_DO_WITH_GR(pGpu, pKernelGraphics, 4036 pParams->bEnable = kgraphicsIsCtxswLoggingEnabled(pGpu, pKernelGraphics)); 4037 4038 return status; 4039 } 4040 4041 NV_STATUS 4042 subdeviceCtrlCmdGrInternalSetFecsTraceRdOffset_IMPL 4043 ( 4044 Subdevice *pSubdevice, 4045 NV2080_CTRL_INTERNAL_GR_SET_FECS_TRACE_RD_OFFSET_PARAMS *pParams 4046 ) 4047 { 4048 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 4049 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 4050 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 4051 NV_STATUS status = NV_OK; 4052 KernelGraphics *pKernelGraphics; 4053 4054 LOCK_ASSERT_AND_RETURN(rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 4055 4056 NV_CHECK_OK_OR_RETURN( 4057 LEVEL_ERROR, 4058 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &pParams->grRouteInfo, 4059 &pKernelGraphics)); 4060 4061 KGR_DO_WITH_GR(pGpu, pKernelGraphics, 4062 kgraphicsSetFecsTraceRdOffset_HAL(pGpu, pKernelGraphics, pParams->offset)); 4063 4064 return status; 4065 } 4066 4067 NV_STATUS 4068 subdeviceCtrlCmdGrInternalGetFecsTraceRdOffset_IMPL 4069 ( 4070 Subdevice *pSubdevice, 4071 NV2080_CTRL_INTERNAL_GR_GET_FECS_TRACE_RD_OFFSET_PARAMS *pParams 4072 ) 4073 { 4074 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 4075 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 4076 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 4077 NV_STATUS status = NV_OK; 4078 KernelGraphics *pKernelGraphics; 4079 4080 LOCK_ASSERT_AND_RETURN(rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 4081 4082 NV_CHECK_OK_OR_RETURN( 4083 LEVEL_ERROR, 4084 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &pParams->grRouteInfo, 4085 &pKernelGraphics)); 4086 4087 KGR_DO_WITH_GR(pGpu, pKernelGraphics, 4088 pParams->offset = kgraphicsGetFecsTraceRdOffset_HAL(pGpu, pKernelGraphics)); 4089 4090 return status; 4091 } 4092 4093 NV_STATUS 4094 subdeviceCtrlCmdGrInternalSetFecsTraceWrOffset_IMPL 4095 ( 4096 Subdevice *pSubdevice, 4097 NV2080_CTRL_INTERNAL_GR_SET_FECS_TRACE_WR_OFFSET_PARAMS *pParams 4098 ) 4099 { 4100 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice); 4101 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 4102 Device *pDevice = GPU_RES_GET_DEVICE(pSubdevice); 4103 NV_STATUS status = NV_OK; 4104 KernelGraphics *pKernelGraphics; 4105 4106 LOCK_ASSERT_AND_RETURN(rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 4107 4108 NV_CHECK_OK_OR_RETURN( 4109 LEVEL_ERROR, 4110 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &pParams->grRouteInfo, 4111 &pKernelGraphics)); 4112 4113 KGR_DO_WITH_GR(pGpu, pKernelGraphics, 4114 kgraphicsSetFecsTraceWrOffset_HAL(pGpu, pKernelGraphics, pParams->offset)); 4115 4116 return status; 4117 } 4118