1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include "kernel/gpu/fifo/kernel_fifo.h" 25 #include "kernel/gpu/fifo/kernel_channel.h" 26 #include "kernel/gpu/fifo/kernel_channel_group.h" 27 #include "kernel/gpu/fifo/kernel_channel_group_api.h" 28 #include "kernel/gpu/fifo/kernel_sched_mgr.h" 29 #include "virtualization/kernel_vgpu_mgr.h" 30 #include "rmapi/rs_utils.h" 31 #include "rmapi/client.h" 32 #include "gpu/subdevice/subdevice.h" 33 34 #include "kernel/core/locks.h" 35 #include "lib/base_utils.h" 36 37 #include "gpu/mmu/kern_gmmu.h" 38 #include "vgpu/rpc.h" 39 #include "vgpu/vgpu_events.h" 40 #include "nvRmReg.h" 41 42 #include "class/cl0080.h" 43 #include "class/cl2080.h" 44 #include "class/cl208f.h" 45 #include "class/clc572.h" 46 47 #include "ctrl/ctrl0080/ctrl0080fifo.h" 48 49 #define KFIFO_EHEAP_OWNER NvU32_BUILD('f','i','f','o') 50 51 static EHeapOwnershipComparator _kfifoUserdOwnerComparator; 52 53 static NV_STATUS _kfifoChidMgrAllocChidHeaps(OBJGPU *pGpu, 54 KernelFifo *pKernelFifo, 55 CHID_MGR *pChidMgr); 56 57 static NV_STATUS _kfifoChidMgrAllocVChidHeapPointers(OBJGPU *pGpu, CHID_MGR *pChidMgr); 58 59 static NV_STATUS _kfifoChidMgrInitChannelGroupMgr(OBJGPU *pGpu, CHID_MGR *pChidMgr); 60 61 static void _kfifoChidMgrDestroyChidHeaps(CHID_MGR *pChidMgr); 62 63 static void _kfifoChidMgrDestroyChannelGroupMgr(CHID_MGR *pChidMgr); 64 65 static NV_STATUS _kfifoChidMgrFreeIsolationId(CHID_MGR *pChidMgr, NvU32 ChID); 66 67 NV_STATUS 68 kfifoChidMgrConstruct_IMPL 69 ( 70 OBJGPU *pGpu, 71 KernelFifo *pKernelFifo 72 ) 73 { 74 NV_STATUS status = NV_OK; 75 NvU32 i; 76 NvU32 numEngines; 77 78 // 79 // Allocate memory for the array of CHID_MGR pointers. Since this is an 80 // array, we allocate memory for pointers unto maxNumRunlists. We will only 81 // allocate objects for the valid ones. 82 // 83 if (kfifoIsPerRunlistChramEnabled(pKernelFifo)) 84 { 85 // 86 // Construct the engine list if it isn't already constructed (internally 87 // checks if it was already constructed) 88 // 89 NV_ASSERT_OK_OR_RETURN(kfifoConstructEngineList_HAL(pGpu, pKernelFifo)); 90 pKernelFifo->numChidMgrs = kfifoGetMaxNumRunlists_HAL(pGpu, pKernelFifo); 91 } 92 else 93 pKernelFifo->numChidMgrs = 1; 94 95 if (pKernelFifo->numChidMgrs > MAX_NUM_RUNLISTS) 96 { 97 // 98 // This only currently defines the size of our bitvector 99 // pKernelFifo->chidMgrValid. Catch this case if HW expands beyond this so we 100 // can increase the size allocated to the bitvector 101 // 102 NV_PRINTF(LEVEL_ERROR, "numChidMgrs 0x%x exceeds MAX_NUM_RUNLISTS\n", 103 pKernelFifo->numChidMgrs); 104 DBG_BREAKPOINT(); 105 return NV_ERR_BUFFER_TOO_SMALL; 106 } 107 108 pKernelFifo->ppChidMgr = portMemAllocNonPaged(sizeof(CHID_MGR *) * pKernelFifo->numChidMgrs); 109 if (pKernelFifo->ppChidMgr == NULL) 110 { 111 status = NV_ERR_NO_MEMORY; 112 pKernelFifo->ppChidMgr = NULL; 113 NV_PRINTF(LEVEL_ERROR, "Failed to allocate pFifo->pChidMgr\n"); 114 DBG_BREAKPOINT(); 115 return status; 116 } 117 portMemSet(pKernelFifo->ppChidMgr, 0, sizeof(CHID_MGR *) * pKernelFifo->numChidMgrs); 118 119 // Initialize the valid mask 120 if (kfifoIsPerRunlistChramEnabled(pKernelFifo)) 121 { 122 numEngines = kfifoGetNumEngines_HAL(pGpu, pKernelFifo); 123 for (i = 0; i < numEngines; i++) 124 { 125 NvU32 runlistId; 126 status = kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo, 127 ENGINE_INFO_TYPE_INVALID, i, 128 ENGINE_INFO_TYPE_RUNLIST, &runlistId); 129 if (status == NV_OK) 130 bitVectorSet(&pKernelFifo->chidMgrValid, runlistId); 131 else 132 { 133 NV_PRINTF(LEVEL_ERROR, "Translation to runlistId failed for engine %d\n", i); 134 DBG_BREAKPOINT(); 135 goto fail; 136 } 137 } 138 } 139 else 140 { 141 bitVectorSet(&pKernelFifo->chidMgrValid, 0); // We only have 1 chidmgr 142 } 143 144 // Allocate memory for each CHID_MGR and its members (only the valid ones) 145 for (i = 0; i < pKernelFifo->numChidMgrs; i++) 146 { 147 if (!bitVectorTest(&pKernelFifo->chidMgrValid, i)) 148 continue; 149 150 pKernelFifo->ppChidMgr[i] = portMemAllocNonPaged(sizeof(CHID_MGR)); 151 if (pKernelFifo->ppChidMgr[i] == NULL) 152 { 153 status = NV_ERR_NO_MEMORY; 154 NV_PRINTF(LEVEL_ERROR, "Failed to allocate pFifo->pChidMgr[%d]\n", i); 155 DBG_BREAKPOINT(); 156 goto fail; 157 } 158 portMemSet(pKernelFifo->ppChidMgr[i], 0, sizeof(CHID_MGR)); 159 160 pKernelFifo->ppChidMgr[i]->runlistId = i; 161 162 pKernelFifo->ppChidMgr[i]->pChanGrpTree = portMemAllocNonPaged(sizeof(KernelChannelGroupMap)); 163 mapInitIntrusive(pKernelFifo->ppChidMgr[i]->pChanGrpTree); 164 165 status = _kfifoChidMgrAllocChidHeaps(pGpu, pKernelFifo, pKernelFifo->ppChidMgr[i]); 166 if (status != NV_OK) 167 { 168 NV_PRINTF(LEVEL_ERROR, "Error allocating FifoDataHeap in " 169 "pChidMgr. Status = %s (0x%x)\n", 170 nvstatusToString(status), status); 171 DBG_BREAKPOINT(); 172 goto fail; 173 } 174 175 status = _kfifoChidMgrInitChannelGroupMgr(pGpu, pKernelFifo->ppChidMgr[i]); 176 if (status != NV_OK) 177 goto fail; 178 } 179 180 181 return status; 182 183 fail: 184 kfifoChidMgrDestruct(pKernelFifo); 185 return status; 186 } 187 188 void 189 kfifoChidMgrDestruct_IMPL 190 ( 191 KernelFifo *pKernelFifo 192 ) 193 { 194 NvU32 i; 195 196 for (i = 0; i < pKernelFifo->numChidMgrs; i++) 197 { 198 if (pKernelFifo->ppChidMgr[i] != NULL) 199 { 200 mapDestroy(pKernelFifo->ppChidMgr[i]->pChanGrpTree); 201 portMemFree(pKernelFifo->ppChidMgr[i]->pChanGrpTree); 202 _kfifoChidMgrDestroyChidHeaps(pKernelFifo->ppChidMgr[i]); 203 _kfifoChidMgrDestroyChannelGroupMgr(pKernelFifo->ppChidMgr[i]); 204 portMemFree(pKernelFifo->ppChidMgr[i]); 205 pKernelFifo->ppChidMgr[i] = NULL; 206 } 207 } 208 209 portMemFree(pKernelFifo->ppChidMgr); 210 pKernelFifo->ppChidMgr = NULL; 211 bitVectorClrAll(&pKernelFifo->chidMgrValid); 212 pKernelFifo->numChidMgrs = 0; 213 } 214 215 /* 216 * @brief Allocate and initialize the virtual ChId heap pointers 217 */ 218 static NV_STATUS 219 _kfifoChidMgrAllocVChidHeapPointers 220 ( 221 OBJGPU *pGpu, 222 CHID_MGR *pChidMgr 223 ) 224 { 225 NV_STATUS status = NV_OK; 226 NvU32 i; 227 228 if (IS_VIRTUAL(pGpu)) 229 { 230 return NV_OK; 231 } 232 233 if (gpuIsSriovEnabled(pGpu)) 234 { 235 // 236 // For Virtual Channel Heap 237 // Allocate Memory for Heap Object pointers 238 // 239 pChidMgr->ppVirtualChIDHeap = portMemAllocNonPaged(sizeof(OBJEHEAP *) * (VMMU_MAX_GFID)); 240 if (pChidMgr->ppVirtualChIDHeap == NULL) 241 { 242 NV_PRINTF(LEVEL_ERROR, 243 "Error allocating memory for virtual channel heap pointers\n"); 244 return NV_ERR_NO_MEMORY; 245 } 246 247 // initialize 248 for (i = 0; i < VMMU_MAX_GFID; i++) 249 { 250 pChidMgr->ppVirtualChIDHeap[i] = NULL; 251 } 252 } 253 return status; 254 } 255 256 257 /* 258 * @brief Allocates & initializes ChID heaps 259 */ 260 static NV_STATUS 261 _kfifoChidMgrAllocChidHeaps 262 ( 263 OBJGPU *pGpu, 264 KernelFifo *pKernelFifo, 265 CHID_MGR *pChidMgr 266 ) 267 { 268 NV_STATUS status = NV_OK; 269 270 if (pChidMgr->numChannels == 0) 271 { 272 if (kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr) == 0) 273 { 274 NV_PRINTF(LEVEL_ERROR, "pChidMgr->numChannels is 0\n"); 275 DBG_BREAKPOINT(); 276 return NV_ERR_INVALID_STATE; 277 } 278 } 279 280 pChidMgr->pFifoDataHeap = portMemAllocNonPaged(sizeof(*pChidMgr->pFifoDataHeap)); 281 if (pChidMgr->pFifoDataHeap == NULL) 282 { 283 status = NV_ERR_NO_MEMORY; 284 NV_PRINTF(LEVEL_ERROR, 285 "Error in Allocating memory for pFifoDataHeap! Status = %s (0x%x)\n", 286 nvstatusToString(status), status); 287 return status; 288 } 289 constructObjEHeap(pChidMgr->pFifoDataHeap, 0, pChidMgr->numChannels, 290 sizeof(KernelChannel *), 0); 291 292 if (kfifoIsChidHeapEnabled(pKernelFifo)) 293 { 294 NvU32 userdBar1Size; 295 NvU32 numChannels = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr); 296 NvU32 subProcessIsolation = 1; 297 298 pChidMgr->pGlobalChIDHeap = portMemAllocNonPaged(sizeof(OBJEHEAP)); 299 if (pChidMgr->pGlobalChIDHeap == NULL) 300 { 301 NV_PRINTF(LEVEL_ERROR, 302 "Error in Allocating memory for global ChID heap!\n"); 303 return NV_ERR_NO_MEMORY; 304 } 305 constructObjEHeap(pChidMgr->pGlobalChIDHeap, 0, numChannels, 306 sizeof(PFIFO_ISOLATIONID), 0); 307 308 // 309 // Enable USERD allocation isolation. USERD allocated by different clients 310 // should not be in the same page 311 // 312 kfifoGetUserdSizeAlign_HAL(pKernelFifo, &userdBar1Size, NULL); 313 NvBool bIsolationEnabled = (pKernelFifo->bUsePerRunlistChram && pKernelFifo->bDisableChidIsolation) ? NV_FALSE : NV_TRUE; 314 pChidMgr->pGlobalChIDHeap->eheapSetOwnerIsolation(pChidMgr->pGlobalChIDHeap, 315 bIsolationEnabled, 316 RM_PAGE_SIZE / userdBar1Size); 317 318 // Disable USERD allocation isolation for guest if disabled from vmioplugin 319 { 320 // In this case subProcessIsolation is always 0 321 if (IS_GSP_CLIENT(pGpu)) 322 { 323 subProcessIsolation = 0; 324 } 325 } 326 if (!subProcessIsolation) 327 { 328 pChidMgr->pGlobalChIDHeap->eheapSetOwnerIsolation( 329 pChidMgr->pGlobalChIDHeap, 330 NV_FALSE, 331 RM_PAGE_SIZE / userdBar1Size); 332 #if (defined(_WIN32) || defined(_WIN64) || defined(NV_UNIX)) && !RMCFG_FEATURE_MODS_FEATURES 333 NV_PRINTF(LEVEL_INFO, 334 "Sub Process channel isolation disabled by vGPU plugin\n"); 335 #endif 336 } 337 338 status = _kfifoChidMgrAllocVChidHeapPointers(pGpu, pChidMgr); 339 } 340 341 return status; 342 } 343 344 static void 345 _kfifoChidMgrDestroyChidHeaps 346 ( 347 CHID_MGR *pChidMgr 348 ) 349 { 350 if (pChidMgr->pFifoDataHeap != NULL) 351 { 352 pChidMgr->pFifoDataHeap->eheapDestruct(pChidMgr->pFifoDataHeap); 353 portMemFree(pChidMgr->pFifoDataHeap); 354 pChidMgr->pFifoDataHeap = NULL; 355 } 356 if (pChidMgr->pGlobalChIDHeap != NULL) 357 { 358 pChidMgr->pGlobalChIDHeap->eheapDestruct(pChidMgr->pGlobalChIDHeap); 359 portMemFree(pChidMgr->pGlobalChIDHeap); 360 pChidMgr->pGlobalChIDHeap = NULL; 361 } 362 363 portMemFree(pChidMgr->ppVirtualChIDHeap); 364 pChidMgr->ppVirtualChIDHeap = NULL; 365 } 366 367 368 static NV_STATUS 369 _kfifoChidMgrInitChannelGroupMgr 370 ( 371 OBJGPU *pGpu, 372 CHID_MGR *pChidMgr 373 ) 374 { 375 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 376 FIFO_HW_ID *pFifoHwID = &pChidMgr->channelGrpMgr; 377 NvU32 allocSize; 378 NvU32 numChannelGroups = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr); 379 380 if (numChannelGroups == 0) 381 { 382 return NV_OK; 383 } 384 385 // Rounds up to dword alignemnt, then converts bits to bytes. 386 allocSize = RM_ALIGN_UP(numChannelGroups, 32)/8; 387 388 pFifoHwID->pHwIdInUse = portMemAllocNonPaged(allocSize); 389 if (pFifoHwID->pHwIdInUse == NULL) 390 return NV_ERR_NO_MEMORY; 391 392 // bytes to NvU32[] elements 393 pFifoHwID->hwIdInUseSz = allocSize/4; 394 395 portMemSet(pFifoHwID->pHwIdInUse, 0, allocSize); 396 397 // 398 // If numChannelGroups isn't a multiple of 32 we need to set the bits > numChannelGroups to 399 // 1. Otherwise when we allocate IDs starting at the top we'll allocate 400 // ids >numChannelGroups. 401 // 402 if (numChannelGroups % 32 != 0) 403 { 404 pFifoHwID->pHwIdInUse[numChannelGroups/32] |= ~ ((1<<(numChannelGroups%32))-1); 405 } 406 407 return NV_OK; 408 } 409 410 static void 411 _kfifoChidMgrDestroyChannelGroupMgr 412 ( 413 CHID_MGR *pChidMgr 414 ) 415 { 416 if (pChidMgr->channelGrpMgr.pHwIdInUse) 417 { 418 portMemFree(pChidMgr->channelGrpMgr.pHwIdInUse); 419 pChidMgr->channelGrpMgr.pHwIdInUse = NULL; 420 pChidMgr->channelGrpMgr.hwIdInUseSz = 0; 421 } 422 } 423 424 static NV_STATUS 425 _kfifoChidMgrFreeIsolationId 426 ( 427 CHID_MGR *pChidMgr, 428 NvU32 ChID 429 ) 430 { 431 EMEMBLOCK *pIsolationIdBlock = pChidMgr->pGlobalChIDHeap->eheapGetBlock( 432 pChidMgr->pGlobalChIDHeap, 433 ChID, 434 NV_FALSE); 435 436 NV_ASSERT_OR_RETURN(pIsolationIdBlock, NV_ERR_OBJECT_NOT_FOUND); 437 NV_ASSERT(pIsolationIdBlock->refCount > 0); 438 NV_ASSERT(pIsolationIdBlock->pData != NULL); 439 portMemFree(pIsolationIdBlock->pData); 440 441 pIsolationIdBlock->pData = NULL; 442 443 return NV_OK; 444 } 445 446 /*! 447 * @breif Fifo defined call back comparator to compare eheap block ownership ID 448 * 449 * @param[in] pRequesterID Ownership ID constructed by caller 450 * @param[in] pIsolationID 451 * 452 * @return NV_TRUE if two ownership IDs belong to the same owner 453 */ 454 static NvBool 455 _kfifoUserdOwnerComparator 456 ( 457 void *pRequesterID, 458 void *pIsolationID 459 ) 460 { 461 PFIFO_ISOLATIONID pAllocID = (PFIFO_ISOLATIONID)pRequesterID; 462 PFIFO_ISOLATIONID pBlockID = (PFIFO_ISOLATIONID)pIsolationID; 463 464 // 465 // The block's data will be NULL if the channel has been destroyed but there 466 // is still a refcount on the channel ID. In that case no work can be issued 467 // to that channel ID now or in the future, so we can act as though the 468 // channel does not exist. 469 // 470 if (!pBlockID) 471 return NV_TRUE; 472 473 if ((pAllocID->domain != pBlockID->domain) || 474 (pAllocID->processID != pBlockID->processID) || 475 (pAllocID->subProcessID != pBlockID->subProcessID)) 476 { 477 return NV_FALSE; 478 } 479 else 480 { 481 return NV_TRUE; 482 } 483 } 484 485 /* 486 * @brief Returns the number of vGPU plugin channels. 487 * 488 * Depending on whether this code is executed on the CPU RM or the Physical RM, 489 * different structures are used to retrieve the number. 490 * On the CPU RM or the monolithic RM, KERNEL_HOST_VGPU_DEVICE::numPluginChannels is used, 491 * whereas on the physical RM it's HOST_VGPU_DEVICE::numPluginChannels. 492 */ 493 static NV_STATUS 494 _kfifoGetVgpuPluginChannelsCount 495 ( 496 OBJGPU *pGpu, 497 NvU32 *pNumPluginChannels 498 ) 499 { 500 NV_ASSERT_OR_RETURN(pNumPluginChannels != NULL, NV_ERR_INVALID_ARGUMENT); 501 502 if (!RMCFG_FEATURE_PLATFORM_GSP) 503 { 504 KERNEL_HOST_VGPU_DEVICE *pKernelHostVgpuDevice = NULL; 505 506 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextKernelHostVgpuDevice(pGpu, &pKernelHostVgpuDevice)); 507 NV_ASSERT_OR_RETURN(pKernelHostVgpuDevice != NULL, NV_ERR_OBJECT_NOT_FOUND); 508 509 *pNumPluginChannels = pKernelHostVgpuDevice->numPluginChannels; 510 } 511 512 return NV_OK; 513 } 514 515 /*! 516 * @brief Allocates one Channel ID on heap 517 * 518 * @param[in] OBJGPU GPU Object 519 * @param[in] KernelFifo KernelFifo Object 520 * @param[in] CHID_MGR Channel ID manager 521 * @param[in] chFlag NVOS32_ALLOC_FLAGS_FORCE_MEM_GROWS_DOWN 522 * NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE 523 * NVOS32_ALLOC_FLAGS_FORCE_MEM_GROWS_UP 524 * @param[in] bForceInternalIdx true if requesting specific index within USERD 525 * page 526 * @param[in] internalIdx requested index within USERD page when 527 * bForceInternalIdx true 528 * @param[in] ChID ChID to assign in case of ADDRESS_ALLOCATE 529 * @param[in,out] pKernelChannel The previously allocated KernelChannel structure 530 * 531 * @return NV_OK if allocation is successful 532 * NV_ERR_NO_FREE_FIFOS: allocated channel ID exceeds MAX channels. 533 */ 534 NV_STATUS 535 kfifoChidMgrAllocChid_IMPL 536 ( 537 OBJGPU *pGpu, 538 KernelFifo *pKernelFifo, 539 CHID_MGR *pChidMgr, 540 NvHandle hClient, 541 CHANNEL_HW_ID_ALLOC_MODE chIdFlag, 542 NvBool bForceInternalIdx, 543 NvU32 internalIdx, 544 NvBool bForceUserdPage, 545 NvU32 userdPageIdx, 546 NvU32 ChID, 547 KernelChannel *pKernelChannel 548 ) 549 { 550 NvU64 chSize; 551 NvU32 chFlag = chIdFlag; 552 NvU64 ChID64 = 0; 553 NvU64 subProcessID = 0; 554 NvU64 processID = 0; 555 NvBool bIsSubProcessDisabled = NV_FALSE; 556 RmClient *pClient; 557 NvU32 offsetAlign = 1; 558 NvU32 gfid; 559 PFIFO_ISOLATIONID pIsolationID = NULL; 560 NV_STATUS status; 561 NvU32 numChannels; 562 563 NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_ARGUMENT); 564 565 numChannels = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr); 566 567 switch (chIdFlag) 568 { 569 case CHANNEL_HW_ID_ALLOC_MODE_GROW_DOWN: 570 chFlag = NVOS32_ALLOC_FLAGS_FORCE_MEM_GROWS_DOWN; 571 break; 572 case CHANNEL_HW_ID_ALLOC_MODE_GROW_UP: 573 chFlag = NVOS32_ALLOC_FLAGS_FORCE_MEM_GROWS_UP; 574 break; 575 case CHANNEL_HW_ID_ALLOC_MODE_PROVIDED: 576 ChID64 = ChID; 577 chFlag = NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE; 578 break; 579 default: 580 NV_PRINTF(LEVEL_ERROR, "Invalid channel ID alloc mode %d\n", chFlag); 581 DBG_BREAKPOINT(); 582 return NV_ERR_INVALID_ARGUMENT; 583 } 584 585 // we are allocating only one Channel at a time 586 chSize = 1; 587 588 // Create unique isolation ID for each process 589 if (serverutilGetClientUnderLock(hClient, &pClient) != NV_OK) 590 { 591 NV_PRINTF(LEVEL_ERROR, "Invalid client handle %ux\n", hClient); 592 DBG_BREAKPOINT(); 593 return NV_ERR_INVALID_CLIENT; 594 } 595 596 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &gfid)); 597 598 // SRIOV: In guest plugin context allocate the chid. 599 // In guest RM context allocate the same chid as guest 600 if (IS_GFID_VF(gfid)) 601 { 602 NvU32 numPluginChannels; 603 NvU64 rangeLo, rangeHi, base, size; 604 605 NV_ASSERT_OR_RETURN(pChidMgr->ppVirtualChIDHeap[gfid], 606 NV_ERR_INVALID_STATE); 607 608 NV_ASSERT_OK_OR_RETURN(_kfifoGetVgpuPluginChannelsCount(pGpu, &numPluginChannels)); 609 610 pChidMgr->ppVirtualChIDHeap[gfid]->eheapGetBase( 611 pChidMgr->ppVirtualChIDHeap[gfid], 612 &base); 613 pChidMgr->ppVirtualChIDHeap[gfid]->eheapGetSize( 614 pChidMgr->ppVirtualChIDHeap[gfid], 615 &size); 616 617 rangeLo = base; 618 rangeHi = base + size - 1; 619 620 // Route plugin channels to be allocated at the top 621 NV_ASSERT_OR_RETURN(numPluginChannels < size, 622 NV_ERR_INVALID_PARAMETER); 623 if (pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->bIsCallingContextVgpuPlugin && 624 numPluginChannels > 0) 625 { 626 rangeLo = rangeHi - numPluginChannels + 1; 627 } 628 else 629 { 630 rangeHi = rangeHi - numPluginChannels; 631 } 632 633 status = pChidMgr->ppVirtualChIDHeap[gfid]->eheapSetAllocRange( 634 pChidMgr->ppVirtualChIDHeap[gfid], 635 rangeLo, 636 rangeHi); 637 638 NV_ASSERT_OK_OR_RETURN(status); 639 640 if (bForceUserdPage) 641 { 642 NV_ASSERT_OR_RETURN(!bForceInternalIdx, NV_ERR_INVALID_STATE); 643 ChID64 = ((NvU64)userdPageIdx) * 644 pChidMgr->ppVirtualChIDHeap[gfid]->ownerGranularity + 645 internalIdx; 646 chFlag |= NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE; 647 } 648 else if (bForceInternalIdx) 649 { 650 chFlag |= NVOS32_ALLOC_FLAGS_FORCE_INTERNAL_INDEX; 651 offsetAlign = internalIdx; 652 } 653 654 if (chFlag & NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE) 655 NV_ASSERT_OR_RETURN((ChID64 <= rangeHi) && (ChID64 >= rangeLo), 656 NV_ERR_INVALID_PARAMETER); 657 658 // We'll allocate from the VirtualChIdHeap for the guest 659 status = pChidMgr->ppVirtualChIDHeap[gfid]->eheapAlloc( 660 pChidMgr->ppVirtualChIDHeap[gfid], // This Heap 661 KFIFO_EHEAP_OWNER, // owner 662 &chFlag, // Alloc Flags 663 &ChID64, // Alloc Offset 664 &chSize, // Size 665 offsetAlign, // offsetAlign 666 1, // sizeAlign 667 NULL, // Allocated mem block 668 NULL, // Isolation ID 669 NULL // Fifo defined ownership comparator 670 ); 671 if (status != NV_OK) 672 { 673 NV_PRINTF(LEVEL_ERROR, 674 "Failed to allocate Channel ID 0x%llx %d on heap \n", 675 ChID64, 676 chIdFlag); 677 DBG_BREAKPOINT(); 678 goto fail; 679 } 680 } 681 else 682 { 683 // 684 // Legacy / SRIOV vGPU Host, SRIOV guest, baremetal CPU RM, GSP FW, GSP 685 // client allocate from global heap 686 // 687 pIsolationID = portMemAllocNonPaged(sizeof(FIFO_ISOLATIONID)); 688 NV_ASSERT_OR_RETURN((pIsolationID != NULL), NV_ERR_NO_MEMORY); 689 portMemSet(pIsolationID, 0, sizeof(FIFO_ISOLATIONID)); 690 691 // 692 // Check if the allocation request is from the guest RM or host RM 693 // 694 processID = pClient->ProcID; 695 subProcessID = pClient->SubProcessID; 696 bIsSubProcessDisabled = pClient->bIsSubProcessDisabled; 697 698 if (RMCFG_FEATURE_PLATFORM_GSP || kchannelCheckIsKernel(pKernelChannel)) 699 { 700 // 701 // If not GSPFW: Allocation request is from host RM kernel 702 // If GSPFW: ChID has already been chosen by CPU-RM, but pClient 703 // doesn't have the true processID, so just allow the whole pool. 704 // 705 pIsolationID->domain = HOST_KERNEL; 706 processID = KERNEL_PID; 707 } 708 else 709 { 710 if (0x0 != subProcessID) 711 { 712 // 713 // Allocation request is from the guest RM 714 // 715 if (KERNEL_PID == subProcessID) 716 { 717 pIsolationID->domain = GUEST_KERNEL; 718 } 719 else 720 { 721 pIsolationID->domain = GUEST_USER; 722 } 723 } 724 else 725 { 726 pIsolationID->domain = HOST_USER; 727 } 728 } 729 730 pIsolationID->processID = processID; 731 pIsolationID->subProcessID = subProcessID; 732 733 // 734 // Overwrite isolation ID if guest USERD isolation is disabled 735 // 736 if ((subProcessID != 0x0) && (bIsSubProcessDisabled)) 737 { 738 pIsolationID->domain = GUEST_INSECURE; 739 pIsolationID->subProcessID = KERNEL_PID; 740 } 741 742 /* Channel USERD manipuliation only supported without GFID */ 743 if (bForceUserdPage) 744 { 745 NV_ASSERT_OR_RETURN(!bForceInternalIdx, NV_ERR_INVALID_STATE); 746 ChID64 = ((NvU64)userdPageIdx) * 747 pChidMgr->pGlobalChIDHeap->ownerGranularity + 748 internalIdx; 749 chFlag |= NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE; 750 } 751 else if (bForceInternalIdx) 752 { 753 chFlag |= NVOS32_ALLOC_FLAGS_FORCE_INTERNAL_INDEX; 754 offsetAlign = internalIdx; 755 } 756 757 status = pChidMgr->pGlobalChIDHeap->eheapAlloc( 758 pChidMgr->pGlobalChIDHeap, // This Heap 759 KFIFO_EHEAP_OWNER, // owner 760 &chFlag, // Alloc Flags 761 &ChID64, // Alloc Offset 762 &chSize, // Size 763 offsetAlign, // offsetAlign 764 1, // sizeAlign 765 NULL, // Allocated mem block 766 pIsolationID, // Isolation ID 767 _kfifoUserdOwnerComparator // Fifo defined ownership comparator 768 ); 769 770 if (status != NV_OK) 771 { 772 NV_PRINTF(LEVEL_ERROR, "Failed to allocate Channel ID on heap\n"); 773 DBG_BREAKPOINT(); 774 goto fail; 775 } 776 } 777 778 // 779 // Now allocate at a fixed offset from the pFifoDataHeap once the previous 780 // ID allocation told us which ID to use. 781 // 782 chFlag = NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE; 783 status = pChidMgr->pFifoDataHeap->eheapAlloc( 784 pChidMgr->pFifoDataHeap, // This Heap 785 KFIFO_EHEAP_OWNER, // owner 786 &chFlag, // Alloc Flags 787 &ChID64, // Alloc Offset 788 &chSize, // Size 789 1, // offsetAlign 790 1, // sizeAlign 791 NULL, // Allocated mem block 792 NULL, // Isolation ID 793 NULL // ownership comparator 794 ); 795 796 if (status != NV_OK) 797 { 798 // 799 // Should never happen since we're mirroring the global chid heap, or 800 // pre-reserving space on the global chid heap for SR-IOV capable 801 // systems. 802 // 803 NV_PRINTF(LEVEL_ERROR, "Failed to allocate Channel on fifo data heap\n"); 804 goto fail; 805 } 806 807 ChID = NvU64_LO32(ChID64); 808 809 if (ChID < numChannels) 810 { 811 PEMEMBLOCK pFifoDataBlock = pChidMgr->pFifoDataHeap->eheapGetBlock( 812 pChidMgr->pFifoDataHeap, 813 ChID, 814 NV_FALSE); 815 PEMEMBLOCK pIsolationIdBlock = pChidMgr->pGlobalChIDHeap->eheapGetBlock( 816 pChidMgr->pGlobalChIDHeap, 817 ChID, 818 NV_FALSE); 819 820 if (IS_GFID_PF(gfid)) 821 pIsolationIdBlock->pData = pIsolationID; 822 823 pFifoDataBlock->pData = pKernelChannel; 824 pKernelChannel->ChID = ChID; 825 } 826 else 827 { 828 NV_PRINTF(LEVEL_WARNING, "No allocatable FIFO available.\n"); 829 status = NV_ERR_NO_FREE_FIFOS; 830 goto fail; 831 } 832 return NV_OK; 833 834 fail: 835 // We already know that pIsolationID is non-NULL here. 836 portMemFree(pIsolationID); 837 return status; 838 } 839 840 /* 841 * Retain a channel ID which has already been allocated by 842 * kfifoChidMgrAllocChid. Until released, the HW channel ID will not be 843 * allocated by any new channels even after kfifoChidMgrFreeChid has been 844 * called. 845 */ 846 NV_STATUS 847 kfifoChidMgrRetainChid_IMPL 848 ( 849 OBJGPU *pGpu, 850 KernelFifo *pKernelFifo, 851 CHID_MGR *pChidMgr, 852 NvU32 ChID 853 ) 854 { 855 NvU32 gfid; 856 PEMEMBLOCK pFifoDataBlock = NULL; 857 858 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &gfid)); 859 860 if (IS_GFID_VF(gfid)) 861 { 862 NV_ASSERT_OR_RETURN(pChidMgr->ppVirtualChIDHeap[gfid] != NULL, 863 NV_ERR_INVALID_STATE); 864 PEMEMBLOCK pVirtChIdBlock = pChidMgr->ppVirtualChIDHeap[gfid]->eheapGetBlock( 865 pChidMgr->ppVirtualChIDHeap[gfid], 866 ChID, 867 NV_FALSE); 868 NV_ASSERT_OR_RETURN(pVirtChIdBlock != NULL, NV_ERR_OBJECT_NOT_FOUND); 869 NV_ASSERT(pVirtChIdBlock->refCount > 0); 870 ++pVirtChIdBlock->refCount; 871 } 872 else 873 { 874 NV_ASSERT_OR_RETURN(pChidMgr->pGlobalChIDHeap != NULL, NV_ERR_INVALID_STATE); 875 PEMEMBLOCK pChIdBlock = pChidMgr->pGlobalChIDHeap->eheapGetBlock( 876 pChidMgr->pGlobalChIDHeap, 877 ChID, 878 NV_FALSE); 879 NV_ASSERT_OR_RETURN(pChIdBlock != NULL, NV_ERR_OBJECT_NOT_FOUND); 880 NV_ASSERT(pChIdBlock->refCount > 0); 881 ++pChIdBlock->refCount; 882 } 883 884 NV_ASSERT_OR_RETURN(pChidMgr->pFifoDataHeap != NULL, NV_ERR_INVALID_STATE); 885 pFifoDataBlock = pChidMgr->pFifoDataHeap->eheapGetBlock( 886 pChidMgr->pFifoDataHeap, 887 ChID, 888 NV_FALSE); 889 NV_ASSERT_OR_RETURN(pFifoDataBlock != NULL, NV_ERR_OBJECT_NOT_FOUND); 890 NV_ASSERT(pFifoDataBlock->refCount > 0); 891 ++pFifoDataBlock->refCount; 892 893 return NV_OK; 894 } 895 896 /* 897 * Drop the refcount on the given channel (ID), removing it from pFifo's heap if 898 * its refcount reaches 0. 899 */ 900 NV_STATUS 901 kfifoChidMgrReleaseChid_IMPL 902 ( 903 OBJGPU *pGpu, 904 KernelFifo *pKernelFifo, 905 CHID_MGR *pChidMgr, 906 NvU32 ChID 907 ) 908 { 909 NvU32 gfid; 910 911 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &gfid)); 912 913 if (IS_GFID_VF(gfid)) 914 { 915 NV_ASSERT_OR_RETURN(pChidMgr->ppVirtualChIDHeap[gfid] != NULL, NV_ERR_INVALID_STATE); 916 NV_ASSERT_OK(pChidMgr->ppVirtualChIDHeap[gfid]->eheapFree(pChidMgr->ppVirtualChIDHeap[gfid], ChID)); 917 } 918 else 919 { 920 NV_ASSERT_OR_RETURN(pChidMgr->pGlobalChIDHeap != NULL, NV_ERR_INVALID_STATE); 921 NV_ASSERT_OK(pChidMgr->pGlobalChIDHeap->eheapFree(pChidMgr->pGlobalChIDHeap, ChID)); 922 } 923 924 NV_ASSERT_OR_RETURN(pChidMgr->pFifoDataHeap != NULL, NV_ERR_INVALID_STATE); 925 NV_ASSERT_OK_OR_RETURN(pChidMgr->pFifoDataHeap->eheapFree(pChidMgr->pFifoDataHeap, ChID)); 926 927 return NV_OK; 928 } 929 930 /* 931 * Removes the association between pKernelChannel and its channel ID. Note that this 932 * will not remove the channel ID itself from pFifo's heap if 933 * fifoHeapRetainChannelId has been called. 934 */ 935 NV_STATUS 936 kfifoChidMgrFreeChid_IMPL 937 ( 938 OBJGPU *pGpu, 939 KernelFifo *pKernelFifo, 940 CHID_MGR *pChidMgr, 941 NvU32 ChID 942 ) 943 { 944 EMEMBLOCK *pFifoDataBlock; 945 NV_STATUS status; 946 NvU32 gfid; 947 948 // 949 // This channel is going away, so clear its pointer from the channel ID's heap 950 // block. 951 // 952 pFifoDataBlock = pChidMgr->pFifoDataHeap->eheapGetBlock( 953 pChidMgr->pFifoDataHeap, 954 ChID, 955 NV_FALSE); 956 NV_ASSERT(pFifoDataBlock->refCount > 0); 957 pFifoDataBlock->pData = NULL; 958 959 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &gfid)); 960 961 if (IS_GFID_PF(gfid)) 962 { 963 // 964 // This marks the channel ID as orphaned and causes it to be ignored for 965 // isolation purposes. This only matters if there will still be a reference 966 // on the ID after we release ours below. 967 // 968 status = _kfifoChidMgrFreeIsolationId(pChidMgr, ChID); 969 if(status != NV_OK) 970 { 971 NV_PRINTF(LEVEL_ERROR, 972 "Failed to free IsolationId. Status = 0x%x\n", status); 973 DBG_BREAKPOINT(); 974 return status; 975 } 976 } 977 978 return kfifoChidMgrReleaseChid(pGpu, pKernelFifo, pChidMgr, ChID); 979 } 980 981 /** 982 * @brief Reserve a contiguous set of SCHIDs from the end of our CHID heap for 983 * the given GFID 984 * 985 986 * @param[in] pChidMgr CHID_MGR pointer 987 * @param[in] numChannels Number of SCHIDs to reserve 988 * @param[in] pHostVgpuDevice HOST_VGPU_DEVICE 989 * 990 * @return NV_OK if success 991 */ 992 NV_STATUS 993 kfifoChidMgrReserveSystemChids_IMPL 994 ( 995 OBJGPU *pGpu, 996 KernelFifo *pKernelFifo, 997 CHID_MGR *pChidMgr, 998 NvU32 numChannels, 999 NvU32 flags, 1000 NvU32 gfid, 1001 NvU32 *pChidOffset, 1002 NvHandle hMigClient, 1003 NvU32 engineFifoListNumEntries, 1004 FIFO_ENGINE_LIST *pEngineFifoList 1005 ) 1006 { 1007 NV_STATUS status = NV_OK; 1008 NvU64 chSize; 1009 NvU64 offset = 0; 1010 PFIFO_ISOLATIONID pIsolationID = NULL; 1011 PEMEMBLOCK pIsolationIdBlock; 1012 NvU32 userdBar1Size; 1013 1014 if (IS_VIRTUAL(pGpu)) 1015 { 1016 // Not supported on guest or when SRIOV is disabled 1017 return NV_ERR_NOT_SUPPORTED; 1018 } 1019 1020 pIsolationID = portMemAllocNonPaged(sizeof(FIFO_ISOLATIONID)); 1021 NV_ASSERT_OR_RETURN((pIsolationID != NULL), NV_ERR_NO_MEMORY); 1022 portMemSet(pIsolationID, 0, sizeof(FIFO_ISOLATIONID)); 1023 1024 chSize = numChannels; 1025 1026 status = pChidMgr->pGlobalChIDHeap->eheapAlloc( 1027 pChidMgr->pGlobalChIDHeap, // This Heap 1028 KFIFO_EHEAP_OWNER, // owner 1029 &flags, // Alloc Flags 1030 &offset, // Alloc Offset 1031 &chSize, // Size 1032 1, // offsetAlign 1033 1, // sizeAlign 1034 NULL, // Allocated mem block 1035 pIsolationID, // IsolationID 1036 _kfifoUserdOwnerComparator // Fifo defined ownership comparator 1037 ); 1038 1039 if(status != NV_OK) 1040 { 1041 NV_PRINTF(LEVEL_ERROR, "Failed to reserve channel IDs. Status = 0x%x\n", status); 1042 DBG_BREAKPOINT(); 1043 1044 // 1045 // Free the allocated memory and return early. After this, all failure 1046 // points can goto the common cleanup label 1047 // 1048 portMemFree(pIsolationID); 1049 return status; 1050 } 1051 1052 pIsolationIdBlock = pChidMgr->pGlobalChIDHeap->eheapGetBlock( 1053 pChidMgr->pGlobalChIDHeap, 1054 offset, 1055 NV_FALSE); 1056 if (pIsolationIdBlock == NULL) 1057 { 1058 // Something bad happened. This should not fail if allocation succeeded 1059 NV_PRINTF(LEVEL_ERROR, "Could not fetch block from eheap\n"); 1060 DBG_BREAKPOINT(); 1061 goto cleanup; 1062 } 1063 pIsolationIdBlock->pData = pIsolationID; 1064 1065 status = kfifoSetChidOffset(pGpu, pKernelFifo, pChidMgr, (NvU32)offset, 1066 numChannels, gfid, pChidOffset, hMigClient, 1067 engineFifoListNumEntries, pEngineFifoList); 1068 1069 if (status != NV_OK) 1070 { 1071 NV_PRINTF(LEVEL_ERROR, 1072 "Failed to program the CHID table\n"); 1073 goto cleanup; 1074 } 1075 1076 pChidMgr->ppVirtualChIDHeap[gfid] = portMemAllocNonPaged(sizeof(OBJEHEAP)); 1077 if (pChidMgr->ppVirtualChIDHeap[gfid] == NULL) 1078 { 1079 status = NV_ERR_NO_MEMORY; 1080 NV_PRINTF(LEVEL_ERROR, "Error allocating memory for virtual " 1081 "channel ID heap\n"); 1082 goto cleanup; 1083 } 1084 1085 // 1086 // Construct heap using low as offset and size of numChannels. This heap 1087 // will be used for guest channel ID allocations, but will be in the 1088 // system channel ID space, hence it only manages IDs from offset to 1089 // (offset + numChannels). 1090 // 1091 constructObjEHeap(pChidMgr->ppVirtualChIDHeap[gfid], offset, 1092 (offset + numChannels), 0, 0); 1093 1094 kfifoGetUserdSizeAlign_HAL(pKernelFifo, &userdBar1Size, NULL); 1095 pChidMgr->ppVirtualChIDHeap[gfid]->eheapSetOwnerIsolation( 1096 pChidMgr->ppVirtualChIDHeap[gfid], 1097 NV_FALSE, 1098 RM_PAGE_SIZE/userdBar1Size); 1099 1100 return status; 1101 1102 cleanup: 1103 portMemFree(pChidMgr->ppVirtualChIDHeap[gfid]); 1104 NV_ASSERT(kfifoSetChidOffset(pGpu, pKernelFifo, pChidMgr, 0, 0, 1105 gfid, pChidOffset, hMigClient, 1106 engineFifoListNumEntries, pEngineFifoList) == NV_OK); 1107 NV_ASSERT(pChidMgr->pGlobalChIDHeap->eheapFree(pChidMgr->pGlobalChIDHeap, offset) == NV_OK); 1108 portMemFree(pIsolationID); 1109 return status; 1110 } 1111 1112 /*! Frees a block of contiguous SCHIDs previously reserved for the given GFID */ 1113 NV_STATUS 1114 kfifoChidMgrFreeSystemChids_IMPL 1115 ( 1116 OBJGPU *pGpu, 1117 KernelFifo *pKernelFifo, 1118 CHID_MGR *pChidMgr, 1119 NvU32 gfid, 1120 NvU32 *pChidOffset, 1121 NvHandle hMigClient, 1122 NvU32 engineFifoListNumEntries, 1123 FIFO_ENGINE_LIST *pEngineFifoList 1124 ) 1125 { 1126 NV_STATUS status, tmpStatus; 1127 NvU64 chId; 1128 1129 if (IS_VIRTUAL(pGpu)) 1130 { 1131 // Not supported on guest or when SRIOV is disabled 1132 return NV_ERR_NOT_SUPPORTED; 1133 } 1134 1135 // Get the schid base 1136 pChidMgr->ppVirtualChIDHeap[gfid]->eheapGetBase( 1137 pChidMgr->ppVirtualChIDHeap[gfid], 1138 &chId); 1139 1140 status = _kfifoChidMgrFreeIsolationId(pChidMgr, (NvU32)chId); 1141 if(status != NV_OK) 1142 { 1143 NV_PRINTF(LEVEL_ERROR, 1144 "Failed to free IsolationId. Status = 0x%x\n", 1145 status); 1146 DBG_BREAKPOINT(); 1147 return status; 1148 } 1149 1150 status = pChidMgr->pGlobalChIDHeap->eheapFree(pChidMgr->pGlobalChIDHeap, chId); 1151 if(status != NV_OK) 1152 { 1153 NV_PRINTF(LEVEL_ERROR, 1154 "Failed to free channel IDs. Status = 0x%x\n", 1155 status); 1156 DBG_BREAKPOINT(); 1157 1158 // 1159 // March on anyway to program the ChId table. We'll return an error 1160 // if we get here though. 1161 // 1162 } 1163 1164 tmpStatus = kfifoSetChidOffset(pGpu, pKernelFifo, pChidMgr, 0, 0, 1165 gfid, pChidOffset, hMigClient, 1166 engineFifoListNumEntries, pEngineFifoList); 1167 if (tmpStatus != NV_OK) 1168 { 1169 NV_PRINTF(LEVEL_ERROR, 1170 "Failed to program the CHID table\n"); 1171 DBG_BREAKPOINT(); 1172 return tmpStatus; 1173 } 1174 1175 pChidMgr->ppVirtualChIDHeap[gfid]->eheapDestruct(pChidMgr->ppVirtualChIDHeap[gfid]); 1176 portMemFree(pChidMgr->ppVirtualChIDHeap[gfid]); 1177 pChidMgr->ppVirtualChIDHeap[gfid] = NULL; 1178 1179 return status; 1180 } 1181 1182 NvU32 1183 kfifoChidMgrGetNumChannels_IMPL 1184 ( 1185 OBJGPU *pGpu, 1186 KernelFifo *pKernelFifo, 1187 CHID_MGR *pChidMgr 1188 ) 1189 { 1190 // Cache ChidMgr's numChannels if not set 1191 if (pChidMgr->numChannels == 0) 1192 { 1193 NvU32 numChannels = kfifoRunlistQueryNumChannels_HAL(pGpu, pKernelFifo, 1194 pChidMgr->runlistId); 1195 1196 if (pKernelFifo->bNumChannelsOverride) 1197 { 1198 pChidMgr->numChannels = NV_MIN(pKernelFifo->numChannelsOverride, numChannels); 1199 } 1200 else 1201 { 1202 pChidMgr->numChannels = numChannels; 1203 } 1204 1205 // Once we have set calculated value disable any overrides. 1206 pKernelFifo->bNumChannelsOverride = 0; 1207 } 1208 1209 return pChidMgr->numChannels; 1210 } 1211 1212 NvU32 1213 kfifoRunlistQueryNumChannels_KERNEL 1214 ( 1215 OBJGPU *pGpu, 1216 KernelFifo *pKernelFifo, 1217 NvU32 runlistId 1218 ) 1219 { 1220 NvU32 numChannels = 0; 1221 NvU32 status; 1222 1223 // Do internal control call and set numChannels 1224 if (IS_GSP_CLIENT(pGpu)) 1225 { 1226 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); 1227 NV2080_CTRL_INTERNAL_FIFO_GET_NUM_CHANNELS_PARAMS numChannelsParams = {0}; 1228 1229 numChannelsParams.runlistId = runlistId; 1230 1231 status = pRmApi->Control(pRmApi, 1232 pGpu->hInternalClient, 1233 pGpu->hInternalSubdevice, 1234 NV2080_CTRL_CMD_INTERNAL_FIFO_GET_NUM_CHANNELS, 1235 &numChannelsParams, 1236 sizeof(NV2080_CTRL_INTERNAL_FIFO_GET_NUM_CHANNELS_PARAMS)); 1237 if (status != NV_OK) 1238 { 1239 DBG_BREAKPOINT(); 1240 return 0; 1241 } 1242 1243 numChannels = numChannelsParams.numChannels; 1244 } 1245 1246 NV_ASSERT(numChannels > 0); 1247 1248 return numChannels; 1249 } 1250 1251 /** 1252 * @brief reserves a hardware channel slot for a channel group 1253 * 1254 * Only responsible for indicating a hardware channel is in use, does not set 1255 * any other software state. 1256 * 1257 * This function is not called in broadcast mode 1258 * 1259 * @param pGpu 1260 * @param pKernelFifo 1261 * @param pChidMgr 1262 * @param[out] grpID 1263 */ 1264 NV_STATUS 1265 kfifoChidMgrAllocChannelGroupHwID_IMPL 1266 ( 1267 OBJGPU *pGpu, 1268 KernelFifo *pKernelFifo, 1269 CHID_MGR *pChidMgr, 1270 NvU32 *pChGrpID 1271 ) 1272 { 1273 NvU32 maxChannelGroups; 1274 1275 if (pChGrpID == NULL) 1276 return NV_ERR_INVALID_ARGUMENT; 1277 1278 maxChannelGroups = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr); 1279 if (maxChannelGroups == 0) 1280 { 1281 NV_PRINTF(LEVEL_ERROR, "Zero max channel groups!!!\n"); 1282 return NV_ERR_INVALID_ARGUMENT; 1283 } 1284 1285 // Find the least unused grpID 1286 *pChGrpID = nvBitFieldLSZero(pChidMgr->channelGrpMgr.pHwIdInUse, 1287 pChidMgr->channelGrpMgr.hwIdInUseSz); 1288 1289 if (*pChGrpID < maxChannelGroups) 1290 { 1291 nvBitFieldSet(pChidMgr->channelGrpMgr.pHwIdInUse, 1292 pChidMgr->channelGrpMgr.hwIdInUseSz, *pChGrpID, NV_TRUE); 1293 } 1294 else 1295 { 1296 *pChGrpID = maxChannelGroups; 1297 NV_PRINTF(LEVEL_ERROR, "No allocatable FIFO available.\n"); 1298 return NV_ERR_NO_FREE_FIFOS; 1299 } 1300 return NV_OK; 1301 } 1302 1303 1304 /** 1305 * @brief Releases a hardware channel group ID. 1306 * 1307 * Not responsible for freeing any software state beyond that which indicates a 1308 * hardware channel is in use. 1309 * 1310 * This function is not called in broadcast mode 1311 * 1312 * @param pGpu 1313 * @param pFifo 1314 * @param pChidMgr 1315 * @param chGrpID 1316 */ 1317 NV_STATUS 1318 kfifoChidMgrFreeChannelGroupHwID_IMPL 1319 ( 1320 OBJGPU *pGpu, 1321 KernelFifo *pKernelFifo, 1322 CHID_MGR *pChidMgr, 1323 NvU32 chGrpID 1324 ) 1325 { 1326 NvU32 maxChannelGroups; 1327 1328 maxChannelGroups = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr); 1329 if (maxChannelGroups == 0) 1330 { 1331 NV_PRINTF(LEVEL_ERROR, "Zero max channel groups!!!\n"); 1332 return NV_ERR_INVALID_ARGUMENT; 1333 } 1334 NV_ASSERT_OR_RETURN(chGrpID < maxChannelGroups, NV_ERR_INVALID_ARGUMENT); 1335 1336 // 1337 // Look for the channel group, check to make sure it's InUse bit is set 1338 // and then clear it to indicate the grpID is no longer in use 1339 // 1340 NV_ASSERT(nvBitFieldTest(pChidMgr->channelGrpMgr.pHwIdInUse, 1341 pChidMgr->channelGrpMgr.hwIdInUseSz, chGrpID)); 1342 nvBitFieldSet(pChidMgr->channelGrpMgr.pHwIdInUse, 1343 pChidMgr->channelGrpMgr.hwIdInUseSz, chGrpID, NV_FALSE); 1344 1345 return NV_OK; 1346 } 1347 1348 CHID_MGR * 1349 kfifoGetChidMgr_IMPL 1350 ( 1351 OBJGPU *pGpu, 1352 KernelFifo *pKernelFifo, 1353 NvU32 runlistId 1354 ) 1355 { 1356 if (!kfifoIsPerRunlistChramEnabled(pKernelFifo)) 1357 { 1358 // We only have 1 chidmgr when we don't have a per-runlist channel RAM 1359 if ((pKernelFifo->numChidMgrs != 1) || 1360 (pKernelFifo->ppChidMgr == NULL) || 1361 !bitVectorTest(&pKernelFifo->chidMgrValid, 0)) 1362 { 1363 return NULL; 1364 } 1365 return pKernelFifo->ppChidMgr[0]; 1366 } 1367 else 1368 { 1369 if (runlistId >= pKernelFifo->numChidMgrs) 1370 { 1371 return NULL; 1372 } 1373 // 1374 // It is valid to return a NULL value as long as runlistId is less than 1375 // maxNumRunlists since it is possible that not everything in the range 1376 // [0, numChidMgrs) represents a valid runlistId. The onus is on the 1377 // caller to check for NULL and only then use the CHIDMGR pointer 1378 // 1379 return pKernelFifo->ppChidMgr[runlistId]; 1380 } 1381 } 1382 1383 /*! Gets associated CHIDMGR object for given FIFO_ENGINE_INFO_TYPE and value */ 1384 NV_STATUS 1385 kfifoGetChidMgrFromType_IMPL 1386 ( 1387 OBJGPU *pGpu, 1388 KernelFifo *pKernelFifo, 1389 NvU32 engineInfoType, 1390 NvU32 val, 1391 CHID_MGR **ppChidMgr 1392 ) 1393 { 1394 NV_STATUS status = NV_OK; 1395 NvU32 runlistId; 1396 1397 NV_CHECK_OR_RETURN(LEVEL_INFO, ppChidMgr != NULL, NV_ERR_INVALID_ARGUMENT); 1398 1399 // Initialize the pointer to NULL, in case we fail and return early 1400 *ppChidMgr = NULL; 1401 1402 status = kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo, 1403 engineInfoType, val, 1404 ENGINE_INFO_TYPE_RUNLIST, &runlistId); 1405 NV_CHECK_OR_RETURN(LEVEL_INFO, NV_OK == status, status); 1406 1407 *ppChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, runlistId); 1408 1409 return NV_OK; 1410 } 1411 1412 /*! 1413 * @brief Fetch pKernelChannel based on chidmgr and chid. 1414 * 1415 * This look-up uses the chid heap. It should find the first allocation of the channel, 1416 * which is useful if the handle is duped to another client. 1417 * 1418 * @param[in] pGpu 1419 * @param[in] pKernelFifo 1420 * @param[in] pChidMgr the ChIDMgr (per-runlist) 1421 * @param[in] ChID the ChID 1422 * 1423 * @return the KernelChannel * or NULL 1424 */ 1425 KernelChannel * 1426 kfifoChidMgrGetKernelChannel_IMPL 1427 ( 1428 OBJGPU *pGpu, 1429 KernelFifo *pKernelFifo, 1430 CHID_MGR *pChidMgr, 1431 NvU32 ChID 1432 ) 1433 { 1434 EMEMBLOCK *pFifoDataBlock; 1435 NvU32 numChannels; 1436 1437 NV_ASSERT_OR_RETURN(pChidMgr != NULL, NULL); 1438 // Lite mode channels don't have KernelChannel yet 1439 NV_ASSERT_OR_RETURN(!kfifoIsLiteModeEnabled_HAL(pGpu, pKernelFifo), NULL); 1440 1441 numChannels = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr); 1442 if (ChID >= numChannels) 1443 { 1444 return NULL; 1445 } 1446 1447 pFifoDataBlock = pChidMgr->pFifoDataHeap->eheapGetBlock( 1448 pChidMgr->pFifoDataHeap, 1449 ChID, 1450 NV_FALSE); 1451 if (pFifoDataBlock != NULL) 1452 { 1453 return (KernelChannel *)pFifoDataBlock->pData; 1454 } 1455 1456 return NULL; 1457 } 1458 1459 /*! Gets channel group data corresponding to grpID */ 1460 KernelChannelGroup * 1461 kfifoChidMgrGetKernelChannelGroup_IMPL 1462 ( 1463 OBJGPU *pGpu, 1464 KernelFifo *pKernelFifo, 1465 CHID_MGR *pChidMgr, 1466 NvU32 grpID 1467 ) 1468 { 1469 KernelChannelGroup *pKernelChannelGroup = NULL; 1470 1471 pKernelChannelGroup = mapFind(pChidMgr->pChanGrpTree, grpID); 1472 if (pKernelChannelGroup == NULL) 1473 { 1474 NV_PRINTF(LEVEL_INFO, "Can't find channel group %d\n", grpID); 1475 } 1476 1477 return pKernelChannelGroup; 1478 } 1479 1480 /*! 1481 * @brief Gets channel group data corresponding to grpID 1482 * 1483 * This function is not called in broadcast mode 1484 * 1485 * @param pGpu 1486 * @param pFifo 1487 * @param[in] grpID 1488 * @param[in] runlistID pass CHIDMGR_RUNLIST_ID_LEGACY if not known 1489 * 1490 * @returns KernelChannelGroup * on success 1491 * NULL if channel group is not found 1492 */ 1493 KernelChannelGroup * 1494 kfifoGetChannelGroup_IMPL 1495 ( 1496 OBJGPU *pGpu, 1497 KernelFifo *pKernelFifo, 1498 NvU32 grpID, 1499 NvU32 runlistID 1500 ) 1501 { 1502 CHID_MGR *pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, runlistID); 1503 1504 return kfifoChidMgrGetKernelChannelGroup(pGpu, pKernelFifo, pChidMgr, grpID); 1505 } 1506 1507 /*! Gets total number of channel groups in use */ 1508 NvU32 1509 kfifoGetChannelGroupsInUse_IMPL 1510 ( 1511 OBJGPU *pGpu, 1512 KernelFifo *pKernelFifo 1513 ) 1514 { 1515 NvU32 numChannelGroups = 0; 1516 NvU32 numChannelGroupsInUse = 0; 1517 NvU32 chGrpID, i; 1518 1519 for (i = 0; i < pKernelFifo->numChidMgrs; i++) 1520 { 1521 if (pKernelFifo->ppChidMgr[i] != NULL) 1522 { 1523 numChannelGroups = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, 1524 pKernelFifo->ppChidMgr[i]); 1525 1526 for (chGrpID = 0; chGrpID < numChannelGroups; chGrpID++) 1527 { 1528 if (nvBitFieldTest(pKernelFifo->ppChidMgr[i]->channelGrpMgr.pHwIdInUse, 1529 pKernelFifo->ppChidMgr[i]->channelGrpMgr.hwIdInUseSz, 1530 chGrpID)) 1531 { 1532 numChannelGroupsInUse++; 1533 } 1534 } 1535 } 1536 } 1537 return numChannelGroupsInUse; 1538 } 1539 1540 /*! Gets total number of channel groups in use per engine */ 1541 NvU32 1542 kfifoGetRunlistChannelGroupsInUse_IMPL 1543 ( 1544 OBJGPU *pGpu, 1545 KernelFifo *pKernelFifo, 1546 NvU32 runlistId 1547 ) 1548 { 1549 NvU32 numChannelGroups = 0; 1550 NvU32 numChannelGroupsInUse = 0; 1551 NvU32 chGrpID; 1552 CHID_MGR *pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, runlistId); 1553 1554 numChannelGroups = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr); 1555 for (chGrpID = 0; chGrpID < numChannelGroups; chGrpID++) 1556 { 1557 if (nvBitFieldTest(pChidMgr->channelGrpMgr.pHwIdInUse, 1558 pChidMgr->channelGrpMgr.hwIdInUseSz, 1559 chGrpID)) 1560 { 1561 numChannelGroupsInUse++; 1562 } 1563 } 1564 return numChannelGroupsInUse; 1565 } 1566 1567 /** 1568 * @brief Sets the timeslice for the specified channel group. 1569 * 1570 * @returns NV_OK if success, appropriate error otherwise 1571 */ 1572 NV_STATUS 1573 kfifoChannelGroupSetTimeslice_IMPL 1574 ( 1575 OBJGPU *pGpu, 1576 KernelFifo *pKernelFifo, 1577 KernelChannelGroup *pKernelChannelGroup, 1578 NvU64 timesliceUs, 1579 NvBool bSkipSubmit 1580 ) 1581 { 1582 NV_STATUS status = NV_OK; 1583 1584 NV_PRINTF(LEVEL_INFO, "Setting TSG %d Timeslice to %lldus\n", 1585 pKernelChannelGroup->grpID, timesliceUs); 1586 1587 if (timesliceUs < kfifoRunlistGetMinTimeSlice_HAL(pKernelFifo)) 1588 { 1589 NV_PRINTF(LEVEL_ERROR, 1590 "Setting Timeslice to %lldus not allowed. Min value is %lldus\n", 1591 timesliceUs, kfifoRunlistGetMinTimeSlice_HAL(pKernelFifo)); 1592 return NV_ERR_NOT_SUPPORTED; 1593 } 1594 1595 pKernelChannelGroup->timesliceUs = timesliceUs; 1596 1597 NV_ASSERT_OK_OR_RETURN(kfifoChannelGroupSetTimesliceSched(pGpu, 1598 pKernelFifo, 1599 pKernelChannelGroup, 1600 timesliceUs, 1601 bSkipSubmit)); 1602 1603 return status; 1604 } 1605 1606 void 1607 kfifoFillMemInfo_IMPL 1608 ( 1609 KernelFifo *pKernelFifo, 1610 MEMORY_DESCRIPTOR *pMemDesc, 1611 NV2080_CTRL_FIFO_MEM_INFO *pMemory 1612 ) 1613 { 1614 if (pMemDesc == NULL) 1615 { 1616 pMemory->aperture = NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_APERTURE_INVALID; 1617 NV_PRINTF(LEVEL_ERROR, "kfifoFillMemInfo: pMemDesc = NULL\n"); 1618 } 1619 else 1620 { 1621 if (memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM) 1622 { 1623 pMemory->aperture = NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_APERTURE_VIDMEM; 1624 } 1625 else if (memdescGetAddressSpace(pMemDesc) == ADDR_SYSMEM) 1626 { 1627 if (memdescGetCpuCacheAttrib(pMemDesc) == NV_MEMORY_CACHED) 1628 { 1629 pMemory->aperture = NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_APERTURE_SYSMEM_COH; 1630 } 1631 else if (memdescGetCpuCacheAttrib(pMemDesc) == NV_MEMORY_UNCACHED) 1632 { 1633 pMemory->aperture = NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_APERTURE_SYSMEM_NCOH; 1634 } 1635 else 1636 { 1637 NV_PRINTF(LEVEL_ERROR, 1638 "kfifoFillMemInfo: Unknown cache attribute for sysmem aperture\n"); 1639 NV_ASSERT(NV_FALSE); 1640 } 1641 } 1642 pMemory->base = memdescGetPhysAddr(pMemDesc, AT_GPU, 0); 1643 pMemory->size = pMemDesc->Size; 1644 } 1645 } 1646 1647 void 1648 kfifoGetChannelIterator_IMPL 1649 ( 1650 OBJGPU *pGpu, 1651 KernelFifo *pKernelFifo, 1652 CHANNEL_ITERATOR *pIt 1653 ) 1654 { 1655 portMemSet(pIt, 0, sizeof(*pIt)); 1656 pIt->physicalChannelID = 0; 1657 pIt->pFifoDataBlock = NULL; 1658 pIt->runlistId = 0; 1659 pIt->numRunlists = 1; 1660 if (kfifoIsPerRunlistChramEnabled(pKernelFifo)) 1661 { 1662 pIt->numRunlists = kfifoGetMaxNumRunlists_HAL(pGpu, pKernelFifo); 1663 } 1664 } 1665 1666 /** 1667 * @brief Returns the next KernelChannel from the iterator. 1668 * 1669 * Iterates over runlist IDs and ChIDs and returns the next KernelChannel found 1670 * on the heap, if any. 1671 * 1672 * (error guaranteed if pointer is NULL; non-NULL pointer guaranteed if NV_OK) 1673 * 1674 * @param[in] pGpu 1675 * @param[in] pKernelFifo 1676 * @param[in] pIt the channel iterator 1677 * @param[out] ppKernelChannel returns a KernelChannel * 1678 * 1679 * @return NV_OK if the returned pointer is valid or error 1680 */ 1681 NV_STATUS kfifoGetNextKernelChannel_IMPL 1682 ( 1683 OBJGPU *pGpu, 1684 KernelFifo *pKernelFifo, 1685 CHANNEL_ITERATOR *pIt, 1686 KernelChannel **ppKernelChannel 1687 ) 1688 { 1689 KernelChannel *pKernelChannel; 1690 1691 if (ppKernelChannel == NULL) 1692 return NV_ERR_INVALID_ARGUMENT; 1693 1694 *ppKernelChannel = NULL; 1695 1696 while (pIt->runlistId < pIt->numRunlists) 1697 { 1698 CHID_MGR *pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, pIt->runlistId); 1699 1700 if (pChidMgr == NULL) 1701 { 1702 pIt->runlistId++; 1703 continue; 1704 } 1705 1706 pIt->numChannels = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr); 1707 1708 if (pIt->pFifoDataBlock == NULL) 1709 { 1710 pIt->pFifoDataBlock = pChidMgr->pFifoDataHeap->eheapGetBlock( 1711 pChidMgr->pFifoDataHeap, 1712 pIt->physicalChannelID, 1713 NV_TRUE); 1714 } 1715 1716 while (pIt->physicalChannelID < pIt->numChannels) 1717 { 1718 if (pIt->pFifoDataBlock->owner == NVOS32_BLOCK_TYPE_FREE) 1719 { 1720 pIt->physicalChannelID = pIt->pFifoDataBlock->end + 1; 1721 } 1722 else 1723 { 1724 pIt->physicalChannelID++; 1725 pKernelChannel = (KernelChannel *)pIt->pFifoDataBlock->pData; 1726 1727 // 1728 // This iterator can be used during an interrupt, when a KernelChannel may 1729 // be in the process of being destroyed. If a KernelChannel expects a pChannel 1730 // but does not have one, it means it's being destroyed and we don't want to 1731 // return it. 1732 // 1733 if (pKernelChannel != NULL && kchannelIsValid_HAL(pKernelChannel)) 1734 { 1735 // Prepare iterator to check next block in pChidMgr->pFifoDataHeap 1736 pIt->pFifoDataBlock = pIt->pFifoDataBlock->next; 1737 *ppKernelChannel = pKernelChannel; 1738 return NV_OK; 1739 } 1740 } 1741 1742 // Check next block in pChidMgr->pFifoDataHeap 1743 pIt->pFifoDataBlock = pIt->pFifoDataBlock->next; 1744 } 1745 1746 pIt->runlistId++; 1747 // Reset iterator for next runlist 1748 pIt->physicalChannelID = 0; 1749 pIt->pFifoDataBlock = NULL; 1750 } 1751 1752 return NV_ERR_OBJECT_NOT_FOUND; 1753 } 1754 1755 /*! 1756 * @brief Localize the engine info received from the host 1757 * 1758 * The host and the guest can run in different version of drivers, the guest driver 1759 * can not directly use the raw MC_ENGINE_IDX values from the host. 1760 * RM does not guarantee those values are consistent cross branches. 1761 * 1762 * To keep the compatibility between different version of branches, this function reconstructs 1763 * of RM_ENGINE_TYPE, MC_ENGINE_IDX values based on NV2080_ENGINE_TYPE 1764 * 1765 * @param[in] pGpu OBJGPU pointer 1766 * @param[in] pFifo KernelFifo pointer 1767 * @param[in/out] pEngine Pointer to engine info table to update 1768 * 1769 */ 1770 static void 1771 _kfifoLocalizeGuestEngineData 1772 ( 1773 OBJGPU *pGpu, 1774 KernelFifo *pKernelFifo, 1775 ENGINE_INFO *pEngineInfo 1776 ) 1777 { 1778 const FIFO_GUEST_ENGINE_TABLE *guestEngineTable; 1779 NvU32 guestEngineTableSz; 1780 NvU32 nv2080EngineType; 1781 NvU32 engineIdx; 1782 NvU32 newEngineIdx; 1783 NvU32 guestTableIdx; 1784 1785 // This function should only be called in a vgpu guest RM 1786 NV_ASSERT_OR_RETURN_VOID(IS_VIRTUAL(pGpu)); 1787 1788 guestEngineTable = kfifoGetGuestEngineLookupTable(&guestEngineTableSz); 1789 1790 newEngineIdx = 0; 1791 1792 for (engineIdx = 0; engineIdx < pEngineInfo->engineInfoListSize; engineIdx++) 1793 { 1794 FIFO_ENGINE_LIST *pEngine = &pEngineInfo->engineInfoList[engineIdx]; 1795 1796 // The actual data in engineData[ENGINE_INFO_TYPE_RM_ENGINE_TYPE] is NV2080 ENGINE TYPE. 1797 nv2080EngineType = pEngine->engineData[ENGINE_INFO_TYPE_RM_ENGINE_TYPE]; 1798 1799 for (guestTableIdx = 0; guestTableIdx < guestEngineTableSz; guestTableIdx++) 1800 { 1801 // Find the engine type supported by the guest 1802 if (guestEngineTable[guestTableIdx].nv2080EngineType == nv2080EngineType) 1803 break; 1804 } 1805 1806 if (guestTableIdx < guestEngineTableSz) 1807 { 1808 // Update the MC for the guest 1809 pEngine->engineData[ENGINE_INFO_TYPE_MC] = guestEngineTable[guestTableIdx].mcIdx; 1810 1811 // Update it with correct engine type 1812 pEngine->engineData[ENGINE_INFO_TYPE_RM_ENGINE_TYPE] = gpuGetRmEngineType(nv2080EngineType); 1813 1814 if (newEngineIdx != engineIdx) 1815 { 1816 // 1817 // Move the engine info up to make sure the engine info table only contains data for 1818 // guest supported engine types. 1819 // 1820 portMemCopy(&pEngineInfo->engineInfoList[newEngineIdx], sizeof(FIFO_ENGINE_LIST), 1821 &pEngineInfo->engineInfoList[engineIdx], sizeof(FIFO_ENGINE_LIST)); 1822 } 1823 1824 newEngineIdx++; 1825 } 1826 } 1827 1828 pEngineInfo->engineInfoListSize = newEngineIdx; 1829 } 1830 1831 /*! 1832 * @brief Performs an RPC into Host RM to read its device info table. 1833 * 1834 * This is necessary because in virtual environments, we cannot directly read 1835 * the device info table, and do not have the physical GPU partitioning 1836 * information to determine which engines belong to this guest, so we have Host 1837 * RM do the filtering and send us the filtered table. 1838 * 1839 * @param[in] pGpu 1840 * @param[in] pKernelFifo 1841 * 1842 * @return NV_OK if succcessful, 1843 * NV_ERR_NOT_SUPPORTED if Host RM calls this interface 1844 * NV_ERR_INVALID_STATE if host supplied invalid data 1845 * NV_STATUS supplied by RPC response from Host 1846 */ 1847 1848 NV_STATUS 1849 kfifoGetHostDeviceInfoTable_KERNEL 1850 ( 1851 OBJGPU *pGpu, 1852 KernelFifo *pKernelFifo, 1853 ENGINE_INFO *pEngineInfo, 1854 NvHandle hMigClient 1855 ) 1856 { 1857 NV_STATUS status = NV_OK; 1858 NvHandle hClient = NV01_NULL_OBJECT; 1859 NvHandle hObject = NV01_NULL_OBJECT; 1860 NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_PARAMS *pParams; 1861 NV2080_CTRL_FIFO_DEVICE_ENTRY *pHostEntries; 1862 NvU32 numEntries; 1863 NvU32 device; 1864 NvU32 entry; 1865 NvU32 numRunlists; 1866 NvU32 maxRunlistId; 1867 NvU32 maxPbdmaId; 1868 NvU32 minPbdmaFaultId; 1869 NvU32 i; 1870 struct { 1871 NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_PARAMS params; 1872 NV2080_CTRL_FIFO_DEVICE_ENTRY entries[NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_MAX_DEVICES]; 1873 } *pLocals; 1874 1875 1876 NV_ASSERT_OR_RETURN(IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu), 1877 NV_ERR_NOT_SUPPORTED); 1878 1879 // RPC call for GSP will throw INVALID_CLIENT error with NULL handles 1880 if (IS_GSP_CLIENT(pGpu)) 1881 { 1882 if (!IS_MIG_IN_USE(pGpu)) 1883 { 1884 hClient = pGpu->hInternalClient; 1885 hObject = pGpu->hInternalSubdevice; 1886 } 1887 else 1888 { 1889 hClient = hMigClient; 1890 if ((status = CliGetSubDeviceHandleFromGpu(hClient, pGpu, &hObject)) != NV_OK) 1891 { 1892 NV_PRINTF(LEVEL_ERROR, "Unable to get subdevice handle.\n"); 1893 return NV_ERR_INVALID_STATE; 1894 } 1895 } 1896 } 1897 1898 // Allocate pHostEntries and params on the heap to avoid stack overflow 1899 pLocals = portMemAllocNonPaged(sizeof(*pLocals)); 1900 NV_ASSERT_OR_RETURN((pLocals != NULL), NV_ERR_NO_MEMORY); 1901 1902 pParams = &pLocals->params; 1903 pHostEntries = pLocals->entries; 1904 1905 // 1906 // Read device info table entries from Host RM until Host indicates that 1907 // there are no more valid entries in the table (by setting bMore flag) 1908 // 1909 numEntries = 0; 1910 for (device = 0; 1911 device < NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_MAX_DEVICES; 1912 device += NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_MAX_ENTRIES) 1913 { 1914 portMemSet(pParams, 0x0, sizeof(*pParams)); 1915 pParams->baseIndex = device; 1916 1917 NV_RM_RPC_CONTROL(pGpu, 1918 hClient, 1919 hObject, 1920 NV2080_CTRL_CMD_FIFO_GET_DEVICE_INFO_TABLE, 1921 pParams, 1922 sizeof(*pParams), 1923 status); 1924 1925 if (status != NV_OK) 1926 goto cleanup; 1927 1928 // Assert that host RM didn't tell us an invalid number of entries 1929 if (pParams->numEntries > 1930 NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_MAX_ENTRIES) 1931 { 1932 DBG_BREAKPOINT(); 1933 status = NV_ERR_INVALID_STATE; 1934 goto cleanup; 1935 } 1936 1937 portMemCopy(&pHostEntries[device], pParams->numEntries * 1938 sizeof(NV2080_CTRL_FIFO_DEVICE_ENTRY), pParams->entries, pParams->numEntries * 1939 sizeof(NV2080_CTRL_FIFO_DEVICE_ENTRY)); 1940 1941 numEntries += pParams->numEntries; 1942 1943 if (!pParams->bMore) 1944 { 1945 break; 1946 } 1947 } 1948 1949 pEngineInfo->engineInfoListSize = numEntries; 1950 pEngineInfo->engineInfoList = portMemAllocNonPaged(sizeof(*pEngineInfo->engineInfoList) * 1951 pEngineInfo->engineInfoListSize); 1952 if (pEngineInfo->engineInfoList == NULL) 1953 { 1954 NV_CHECK(LEVEL_ERROR, pEngineInfo->engineInfoList != NULL); 1955 status = NV_ERR_NO_MEMORY; 1956 goto cleanup; 1957 } 1958 1959 // Copy each entry from the table 1960 numRunlists = 0; 1961 maxRunlistId = 0; 1962 maxPbdmaId = 0; 1963 minPbdmaFaultId = NV_U32_MAX; 1964 for (entry = 0; entry < numEntries; ++entry) 1965 { 1966 portMemCopy(pEngineInfo->engineInfoList[entry].engineData, 1967 ENGINE_INFO_TYPE_INVALID * sizeof(*(pEngineInfo->engineInfoList[entry].engineData)), 1968 pHostEntries[entry].engineData, 1969 ENGINE_INFO_TYPE_INVALID * sizeof(*(pEngineInfo->engineInfoList[entry].engineData))); 1970 1971 pEngineInfo->engineInfoList[entry].numPbdmas = pHostEntries[entry].numPbdmas; 1972 portMemCopy(pEngineInfo->engineInfoList[entry].pbdmaIds, 1973 FIFO_ENGINE_MAX_NUM_PBDMA * sizeof(*(pEngineInfo->engineInfoList[entry].pbdmaIds)), 1974 pHostEntries[entry].pbdmaIds, 1975 FIFO_ENGINE_MAX_NUM_PBDMA * sizeof(*(pEngineInfo->engineInfoList[entry].pbdmaIds))); 1976 1977 portMemCopy(pEngineInfo->engineInfoList[entry].pbdmaFaultIds, 1978 FIFO_ENGINE_MAX_NUM_PBDMA * sizeof(*(pEngineInfo->engineInfoList[entry].pbdmaFaultIds)), 1979 pHostEntries[entry].pbdmaFaultIds, 1980 FIFO_ENGINE_MAX_NUM_PBDMA * sizeof(*(pEngineInfo->engineInfoList[entry].pbdmaFaultIds))); 1981 1982 portStringCopy((char *)pEngineInfo->engineInfoList[entry].engineName, 1983 sizeof(pEngineInfo->engineInfoList[entry].engineName), 1984 (char *)pHostEntries[entry].engineName, 1985 FIFO_ENGINE_NAME_MAX_SIZE); 1986 1987 if (0 != pEngineInfo->engineInfoList[entry].engineData[ENGINE_INFO_TYPE_IS_ENGINE]) 1988 { 1989 numRunlists++; 1990 } 1991 maxRunlistId = NV_MAX(maxRunlistId, 1992 pHostEntries[entry].engineData[ENGINE_INFO_TYPE_RUNLIST]); 1993 1994 for (i = 0; i < pEngineInfo->engineInfoList[entry].numPbdmas; i++) 1995 { 1996 maxPbdmaId = NV_MAX(maxPbdmaId, pEngineInfo->engineInfoList[entry].pbdmaIds[i]); 1997 1998 // 1999 // SW engine while being constructed does not populate any PBDMA Fault IDs. 2000 // Hence, skipping it. 2001 // 2002 if (pEngineInfo->engineInfoList[entry].engineData[ENGINE_INFO_TYPE_ENG_DESC] != ENG_SW) 2003 { 2004 minPbdmaFaultId = NV_MIN(minPbdmaFaultId, pEngineInfo->engineInfoList[entry].pbdmaFaultIds[i]); 2005 } 2006 } 2007 } 2008 2009 if (IS_VIRTUAL(pGpu)) 2010 { 2011 _kfifoLocalizeGuestEngineData(pGpu, pKernelFifo, pEngineInfo); 2012 } 2013 2014 // 2015 // Host RM sends back a copy of their devinfo table, which includes the SW 2016 // engine. This engine has no runlist, so decrement the runlist count. 2017 // 2018 if (numRunlists > 0) 2019 { 2020 --numRunlists; 2021 } 2022 2023 pEngineInfo->numRunlists = numRunlists; 2024 pEngineInfo->maxNumRunlists = maxRunlistId + 1; 2025 pEngineInfo->maxNumPbdmas = maxPbdmaId + 1; 2026 pEngineInfo->basePbdmaFaultId = minPbdmaFaultId; 2027 2028 cleanup: 2029 portMemFree(pLocals); 2030 2031 return status; 2032 } 2033 2034 /*! 2035 * @brief Constructs EngineInfo List 2036 * 2037 * @param[in] pGpu 2038 * @param[in] pKernelFifo 2039 * 2040 * @return NV_OK if succcessful, 2041 * NV_STATUS supplied by HALs called 2042 */ 2043 NV_STATUS 2044 kfifoConstructEngineList_KERNEL 2045 ( 2046 OBJGPU *pGpu, 2047 KernelFifo *pKernelFifo 2048 ) 2049 { 2050 ENGINE_INFO *pEngineInfo = &pKernelFifo->engineInfo; 2051 2052 // Return early if EngineList is already constructed 2053 if (pEngineInfo->engineInfoList != NULL) 2054 return NV_OK; 2055 2056 if (IS_GSP_CLIENT(pGpu)) 2057 { 2058 NV_ASSERT_OK_OR_RETURN(gpuConstructDeviceInfoTable_HAL(pGpu)); 2059 } 2060 2061 NV_ASSERT_OK_OR_RETURN(kfifoGetHostDeviceInfoTable_HAL(pGpu, pKernelFifo, pEngineInfo, 0)); 2062 2063 return NV_OK; 2064 } 2065 2066 /** 2067 * @brief Create a list of channels. 2068 * 2069 * @param pGpu 2070 * @param pKernelFifo 2071 * @param pList 2072 */ 2073 NV_STATUS 2074 kfifoChannelListCreate_IMPL 2075 ( 2076 OBJGPU *pGpu, 2077 KernelFifo *pKernelFifo, 2078 CHANNEL_LIST **ppList 2079 ) 2080 { 2081 if (!ppList) 2082 return NV_ERR_INVALID_ARGUMENT; 2083 2084 *ppList = portMemAllocNonPaged(sizeof(CHANNEL_LIST)); 2085 NV_ASSERT_OR_RETURN((*ppList != NULL), NV_ERR_NO_MEMORY); 2086 2087 (*ppList)->pHead = NULL; 2088 (*ppList)->pTail = NULL; 2089 2090 return NV_OK; 2091 } 2092 2093 /** 2094 * @brief Append a channel to a channel list. 2095 * 2096 * @param pGpu 2097 * @param pKernelFifo 2098 * @param pKernelChannel 2099 * @param pList 2100 */ 2101 2102 NV_STATUS 2103 kfifoChannelListAppend_IMPL 2104 ( 2105 OBJGPU *pGpu, 2106 KernelFifo *pKernel, 2107 KernelChannel *pKernelChannel, 2108 CHANNEL_LIST *pList 2109 ) 2110 { 2111 PCHANNEL_NODE pNewNode = NULL; 2112 2113 if (!pKernelChannel || !pList) 2114 return NV_ERR_INVALID_ARGUMENT; 2115 2116 pNewNode = portMemAllocNonPaged(sizeof(CHANNEL_NODE)); 2117 NV_ASSERT_OR_RETURN((pNewNode != NULL), NV_ERR_NO_MEMORY); 2118 2119 pNewNode->pKernelChannel = pKernelChannel; 2120 pKernelChannel->refCount++; 2121 2122 pNewNode->pNext = NULL; 2123 2124 // Searching based on the ChID 2125 if (pList->pTail) 2126 { 2127 pList->pTail->pNext = pNewNode; 2128 pList->pTail = pNewNode; 2129 } 2130 else 2131 { 2132 pList->pHead = pNewNode; 2133 pList->pTail = pNewNode; 2134 } 2135 2136 return NV_OK; 2137 } 2138 2139 /** 2140 * @brief remove channel from the given channel list 2141 * look for duplicates. 2142 * 2143 * @param pGpu 2144 * @param pKernelFifo 2145 * @param pKernelChannel 2146 * @param pList 2147 */ 2148 NV_STATUS 2149 kfifoChannelListRemove_IMPL 2150 ( 2151 OBJGPU *pGpu, 2152 KernelFifo *pKernelFifo, 2153 KernelChannel *pKernelChannel, 2154 CHANNEL_LIST *pList 2155 ) 2156 { 2157 PCHANNEL_NODE pNewNode = NULL; 2158 PCHANNEL_NODE pPrevNode = NULL; 2159 PCHANNEL_NODE pTempNode = NULL; 2160 NvBool bFoundOnce = NV_FALSE; 2161 NV_STATUS status = NV_OK; 2162 2163 if (!pKernelChannel) 2164 return NV_ERR_INVALID_ARGUMENT; 2165 2166 if (!pList) 2167 return NV_ERR_INVALID_ARGUMENT; 2168 2169 pNewNode = pList->pHead; 2170 pPrevNode = NULL; 2171 2172 while (pNewNode) 2173 { 2174 2175 if (pKernelChannel != pNewNode->pKernelChannel) 2176 { 2177 pPrevNode = pNewNode; 2178 pNewNode = pNewNode->pNext; 2179 continue; 2180 } 2181 2182 // Deleting first node 2183 if (pList->pHead == pNewNode) 2184 { 2185 pList->pHead = pNewNode->pNext; 2186 } 2187 2188 // Deleting tail node 2189 if (pList->pTail == pNewNode) 2190 { 2191 pList->pTail = pPrevNode; 2192 } 2193 2194 // First node does not have previous node. 2195 if (pPrevNode) 2196 { 2197 pPrevNode->pNext = pNewNode->pNext; 2198 } 2199 2200 pTempNode = pNewNode; 2201 pNewNode = pNewNode->pNext; 2202 portMemFree(pTempNode); 2203 2204 bFoundOnce = NV_TRUE; 2205 2206 if (0 == pKernelChannel->refCount) 2207 { 2208 NV_PRINTF(LEVEL_ERROR, "RefCount for channel is not right!!!\n"); 2209 DBG_BREAKPOINT(); 2210 status = NV_ERR_GENERIC; 2211 break; 2212 } 2213 2214 pKernelChannel->refCount--; 2215 } 2216 2217 2218 if (!bFoundOnce) 2219 { 2220 NV_PRINTF(LEVEL_INFO, 2221 "Can't find channel in channelGroupList (Normal during RC Recovery on " 2222 "GK110+ or if software scheduling is enabled).\n"); 2223 2224 status = NV_ERR_INVALID_CHANNEL; 2225 } 2226 2227 return status; 2228 } 2229 2230 /** 2231 * @brief Destroy channel list. 2232 * 2233 * @param pGpu 2234 * @param pKernelFifo 2235 * @param pList 2236 */ 2237 NV_STATUS 2238 kfifoChannelListDestroy_IMPL 2239 ( 2240 OBJGPU *pGpu, 2241 KernelFifo *pKernel, 2242 CHANNEL_LIST *pList 2243 ) 2244 { 2245 PCHANNEL_NODE pTempNode; 2246 2247 if (!pList) 2248 return NV_OK; 2249 2250 while (pList->pHead) 2251 { 2252 pTempNode = pList->pHead; 2253 2254 NV_ASSERT_OR_RETURN(pTempNode->pKernelChannel && pTempNode->pKernelChannel->refCount, NV_ERR_INVALID_STATE); 2255 2256 pTempNode->pKernelChannel->refCount--; 2257 2258 pList->pHead = pTempNode->pNext; 2259 portMemFree(pTempNode); 2260 } 2261 2262 portMemFree(pList); 2263 2264 return NV_OK; 2265 } 2266 2267 /*! 2268 * @brief Determines whether provided engines have any channels/contexts assigned 2269 * 2270 * @param[IN] pGpu OBJGPU 2271 * @param[IN] pKernelFifo KernelFifo 2272 * @param[IN] pEngines Which engines to check (RM_ENGINE_TYPE_***) 2273 * @param[IN] engineCount Number of engines to check 2274 * 2275 * @return Returns NV_TRUE if any provided engines are active 2276 */ 2277 NvBool 2278 kfifoEngineListHasChannel_IMPL 2279 ( 2280 OBJGPU *pGpu, 2281 KernelFifo *pKernelFifo, 2282 RM_ENGINE_TYPE *pEngines, 2283 NvU32 engineCount 2284 ) 2285 { 2286 KernelChannel *pKernelChannel; 2287 CHANNEL_ITERATOR it; 2288 NvU32 i; 2289 2290 NV_ASSERT_OR_RETURN((pEngines != NULL) && (engineCount > 0), NV_TRUE); 2291 2292 // Find any channels or contexts on passed engines 2293 kfifoGetChannelIterator(pGpu, pKernelFifo, &it); 2294 while (kchannelGetNextKernelChannel(pGpu, &it, &pKernelChannel) == NV_OK) 2295 { 2296 NV_ASSERT_OR_ELSE(pKernelChannel != NULL, continue); 2297 2298 // If the client supplied the engine type, directly check it 2299 if (RM_ENGINE_TYPE_IS_VALID(kchannelGetEngineType(pKernelChannel))) 2300 { 2301 for (i = 0; i < engineCount; ++i) 2302 { 2303 if (kchannelGetEngineType(pKernelChannel) == pEngines[i]) 2304 { 2305 NV_PRINTF(LEVEL_ERROR, 2306 "Found channel on engine 0x%x owned by 0x%x\n", 2307 kchannelGetEngineType(pKernelChannel), RES_GET_CLIENT_HANDLE(pKernelChannel)); 2308 2309 return NV_TRUE; 2310 } 2311 } 2312 } 2313 else 2314 { 2315 NvU32 runlistId; 2316 2317 // Ideally valid engine Id should always be set in channel if this property is enabled 2318 NV_ASSERT_OR_RETURN(!kfifoIsPerRunlistChramEnabled(pKernelFifo), NV_TRUE); 2319 2320 // 2321 // If runlist Id for channel is set then check if it matches with any of the engines 2322 // If channel is not associated with any engine then there is a chance 2323 // it can be created on one of the engines we care about. 2324 // 2325 if (kchannelIsRunlistSet(pGpu, pKernelChannel)) 2326 { 2327 for (i = 0; i < engineCount; ++i) 2328 { 2329 NV_ASSERT_OR_RETURN((kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo, 2330 ENGINE_INFO_TYPE_RM_ENGINE_TYPE, (NvU32)pEngines[i], 2331 ENGINE_INFO_TYPE_RUNLIST, &runlistId) == NV_OK), NV_TRUE); 2332 if (kchannelGetRunlistId(pKernelChannel) == runlistId) 2333 { 2334 NV_PRINTF(LEVEL_ERROR, 2335 "Found channel on runlistId 0x%x owned by 0x%x\n", 2336 kchannelGetRunlistId(pKernelChannel), RES_GET_CLIENT_HANDLE(pKernelChannel)); 2337 2338 return NV_TRUE; 2339 } 2340 } 2341 } 2342 else 2343 { 2344 NV_PRINTF(LEVEL_ERROR, 2345 "Found channel owned by 0x%x that can be associated to any engine\n", 2346 RES_GET_CLIENT_HANDLE(pKernelChannel)); 2347 2348 return NV_TRUE; 2349 } 2350 } 2351 } 2352 2353 return NV_FALSE; 2354 } 2355 2356 /** 2357 * @brief Maximum number of subcontexts for a non-legacy mode TSG 2358 */ 2359 CTX_BUF_POOL_INFO * 2360 kfifoGetRunlistBufPool_IMPL 2361 ( 2362 OBJGPU *pGpu, 2363 KernelFifo *pKernelFifo, 2364 RM_ENGINE_TYPE rmEngineType 2365 ) 2366 { 2367 return pKernelFifo->pRunlistBufPool[rmEngineType]; 2368 } 2369 2370 /** 2371 * @brief Get size and alignment requirements for runlist buffers 2372 * 2373 * @param[in] pGpu Pointer to OBJGPU 2374 * @param[in] pKernelFifo Pointer to KernelFifo 2375 * @param[in] runlistId Runlist ID 2376 * @param[in] bTsgSupported Is TSG supported 2377 * @param[in] maxRunlistEntries Max entries to be supported in a runlist 2378 * @param[out] pSize Size of runlist buffer 2379 * @param[out] pAlignment Alignment for runlist buffer 2380 */ 2381 NV_STATUS 2382 kfifoGetRunlistBufInfo_IMPL 2383 ( 2384 OBJGPU *pGpu, 2385 KernelFifo *pKernelFifo, 2386 NvU32 runlistId, 2387 NvBool bTsgSupported, 2388 NvU32 maxRunlistEntries, 2389 NvU64 *pSize, 2390 NvU64 *pAlignment 2391 ) 2392 { 2393 NvU32 runlistEntrySize = 0; 2394 NvU32 maxRunlistEntriesSupported = 0; 2395 CHID_MGR *pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, runlistId); 2396 2397 NV_ASSERT_OR_RETURN(pSize != NULL, NV_ERR_INVALID_ARGUMENT); 2398 NV_ASSERT_OR_RETURN(pAlignment != NULL, NV_ERR_INVALID_ARGUMENT); 2399 2400 if (kfifoIsPerRunlistChramEnabled(pKernelFifo)) 2401 { 2402 NV_ASSERT_OR_RETURN(pChidMgr != NULL, NV_ERR_INVALID_ARGUMENT); 2403 // 2404 // We assume worst case of one TSG wrapper per channel, and 2405 // the number of TSGs + number of channels is how we get 2406 // the 2 x number of fifos. 2407 // 2408 maxRunlistEntriesSupported = 2 * kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr); 2409 } 2410 else 2411 { 2412 maxRunlistEntriesSupported = kfifoGetMaxChannelsInSystem(pGpu, pKernelFifo); 2413 maxRunlistEntriesSupported += (bTsgSupported ? 2414 kfifoGetMaxChannelGroupsInSystem(pGpu, pKernelFifo) 2415 : 0); 2416 } 2417 2418 NV_ASSERT_OR_RETURN(maxRunlistEntries <= maxRunlistEntriesSupported, NV_ERR_INVALID_ARGUMENT); 2419 2420 if (maxRunlistEntries == 0) 2421 { 2422 maxRunlistEntries = maxRunlistEntriesSupported; 2423 } 2424 2425 runlistEntrySize = kfifoRunlistGetEntrySize_HAL(pKernelFifo); 2426 *pSize = (NvU64)runlistEntrySize * maxRunlistEntries; 2427 2428 *pAlignment = NVBIT64(kfifoRunlistGetBaseShift_HAL(pKernelFifo)); 2429 return NV_OK; 2430 } 2431 2432 /*! 2433 * @brief Gets total number of channels supported by the system 2434 */ 2435 NvU32 2436 kfifoGetMaxChannelsInSystem_IMPL 2437 ( 2438 OBJGPU *pGpu, 2439 KernelFifo *pKernelFifo 2440 ) 2441 { 2442 NvU32 numChannels = 0; 2443 NvU32 i; 2444 2445 for (i = 0; i < pKernelFifo->numChidMgrs; i++) 2446 { 2447 if (pKernelFifo->ppChidMgr[i] != NULL) 2448 { 2449 numChannels += kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pKernelFifo->ppChidMgr[i]); 2450 } 2451 } 2452 return numChannels; 2453 } 2454 2455 /*! 2456 * @brief Gets total number of channel groups supported by the system 2457 */ 2458 NvU32 2459 kfifoGetMaxChannelGroupsInSystem_IMPL 2460 ( 2461 OBJGPU *pGpu, 2462 KernelFifo *pKernelFifo 2463 ) 2464 { 2465 // Max channel groups is the same as max channels 2466 return kfifoGetMaxChannelsInSystem(pGpu, pKernelFifo); 2467 } 2468 2469 /*! 2470 * @brief Get runlist buffer allocation params 2471 * 2472 * @param[in] pGpu 2473 * @param[out] *pAperture Aperture to use for runlist buffer allocation 2474 * @param[out] *pAttr Attributes to use for runlits buffer allocation 2475 * @param[out] *pAllocFlags Allocation flags to use for runlist buffer allocation 2476 */ 2477 void 2478 kfifoRunlistGetBufAllocParams_IMPL 2479 ( 2480 OBJGPU *pGpu, 2481 NV_ADDRESS_SPACE *pAperture, 2482 NvU32 *pAttr, 2483 NvU64 *pAllocFlags 2484 ) 2485 { 2486 *pAperture = ADDR_FBMEM; 2487 *pAttr = NV_MEMORY_WRITECOMBINED; 2488 2489 memdescOverrideInstLoc(DRF_VAL(_REG_STR_RM, _INST_LOC, _RUNLIST, pGpu->instLocOverrides), 2490 "runlist", pAperture, pAttr); 2491 2492 *pAllocFlags = FLD_TEST_DRF(_REG_STR_RM, _INST_VPR, _RUNLIST, _TRUE, pGpu->instVprOverrides) 2493 ? MEMDESC_ALLOC_FLAGS_PROTECTED : MEMDESC_FLAGS_NONE; 2494 } 2495 2496 /*! 2497 * @brief Allocate Runlist buffers for a single runlistId 2498 * 2499 * @param[in] pGpu 2500 * @param[in] pKernelFifo 2501 * @param[in] bSupportTsg Will this runlist support TSGs? 2502 * @param[in] aperture NV_ADDRESS_SPACE requested 2503 * @param[in] runlistId runlistId to allocate buffer for 2504 * @param[in] attr CPU cacheability requested 2505 * @param[in] allocFlags MEMDESC_FLAGS_* 2506 * @param[in] maxRunlistEntries Can pass zero to determine in function 2507 * @param[in] bHWRL Is this runlist a HW runlist? (verif feature specific) 2508 * @param[out] ppMemDesc memdesc created/allocated by function 2509 */ 2510 NV_STATUS 2511 kfifoRunlistAllocBuffers_IMPL 2512 ( 2513 OBJGPU *pGpu, 2514 KernelFifo *pKernelFifo, 2515 NvBool bSupportTsg, 2516 NV_ADDRESS_SPACE aperture, 2517 NvU32 runlistId, 2518 NvU32 attr, 2519 NvU64 allocFlags, 2520 NvU64 maxRunlistEntries, 2521 NvBool bHWRL, 2522 MEMORY_DESCRIPTOR **ppMemDesc 2523 ) 2524 { 2525 NV_STATUS status = NV_OK; 2526 NvU64 runlistSz = 0; 2527 NvU64 runlistAlign = 0; 2528 NvU32 counter; 2529 2530 status = kfifoGetRunlistBufInfo(pGpu, pKernelFifo, runlistId, bSupportTsg, 2531 maxRunlistEntries, &runlistSz, &runlistAlign); 2532 if (status != NV_OK) 2533 { 2534 NV_PRINTF(LEVEL_ERROR, "failed to get runlist buffer info 0x%08x\n", 2535 status); 2536 DBG_BREAKPOINT(); 2537 goto failed; 2538 } 2539 2540 for (counter = 0; counter < NUM_BUFFERS_PER_RUNLIST; ++counter) 2541 { 2542 ppMemDesc[counter] = NULL; 2543 2544 status = memdescCreate(&ppMemDesc[counter], pGpu, runlistSz, runlistAlign, 2545 NV_TRUE, aperture, attr, allocFlags); 2546 if (status != NV_OK) 2547 { 2548 NV_PRINTF(LEVEL_ERROR, 2549 "Runlist buffer memdesc create failed 0x%08x\n", status); 2550 DBG_BREAKPOINT(); 2551 goto failed; 2552 } 2553 2554 // If flag is set then allocate runlist from ctx buf pool 2555 if (allocFlags & MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL) 2556 { 2557 RM_ENGINE_TYPE rmEngineType; 2558 CTX_BUF_POOL_INFO *pCtxBufPool = NULL; 2559 status = kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo, ENGINE_INFO_TYPE_RUNLIST, 2560 runlistId, ENGINE_INFO_TYPE_RM_ENGINE_TYPE, (NvU32 *)&rmEngineType); 2561 if (status != NV_OK) 2562 { 2563 NV_PRINTF(LEVEL_ERROR, 2564 "Failed to translate runlistId 0x%x to NV2080 engine type\n", runlistId); 2565 DBG_BREAKPOINT(); 2566 goto failed; 2567 } 2568 status = ctxBufPoolGetGlobalPool(pGpu, CTX_BUF_ID_RUNLIST, rmEngineType, &pCtxBufPool); 2569 if (status != NV_OK) 2570 { 2571 NV_PRINTF(LEVEL_ERROR, 2572 "Failed to get ctx buf pool for engine type 0x%x (0x%x)\n", 2573 gpuGetNv2080EngineType(rmEngineType), rmEngineType); 2574 DBG_BREAKPOINT(); 2575 goto failed; 2576 } 2577 status = memdescSetCtxBufPool(ppMemDesc[counter], pCtxBufPool); 2578 if (status != NV_OK) 2579 { 2580 NV_PRINTF(LEVEL_ERROR, 2581 "Failed to set ctx buf pool for runlistId 0x%x\n", runlistId); 2582 DBG_BREAKPOINT(); 2583 goto failed; 2584 } 2585 } 2586 2587 status = memdescAlloc(ppMemDesc[counter]); 2588 if (status != NV_OK) 2589 { 2590 NV_PRINTF(LEVEL_ERROR, "Runlist buffer mem alloc failed 0x%08x\n", 2591 status); 2592 DBG_BREAKPOINT(); 2593 goto failed; 2594 } 2595 } 2596 2597 return NV_OK; 2598 2599 failed: 2600 for (counter = 0; counter < NUM_BUFFERS_PER_RUNLIST; counter++) 2601 { 2602 if (ppMemDesc[counter]) 2603 { 2604 memdescFree(ppMemDesc[counter]); 2605 memdescDestroy(ppMemDesc[counter]); 2606 ppMemDesc[counter] = NULL; 2607 } 2608 } 2609 return status; 2610 } 2611 2612 NvU32 2613 kfifoGetMaxSubcontextFromGr_KERNEL 2614 ( 2615 OBJGPU *pGpu, 2616 KernelFifo *pKernelFifo 2617 ) 2618 { 2619 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 2620 2621 NV_ASSERT_OR_RETURN(pKernelGraphicsManager != NULL, 0); 2622 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized, 0); 2623 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo != NULL, 0); 2624 2625 return pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo->infoList[NV0080_CTRL_GR_INFO_INDEX_MAX_SUBCONTEXT_COUNT].data; 2626 } 2627 2628 NvU32 2629 kfifoReturnPushbufferCaps_IMPL 2630 ( 2631 OBJGPU *pGpu, 2632 KernelFifo *pKernelFifo 2633 ) 2634 { 2635 NvU32 kfifoBitMask = 0; 2636 2637 // PCI is always supported 2638 kfifoBitMask = PCI_PB_ALLOWED; 2639 2640 if (!gpuIsUnifiedMemorySpaceEnabled(pGpu)) 2641 { 2642 kfifoBitMask |= VID_PB_ALLOWED; 2643 } 2644 2645 return kfifoBitMask; 2646 } 2647 2648 /*! 2649 * @brief kfifoGetDeviceCaps 2650 * 2651 * This routine gets cap bits in unicast. If pbCapsInitialized is passed as 2652 * NV_FALSE, the caps will be copied into pKfifoCaps without OR/ANDING. 2653 * Otherwise, the caps bits for the current GPU will be ORed/ANDed together with 2654 * pKfifoCaps to create a single set of caps that accurately represents the 2655 * functionality of the device. 2656 */ 2657 void kfifoGetDeviceCaps_IMPL 2658 ( 2659 OBJGPU *pGpu, 2660 KernelFifo *pKernelFifo, 2661 NvU8 *pKfifoCaps, 2662 NvBool bCapsInitialized 2663 ) 2664 { 2665 NvU8 tempCaps[NV0080_CTRL_FIFO_CAPS_TBL_SIZE]; 2666 NvU8 temp; 2667 NvU32 kfifoBitMask; 2668 2669 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2670 2671 portMemSet(tempCaps, 0, NV0080_CTRL_FIFO_CAPS_TBL_SIZE); 2672 2673 kfifoBitMask = kfifoReturnPushbufferCaps(pGpu, pKernelFifo); 2674 2675 if (kfifoBitMask & PCI_PB_ALLOWED) 2676 RMCTRL_SET_CAP(tempCaps, NV0080_CTRL_FIFO_CAPS, _SUPPORT_PCI_PB); 2677 if (kfifoBitMask & VID_PB_ALLOWED) 2678 RMCTRL_SET_CAP(tempCaps, NV0080_CTRL_FIFO_CAPS, _SUPPORT_VID_PB); 2679 2680 if ((IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu)) && 2681 !gpuIsPipelinedPteMemEnabled(pGpu)) 2682 RMCTRL_SET_CAP(tempCaps, NV0080_CTRL_FIFO_CAPS, _NO_PIPELINED_PTE_BLIT); 2683 2684 if (kfifoIsUserdInSystemMemory(pKernelFifo)) 2685 RMCTRL_SET_CAP(tempCaps, NV0080_CTRL_FIFO_CAPS, _USERD_IN_SYSMEM); 2686 2687 if (kfifoIsUserdMapDmaSupported(pKernelFifo)) 2688 RMCTRL_SET_CAP(tempCaps, NV0080_CTRL_FIFO_CAPS, _GPU_MAP_CHANNEL); 2689 2690 if (kfifoHostHasLbOverflow(pKernelFifo)) 2691 RMCTRL_SET_CAP(tempCaps, NV0080_CTRL_FIFO_CAPS, _HAS_HOST_LB_OVERFLOW_BUG_1667921); 2692 2693 if (kfifoIsSubcontextSupported(pKernelFifo)) 2694 RMCTRL_SET_CAP(tempCaps, NV0080_CTRL_FIFO_CAPS, _MULTI_VAS_PER_CHANGRP); 2695 2696 if (kfifoIsWddmInterleavingPolicyEnabled(pKernelFifo)) 2697 RMCTRL_SET_CAP(tempCaps, NV0080_CTRL_FIFO_CAPS, _SUPPORT_WDDM_INTERLEAVING); 2698 2699 // if this is the first GPU in the device, then start with it's caps 2700 if (bCapsInitialized == NV_FALSE) 2701 { 2702 portMemCopy(pKfifoCaps, NV0080_CTRL_FIFO_CAPS_TBL_SIZE, 2703 tempCaps, NV0080_CTRL_FIFO_CAPS_TBL_SIZE); 2704 return; 2705 } 2706 2707 RMCTRL_AND_CAP(pKfifoCaps, tempCaps, temp, 2708 NV0080_CTRL_FIFO_CAPS, _SUPPORT_PCI_PB); 2709 RMCTRL_AND_CAP(pKfifoCaps, tempCaps, temp, 2710 NV0080_CTRL_FIFO_CAPS, _SUPPORT_VID_PB); 2711 RMCTRL_AND_CAP(pKfifoCaps, tempCaps, temp, 2712 NV0080_CTRL_FIFO_CAPS, _GPU_MAP_CHANNEL); 2713 2714 RMCTRL_OR_CAP(pKfifoCaps, tempCaps, temp, 2715 NV0080_CTRL_FIFO_CAPS, _MULTI_VAS_PER_CHANGRP); 2716 2717 RMCTRL_OR_CAP(pKfifoCaps, tempCaps, temp, 2718 NV0080_CTRL_FIFO_CAPS, _HAS_HOST_LB_OVERFLOW_BUG_1667921); 2719 2720 RMCTRL_OR_CAP(pKfifoCaps, tempCaps, temp, 2721 NV0080_CTRL_FIFO_CAPS, _SUPPORT_WDDM_INTERLEAVING); 2722 return; 2723 } 2724 2725 /*! 2726 * @brief Add handlers for scheduling enable and/or disable. 2727 * 2728 * @param[in] pGpu OBJGPU pointer 2729 * @param[in] pKernelFifo KernelFifo pointer 2730 * @param[in] pPostSchedulingEnableHandler No action if NULL 2731 * @param[in] pPostSchedulingEnableHandlerData Data to pass to 2732 * @p pPostSchedulingEnableHandler 2733 * @param[in] pPreSchedulingDisableHandler No action if NULL 2734 * @param[in] pPreSchedulingDisableHandlerData Data to pass to 2735 * @p pPreSchedulingDisableHandler 2736 * 2737 * @returns NV_OK if successfully processed both handlers 2738 * NV_WARN_NOTHING_TO_DO if: - Both handlers are NULL 2739 * - Both handlers are already installed 2740 * NV_ERR_INVALID_STATE if one handler is already installed, but not both 2741 */ 2742 NV_STATUS 2743 kfifoAddSchedulingHandler_IMPL 2744 ( 2745 OBJGPU *pGpu, 2746 KernelFifo *pKernelFifo, 2747 PFifoSchedulingHandler pPostSchedulingEnableHandler, 2748 void *pPostSchedulingEnableHandlerData, 2749 PFifoSchedulingHandler pPreSchedulingDisableHandler, 2750 void *pPreSchedulingDisableHandlerData 2751 ) 2752 { 2753 FifoSchedulingHandlerEntry *pEntry; 2754 NvBool bPostHandlerAlreadyPresent = NV_FALSE; 2755 NvBool bPreHandlerAlreadyPresent = NV_FALSE; 2756 FifoSchedulingHandlerEntry postEntry; 2757 FifoSchedulingHandlerEntry preEntry; 2758 2759 NV_CHECK_OR_RETURN(LEVEL_SILENT, 2760 (pPostSchedulingEnableHandler != NULL) || 2761 (pPreSchedulingDisableHandler != NULL), 2762 NV_WARN_NOTHING_TO_DO); 2763 2764 // Check for already installed handler if non-NULL 2765 if (pPostSchedulingEnableHandler != NULL) 2766 { 2767 for (pEntry = listHead(&pKernelFifo->postSchedulingEnableHandlerList); 2768 pEntry != NULL; 2769 pEntry = listNext(&pKernelFifo->postSchedulingEnableHandlerList, pEntry)) 2770 { 2771 if (pEntry->pCallback == pPostSchedulingEnableHandler && 2772 pEntry->pCallbackParam == pPostSchedulingEnableHandlerData) 2773 { 2774 bPostHandlerAlreadyPresent = NV_TRUE; 2775 break; 2776 } 2777 } 2778 } 2779 2780 // Check for already installed handler if non-NULL 2781 if (pPreSchedulingDisableHandler != NULL) 2782 { 2783 for (pEntry = listHead(&pKernelFifo->preSchedulingDisableHandlerList); 2784 pEntry != NULL; 2785 pEntry = listNext(&pKernelFifo->preSchedulingDisableHandlerList, pEntry)) 2786 { 2787 if (pEntry->pCallback == pPreSchedulingDisableHandler && 2788 pEntry->pCallbackParam == pPreSchedulingDisableHandlerData) 2789 { 2790 bPreHandlerAlreadyPresent = NV_TRUE; 2791 break; 2792 } 2793 } 2794 } 2795 2796 // 2797 // If we are installing both handlers, and one is already present, but not 2798 // the other, we will do nothing, so assert loudly in that case 2799 // 2800 if ((pPostSchedulingEnableHandler != NULL) && (pPreSchedulingDisableHandler != NULL)) 2801 { 2802 NV_ASSERT_OR_RETURN(!(bPostHandlerAlreadyPresent ^ bPreHandlerAlreadyPresent), 2803 NV_ERR_INVALID_STATE); 2804 } 2805 2806 // Return early unless all non-null handlers are not already installed 2807 NV_CHECK_OR_RETURN(LEVEL_SILENT, 2808 !bPostHandlerAlreadyPresent && !bPreHandlerAlreadyPresent, 2809 NV_WARN_NOTHING_TO_DO); 2810 2811 // Add handler entry to list unless NULL 2812 if (pPostSchedulingEnableHandler != NULL) 2813 { 2814 postEntry.pCallback = pPostSchedulingEnableHandler; 2815 postEntry.pCallbackParam = pPostSchedulingEnableHandlerData; 2816 postEntry.bHandled = NV_FALSE; 2817 NV_ASSERT_OR_RETURN(listPrependValue(&pKernelFifo->postSchedulingEnableHandlerList, &postEntry), 2818 NV_ERR_NO_MEMORY); 2819 } 2820 2821 // Add handler entry to list unless NULL 2822 if (pPreSchedulingDisableHandler != NULL) 2823 { 2824 preEntry.pCallback = pPreSchedulingDisableHandler; 2825 preEntry.pCallbackParam = pPreSchedulingDisableHandlerData; 2826 preEntry.bHandled = NV_FALSE; 2827 NV_ASSERT_OR_RETURN(listPrependValue(&pKernelFifo->preSchedulingDisableHandlerList, &preEntry), 2828 NV_ERR_NO_MEMORY); 2829 } 2830 2831 return NV_OK; 2832 } 2833 2834 /*! 2835 * @brief Remove handlers for scheduling enable and/or disable. 2836 * 2837 * @param[in] pGpu OBJGPU pointer 2838 * @param[in] pKernelFifo KernelFifo pointer 2839 * @param[in] pPostSchedulingEnableHandler No action if NULL 2840 * @param[in] pPostSchedulingEnableHandlerData Data argument set for the 2841 * handler. 2842 * @param[in] pPreSchedulingDisableHandler No action if NULL 2843 * @param[in] pPreSchedulingDisableHandlerData Data argument set for the 2844 * handler. 2845 */ 2846 void 2847 kfifoRemoveSchedulingHandler_IMPL 2848 ( 2849 OBJGPU *pGpu, 2850 KernelFifo *pKernelFifo, 2851 PFifoSchedulingHandler pPostSchedulingEnableHandler, 2852 void *pPostSchedulingEnableHandlerData, 2853 PFifoSchedulingHandler pPreSchedulingDisableHandler, 2854 void *pPreSchedulingDisableHandlerData 2855 ) 2856 { 2857 FifoSchedulingHandlerEntry *pEntry; 2858 FifoSchedulingHandlerEntry *pTemp; 2859 2860 // Search for the post handler in the post handler list and remove it if present 2861 pEntry = listHead(&pKernelFifo->postSchedulingEnableHandlerList); 2862 while (pEntry != NULL) 2863 { 2864 pTemp = listNext(&pKernelFifo->postSchedulingEnableHandlerList, pEntry); 2865 2866 if (pEntry->pCallback == pPostSchedulingEnableHandler && 2867 pEntry->pCallbackParam == pPostSchedulingEnableHandlerData) 2868 { 2869 listRemove(&pKernelFifo->postSchedulingEnableHandlerList, pEntry); 2870 } 2871 2872 pEntry = pTemp; 2873 } 2874 2875 // Search for the pre handler in the pre handler list and remove it if present 2876 pEntry = listHead(&pKernelFifo->preSchedulingDisableHandlerList); 2877 while (pEntry != NULL) 2878 { 2879 pTemp = listNext(&pKernelFifo->preSchedulingDisableHandlerList, pEntry); 2880 2881 if (pEntry->pCallback == pPreSchedulingDisableHandler && 2882 pEntry->pCallbackParam == pPreSchedulingDisableHandlerData) 2883 { 2884 listRemove(&pKernelFifo->preSchedulingDisableHandlerList, pEntry); 2885 } 2886 2887 pEntry = pTemp; 2888 } 2889 } 2890 2891 2892 /*! 2893 * @brief Notify handlers that scheduling has been enabled. 2894 * 2895 * @param[in] pGpu OBJGPU pointer 2896 * @param[in] pKernelFifo KernelFifo pointer 2897 * 2898 * @returns NV_STATUS 2899 */ 2900 NV_STATUS 2901 kfifoTriggerPostSchedulingEnableCallback_IMPL 2902 ( 2903 OBJGPU *pGpu, 2904 KernelFifo *pKernelFifo 2905 ) 2906 { 2907 NV_STATUS status = NV_OK; 2908 FifoSchedulingHandlerEntry *pEntry; 2909 NvBool bRetry = NV_FALSE; 2910 2911 for (pEntry = listHead(&pKernelFifo->postSchedulingEnableHandlerList); 2912 pEntry != NULL; 2913 pEntry = listNext(&pKernelFifo->postSchedulingEnableHandlerList, pEntry)) 2914 { 2915 NV_ASSERT_OR_ELSE(pEntry->pCallback != NULL, 2916 status = NV_ERR_INVALID_STATE; break;); 2917 2918 pEntry->bHandled = NV_FALSE; 2919 status = pEntry->pCallback(pGpu, pEntry->pCallbackParam); 2920 2921 // Retry mechanism: Some callbacks depend on other callbacks in this list. 2922 bRetry = bRetry || (status == NV_WARN_MORE_PROCESSING_REQUIRED); 2923 2924 if (status == NV_WARN_MORE_PROCESSING_REQUIRED) 2925 // Quash retry status 2926 status = NV_OK; 2927 else if (status == NV_OK) 2928 // Successfully handled, no need to retry 2929 pEntry->bHandled = NV_TRUE; 2930 else 2931 // Actual error, abort 2932 break; 2933 } 2934 2935 // If we hit an actual error or completed everything successfully, return early. 2936 if ((status != NV_OK) || !bRetry) 2937 return status; 2938 2939 // Second pass, retry anything that asked nicely to be deferred 2940 for (pEntry = listHead(&pKernelFifo->postSchedulingEnableHandlerList); 2941 pEntry != NULL; 2942 pEntry = listNext(&pKernelFifo->postSchedulingEnableHandlerList, pEntry)) 2943 { 2944 NV_ASSERT_OR_ELSE(pEntry->pCallback != NULL, 2945 status = NV_ERR_INVALID_STATE; break;); 2946 2947 // Skip anything that was completed successfully 2948 if (pEntry->bHandled) 2949 continue; 2950 2951 NV_CHECK_OK_OR_ELSE(status, LEVEL_ERROR, 2952 pEntry->pCallback(pGpu, pEntry->pCallbackParam), 2953 break; ); 2954 } 2955 2956 return status; 2957 } 2958 2959 2960 /*! 2961 * @brief Notify handlers that scheduling will soon be disabled. 2962 * 2963 * @param[in] pGpu OBJGPU pointer 2964 * @param[in] pKernelFifo KernelFifo pointer 2965 * 2966 * @returns NV_STATUS 2967 */ 2968 NV_STATUS 2969 kfifoTriggerPreSchedulingDisableCallback_IMPL 2970 ( 2971 OBJGPU *pGpu, 2972 KernelFifo *pKernelFifo 2973 ) 2974 { 2975 NV_STATUS status = NV_OK; 2976 FifoSchedulingHandlerEntry *pEntry; 2977 NvBool bRetry = NV_FALSE; 2978 2979 // First pass 2980 for (pEntry = listHead(&pKernelFifo->preSchedulingDisableHandlerList); 2981 pEntry != NULL; 2982 pEntry = listNext(&pKernelFifo->preSchedulingDisableHandlerList, pEntry)) 2983 { 2984 NV_ASSERT_OR_ELSE(pEntry->pCallback != NULL, 2985 status = NV_ERR_INVALID_STATE; break;); 2986 2987 pEntry->bHandled = NV_FALSE; 2988 status = pEntry->pCallback(pGpu, pEntry->pCallbackParam); 2989 2990 // Retry mechanism: Some callbacks depend on other callbacks in this list. 2991 bRetry = bRetry || (status == NV_WARN_MORE_PROCESSING_REQUIRED); 2992 2993 if (status == NV_WARN_MORE_PROCESSING_REQUIRED) 2994 // Quash retry status 2995 status = NV_OK; 2996 else if (status == NV_OK) 2997 // Successfully handled, no need to retry 2998 pEntry->bHandled = NV_TRUE; 2999 else 3000 // Actual error, abort 3001 break; 3002 } 3003 3004 // If we hit an actual error or completed everything successfully, return early. 3005 if ((status != NV_OK) || !bRetry) 3006 return status; 3007 3008 // Second pass, retry anything that asked nicely to be deferred 3009 for (pEntry = listHead(&pKernelFifo->preSchedulingDisableHandlerList); 3010 pEntry != NULL; 3011 pEntry = listNext(&pKernelFifo->preSchedulingDisableHandlerList, pEntry)) 3012 { 3013 NV_ASSERT_OR_ELSE(pEntry->pCallback != NULL, 3014 status = NV_ERR_INVALID_STATE; break;); 3015 3016 // Skip anything that was completed successfully 3017 if (pEntry->bHandled) 3018 continue; 3019 3020 NV_CHECK_OK_OR_ELSE(status, LEVEL_ERROR, 3021 pEntry->pCallback(pGpu, pEntry->pCallbackParam), 3022 break; ); 3023 } 3024 3025 return status; 3026 } 3027 3028 /** 3029 * @brief Gets vChid corresponding to a sChid 3030 */ 3031 NV_STATUS 3032 kfifoGetVChIdForSChId_FWCLIENT 3033 ( 3034 OBJGPU *pGpu, 3035 KernelFifo *pKernelFifo, 3036 NvU32 sChId, 3037 NvU32 gfid, 3038 NvU32 engineId, 3039 NvU32 *pVChid 3040 ) 3041 { 3042 KERNEL_HOST_VGPU_DEVICE *pKernelHostVgpuDevice = NULL; 3043 3044 NV_ASSERT_OR_RETURN(pVChid != NULL, NV_ERR_INVALID_ARGUMENT); 3045 *pVChid = sChId; 3046 3047 NV_CHECK_OR_RETURN(LEVEL_INFO, IS_GFID_VF(gfid), NV_OK); 3048 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextKernelHostVgpuDevice(pGpu, &pKernelHostVgpuDevice)); 3049 NV_ASSERT_OR_RETURN(pKernelHostVgpuDevice->gfid == gfid, NV_ERR_INVALID_ARGUMENT); 3050 NV_ASSERT_OR_RETURN(engineId < (sizeof(pKernelHostVgpuDevice->chidOffset) / sizeof(pKernelHostVgpuDevice->chidOffset[0])), 3051 NV_ERR_INVALID_STATE); 3052 NV_ASSERT_OR_RETURN(pKernelHostVgpuDevice->chidOffset[engineId] != 0, NV_ERR_INVALID_STATE); 3053 3054 NV_ASSERT_OR_RETURN(sChId >= pKernelHostVgpuDevice->chidOffset[engineId], NV_ERR_INVALID_ARGUMENT); 3055 3056 *pVChid = sChId - pKernelHostVgpuDevice->chidOffset[engineId]; 3057 3058 return NV_OK; 3059 } 3060 3061 /** 3062 * @brief cache vChid <-> sChid offset 3063 */ 3064 NV_STATUS 3065 kfifoSetChidOffset_IMPL 3066 ( 3067 OBJGPU *pGpu, 3068 KernelFifo *pKernelFifo, 3069 CHID_MGR *pChidMgr, 3070 NvU32 offset, 3071 NvU32 numChannels, 3072 NvU32 gfid, 3073 NvU32 *pChidOffset, 3074 NvU32 hMigClient, 3075 NvU32 engineFifoListNumEntries, 3076 FIFO_ENGINE_LIST *pEngineFifoList 3077 ) 3078 { 3079 NV_STATUS status = NV_OK; 3080 RM_ENGINE_TYPE *pEngineIds = NULL; 3081 NvU32 maxEngines = kfifoGetNumEngines_HAL(pGpu, pKernelFifo); 3082 NvU32 numEngines, i; 3083 3084 NV_ASSERT_OR_RETURN(pChidMgr != NULL, NV_ERR_INVALID_ARGUMENT); 3085 NV_ASSERT_OR_RETURN(pChidOffset != NULL, NV_ERR_INVALID_ARGUMENT); 3086 3087 pEngineIds = portMemAllocNonPaged(sizeof(RM_ENGINE_TYPE) * maxEngines); 3088 NV_ASSERT_OR_RETURN((pEngineIds != NULL), NV_ERR_NO_MEMORY); 3089 portMemSet(pEngineIds, 0, sizeof(RM_ENGINE_TYPE) * maxEngines); 3090 3091 NV_ASSERT_OK_OR_GOTO(status, kfifoGetEngineListForRunlist(pGpu, pKernelFifo, pChidMgr->runlistId, pEngineIds, &numEngines), cleanup) 3092 3093 for (i = 0; i < numEngines; i++) 3094 { 3095 NV_ASSERT_OR_ELSE(NV2080_ENGINE_TYPE_IS_VALID(pEngineIds[i]), status = NV_ERR_INVALID_STATE; goto cleanup); 3096 pChidOffset[pEngineIds[i]] = offset; 3097 } 3098 3099 NV_ASSERT_OK_OR_GOTO(status, kfifoProgramChIdTable_HAL(pGpu, pKernelFifo, pChidMgr, offset, numChannels, gfid, 3100 hMigClient, engineFifoListNumEntries, pEngineFifoList), cleanup); 3101 3102 cleanup: 3103 3104 portMemFree(pEngineIds); 3105 3106 return status; 3107 } 3108 3109 /* 3110 * @brief Gets a list of engine ids that use this runlist 3111 * 3112 * @param[in] runlistId Runlist id 3113 * @param[out] pOutEngineIds List of engineids 3114 * @param[out] pNumEngines # of entries in pOutEngines 3115 */ 3116 NV_STATUS 3117 kfifoGetEngineListForRunlist_IMPL 3118 ( 3119 OBJGPU *pGpu, 3120 KernelFifo *pKernelFifo, 3121 NvU32 runlistId, 3122 RM_ENGINE_TYPE *pOutEngineIds, 3123 NvU32 *pNumEngines 3124 ) 3125 { 3126 NV_STATUS status = NV_OK; 3127 NvU32 numEngines = kfifoGetNumEngines_HAL(pGpu, pKernelFifo); 3128 NvU32 i; 3129 3130 // Sanity check the input 3131 NV_CHECK_OR_RETURN(LEVEL_ERROR, pOutEngineIds != NULL, NV_ERR_INVALID_ARGUMENT); 3132 NV_CHECK_OR_RETURN(LEVEL_ERROR, pNumEngines != NULL, NV_ERR_INVALID_ARGUMENT); 3133 3134 *pNumEngines = 0; 3135 NV_PRINTF(LEVEL_INFO, "Engine list for runlistId 0x%x:\n", runlistId); 3136 3137 for (i = 0; i < numEngines; i++) 3138 { 3139 RM_ENGINE_TYPE rmEngineType; 3140 NvU32 thisRunlistId; 3141 3142 NV_ASSERT_OK_OR_GOTO(status, 3143 kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo, 3144 ENGINE_INFO_TYPE_INVALID, 3145 i, 3146 ENGINE_INFO_TYPE_RUNLIST, 3147 &thisRunlistId), done); 3148 if (runlistId == thisRunlistId) 3149 { 3150 NV_ASSERT_OK_OR_GOTO(status, 3151 kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo, 3152 ENGINE_INFO_TYPE_INVALID, 3153 i, 3154 ENGINE_INFO_TYPE_RM_ENGINE_TYPE, 3155 (NvU32 *)&rmEngineType), done); 3156 pOutEngineIds[(*pNumEngines)++] = rmEngineType; 3157 3158 NV_PRINTF(LEVEL_INFO, "Engine name: %s\n", 3159 kfifoGetEngineName_HAL(pKernelFifo, ENGINE_INFO_TYPE_RM_ENGINE_TYPE, 3160 (NvU32)rmEngineType)); 3161 } 3162 } 3163 done: 3164 if ((status != NV_OK) && (*pNumEngines != 0)) 3165 { 3166 portMemSet(pOutEngineIds, 0, sizeof(NvU32) * (*pNumEngines)); 3167 *pNumEngines = 0; 3168 } 3169 return status; 3170 } 3171 3172 /** 3173 * @brief Return bitmask of currently allocated channels for a given runlist 3174 * 3175 * @param[in] bitMaskSize @p pBitMask size in bytes 3176 * 3177 */ 3178 NV_STATUS 3179 kfifoGetAllocatedChannelMask_IMPL 3180 ( 3181 OBJGPU *pGpu, 3182 KernelFifo *pKernelFifo, 3183 NvU32 runlistId, 3184 NvU32 *pBitMask, 3185 NvLength bitMaskSize 3186 ) 3187 { 3188 CHID_MGR *pChidMgr; 3189 NvU32 chId; 3190 NvU32 numChannels; 3191 3192 NV_ASSERT(pBitMask != NULL); 3193 portMemSet(pBitMask, 0, bitMaskSize); 3194 3195 NV_ASSERT_OR_RETURN(bitMaskSize % sizeof(NvU32) == 0, 3196 NV_ERR_INVALID_ARGUMENT); 3197 3198 if (!kfifoIsPerRunlistChramEnabled(pKernelFifo)) 3199 { 3200 if (runlistId > 0) 3201 { 3202 return NV_ERR_OUT_OF_RANGE; 3203 } 3204 else 3205 { 3206 runlistId = CHIDMGR_RUNLIST_ID_LEGACY; 3207 } 3208 } 3209 else 3210 { 3211 if (!(runlistId < kfifoGetMaxNumRunlists_HAL(pGpu, pKernelFifo))) 3212 { 3213 return NV_ERR_OUT_OF_RANGE; 3214 } 3215 } 3216 3217 pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, runlistId); 3218 if (pChidMgr == NULL) 3219 { 3220 // 3221 // This runlist is not valid. This is not an error since it might be 3222 // possible for some runlists between [0, maxRunlists) to be invalid. 3223 // Simply return that it has no channels to simplify clients iterating 3224 // over all runlists. 3225 // 3226 numChannels = 0; 3227 } 3228 else 3229 { 3230 numChannels = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr); 3231 } 3232 3233 if (((numChannels + 7) / 8) > bitMaskSize) 3234 { 3235 return NV_ERR_BUFFER_TOO_SMALL; 3236 } 3237 3238 for (chId = 0; chId < numChannels; chId++) 3239 { 3240 KernelChannel *pKernelChannel; 3241 pKernelChannel = kfifoChidMgrGetKernelChannel(pGpu, pKernelFifo, 3242 pChidMgr, 3243 chId); 3244 if (pKernelChannel != NULL) 3245 { 3246 NV_BITMASK32_SET(pBitMask, chId); 3247 } 3248 } 3249 3250 return NV_OK; 3251 } 3252 3253 /*! 3254 * Get host channel class 3255 */ 3256 NvU32 3257 kfifoGetChannelClassId_IMPL 3258 ( 3259 OBJGPU *pGpu, 3260 KernelFifo *pKernelFifo 3261 ) 3262 { 3263 NvU32 numClasses; 3264 NvU32 *pClassList = NULL; 3265 CLI_CHANNEL_CLASS_INFO classInfo; 3266 NvU32 i; 3267 NvU32 class = 0; 3268 3269 NV_ASSERT_OR_RETURN(NV_OK == gpuGetClassList(pGpu, &numClasses, NULL, ENG_KERNEL_FIFO), 0); 3270 NV_ASSERT_OR_RETURN(numClasses > 0, 0); 3271 pClassList = portMemAllocNonPaged(sizeof(NvU32) * numClasses); 3272 NV_ASSERT_OR_RETURN((pClassList != NULL), 0); 3273 3274 if (NV_OK == gpuGetClassList(pGpu, &numClasses, pClassList, ENG_KERNEL_FIFO)) 3275 { 3276 for (i = 0; i < numClasses; i++) 3277 { 3278 if (pClassList[i] == PHYSICAL_CHANNEL_GPFIFO) 3279 { 3280 // Skip the physical channel class 3281 continue; 3282 } 3283 CliGetChannelClassInfo(pClassList[i], &classInfo); 3284 if (classInfo.classType == CHANNEL_CLASS_TYPE_GPFIFO) 3285 class = NV_MAX(class, pClassList[i]); 3286 } 3287 } 3288 3289 NV_ASSERT(class); 3290 portMemFree(pClassList); 3291 return class; 3292 } 3293 3294 /** 3295 * @brief Return the FIFO_GUEST_ENGINE_TABLE 3296 * 3297 * @param[out] pEngLookupTblSize To get the table size 3298 * 3299 * @return a pointer to FIFO_GUEST_ENGINE_TABLE 3300 * 3301 */ 3302 const FIFO_GUEST_ENGINE_TABLE * 3303 kfifoGetGuestEngineLookupTable_IMPL 3304 ( 3305 NvU32 *pEngLookupTblSize 3306 ) 3307 { 3308 // 3309 // This table is used for a guest RM to reconstruct the engine list data 3310 // received from the host RM. The host and guest RM can be running on 3311 // different versions. 3312 // 3313 // This table does not need to be HALified. It need to match NV2080_ENGINE_TYPE 3314 // defined in cl2080_notification.h 3315 // 3316 const static FIFO_GUEST_ENGINE_TABLE guestEngineLookupTable[] = 3317 { 3318 // nv2080EngineType mcIdx 3319 {NV2080_ENGINE_TYPE_GR0, MC_ENGINE_IDX_GR0}, 3320 {NV2080_ENGINE_TYPE_GR1, MC_ENGINE_IDX_GR1}, 3321 {NV2080_ENGINE_TYPE_GR2, MC_ENGINE_IDX_GR2}, 3322 {NV2080_ENGINE_TYPE_GR3, MC_ENGINE_IDX_GR3}, 3323 {NV2080_ENGINE_TYPE_GR4, MC_ENGINE_IDX_GR4}, 3324 {NV2080_ENGINE_TYPE_GR5, MC_ENGINE_IDX_GR5}, 3325 {NV2080_ENGINE_TYPE_GR6, MC_ENGINE_IDX_GR6}, 3326 {NV2080_ENGINE_TYPE_GR7, MC_ENGINE_IDX_GR7}, 3327 {NV2080_ENGINE_TYPE_COPY0, MC_ENGINE_IDX_CE0}, 3328 {NV2080_ENGINE_TYPE_COPY1, MC_ENGINE_IDX_CE1}, 3329 {NV2080_ENGINE_TYPE_COPY2, MC_ENGINE_IDX_CE2}, 3330 {NV2080_ENGINE_TYPE_COPY3, MC_ENGINE_IDX_CE3}, 3331 {NV2080_ENGINE_TYPE_COPY4, MC_ENGINE_IDX_CE4}, 3332 {NV2080_ENGINE_TYPE_COPY5, MC_ENGINE_IDX_CE5}, 3333 {NV2080_ENGINE_TYPE_COPY6, MC_ENGINE_IDX_CE6}, 3334 {NV2080_ENGINE_TYPE_COPY7, MC_ENGINE_IDX_CE7}, 3335 {NV2080_ENGINE_TYPE_COPY8, MC_ENGINE_IDX_CE8}, 3336 {NV2080_ENGINE_TYPE_COPY9, MC_ENGINE_IDX_CE9}, 3337 {NV2080_ENGINE_TYPE_NVENC0, MC_ENGINE_IDX_MSENC}, 3338 {NV2080_ENGINE_TYPE_NVENC1, MC_ENGINE_IDX_MSENC1}, 3339 {NV2080_ENGINE_TYPE_NVENC2, MC_ENGINE_IDX_MSENC2}, 3340 {NV2080_ENGINE_TYPE_NVDEC0, MC_ENGINE_IDX_NVDEC0}, 3341 {NV2080_ENGINE_TYPE_NVDEC1, MC_ENGINE_IDX_NVDEC1}, 3342 {NV2080_ENGINE_TYPE_NVDEC2, MC_ENGINE_IDX_NVDEC2}, 3343 {NV2080_ENGINE_TYPE_NVDEC3, MC_ENGINE_IDX_NVDEC3}, 3344 {NV2080_ENGINE_TYPE_NVDEC4, MC_ENGINE_IDX_NVDEC4}, 3345 {NV2080_ENGINE_TYPE_NVDEC5, MC_ENGINE_IDX_NVDEC5}, 3346 {NV2080_ENGINE_TYPE_NVDEC6, MC_ENGINE_IDX_NVDEC6}, 3347 {NV2080_ENGINE_TYPE_NVDEC7, MC_ENGINE_IDX_NVDEC7}, 3348 {NV2080_ENGINE_TYPE_SW, MC_ENGINE_IDX_NULL}, 3349 {NV2080_ENGINE_TYPE_SEC2, MC_ENGINE_IDX_SEC2}, 3350 {NV2080_ENGINE_TYPE_NVJPEG0, MC_ENGINE_IDX_NVJPEG0}, 3351 {NV2080_ENGINE_TYPE_NVJPEG1, MC_ENGINE_IDX_NVJPEG1}, 3352 {NV2080_ENGINE_TYPE_NVJPEG2, MC_ENGINE_IDX_NVJPEG2}, 3353 {NV2080_ENGINE_TYPE_NVJPEG3, MC_ENGINE_IDX_NVJPEG3}, 3354 {NV2080_ENGINE_TYPE_NVJPEG4, MC_ENGINE_IDX_NVJPEG4}, 3355 {NV2080_ENGINE_TYPE_NVJPEG5, MC_ENGINE_IDX_NVJPEG5}, 3356 {NV2080_ENGINE_TYPE_NVJPEG6, MC_ENGINE_IDX_NVJPEG6}, 3357 {NV2080_ENGINE_TYPE_NVJPEG7, MC_ENGINE_IDX_NVJPEG7}, 3358 {NV2080_ENGINE_TYPE_OFA, MC_ENGINE_IDX_OFA0}, 3359 }; 3360 3361 // 3362 // To trap NV2080_ENGINE_TYPE expansions. 3363 // Please update the table guestEngineLookupTable if this assertion is triggered. 3364 // 3365 ct_assert(NV2080_ENGINE_TYPE_LAST == 0x0000003e); 3366 3367 *pEngLookupTblSize = NV_ARRAY_ELEMENTS(guestEngineLookupTable); 3368 3369 return guestEngineLookupTable; 3370 }; 3371 3372