1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include "kernel/gpu/fifo/kernel_fifo.h" 25 #include "kernel/gpu/fifo/kernel_channel.h" 26 #include "kernel/gpu/fifo/kernel_channel_group.h" 27 #include "kernel/gpu/fifo/kernel_channel_group_api.h" 28 #include "kernel/gpu/fifo/kernel_sched_mgr.h" 29 #include "virtualization/kernel_vgpu_mgr.h" 30 #include "rmapi/rs_utils.h" 31 #include "rmapi/client.h" 32 #include "gpu/subdevice/subdevice.h" 33 34 #include "kernel/core/locks.h" 35 #include "lib/base_utils.h" 36 37 #include "gpu/mmu/kern_gmmu.h" 38 #include "vgpu/rpc.h" 39 #include "vgpu/vgpu_events.h" 40 #include "nvRmReg.h" 41 42 #include "class/cl0080.h" 43 #include "class/cl2080.h" 44 #include "class/cl208f.h" 45 #include "class/clc572.h" 46 47 #include "ctrl/ctrl0080/ctrl0080fifo.h" 48 49 #define KFIFO_EHEAP_OWNER NvU32_BUILD('f','i','f','o') 50 51 static EHeapOwnershipComparator _kfifoUserdOwnerComparator; 52 53 static NV_STATUS _kfifoChidMgrAllocChidHeaps(OBJGPU *pGpu, 54 KernelFifo *pKernelFifo, 55 CHID_MGR *pChidMgr); 56 57 static NV_STATUS _kfifoChidMgrAllocVChidHeapPointers(OBJGPU *pGpu, CHID_MGR *pChidMgr); 58 59 static NV_STATUS _kfifoChidMgrInitChannelGroupMgr(OBJGPU *pGpu, CHID_MGR *pChidMgr); 60 61 static void _kfifoChidMgrDestroyChidHeaps(CHID_MGR *pChidMgr); 62 63 static void _kfifoChidMgrDestroyChannelGroupMgr(CHID_MGR *pChidMgr); 64 65 static NV_STATUS _kfifoChidMgrFreeIsolationId(CHID_MGR *pChidMgr, NvU32 ChID); 66 67 68 NvU32 kfifoGetNumEschedDrivenEngines_IMPL 69 ( 70 KernelFifo *pKernelFifo 71 ) 72 { 73 const ENGINE_INFO *pEngineInfo; 74 NV_ASSERT(kfifoGetNumEngines_HAL(ENG_GET_GPU(pKernelFifo), pKernelFifo) > 75 0); 76 pEngineInfo = kfifoGetEngineInfo(pKernelFifo); 77 return pEngineInfo->numRunlists; 78 } 79 80 81 NV_STATUS 82 kfifoChidMgrConstruct_IMPL 83 ( 84 OBJGPU *pGpu, 85 KernelFifo *pKernelFifo 86 ) 87 { 88 NV_STATUS status = NV_OK; 89 NvU32 i; 90 NvU32 numEngines; 91 92 // 93 // Allocate memory for the array of CHID_MGR pointers. Since this is an 94 // array, we allocate memory for pointers unto maxNumRunlists. We will only 95 // allocate objects for the valid ones. 96 // 97 if (kfifoIsPerRunlistChramEnabled(pKernelFifo)) 98 { 99 // 100 // Construct the engine list if it isn't already constructed (internally 101 // checks if it was already constructed) 102 // 103 NV_ASSERT_OK_OR_RETURN(kfifoConstructEngineList_HAL(pGpu, pKernelFifo)); 104 pKernelFifo->numChidMgrs = kfifoGetMaxNumRunlists_HAL(pGpu, pKernelFifo); 105 } 106 else 107 pKernelFifo->numChidMgrs = 1; 108 109 if (pKernelFifo->numChidMgrs > MAX_NUM_RUNLISTS) 110 { 111 // 112 // This only currently defines the size of our bitvector 113 // pKernelFifo->chidMgrValid. Catch this case if HW expands beyond this so we 114 // can increase the size allocated to the bitvector 115 // 116 NV_PRINTF(LEVEL_ERROR, "numChidMgrs 0x%x exceeds MAX_NUM_RUNLISTS\n", 117 pKernelFifo->numChidMgrs); 118 DBG_BREAKPOINT(); 119 return NV_ERR_BUFFER_TOO_SMALL; 120 } 121 122 pKernelFifo->ppChidMgr = portMemAllocNonPaged(sizeof(CHID_MGR *) * pKernelFifo->numChidMgrs); 123 if (pKernelFifo->ppChidMgr == NULL) 124 { 125 status = NV_ERR_NO_MEMORY; 126 pKernelFifo->ppChidMgr = NULL; 127 NV_PRINTF(LEVEL_ERROR, "Failed to allocate pFifo->pChidMgr\n"); 128 DBG_BREAKPOINT(); 129 return status; 130 } 131 portMemSet(pKernelFifo->ppChidMgr, 0, sizeof(CHID_MGR *) * pKernelFifo->numChidMgrs); 132 133 // Initialize the valid mask 134 if (kfifoIsPerRunlistChramEnabled(pKernelFifo)) 135 { 136 numEngines = kfifoGetNumEngines_HAL(pGpu, pKernelFifo); 137 for (i = 0; i < numEngines; i++) 138 { 139 NvU32 runlistId; 140 status = kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo, 141 ENGINE_INFO_TYPE_INVALID, i, 142 ENGINE_INFO_TYPE_RUNLIST, &runlistId); 143 if (status == NV_OK) 144 bitVectorSet(&pKernelFifo->chidMgrValid, runlistId); 145 else 146 { 147 NV_PRINTF(LEVEL_ERROR, "Translation to runlistId failed for engine %d\n", i); 148 DBG_BREAKPOINT(); 149 goto fail; 150 } 151 } 152 } 153 else 154 { 155 bitVectorSet(&pKernelFifo->chidMgrValid, 0); // We only have 1 chidmgr 156 } 157 158 // Allocate memory for each CHID_MGR and its members (only the valid ones) 159 for (i = 0; i < pKernelFifo->numChidMgrs; i++) 160 { 161 if (!bitVectorTest(&pKernelFifo->chidMgrValid, i)) 162 continue; 163 164 pKernelFifo->ppChidMgr[i] = portMemAllocNonPaged(sizeof(CHID_MGR)); 165 if (pKernelFifo->ppChidMgr[i] == NULL) 166 { 167 status = NV_ERR_NO_MEMORY; 168 NV_PRINTF(LEVEL_ERROR, "Failed to allocate pFifo->pChidMgr[%d]\n", i); 169 DBG_BREAKPOINT(); 170 goto fail; 171 } 172 portMemSet(pKernelFifo->ppChidMgr[i], 0, sizeof(CHID_MGR)); 173 174 pKernelFifo->ppChidMgr[i]->runlistId = i; 175 176 pKernelFifo->ppChidMgr[i]->pChanGrpTree = portMemAllocNonPaged(sizeof(KernelChannelGroupMap)); 177 mapInitIntrusive(pKernelFifo->ppChidMgr[i]->pChanGrpTree); 178 179 status = _kfifoChidMgrAllocChidHeaps(pGpu, pKernelFifo, pKernelFifo->ppChidMgr[i]); 180 if (status != NV_OK) 181 { 182 NV_PRINTF(LEVEL_ERROR, "Error allocating FifoDataHeap in " 183 "pChidMgr. Status = %s (0x%x)\n", 184 nvstatusToString(status), status); 185 DBG_BREAKPOINT(); 186 goto fail; 187 } 188 189 status = _kfifoChidMgrInitChannelGroupMgr(pGpu, pKernelFifo->ppChidMgr[i]); 190 if (status != NV_OK) 191 goto fail; 192 } 193 194 195 return status; 196 197 fail: 198 kfifoChidMgrDestruct(pKernelFifo); 199 return status; 200 } 201 202 void 203 kfifoChidMgrDestruct_IMPL 204 ( 205 KernelFifo *pKernelFifo 206 ) 207 { 208 NvU32 i; 209 210 for (i = 0; i < pKernelFifo->numChidMgrs; i++) 211 { 212 if (pKernelFifo->ppChidMgr[i] != NULL) 213 { 214 mapDestroy(pKernelFifo->ppChidMgr[i]->pChanGrpTree); 215 portMemFree(pKernelFifo->ppChidMgr[i]->pChanGrpTree); 216 _kfifoChidMgrDestroyChidHeaps(pKernelFifo->ppChidMgr[i]); 217 _kfifoChidMgrDestroyChannelGroupMgr(pKernelFifo->ppChidMgr[i]); 218 portMemFree(pKernelFifo->ppChidMgr[i]); 219 pKernelFifo->ppChidMgr[i] = NULL; 220 } 221 } 222 223 portMemFree(pKernelFifo->ppChidMgr); 224 pKernelFifo->ppChidMgr = NULL; 225 bitVectorClrAll(&pKernelFifo->chidMgrValid); 226 pKernelFifo->numChidMgrs = 0; 227 } 228 229 /* 230 * @brief Allocate and initialize the virtual ChId heap pointers 231 */ 232 static NV_STATUS 233 _kfifoChidMgrAllocVChidHeapPointers 234 ( 235 OBJGPU *pGpu, 236 CHID_MGR *pChidMgr 237 ) 238 { 239 NV_STATUS status = NV_OK; 240 NvU32 i; 241 242 if (IS_VIRTUAL(pGpu)) 243 { 244 return NV_OK; 245 } 246 247 if (gpuIsSriovEnabled(pGpu)) 248 { 249 // 250 // For Virtual Channel Heap 251 // Allocate Memory for Heap Object pointers 252 // 253 pChidMgr->ppVirtualChIDHeap = portMemAllocNonPaged(sizeof(OBJEHEAP *) * (VMMU_MAX_GFID)); 254 if (pChidMgr->ppVirtualChIDHeap == NULL) 255 { 256 NV_PRINTF(LEVEL_ERROR, 257 "Error allocating memory for virtual channel heap pointers\n"); 258 return NV_ERR_NO_MEMORY; 259 } 260 261 // initialize 262 for (i = 0; i < VMMU_MAX_GFID; i++) 263 { 264 pChidMgr->ppVirtualChIDHeap[i] = NULL; 265 } 266 } 267 return status; 268 } 269 270 271 /* 272 * @brief Allocates & initializes ChID heaps 273 */ 274 static NV_STATUS 275 _kfifoChidMgrAllocChidHeaps 276 ( 277 OBJGPU *pGpu, 278 KernelFifo *pKernelFifo, 279 CHID_MGR *pChidMgr 280 ) 281 { 282 NV_STATUS status = NV_OK; 283 284 if (pChidMgr->numChannels == 0) 285 { 286 if (kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr) == 0) 287 { 288 NV_PRINTF(LEVEL_ERROR, "pChidMgr->numChannels is 0\n"); 289 DBG_BREAKPOINT(); 290 return NV_ERR_INVALID_STATE; 291 } 292 } 293 294 pChidMgr->pFifoDataHeap = portMemAllocNonPaged(sizeof(*pChidMgr->pFifoDataHeap)); 295 if (pChidMgr->pFifoDataHeap == NULL) 296 { 297 status = NV_ERR_NO_MEMORY; 298 NV_PRINTF(LEVEL_ERROR, 299 "Error in Allocating memory for pFifoDataHeap! Status = %s (0x%x)\n", 300 nvstatusToString(status), status); 301 return status; 302 } 303 constructObjEHeap(pChidMgr->pFifoDataHeap, 0, pChidMgr->numChannels, 304 sizeof(KernelChannel *), 0); 305 306 if (kfifoIsChidHeapEnabled(pKernelFifo)) 307 { 308 NvU32 userdBar1Size; 309 NvU32 numChannels = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr); 310 NvU32 subProcessIsolation = 1; 311 312 pChidMgr->pGlobalChIDHeap = portMemAllocNonPaged(sizeof(OBJEHEAP)); 313 if (pChidMgr->pGlobalChIDHeap == NULL) 314 { 315 NV_PRINTF(LEVEL_ERROR, 316 "Error in Allocating memory for global ChID heap!\n"); 317 return NV_ERR_NO_MEMORY; 318 } 319 constructObjEHeap(pChidMgr->pGlobalChIDHeap, 0, numChannels, 320 sizeof(PFIFO_ISOLATIONID), 0); 321 322 // 323 // Enable USERD allocation isolation. USERD allocated by different clients 324 // should not be in the same page 325 // 326 kfifoGetUserdSizeAlign_HAL(pKernelFifo, &userdBar1Size, NULL); 327 NvBool bIsolationEnabled = (pKernelFifo->bUsePerRunlistChram && pKernelFifo->bDisableChidIsolation) ? NV_FALSE : NV_TRUE; 328 pChidMgr->pGlobalChIDHeap->eheapSetOwnerIsolation(pChidMgr->pGlobalChIDHeap, 329 bIsolationEnabled, 330 RM_PAGE_SIZE / userdBar1Size); 331 332 // Disable USERD allocation isolation for guest if disabled from vmioplugin 333 { 334 // In this case subProcessIsolation is always 0 335 if (IS_GSP_CLIENT(pGpu)) 336 { 337 subProcessIsolation = 0; 338 } 339 } 340 if (!subProcessIsolation) 341 { 342 pChidMgr->pGlobalChIDHeap->eheapSetOwnerIsolation( 343 pChidMgr->pGlobalChIDHeap, 344 NV_FALSE, 345 RM_PAGE_SIZE / userdBar1Size); 346 #if (defined(_WIN32) || defined(_WIN64) || defined(NV_UNIX)) && !RMCFG_FEATURE_MODS_FEATURES 347 NV_PRINTF(LEVEL_INFO, 348 "Sub Process channel isolation disabled by vGPU plugin\n"); 349 #endif 350 } 351 352 status = _kfifoChidMgrAllocVChidHeapPointers(pGpu, pChidMgr); 353 } 354 355 return status; 356 } 357 358 static void 359 _kfifoChidMgrDestroyChidHeaps 360 ( 361 CHID_MGR *pChidMgr 362 ) 363 { 364 if (pChidMgr->pFifoDataHeap != NULL) 365 { 366 pChidMgr->pFifoDataHeap->eheapDestruct(pChidMgr->pFifoDataHeap); 367 portMemFree(pChidMgr->pFifoDataHeap); 368 pChidMgr->pFifoDataHeap = NULL; 369 } 370 if (pChidMgr->pGlobalChIDHeap != NULL) 371 { 372 pChidMgr->pGlobalChIDHeap->eheapDestruct(pChidMgr->pGlobalChIDHeap); 373 portMemFree(pChidMgr->pGlobalChIDHeap); 374 pChidMgr->pGlobalChIDHeap = NULL; 375 } 376 377 portMemFree(pChidMgr->ppVirtualChIDHeap); 378 pChidMgr->ppVirtualChIDHeap = NULL; 379 } 380 381 382 static NV_STATUS 383 _kfifoChidMgrInitChannelGroupMgr 384 ( 385 OBJGPU *pGpu, 386 CHID_MGR *pChidMgr 387 ) 388 { 389 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 390 FIFO_HW_ID *pFifoHwID = &pChidMgr->channelGrpMgr; 391 NvU32 allocSize; 392 NvU32 numChannelGroups = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr); 393 394 if (numChannelGroups == 0) 395 { 396 return NV_OK; 397 } 398 399 // Rounds up to dword alignemnt, then converts bits to bytes. 400 allocSize = RM_ALIGN_UP(numChannelGroups, 32)/8; 401 402 pFifoHwID->pHwIdInUse = portMemAllocNonPaged(allocSize); 403 if (pFifoHwID->pHwIdInUse == NULL) 404 return NV_ERR_NO_MEMORY; 405 406 // bytes to NvU32[] elements 407 pFifoHwID->hwIdInUseSz = allocSize/4; 408 409 portMemSet(pFifoHwID->pHwIdInUse, 0, allocSize); 410 411 // 412 // If numChannelGroups isn't a multiple of 32 we need to set the bits > numChannelGroups to 413 // 1. Otherwise when we allocate IDs starting at the top we'll allocate 414 // ids >numChannelGroups. 415 // 416 if (numChannelGroups % 32 != 0) 417 { 418 pFifoHwID->pHwIdInUse[numChannelGroups/32] |= ~ ((1<<(numChannelGroups%32))-1); 419 } 420 421 return NV_OK; 422 } 423 424 static void 425 _kfifoChidMgrDestroyChannelGroupMgr 426 ( 427 CHID_MGR *pChidMgr 428 ) 429 { 430 if (pChidMgr->channelGrpMgr.pHwIdInUse) 431 { 432 portMemFree(pChidMgr->channelGrpMgr.pHwIdInUse); 433 pChidMgr->channelGrpMgr.pHwIdInUse = NULL; 434 pChidMgr->channelGrpMgr.hwIdInUseSz = 0; 435 } 436 } 437 438 static NV_STATUS 439 _kfifoChidMgrFreeIsolationId 440 ( 441 CHID_MGR *pChidMgr, 442 NvU32 ChID 443 ) 444 { 445 EMEMBLOCK *pIsolationIdBlock = pChidMgr->pGlobalChIDHeap->eheapGetBlock( 446 pChidMgr->pGlobalChIDHeap, 447 ChID, 448 NV_FALSE); 449 450 NV_ASSERT_OR_RETURN(pIsolationIdBlock, NV_ERR_OBJECT_NOT_FOUND); 451 NV_ASSERT(pIsolationIdBlock->refCount > 0); 452 NV_ASSERT(pIsolationIdBlock->pData != NULL); 453 portMemFree(pIsolationIdBlock->pData); 454 455 pIsolationIdBlock->pData = NULL; 456 457 return NV_OK; 458 } 459 460 /*! 461 * @breif Fifo defined call back comparator to compare eheap block ownership ID 462 * 463 * @param[in] pRequesterID Ownership ID constructed by caller 464 * @param[in] pIsolationID 465 * 466 * @return NV_TRUE if two ownership IDs belong to the same owner 467 */ 468 static NvBool 469 _kfifoUserdOwnerComparator 470 ( 471 void *pRequesterID, 472 void *pIsolationID 473 ) 474 { 475 PFIFO_ISOLATIONID pAllocID = (PFIFO_ISOLATIONID)pRequesterID; 476 PFIFO_ISOLATIONID pBlockID = (PFIFO_ISOLATIONID)pIsolationID; 477 478 // 479 // The block's data will be NULL if the channel has been destroyed but there 480 // is still a refcount on the channel ID. In that case no work can be issued 481 // to that channel ID now or in the future, so we can act as though the 482 // channel does not exist. 483 // 484 if (!pBlockID) 485 return NV_TRUE; 486 487 if ((pAllocID->domain != pBlockID->domain) || 488 (pAllocID->processID != pBlockID->processID) || 489 (pAllocID->subProcessID != pBlockID->subProcessID)) 490 { 491 return NV_FALSE; 492 } 493 else 494 { 495 return NV_TRUE; 496 } 497 } 498 499 /* 500 * @brief Returns the number of vGPU plugin channels. 501 * 502 * Depending on whether this code is executed on the CPU RM or the Physical RM, 503 * different structures are used to retrieve the number. 504 * On the CPU RM or the monolithic RM, KERNEL_HOST_VGPU_DEVICE::numPluginChannels is used, 505 * whereas on the physical RM it's HOST_VGPU_DEVICE::numPluginChannels. 506 */ 507 static NV_STATUS 508 _kfifoGetVgpuPluginChannelsCount 509 ( 510 OBJGPU *pGpu, 511 NvU32 *pNumPluginChannels 512 ) 513 { 514 NV_ASSERT_OR_RETURN(pNumPluginChannels != NULL, NV_ERR_INVALID_ARGUMENT); 515 516 if (!RMCFG_FEATURE_PLATFORM_GSP) 517 { 518 KERNEL_HOST_VGPU_DEVICE *pKernelHostVgpuDevice = NULL; 519 520 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextKernelHostVgpuDevice(pGpu, &pKernelHostVgpuDevice)); 521 NV_ASSERT_OR_RETURN(pKernelHostVgpuDevice != NULL, NV_ERR_OBJECT_NOT_FOUND); 522 523 *pNumPluginChannels = pKernelHostVgpuDevice->numPluginChannels; 524 } 525 526 return NV_OK; 527 } 528 529 /*! 530 * @brief Allocates one Channel ID on heap 531 * 532 * @param[in] OBJGPU GPU Object 533 * @param[in] KernelFifo KernelFifo Object 534 * @param[in] CHID_MGR Channel ID manager 535 * @param[in] chFlag NVOS32_ALLOC_FLAGS_FORCE_MEM_GROWS_DOWN 536 * NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE 537 * NVOS32_ALLOC_FLAGS_FORCE_MEM_GROWS_UP 538 * @param[in] bForceInternalIdx true if requesting specific index within USERD 539 * page 540 * @param[in] internalIdx requested index within USERD page when 541 * bForceInternalIdx true 542 * @param[in] ChID ChID to assign in case of ADDRESS_ALLOCATE 543 * @param[in,out] pKernelChannel The previously allocated KernelChannel structure 544 * 545 * @return NV_OK if allocation is successful 546 * NV_ERR_NO_FREE_FIFOS: allocated channel ID exceeds MAX channels. 547 */ 548 NV_STATUS 549 kfifoChidMgrAllocChid_IMPL 550 ( 551 OBJGPU *pGpu, 552 KernelFifo *pKernelFifo, 553 CHID_MGR *pChidMgr, 554 NvHandle hClient, 555 CHANNEL_HW_ID_ALLOC_MODE chIdFlag, 556 NvBool bForceInternalIdx, 557 NvU32 internalIdx, 558 NvBool bForceUserdPage, 559 NvU32 userdPageIdx, 560 NvU32 ChID, 561 KernelChannel *pKernelChannel 562 ) 563 { 564 NvU64 chSize; 565 NvU32 chFlag = chIdFlag; 566 NvU64 ChID64 = 0; 567 NvU64 subProcessID = 0; 568 NvU64 processID = 0; 569 NvBool bIsSubProcessDisabled = NV_FALSE; 570 RmClient *pClient; 571 NvU32 offsetAlign = 1; 572 NvU32 gfid; 573 PFIFO_ISOLATIONID pIsolationID = NULL; 574 NV_STATUS status; 575 NvU32 numChannels; 576 577 NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_ARGUMENT); 578 579 numChannels = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr); 580 581 switch (chIdFlag) 582 { 583 case CHANNEL_HW_ID_ALLOC_MODE_GROW_DOWN: 584 chFlag = NVOS32_ALLOC_FLAGS_FORCE_MEM_GROWS_DOWN; 585 break; 586 case CHANNEL_HW_ID_ALLOC_MODE_GROW_UP: 587 chFlag = NVOS32_ALLOC_FLAGS_FORCE_MEM_GROWS_UP; 588 break; 589 case CHANNEL_HW_ID_ALLOC_MODE_PROVIDED: 590 ChID64 = ChID; 591 chFlag = NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE; 592 break; 593 default: 594 NV_PRINTF(LEVEL_ERROR, "Invalid channel ID alloc mode %d\n", chFlag); 595 DBG_BREAKPOINT(); 596 return NV_ERR_INVALID_ARGUMENT; 597 } 598 599 // we are allocating only one Channel at a time 600 chSize = 1; 601 602 // Create unique isolation ID for each process 603 if (serverutilGetClientUnderLock(hClient, &pClient) != NV_OK) 604 { 605 NV_PRINTF(LEVEL_ERROR, "Invalid client handle %ux\n", hClient); 606 DBG_BREAKPOINT(); 607 return NV_ERR_INVALID_CLIENT; 608 } 609 610 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &gfid)); 611 612 // SRIOV: In guest plugin context allocate the chid. 613 // In guest RM context allocate the same chid as guest 614 if (IS_GFID_VF(gfid)) 615 { 616 NvU32 numPluginChannels; 617 NvU64 rangeLo, rangeHi, base, size; 618 619 NV_ASSERT_OR_RETURN(pChidMgr->ppVirtualChIDHeap[gfid], 620 NV_ERR_INVALID_STATE); 621 622 NV_ASSERT_OK_OR_RETURN(_kfifoGetVgpuPluginChannelsCount(pGpu, &numPluginChannels)); 623 624 pChidMgr->ppVirtualChIDHeap[gfid]->eheapGetBase( 625 pChidMgr->ppVirtualChIDHeap[gfid], 626 &base); 627 pChidMgr->ppVirtualChIDHeap[gfid]->eheapGetSize( 628 pChidMgr->ppVirtualChIDHeap[gfid], 629 &size); 630 631 rangeLo = base; 632 rangeHi = base + size - 1; 633 634 // Route plugin channels to be allocated at the top 635 NV_ASSERT_OR_RETURN(numPluginChannels < size, 636 NV_ERR_INVALID_PARAMETER); 637 if (pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->bIsCallingContextVgpuPlugin && 638 numPluginChannels > 0) 639 { 640 rangeLo = rangeHi - numPluginChannels + 1; 641 } 642 else 643 { 644 rangeHi = rangeHi - numPluginChannels; 645 } 646 647 status = pChidMgr->ppVirtualChIDHeap[gfid]->eheapSetAllocRange( 648 pChidMgr->ppVirtualChIDHeap[gfid], 649 rangeLo, 650 rangeHi); 651 652 NV_ASSERT_OK_OR_RETURN(status); 653 654 if (bForceUserdPage) 655 { 656 NV_ASSERT_OR_RETURN(!bForceInternalIdx, NV_ERR_INVALID_STATE); 657 ChID64 = ((NvU64)userdPageIdx) * 658 pChidMgr->ppVirtualChIDHeap[gfid]->ownerGranularity + 659 internalIdx; 660 chFlag |= NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE; 661 } 662 else if (bForceInternalIdx) 663 { 664 chFlag |= NVOS32_ALLOC_FLAGS_FORCE_INTERNAL_INDEX; 665 offsetAlign = internalIdx; 666 } 667 668 if (chFlag & NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE) 669 NV_ASSERT_OR_RETURN((ChID64 <= rangeHi) && (ChID64 >= rangeLo), 670 NV_ERR_INVALID_PARAMETER); 671 672 // We'll allocate from the VirtualChIdHeap for the guest 673 status = pChidMgr->ppVirtualChIDHeap[gfid]->eheapAlloc( 674 pChidMgr->ppVirtualChIDHeap[gfid], // This Heap 675 KFIFO_EHEAP_OWNER, // owner 676 &chFlag, // Alloc Flags 677 &ChID64, // Alloc Offset 678 &chSize, // Size 679 offsetAlign, // offsetAlign 680 1, // sizeAlign 681 NULL, // Allocated mem block 682 NULL, // Isolation ID 683 NULL // Fifo defined ownership comparator 684 ); 685 if (status != NV_OK) 686 { 687 NV_PRINTF(LEVEL_ERROR, 688 "Failed to allocate Channel ID 0x%llx %d on heap \n", 689 ChID64, 690 chIdFlag); 691 DBG_BREAKPOINT(); 692 goto fail; 693 } 694 } 695 else 696 { 697 // 698 // Legacy / SRIOV vGPU Host, SRIOV guest, baremetal CPU RM, GSP FW, GSP 699 // client allocate from global heap 700 // 701 pIsolationID = portMemAllocNonPaged(sizeof(FIFO_ISOLATIONID)); 702 NV_ASSERT_OR_RETURN((pIsolationID != NULL), NV_ERR_NO_MEMORY); 703 portMemSet(pIsolationID, 0, sizeof(FIFO_ISOLATIONID)); 704 705 // 706 // Check if the allocation request is from the guest RM or host RM 707 // 708 processID = pClient->ProcID; 709 subProcessID = pClient->SubProcessID; 710 bIsSubProcessDisabled = pClient->bIsSubProcessDisabled; 711 712 if (RMCFG_FEATURE_PLATFORM_GSP || kchannelCheckIsKernel(pKernelChannel)) 713 { 714 // 715 // If not GSPFW: Allocation request is from host RM kernel 716 // If GSPFW: ChID has already been chosen by CPU-RM, but pClient 717 // doesn't have the true processID, so just allow the whole pool. 718 // 719 pIsolationID->domain = HOST_KERNEL; 720 processID = KERNEL_PID; 721 } 722 else 723 { 724 if (0x0 != subProcessID) 725 { 726 // 727 // Allocation request is from the guest RM 728 // 729 if (KERNEL_PID == subProcessID) 730 { 731 pIsolationID->domain = GUEST_KERNEL; 732 } 733 else 734 { 735 pIsolationID->domain = GUEST_USER; 736 } 737 } 738 else 739 { 740 pIsolationID->domain = HOST_USER; 741 } 742 } 743 744 pIsolationID->processID = processID; 745 pIsolationID->subProcessID = subProcessID; 746 747 // 748 // Overwrite isolation ID if guest USERD isolation is disabled 749 // 750 if ((subProcessID != 0x0) && (bIsSubProcessDisabled)) 751 { 752 pIsolationID->domain = GUEST_INSECURE; 753 pIsolationID->subProcessID = KERNEL_PID; 754 } 755 756 /* Channel USERD manipuliation only supported without GFID */ 757 if (bForceUserdPage) 758 { 759 NV_ASSERT_OR_RETURN(!bForceInternalIdx, NV_ERR_INVALID_STATE); 760 ChID64 = ((NvU64)userdPageIdx) * 761 pChidMgr->pGlobalChIDHeap->ownerGranularity + 762 internalIdx; 763 chFlag |= NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE; 764 } 765 else if (bForceInternalIdx) 766 { 767 chFlag |= NVOS32_ALLOC_FLAGS_FORCE_INTERNAL_INDEX; 768 offsetAlign = internalIdx; 769 } 770 771 status = pChidMgr->pGlobalChIDHeap->eheapAlloc( 772 pChidMgr->pGlobalChIDHeap, // This Heap 773 KFIFO_EHEAP_OWNER, // owner 774 &chFlag, // Alloc Flags 775 &ChID64, // Alloc Offset 776 &chSize, // Size 777 offsetAlign, // offsetAlign 778 1, // sizeAlign 779 NULL, // Allocated mem block 780 pIsolationID, // Isolation ID 781 _kfifoUserdOwnerComparator // Fifo defined ownership comparator 782 ); 783 784 if (status != NV_OK) 785 { 786 NV_PRINTF(LEVEL_ERROR, "Failed to allocate Channel ID on heap\n"); 787 DBG_BREAKPOINT(); 788 goto fail; 789 } 790 } 791 792 // 793 // Now allocate at a fixed offset from the pFifoDataHeap once the previous 794 // ID allocation told us which ID to use. 795 // 796 chFlag = NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE; 797 status = pChidMgr->pFifoDataHeap->eheapAlloc( 798 pChidMgr->pFifoDataHeap, // This Heap 799 KFIFO_EHEAP_OWNER, // owner 800 &chFlag, // Alloc Flags 801 &ChID64, // Alloc Offset 802 &chSize, // Size 803 1, // offsetAlign 804 1, // sizeAlign 805 NULL, // Allocated mem block 806 NULL, // Isolation ID 807 NULL // ownership comparator 808 ); 809 810 if (status != NV_OK) 811 { 812 // 813 // Should never happen since we're mirroring the global chid heap, or 814 // pre-reserving space on the global chid heap for SR-IOV capable 815 // systems. 816 // 817 NV_PRINTF(LEVEL_ERROR, "Failed to allocate Channel on fifo data heap\n"); 818 goto fail; 819 } 820 821 ChID = NvU64_LO32(ChID64); 822 823 if (ChID < numChannels) 824 { 825 PEMEMBLOCK pFifoDataBlock = pChidMgr->pFifoDataHeap->eheapGetBlock( 826 pChidMgr->pFifoDataHeap, 827 ChID, 828 NV_FALSE); 829 PEMEMBLOCK pIsolationIdBlock = pChidMgr->pGlobalChIDHeap->eheapGetBlock( 830 pChidMgr->pGlobalChIDHeap, 831 ChID, 832 NV_FALSE); 833 834 if (IS_GFID_PF(gfid)) 835 pIsolationIdBlock->pData = pIsolationID; 836 837 pFifoDataBlock->pData = pKernelChannel; 838 pKernelChannel->ChID = ChID; 839 } 840 else 841 { 842 NV_PRINTF(LEVEL_WARNING, "No allocatable FIFO available.\n"); 843 status = NV_ERR_NO_FREE_FIFOS; 844 goto fail; 845 } 846 return NV_OK; 847 848 fail: 849 // We already know that pIsolationID is non-NULL here. 850 portMemFree(pIsolationID); 851 return status; 852 } 853 854 /* 855 * Retain a channel ID which has already been allocated by 856 * kfifoChidMgrAllocChid. Until released, the HW channel ID will not be 857 * allocated by any new channels even after kfifoChidMgrFreeChid has been 858 * called. 859 */ 860 NV_STATUS 861 kfifoChidMgrRetainChid_IMPL 862 ( 863 OBJGPU *pGpu, 864 KernelFifo *pKernelFifo, 865 CHID_MGR *pChidMgr, 866 NvU32 ChID 867 ) 868 { 869 NvU32 gfid; 870 PEMEMBLOCK pFifoDataBlock = NULL; 871 872 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &gfid)); 873 874 if (IS_GFID_VF(gfid)) 875 { 876 NV_ASSERT_OR_RETURN(pChidMgr->ppVirtualChIDHeap[gfid] != NULL, 877 NV_ERR_INVALID_STATE); 878 PEMEMBLOCK pVirtChIdBlock = pChidMgr->ppVirtualChIDHeap[gfid]->eheapGetBlock( 879 pChidMgr->ppVirtualChIDHeap[gfid], 880 ChID, 881 NV_FALSE); 882 NV_ASSERT_OR_RETURN(pVirtChIdBlock != NULL, NV_ERR_OBJECT_NOT_FOUND); 883 NV_ASSERT(pVirtChIdBlock->refCount > 0); 884 ++pVirtChIdBlock->refCount; 885 } 886 else 887 { 888 NV_ASSERT_OR_RETURN(pChidMgr->pGlobalChIDHeap != NULL, NV_ERR_INVALID_STATE); 889 PEMEMBLOCK pChIdBlock = pChidMgr->pGlobalChIDHeap->eheapGetBlock( 890 pChidMgr->pGlobalChIDHeap, 891 ChID, 892 NV_FALSE); 893 NV_ASSERT_OR_RETURN(pChIdBlock != NULL, NV_ERR_OBJECT_NOT_FOUND); 894 NV_ASSERT(pChIdBlock->refCount > 0); 895 ++pChIdBlock->refCount; 896 } 897 898 NV_ASSERT_OR_RETURN(pChidMgr->pFifoDataHeap != NULL, NV_ERR_INVALID_STATE); 899 pFifoDataBlock = pChidMgr->pFifoDataHeap->eheapGetBlock( 900 pChidMgr->pFifoDataHeap, 901 ChID, 902 NV_FALSE); 903 NV_ASSERT_OR_RETURN(pFifoDataBlock != NULL, NV_ERR_OBJECT_NOT_FOUND); 904 NV_ASSERT(pFifoDataBlock->refCount > 0); 905 ++pFifoDataBlock->refCount; 906 907 return NV_OK; 908 } 909 910 /* 911 * Drop the refcount on the given channel (ID), removing it from pFifo's heap if 912 * its refcount reaches 0. 913 */ 914 NV_STATUS 915 kfifoChidMgrReleaseChid_IMPL 916 ( 917 OBJGPU *pGpu, 918 KernelFifo *pKernelFifo, 919 CHID_MGR *pChidMgr, 920 NvU32 ChID 921 ) 922 { 923 NvU32 gfid; 924 925 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &gfid)); 926 927 if (IS_GFID_VF(gfid)) 928 { 929 NV_ASSERT_OR_RETURN(pChidMgr->ppVirtualChIDHeap[gfid] != NULL, NV_ERR_INVALID_STATE); 930 NV_ASSERT_OK(pChidMgr->ppVirtualChIDHeap[gfid]->eheapFree(pChidMgr->ppVirtualChIDHeap[gfid], ChID)); 931 } 932 else 933 { 934 NV_ASSERT_OR_RETURN(pChidMgr->pGlobalChIDHeap != NULL, NV_ERR_INVALID_STATE); 935 NV_ASSERT_OK(pChidMgr->pGlobalChIDHeap->eheapFree(pChidMgr->pGlobalChIDHeap, ChID)); 936 } 937 938 NV_ASSERT_OR_RETURN(pChidMgr->pFifoDataHeap != NULL, NV_ERR_INVALID_STATE); 939 NV_ASSERT_OK_OR_RETURN(pChidMgr->pFifoDataHeap->eheapFree(pChidMgr->pFifoDataHeap, ChID)); 940 941 return NV_OK; 942 } 943 944 /* 945 * Removes the association between pKernelChannel and its channel ID. Note that this 946 * will not remove the channel ID itself from pFifo's heap if 947 * fifoHeapRetainChannelId has been called. 948 */ 949 NV_STATUS 950 kfifoChidMgrFreeChid_IMPL 951 ( 952 OBJGPU *pGpu, 953 KernelFifo *pKernelFifo, 954 CHID_MGR *pChidMgr, 955 NvU32 ChID 956 ) 957 { 958 EMEMBLOCK *pFifoDataBlock; 959 NV_STATUS status; 960 NvU32 gfid; 961 962 // 963 // This channel is going away, so clear its pointer from the channel ID's heap 964 // block. 965 // 966 pFifoDataBlock = pChidMgr->pFifoDataHeap->eheapGetBlock( 967 pChidMgr->pFifoDataHeap, 968 ChID, 969 NV_FALSE); 970 NV_ASSERT_OR_RETURN(pFifoDataBlock != NULL, NV_ERR_OBJECT_NOT_FOUND); 971 NV_ASSERT(pFifoDataBlock->refCount > 0); 972 pFifoDataBlock->pData = NULL; 973 974 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &gfid)); 975 976 if (IS_GFID_PF(gfid)) 977 { 978 // 979 // This marks the channel ID as orphaned and causes it to be ignored for 980 // isolation purposes. This only matters if there will still be a reference 981 // on the ID after we release ours below. 982 // 983 status = _kfifoChidMgrFreeIsolationId(pChidMgr, ChID); 984 if(status != NV_OK) 985 { 986 NV_PRINTF(LEVEL_ERROR, 987 "Failed to free IsolationId. Status = 0x%x\n", status); 988 DBG_BREAKPOINT(); 989 return status; 990 } 991 } 992 993 return kfifoChidMgrReleaseChid(pGpu, pKernelFifo, pChidMgr, ChID); 994 } 995 996 /** 997 * @brief Reserve a contiguous set of SCHIDs from the end of our CHID heap for 998 * the given GFID 999 * 1000 1001 * @param[in] pChidMgr CHID_MGR pointer 1002 * @param[in] numChannels Number of SCHIDs to reserve 1003 * @param[in] pHostVgpuDevice HOST_VGPU_DEVICE 1004 * 1005 * @return NV_OK if success 1006 */ 1007 NV_STATUS 1008 kfifoChidMgrReserveSystemChids_IMPL 1009 ( 1010 OBJGPU *pGpu, 1011 KernelFifo *pKernelFifo, 1012 CHID_MGR *pChidMgr, 1013 NvU32 numChannels, 1014 NvU32 flags, 1015 NvU32 gfid, 1016 NvU32 *pChidOffset, 1017 NvU32 *pChannelCount, 1018 NvHandle hMigClient, 1019 NvU32 engineFifoListNumEntries, 1020 FIFO_ENGINE_LIST *pEngineFifoList 1021 ) 1022 { 1023 NV_STATUS status = NV_OK; 1024 NvU64 chSize; 1025 NvU64 offset = 0; 1026 PFIFO_ISOLATIONID pIsolationID = NULL; 1027 PEMEMBLOCK pIsolationIdBlock; 1028 NvU32 userdBar1Size; 1029 1030 if (IS_VIRTUAL(pGpu)) 1031 { 1032 // Not supported on guest or when SRIOV is disabled 1033 return NV_ERR_NOT_SUPPORTED; 1034 } 1035 1036 pIsolationID = portMemAllocNonPaged(sizeof(FIFO_ISOLATIONID)); 1037 NV_ASSERT_OR_RETURN((pIsolationID != NULL), NV_ERR_NO_MEMORY); 1038 portMemSet(pIsolationID, 0, sizeof(FIFO_ISOLATIONID)); 1039 1040 chSize = numChannels; 1041 1042 status = pChidMgr->pGlobalChIDHeap->eheapAlloc( 1043 pChidMgr->pGlobalChIDHeap, // This Heap 1044 KFIFO_EHEAP_OWNER, // owner 1045 &flags, // Alloc Flags 1046 &offset, // Alloc Offset 1047 &chSize, // Size 1048 1, // offsetAlign 1049 1, // sizeAlign 1050 NULL, // Allocated mem block 1051 pIsolationID, // IsolationID 1052 _kfifoUserdOwnerComparator // Fifo defined ownership comparator 1053 ); 1054 1055 if(status != NV_OK) 1056 { 1057 NV_PRINTF(LEVEL_ERROR, "Failed to reserve channel IDs. Status = 0x%x\n", status); 1058 DBG_BREAKPOINT(); 1059 1060 // 1061 // Free the allocated memory and return early. After this, all failure 1062 // points can goto the common cleanup label 1063 // 1064 portMemFree(pIsolationID); 1065 return status; 1066 } 1067 1068 pIsolationIdBlock = pChidMgr->pGlobalChIDHeap->eheapGetBlock( 1069 pChidMgr->pGlobalChIDHeap, 1070 offset, 1071 NV_FALSE); 1072 if (pIsolationIdBlock == NULL) 1073 { 1074 // Something bad happened. This should not fail if allocation succeeded 1075 NV_PRINTF(LEVEL_ERROR, "Could not fetch block from eheap\n"); 1076 DBG_BREAKPOINT(); 1077 goto cleanup; 1078 } 1079 pIsolationIdBlock->pData = pIsolationID; 1080 1081 status = kfifoSetChidOffset(pGpu, pKernelFifo, pChidMgr, (NvU32)offset, 1082 numChannels, gfid, pChidOffset, pChannelCount, 1083 hMigClient, engineFifoListNumEntries, pEngineFifoList); 1084 1085 if (status != NV_OK) 1086 { 1087 NV_PRINTF(LEVEL_ERROR, 1088 "Failed to program the CHID table\n"); 1089 goto cleanup; 1090 } 1091 1092 pChidMgr->ppVirtualChIDHeap[gfid] = portMemAllocNonPaged(sizeof(OBJEHEAP)); 1093 if (pChidMgr->ppVirtualChIDHeap[gfid] == NULL) 1094 { 1095 status = NV_ERR_NO_MEMORY; 1096 NV_PRINTF(LEVEL_ERROR, "Error allocating memory for virtual " 1097 "channel ID heap\n"); 1098 goto cleanup; 1099 } 1100 1101 // 1102 // Construct heap using low as offset and size of numChannels. This heap 1103 // will be used for guest channel ID allocations, but will be in the 1104 // system channel ID space, hence it only manages IDs from offset to 1105 // (offset + numChannels). 1106 // 1107 constructObjEHeap(pChidMgr->ppVirtualChIDHeap[gfid], offset, 1108 (offset + numChannels), 0, 0); 1109 1110 kfifoGetUserdSizeAlign_HAL(pKernelFifo, &userdBar1Size, NULL); 1111 pChidMgr->ppVirtualChIDHeap[gfid]->eheapSetOwnerIsolation( 1112 pChidMgr->ppVirtualChIDHeap[gfid], 1113 NV_FALSE, 1114 RM_PAGE_SIZE/userdBar1Size); 1115 1116 return status; 1117 1118 cleanup: 1119 portMemFree(pChidMgr->ppVirtualChIDHeap[gfid]); 1120 NV_ASSERT(kfifoSetChidOffset(pGpu, pKernelFifo, pChidMgr, 0, 0, 1121 gfid, pChidOffset, pChannelCount, hMigClient, 1122 engineFifoListNumEntries, pEngineFifoList) == NV_OK); 1123 NV_ASSERT(pChidMgr->pGlobalChIDHeap->eheapFree(pChidMgr->pGlobalChIDHeap, offset) == NV_OK); 1124 portMemFree(pIsolationID); 1125 return status; 1126 } 1127 1128 /*! Frees a block of contiguous SCHIDs previously reserved for the given GFID */ 1129 NV_STATUS 1130 kfifoChidMgrFreeSystemChids_IMPL 1131 ( 1132 OBJGPU *pGpu, 1133 KernelFifo *pKernelFifo, 1134 CHID_MGR *pChidMgr, 1135 NvU32 gfid, 1136 NvU32 *pChidOffset, 1137 NvU32 *pChannelCount, 1138 NvHandle hMigClient, 1139 NvU32 engineFifoListNumEntries, 1140 FIFO_ENGINE_LIST *pEngineFifoList 1141 ) 1142 { 1143 NV_STATUS status, tmpStatus; 1144 NvU64 chId; 1145 1146 if (IS_VIRTUAL(pGpu)) 1147 { 1148 // Not supported on guest or when SRIOV is disabled 1149 return NV_ERR_NOT_SUPPORTED; 1150 } 1151 1152 // Get the schid base 1153 pChidMgr->ppVirtualChIDHeap[gfid]->eheapGetBase( 1154 pChidMgr->ppVirtualChIDHeap[gfid], 1155 &chId); 1156 1157 status = _kfifoChidMgrFreeIsolationId(pChidMgr, (NvU32)chId); 1158 if(status != NV_OK) 1159 { 1160 NV_PRINTF(LEVEL_ERROR, 1161 "Failed to free IsolationId. Status = 0x%x\n", 1162 status); 1163 DBG_BREAKPOINT(); 1164 return status; 1165 } 1166 1167 status = pChidMgr->pGlobalChIDHeap->eheapFree(pChidMgr->pGlobalChIDHeap, chId); 1168 if(status != NV_OK) 1169 { 1170 NV_PRINTF(LEVEL_ERROR, 1171 "Failed to free channel IDs. Status = 0x%x\n", 1172 status); 1173 DBG_BREAKPOINT(); 1174 1175 // 1176 // March on anyway to program the ChId table. We'll return an error 1177 // if we get here though. 1178 // 1179 } 1180 1181 tmpStatus = kfifoSetChidOffset(pGpu, pKernelFifo, pChidMgr, 0, 0, 1182 gfid, pChidOffset, pChannelCount, hMigClient, 1183 engineFifoListNumEntries, pEngineFifoList); 1184 if (tmpStatus != NV_OK) 1185 { 1186 NV_PRINTF(LEVEL_ERROR, 1187 "Failed to program the CHID table\n"); 1188 DBG_BREAKPOINT(); 1189 return tmpStatus; 1190 } 1191 1192 pChidMgr->ppVirtualChIDHeap[gfid]->eheapDestruct(pChidMgr->ppVirtualChIDHeap[gfid]); 1193 portMemFree(pChidMgr->ppVirtualChIDHeap[gfid]); 1194 pChidMgr->ppVirtualChIDHeap[gfid] = NULL; 1195 1196 return status; 1197 } 1198 1199 NvU32 1200 kfifoChidMgrGetNumChannels_IMPL 1201 ( 1202 OBJGPU *pGpu, 1203 KernelFifo *pKernelFifo, 1204 CHID_MGR *pChidMgr 1205 ) 1206 { 1207 // Cache ChidMgr's numChannels if not set 1208 if (pChidMgr->numChannels == 0) 1209 { 1210 NvU32 numChannels = kfifoRunlistQueryNumChannels_HAL(pGpu, pKernelFifo, 1211 pChidMgr->runlistId); 1212 1213 if (pKernelFifo->bNumChannelsOverride) 1214 { 1215 pChidMgr->numChannels = NV_MIN(pKernelFifo->numChannelsOverride, numChannels); 1216 } 1217 else 1218 { 1219 pChidMgr->numChannels = numChannels; 1220 } 1221 1222 // Once we have set calculated value disable any overrides. 1223 pKernelFifo->bNumChannelsOverride = 0; 1224 } 1225 1226 return pChidMgr->numChannels; 1227 } 1228 1229 NvU32 1230 kfifoRunlistQueryNumChannels_KERNEL 1231 ( 1232 OBJGPU *pGpu, 1233 KernelFifo *pKernelFifo, 1234 NvU32 runlistId 1235 ) 1236 { 1237 NvU32 numChannels = 0; 1238 NvU32 status; 1239 1240 // Do internal control call and set numChannels 1241 if (IS_GSP_CLIENT(pGpu)) 1242 { 1243 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); 1244 NV2080_CTRL_INTERNAL_FIFO_GET_NUM_CHANNELS_PARAMS numChannelsParams = {0}; 1245 1246 numChannelsParams.runlistId = runlistId; 1247 1248 status = pRmApi->Control(pRmApi, 1249 pGpu->hInternalClient, 1250 pGpu->hInternalSubdevice, 1251 NV2080_CTRL_CMD_INTERNAL_FIFO_GET_NUM_CHANNELS, 1252 &numChannelsParams, 1253 sizeof(NV2080_CTRL_INTERNAL_FIFO_GET_NUM_CHANNELS_PARAMS)); 1254 if (status != NV_OK) 1255 { 1256 DBG_BREAKPOINT(); 1257 return 0; 1258 } 1259 1260 numChannels = numChannelsParams.numChannels; 1261 } 1262 1263 NV_ASSERT(numChannels > 0); 1264 1265 return numChannels; 1266 } 1267 1268 /** 1269 * @brief reserves a hardware channel slot for a channel group 1270 * 1271 * Only responsible for indicating a hardware channel is in use, does not set 1272 * any other software state. 1273 * 1274 * This function is not called in broadcast mode 1275 * 1276 * @param pGpu 1277 * @param pKernelFifo 1278 * @param pChidMgr 1279 * @param[out] grpID 1280 */ 1281 NV_STATUS 1282 kfifoChidMgrAllocChannelGroupHwID_IMPL 1283 ( 1284 OBJGPU *pGpu, 1285 KernelFifo *pKernelFifo, 1286 CHID_MGR *pChidMgr, 1287 NvU32 *pChGrpID 1288 ) 1289 { 1290 NvU32 maxChannelGroups; 1291 1292 if (pChGrpID == NULL) 1293 return NV_ERR_INVALID_ARGUMENT; 1294 1295 maxChannelGroups = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr); 1296 if (maxChannelGroups == 0) 1297 { 1298 NV_PRINTF(LEVEL_ERROR, "Zero max channel groups!!!\n"); 1299 return NV_ERR_INVALID_ARGUMENT; 1300 } 1301 1302 // Find the least unused grpID 1303 *pChGrpID = nvBitFieldLSZero(pChidMgr->channelGrpMgr.pHwIdInUse, 1304 pChidMgr->channelGrpMgr.hwIdInUseSz); 1305 1306 if (*pChGrpID < maxChannelGroups) 1307 { 1308 nvBitFieldSet(pChidMgr->channelGrpMgr.pHwIdInUse, 1309 pChidMgr->channelGrpMgr.hwIdInUseSz, *pChGrpID, NV_TRUE); 1310 } 1311 else 1312 { 1313 *pChGrpID = maxChannelGroups; 1314 NV_PRINTF(LEVEL_ERROR, "No allocatable FIFO available.\n"); 1315 return NV_ERR_NO_FREE_FIFOS; 1316 } 1317 return NV_OK; 1318 } 1319 1320 1321 /** 1322 * @brief Releases a hardware channel group ID. 1323 * 1324 * Not responsible for freeing any software state beyond that which indicates a 1325 * hardware channel is in use. 1326 * 1327 * This function is not called in broadcast mode 1328 * 1329 * @param pGpu 1330 * @param pFifo 1331 * @param pChidMgr 1332 * @param chGrpID 1333 */ 1334 NV_STATUS 1335 kfifoChidMgrFreeChannelGroupHwID_IMPL 1336 ( 1337 OBJGPU *pGpu, 1338 KernelFifo *pKernelFifo, 1339 CHID_MGR *pChidMgr, 1340 NvU32 chGrpID 1341 ) 1342 { 1343 NvU32 maxChannelGroups; 1344 1345 maxChannelGroups = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr); 1346 if (maxChannelGroups == 0) 1347 { 1348 NV_PRINTF(LEVEL_ERROR, "Zero max channel groups!!!\n"); 1349 return NV_ERR_INVALID_ARGUMENT; 1350 } 1351 NV_ASSERT_OR_RETURN(chGrpID < maxChannelGroups, NV_ERR_INVALID_ARGUMENT); 1352 1353 // 1354 // Look for the channel group, check to make sure it's InUse bit is set 1355 // and then clear it to indicate the grpID is no longer in use 1356 // 1357 NV_ASSERT(nvBitFieldTest(pChidMgr->channelGrpMgr.pHwIdInUse, 1358 pChidMgr->channelGrpMgr.hwIdInUseSz, chGrpID)); 1359 nvBitFieldSet(pChidMgr->channelGrpMgr.pHwIdInUse, 1360 pChidMgr->channelGrpMgr.hwIdInUseSz, chGrpID, NV_FALSE); 1361 1362 return NV_OK; 1363 } 1364 1365 CHID_MGR * 1366 kfifoGetChidMgr_IMPL 1367 ( 1368 OBJGPU *pGpu, 1369 KernelFifo *pKernelFifo, 1370 NvU32 runlistId 1371 ) 1372 { 1373 if (!kfifoIsPerRunlistChramEnabled(pKernelFifo)) 1374 { 1375 // We only have 1 chidmgr when we don't have a per-runlist channel RAM 1376 if ((pKernelFifo->numChidMgrs != 1) || 1377 (pKernelFifo->ppChidMgr == NULL) || 1378 !bitVectorTest(&pKernelFifo->chidMgrValid, 0)) 1379 { 1380 return NULL; 1381 } 1382 return pKernelFifo->ppChidMgr[0]; 1383 } 1384 else 1385 { 1386 if (runlistId >= pKernelFifo->numChidMgrs) 1387 { 1388 return NULL; 1389 } 1390 // 1391 // It is valid to return a NULL value as long as runlistId is less than 1392 // maxNumRunlists since it is possible that not everything in the range 1393 // [0, numChidMgrs) represents a valid runlistId. The onus is on the 1394 // caller to check for NULL and only then use the CHIDMGR pointer 1395 // 1396 return pKernelFifo->ppChidMgr[runlistId]; 1397 } 1398 } 1399 1400 /*! Gets associated CHIDMGR object for given FIFO_ENGINE_INFO_TYPE and value */ 1401 NV_STATUS 1402 kfifoGetChidMgrFromType_IMPL 1403 ( 1404 OBJGPU *pGpu, 1405 KernelFifo *pKernelFifo, 1406 NvU32 engineInfoType, 1407 NvU32 val, 1408 CHID_MGR **ppChidMgr 1409 ) 1410 { 1411 NV_STATUS status = NV_OK; 1412 NvU32 runlistId; 1413 1414 NV_CHECK_OR_RETURN(LEVEL_INFO, ppChidMgr != NULL, NV_ERR_INVALID_ARGUMENT); 1415 1416 // Initialize the pointer to NULL, in case we fail and return early 1417 *ppChidMgr = NULL; 1418 1419 status = kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo, 1420 engineInfoType, val, 1421 ENGINE_INFO_TYPE_RUNLIST, &runlistId); 1422 NV_CHECK_OR_RETURN(LEVEL_INFO, NV_OK == status, status); 1423 1424 *ppChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, runlistId); 1425 1426 return NV_OK; 1427 } 1428 1429 /*! 1430 * @brief Fetch pKernelChannel based on chidmgr and chid. 1431 * 1432 * This look-up uses the chid heap. It should find the first allocation of the channel, 1433 * which is useful if the handle is duped to another client. 1434 * 1435 * @param[in] pGpu 1436 * @param[in] pKernelFifo 1437 * @param[in] pChidMgr the ChIDMgr (per-runlist) 1438 * @param[in] ChID the ChID 1439 * 1440 * @return the KernelChannel * or NULL 1441 */ 1442 KernelChannel * 1443 kfifoChidMgrGetKernelChannel_IMPL 1444 ( 1445 OBJGPU *pGpu, 1446 KernelFifo *pKernelFifo, 1447 CHID_MGR *pChidMgr, 1448 NvU32 ChID 1449 ) 1450 { 1451 EMEMBLOCK *pFifoDataBlock; 1452 NvU32 numChannels; 1453 1454 NV_ASSERT_OR_RETURN(pChidMgr != NULL, NULL); 1455 // Lite mode channels don't have KernelChannel yet 1456 NV_ASSERT_OR_RETURN(!kfifoIsLiteModeEnabled_HAL(pGpu, pKernelFifo), NULL); 1457 1458 numChannels = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr); 1459 if (ChID >= numChannels) 1460 { 1461 return NULL; 1462 } 1463 1464 pFifoDataBlock = pChidMgr->pFifoDataHeap->eheapGetBlock( 1465 pChidMgr->pFifoDataHeap, 1466 ChID, 1467 NV_FALSE); 1468 if (pFifoDataBlock != NULL) 1469 { 1470 return (KernelChannel *)pFifoDataBlock->pData; 1471 } 1472 1473 return NULL; 1474 } 1475 1476 /*! Gets channel group data corresponding to grpID */ 1477 KernelChannelGroup * 1478 kfifoChidMgrGetKernelChannelGroup_IMPL 1479 ( 1480 OBJGPU *pGpu, 1481 KernelFifo *pKernelFifo, 1482 CHID_MGR *pChidMgr, 1483 NvU32 grpID 1484 ) 1485 { 1486 KernelChannelGroup *pKernelChannelGroup = NULL; 1487 1488 pKernelChannelGroup = mapFind(pChidMgr->pChanGrpTree, grpID); 1489 if (pKernelChannelGroup == NULL) 1490 { 1491 NV_PRINTF(LEVEL_INFO, "Can't find channel group %d\n", grpID); 1492 } 1493 1494 return pKernelChannelGroup; 1495 } 1496 1497 /*! 1498 * @brief Gets channel group data corresponding to grpID 1499 * 1500 * This function is not called in broadcast mode 1501 * 1502 * @param pGpu 1503 * @param pFifo 1504 * @param[in] grpID 1505 * @param[in] runlistID pass CHIDMGR_RUNLIST_ID_LEGACY if not known 1506 * 1507 * @returns KernelChannelGroup * on success 1508 * NULL if channel group is not found 1509 */ 1510 KernelChannelGroup * 1511 kfifoGetChannelGroup_IMPL 1512 ( 1513 OBJGPU *pGpu, 1514 KernelFifo *pKernelFifo, 1515 NvU32 grpID, 1516 NvU32 runlistID 1517 ) 1518 { 1519 CHID_MGR *pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, runlistID); 1520 1521 return kfifoChidMgrGetKernelChannelGroup(pGpu, pKernelFifo, pChidMgr, grpID); 1522 } 1523 1524 /*! Gets total number of channel groups in use */ 1525 NvU32 1526 kfifoGetChannelGroupsInUse_IMPL 1527 ( 1528 OBJGPU *pGpu, 1529 KernelFifo *pKernelFifo 1530 ) 1531 { 1532 NvU32 numChannelGroups = 0; 1533 NvU32 numChannelGroupsInUse = 0; 1534 NvU32 chGrpID, i; 1535 1536 for (i = 0; i < pKernelFifo->numChidMgrs; i++) 1537 { 1538 if (pKernelFifo->ppChidMgr[i] != NULL) 1539 { 1540 numChannelGroups = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, 1541 pKernelFifo->ppChidMgr[i]); 1542 1543 for (chGrpID = 0; chGrpID < numChannelGroups; chGrpID++) 1544 { 1545 if (nvBitFieldTest(pKernelFifo->ppChidMgr[i]->channelGrpMgr.pHwIdInUse, 1546 pKernelFifo->ppChidMgr[i]->channelGrpMgr.hwIdInUseSz, 1547 chGrpID)) 1548 { 1549 numChannelGroupsInUse++; 1550 } 1551 } 1552 } 1553 } 1554 return numChannelGroupsInUse; 1555 } 1556 1557 /*! Gets total number of channel groups in use per engine */ 1558 NvU32 1559 kfifoGetRunlistChannelGroupsInUse_IMPL 1560 ( 1561 OBJGPU *pGpu, 1562 KernelFifo *pKernelFifo, 1563 NvU32 runlistId 1564 ) 1565 { 1566 NvU32 numChannelGroups = 0; 1567 NvU32 numChannelGroupsInUse = 0; 1568 NvU32 chGrpID; 1569 CHID_MGR *pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, runlistId); 1570 1571 numChannelGroups = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr); 1572 for (chGrpID = 0; chGrpID < numChannelGroups; chGrpID++) 1573 { 1574 if (nvBitFieldTest(pChidMgr->channelGrpMgr.pHwIdInUse, 1575 pChidMgr->channelGrpMgr.hwIdInUseSz, 1576 chGrpID)) 1577 { 1578 numChannelGroupsInUse++; 1579 } 1580 } 1581 return numChannelGroupsInUse; 1582 } 1583 1584 /** 1585 * @brief Sets the timeslice for the specified channel group. 1586 * 1587 * @returns NV_OK if success, appropriate error otherwise 1588 */ 1589 NV_STATUS 1590 kfifoChannelGroupSetTimeslice_IMPL 1591 ( 1592 OBJGPU *pGpu, 1593 KernelFifo *pKernelFifo, 1594 KernelChannelGroup *pKernelChannelGroup, 1595 NvU64 timesliceUs, 1596 NvBool bSkipSubmit 1597 ) 1598 { 1599 NV_STATUS status = NV_OK; 1600 1601 NV_PRINTF(LEVEL_INFO, "Setting TSG %d Timeslice to %lldus\n", 1602 pKernelChannelGroup->grpID, timesliceUs); 1603 1604 if (timesliceUs < kfifoRunlistGetMinTimeSlice_HAL(pKernelFifo)) 1605 { 1606 NV_PRINTF(LEVEL_ERROR, 1607 "Setting Timeslice to %lldus not allowed. Min value is %lldus\n", 1608 timesliceUs, kfifoRunlistGetMinTimeSlice_HAL(pKernelFifo)); 1609 return NV_ERR_NOT_SUPPORTED; 1610 } 1611 1612 pKernelChannelGroup->timesliceUs = timesliceUs; 1613 1614 NV_ASSERT_OK_OR_RETURN(kfifoChannelGroupSetTimesliceSched(pGpu, 1615 pKernelFifo, 1616 pKernelChannelGroup, 1617 timesliceUs, 1618 bSkipSubmit)); 1619 1620 return status; 1621 } 1622 1623 void 1624 kfifoFillMemInfo_IMPL 1625 ( 1626 KernelFifo *pKernelFifo, 1627 MEMORY_DESCRIPTOR *pMemDesc, 1628 NV2080_CTRL_FIFO_MEM_INFO *pMemory 1629 ) 1630 { 1631 if (pMemDesc == NULL) 1632 { 1633 pMemory->aperture = NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_APERTURE_INVALID; 1634 NV_PRINTF(LEVEL_ERROR, "kfifoFillMemInfo: pMemDesc = NULL\n"); 1635 } 1636 else 1637 { 1638 if (memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM) 1639 { 1640 pMemory->aperture = NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_APERTURE_VIDMEM; 1641 } 1642 else if (memdescGetAddressSpace(pMemDesc) == ADDR_SYSMEM) 1643 { 1644 if (memdescGetCpuCacheAttrib(pMemDesc) == NV_MEMORY_CACHED) 1645 { 1646 pMemory->aperture = NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_APERTURE_SYSMEM_COH; 1647 } 1648 else if (memdescGetCpuCacheAttrib(pMemDesc) == NV_MEMORY_UNCACHED) 1649 { 1650 pMemory->aperture = NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_APERTURE_SYSMEM_NCOH; 1651 } 1652 else 1653 { 1654 NV_PRINTF(LEVEL_ERROR, 1655 "kfifoFillMemInfo: Unknown cache attribute for sysmem aperture\n"); 1656 NV_ASSERT(NV_FALSE); 1657 } 1658 } 1659 pMemory->base = memdescGetPhysAddr(pMemDesc, AT_GPU, 0); 1660 pMemory->size = pMemDesc->Size; 1661 } 1662 } 1663 1664 void 1665 kfifoGetChannelIterator_IMPL 1666 ( 1667 OBJGPU *pGpu, 1668 KernelFifo *pKernelFifo, 1669 CHANNEL_ITERATOR *pIt 1670 ) 1671 { 1672 portMemSet(pIt, 0, sizeof(*pIt)); 1673 pIt->physicalChannelID = 0; 1674 pIt->pFifoDataBlock = NULL; 1675 pIt->runlistId = 0; 1676 pIt->numRunlists = 1; 1677 if (kfifoIsPerRunlistChramEnabled(pKernelFifo)) 1678 { 1679 pIt->numRunlists = kfifoGetMaxNumRunlists_HAL(pGpu, pKernelFifo); 1680 } 1681 } 1682 1683 /** 1684 * @brief Returns the next KernelChannel from the iterator. 1685 * 1686 * Iterates over runlist IDs and ChIDs and returns the next KernelChannel found 1687 * on the heap, if any. 1688 * 1689 * (error guaranteed if pointer is NULL; non-NULL pointer guaranteed if NV_OK) 1690 * 1691 * @param[in] pGpu 1692 * @param[in] pKernelFifo 1693 * @param[in] pIt the channel iterator 1694 * @param[out] ppKernelChannel returns a KernelChannel * 1695 * 1696 * @return NV_OK if the returned pointer is valid or error 1697 */ 1698 NV_STATUS kfifoGetNextKernelChannel_IMPL 1699 ( 1700 OBJGPU *pGpu, 1701 KernelFifo *pKernelFifo, 1702 CHANNEL_ITERATOR *pIt, 1703 KernelChannel **ppKernelChannel 1704 ) 1705 { 1706 KernelChannel *pKernelChannel; 1707 1708 if (ppKernelChannel == NULL) 1709 return NV_ERR_INVALID_ARGUMENT; 1710 1711 *ppKernelChannel = NULL; 1712 1713 while (pIt->runlistId < pIt->numRunlists) 1714 { 1715 CHID_MGR *pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, pIt->runlistId); 1716 1717 if (pChidMgr == NULL) 1718 { 1719 pIt->runlistId++; 1720 continue; 1721 } 1722 1723 pIt->numChannels = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr); 1724 1725 if (pIt->pFifoDataBlock == NULL) 1726 { 1727 pIt->pFifoDataBlock = pChidMgr->pFifoDataHeap->eheapGetBlock( 1728 pChidMgr->pFifoDataHeap, 1729 pIt->physicalChannelID, 1730 NV_TRUE); 1731 } 1732 1733 while (pIt->physicalChannelID < pIt->numChannels) 1734 { 1735 if (pIt->pFifoDataBlock->owner == NVOS32_BLOCK_TYPE_FREE) 1736 { 1737 pIt->physicalChannelID = pIt->pFifoDataBlock->end + 1; 1738 } 1739 else 1740 { 1741 pIt->physicalChannelID++; 1742 pKernelChannel = (KernelChannel *)pIt->pFifoDataBlock->pData; 1743 1744 // 1745 // This iterator can be used during an interrupt, when a KernelChannel may 1746 // be in the process of being destroyed. If a KernelChannel expects a pChannel 1747 // but does not have one, it means it's being destroyed and we don't want to 1748 // return it. 1749 // 1750 if (pKernelChannel != NULL && kchannelIsValid_HAL(pKernelChannel)) 1751 { 1752 // Prepare iterator to check next block in pChidMgr->pFifoDataHeap 1753 pIt->pFifoDataBlock = pIt->pFifoDataBlock->next; 1754 *ppKernelChannel = pKernelChannel; 1755 return NV_OK; 1756 } 1757 } 1758 1759 // Check next block in pChidMgr->pFifoDataHeap 1760 pIt->pFifoDataBlock = pIt->pFifoDataBlock->next; 1761 } 1762 1763 pIt->runlistId++; 1764 // Reset iterator for next runlist 1765 pIt->physicalChannelID = 0; 1766 pIt->pFifoDataBlock = NULL; 1767 } 1768 1769 return NV_ERR_OBJECT_NOT_FOUND; 1770 } 1771 1772 /*! 1773 * @brief Localize the engine info received from the host 1774 * 1775 * The host and the guest can run in different version of drivers, the guest driver 1776 * can not directly use the raw MC_ENGINE_IDX values from the host. 1777 * RM does not guarantee those values are consistent cross branches. 1778 * 1779 * To keep the compatibility between different version of branches, this function reconstructs 1780 * of RM_ENGINE_TYPE, MC_ENGINE_IDX values based on NV2080_ENGINE_TYPE 1781 * 1782 * @param[in] pGpu OBJGPU pointer 1783 * @param[in] pFifo KernelFifo pointer 1784 * @param[in/out] pEngine Pointer to engine info table to update 1785 * 1786 */ 1787 static void 1788 _kfifoLocalizeGuestEngineData 1789 ( 1790 OBJGPU *pGpu, 1791 KernelFifo *pKernelFifo, 1792 ENGINE_INFO *pEngineInfo 1793 ) 1794 { 1795 const FIFO_GUEST_ENGINE_TABLE *guestEngineTable; 1796 NvU32 guestEngineTableSz; 1797 NvU32 nv2080EngineType; 1798 NvU32 engineIdx; 1799 NvU32 newEngineIdx; 1800 NvU32 guestTableIdx; 1801 1802 // This function should only be called in a vgpu guest RM 1803 NV_ASSERT_OR_RETURN_VOID(IS_VIRTUAL(pGpu)); 1804 1805 guestEngineTable = kfifoGetGuestEngineLookupTable(&guestEngineTableSz); 1806 1807 newEngineIdx = 0; 1808 1809 for (engineIdx = 0; engineIdx < pEngineInfo->engineInfoListSize; engineIdx++) 1810 { 1811 FIFO_ENGINE_LIST *pEngine = &pEngineInfo->engineInfoList[engineIdx]; 1812 1813 // The actual data in engineData[ENGINE_INFO_TYPE_RM_ENGINE_TYPE] is NV2080 ENGINE TYPE. 1814 nv2080EngineType = pEngine->engineData[ENGINE_INFO_TYPE_RM_ENGINE_TYPE]; 1815 1816 for (guestTableIdx = 0; guestTableIdx < guestEngineTableSz; guestTableIdx++) 1817 { 1818 // Find the engine type supported by the guest 1819 if (guestEngineTable[guestTableIdx].nv2080EngineType == nv2080EngineType) 1820 break; 1821 } 1822 1823 if (guestTableIdx < guestEngineTableSz) 1824 { 1825 // Update the MC for the guest 1826 pEngine->engineData[ENGINE_INFO_TYPE_MC] = guestEngineTable[guestTableIdx].mcIdx; 1827 1828 // Update it with correct engine type 1829 pEngine->engineData[ENGINE_INFO_TYPE_RM_ENGINE_TYPE] = gpuGetRmEngineType(nv2080EngineType); 1830 1831 if (newEngineIdx != engineIdx) 1832 { 1833 // 1834 // Move the engine info up to make sure the engine info table only contains data for 1835 // guest supported engine types. 1836 // 1837 portMemCopy(&pEngineInfo->engineInfoList[newEngineIdx], sizeof(FIFO_ENGINE_LIST), 1838 &pEngineInfo->engineInfoList[engineIdx], sizeof(FIFO_ENGINE_LIST)); 1839 } 1840 1841 newEngineIdx++; 1842 } 1843 } 1844 1845 pEngineInfo->engineInfoListSize = newEngineIdx; 1846 } 1847 1848 /*! 1849 * @brief Performs an RPC into Host RM to read its device info table. 1850 * 1851 * This is necessary because in virtual environments, we cannot directly read 1852 * the device info table, and do not have the physical GPU partitioning 1853 * information to determine which engines belong to this guest, so we have Host 1854 * RM do the filtering and send us the filtered table. 1855 * 1856 * @param[in] pGpu 1857 * @param[in] pKernelFifo 1858 * 1859 * @return NV_OK if succcessful, 1860 * NV_ERR_NOT_SUPPORTED if Host RM calls this interface 1861 * NV_ERR_INVALID_STATE if host supplied invalid data 1862 * NV_STATUS supplied by RPC response from Host 1863 */ 1864 1865 NV_STATUS 1866 kfifoGetHostDeviceInfoTable_KERNEL 1867 ( 1868 OBJGPU *pGpu, 1869 KernelFifo *pKernelFifo, 1870 ENGINE_INFO *pEngineInfo, 1871 NvHandle hMigClient 1872 ) 1873 { 1874 NV_STATUS status = NV_OK; 1875 NvHandle hClient = NV01_NULL_OBJECT; 1876 NvHandle hObject = NV01_NULL_OBJECT; 1877 NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_PARAMS *pParams; 1878 NV2080_CTRL_FIFO_DEVICE_ENTRY *pHostEntries; 1879 NvU32 numEntries; 1880 NvU32 device; 1881 NvU32 entry; 1882 NvU32 numRunlists; 1883 NvU32 maxRunlistId; 1884 NvU32 maxPbdmaId; 1885 NvU32 i; 1886 struct { 1887 NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_PARAMS params; 1888 NV2080_CTRL_FIFO_DEVICE_ENTRY entries[NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_MAX_DEVICES]; 1889 } *pLocals; 1890 1891 1892 NV_ASSERT_OR_RETURN(IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu), 1893 NV_ERR_NOT_SUPPORTED); 1894 1895 // RPC call for GSP will throw INVALID_CLIENT error with NULL handles 1896 if (IS_GSP_CLIENT(pGpu)) 1897 { 1898 if (!IS_MIG_IN_USE(pGpu)) 1899 { 1900 hClient = pGpu->hInternalClient; 1901 hObject = pGpu->hInternalSubdevice; 1902 } 1903 else 1904 { 1905 hClient = hMigClient; 1906 if ((status = CliGetSubDeviceHandleFromGpu(hClient, pGpu, &hObject)) != NV_OK) 1907 { 1908 NV_PRINTF(LEVEL_ERROR, "Unable to get subdevice handle.\n"); 1909 return NV_ERR_INVALID_STATE; 1910 } 1911 } 1912 } 1913 1914 // Allocate pHostEntries and params on the heap to avoid stack overflow 1915 pLocals = portMemAllocNonPaged(sizeof(*pLocals)); 1916 NV_ASSERT_OR_RETURN((pLocals != NULL), NV_ERR_NO_MEMORY); 1917 1918 pParams = &pLocals->params; 1919 pHostEntries = pLocals->entries; 1920 1921 // 1922 // Read device info table entries from Host RM until Host indicates that 1923 // there are no more valid entries in the table (by setting bMore flag) 1924 // 1925 numEntries = 0; 1926 for (device = 0; 1927 device < NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_MAX_DEVICES; 1928 device += NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_MAX_ENTRIES) 1929 { 1930 portMemSet(pParams, 0x0, sizeof(*pParams)); 1931 pParams->baseIndex = device; 1932 1933 NV_RM_RPC_CONTROL(pGpu, 1934 hClient, 1935 hObject, 1936 NV2080_CTRL_CMD_FIFO_GET_DEVICE_INFO_TABLE, 1937 pParams, 1938 sizeof(*pParams), 1939 status); 1940 1941 if (status != NV_OK) 1942 goto cleanup; 1943 1944 // Assert that host RM didn't tell us an invalid number of entries 1945 if (pParams->numEntries > 1946 NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_MAX_ENTRIES) 1947 { 1948 DBG_BREAKPOINT(); 1949 status = NV_ERR_INVALID_STATE; 1950 goto cleanup; 1951 } 1952 1953 portMemCopy(&pHostEntries[device], pParams->numEntries * 1954 sizeof(NV2080_CTRL_FIFO_DEVICE_ENTRY), pParams->entries, pParams->numEntries * 1955 sizeof(NV2080_CTRL_FIFO_DEVICE_ENTRY)); 1956 1957 numEntries += pParams->numEntries; 1958 1959 if (!pParams->bMore) 1960 { 1961 break; 1962 } 1963 } 1964 1965 pEngineInfo->engineInfoListSize = numEntries; 1966 pEngineInfo->engineInfoList = portMemAllocNonPaged(sizeof(*pEngineInfo->engineInfoList) * 1967 pEngineInfo->engineInfoListSize); 1968 if (pEngineInfo->engineInfoList == NULL) 1969 { 1970 NV_CHECK(LEVEL_ERROR, pEngineInfo->engineInfoList != NULL); 1971 status = NV_ERR_NO_MEMORY; 1972 goto cleanup; 1973 } 1974 1975 // Copy each entry from the table 1976 numRunlists = 0; 1977 maxRunlistId = 0; 1978 maxPbdmaId = 0; 1979 for (entry = 0; entry < numEntries; ++entry) 1980 { 1981 portMemCopy(pEngineInfo->engineInfoList[entry].engineData, 1982 ENGINE_INFO_TYPE_INVALID * sizeof(*(pEngineInfo->engineInfoList[entry].engineData)), 1983 pHostEntries[entry].engineData, 1984 ENGINE_INFO_TYPE_INVALID * sizeof(*(pEngineInfo->engineInfoList[entry].engineData))); 1985 1986 pEngineInfo->engineInfoList[entry].numPbdmas = pHostEntries[entry].numPbdmas; 1987 portMemCopy(pEngineInfo->engineInfoList[entry].pbdmaIds, 1988 FIFO_ENGINE_MAX_NUM_PBDMA * sizeof(*(pEngineInfo->engineInfoList[entry].pbdmaIds)), 1989 pHostEntries[entry].pbdmaIds, 1990 FIFO_ENGINE_MAX_NUM_PBDMA * sizeof(*(pEngineInfo->engineInfoList[entry].pbdmaIds))); 1991 1992 portMemCopy(pEngineInfo->engineInfoList[entry].pbdmaFaultIds, 1993 FIFO_ENGINE_MAX_NUM_PBDMA * sizeof(*(pEngineInfo->engineInfoList[entry].pbdmaFaultIds)), 1994 pHostEntries[entry].pbdmaFaultIds, 1995 FIFO_ENGINE_MAX_NUM_PBDMA * sizeof(*(pEngineInfo->engineInfoList[entry].pbdmaFaultIds))); 1996 1997 portStringCopy((char *)pEngineInfo->engineInfoList[entry].engineName, 1998 sizeof(pEngineInfo->engineInfoList[entry].engineName), 1999 (char *)pHostEntries[entry].engineName, 2000 FIFO_ENGINE_NAME_MAX_SIZE); 2001 2002 if (pEngineInfo->engineInfoList[entry].engineData[ENGINE_INFO_TYPE_IS_HOST_DRIVEN_ENGINE] != 0) 2003 { 2004 numRunlists++; 2005 } 2006 maxRunlistId = NV_MAX(maxRunlistId, 2007 pHostEntries[entry].engineData[ENGINE_INFO_TYPE_RUNLIST]); 2008 2009 for (i = 0; i < pEngineInfo->engineInfoList[entry].numPbdmas; i++) 2010 { 2011 maxPbdmaId = NV_MAX(maxPbdmaId, pEngineInfo->engineInfoList[entry].pbdmaIds[i]); 2012 2013 // 2014 // SW engine while being constructed does not populate any PBDMA Fault IDs. 2015 // Hence, skipping it. 2016 // 2017 if (pEngineInfo->engineInfoList[entry].engineData[ENGINE_INFO_TYPE_ENG_DESC] != ENG_SW) 2018 { 2019 bitVectorSet(&pEngineInfo->validEngineIdsForPbdmas, pEngineInfo->engineInfoList[entry].pbdmaFaultIds[i]); 2020 } 2021 } 2022 } 2023 2024 NV_ASSERT_OR_GOTO(kfifoReservePbdmaFaultIds_HAL(pGpu, pKernelFifo, pEngineInfo->engineInfoList, 2025 pEngineInfo->engineInfoListSize) == NV_OK, cleanup); 2026 2027 if (IS_VIRTUAL(pGpu)) 2028 { 2029 _kfifoLocalizeGuestEngineData(pGpu, pKernelFifo, pEngineInfo); 2030 } 2031 2032 pEngineInfo->numRunlists = numRunlists; 2033 pEngineInfo->maxNumRunlists = maxRunlistId + 1; 2034 pEngineInfo->maxNumPbdmas = maxPbdmaId + 1; 2035 2036 cleanup: 2037 portMemFree(pLocals); 2038 2039 return status; 2040 } 2041 2042 /*! 2043 * @brief Constructs EngineInfo List 2044 * 2045 * @param[in] pGpu 2046 * @param[in] pKernelFifo 2047 * 2048 * @return NV_OK if succcessful, 2049 * NV_STATUS supplied by HALs called 2050 */ 2051 NV_STATUS 2052 kfifoConstructEngineList_KERNEL 2053 ( 2054 OBJGPU *pGpu, 2055 KernelFifo *pKernelFifo 2056 ) 2057 { 2058 ENGINE_INFO *pEngineInfo = &pKernelFifo->engineInfo; 2059 2060 // Return early if EngineList is already constructed 2061 if (pEngineInfo->engineInfoList != NULL) 2062 return NV_OK; 2063 2064 if (IS_GSP_CLIENT(pGpu)) 2065 { 2066 NV_ASSERT_OK_OR_RETURN(gpuConstructDeviceInfoTable_HAL(pGpu)); 2067 } 2068 2069 NV_ASSERT_OK_OR_RETURN(kfifoGetHostDeviceInfoTable_HAL(pGpu, pKernelFifo, pEngineInfo, 0)); 2070 2071 return NV_OK; 2072 } 2073 2074 /** 2075 * @brief Create a list of channels. 2076 * 2077 * @param pGpu 2078 * @param pKernelFifo 2079 * @param pList 2080 */ 2081 NV_STATUS 2082 kfifoChannelListCreate_IMPL 2083 ( 2084 OBJGPU *pGpu, 2085 KernelFifo *pKernelFifo, 2086 CHANNEL_LIST **ppList 2087 ) 2088 { 2089 if (!ppList) 2090 return NV_ERR_INVALID_ARGUMENT; 2091 2092 *ppList = portMemAllocNonPaged(sizeof(CHANNEL_LIST)); 2093 NV_ASSERT_OR_RETURN((*ppList != NULL), NV_ERR_NO_MEMORY); 2094 2095 (*ppList)->pHead = NULL; 2096 (*ppList)->pTail = NULL; 2097 2098 return NV_OK; 2099 } 2100 2101 /** 2102 * @brief Append a channel to a channel list. 2103 * 2104 * @param pGpu 2105 * @param pKernelFifo 2106 * @param pKernelChannel 2107 * @param pList 2108 */ 2109 2110 NV_STATUS 2111 kfifoChannelListAppend_IMPL 2112 ( 2113 OBJGPU *pGpu, 2114 KernelFifo *pKernel, 2115 KernelChannel *pKernelChannel, 2116 CHANNEL_LIST *pList 2117 ) 2118 { 2119 PCHANNEL_NODE pNewNode = NULL; 2120 2121 if (!pKernelChannel || !pList) 2122 return NV_ERR_INVALID_ARGUMENT; 2123 2124 pNewNode = portMemAllocNonPaged(sizeof(CHANNEL_NODE)); 2125 NV_ASSERT_OR_RETURN((pNewNode != NULL), NV_ERR_NO_MEMORY); 2126 2127 pNewNode->pKernelChannel = pKernelChannel; 2128 pKernelChannel->refCount++; 2129 2130 pNewNode->pNext = NULL; 2131 2132 // Searching based on the ChID 2133 if (pList->pTail) 2134 { 2135 pList->pTail->pNext = pNewNode; 2136 pList->pTail = pNewNode; 2137 } 2138 else 2139 { 2140 pList->pHead = pNewNode; 2141 pList->pTail = pNewNode; 2142 } 2143 2144 return NV_OK; 2145 } 2146 2147 /** 2148 * @brief remove channel from the given channel list 2149 * look for duplicates. 2150 * 2151 * @param pGpu 2152 * @param pKernelFifo 2153 * @param pKernelChannel 2154 * @param pList 2155 */ 2156 NV_STATUS 2157 kfifoChannelListRemove_IMPL 2158 ( 2159 OBJGPU *pGpu, 2160 KernelFifo *pKernelFifo, 2161 KernelChannel *pKernelChannel, 2162 CHANNEL_LIST *pList 2163 ) 2164 { 2165 PCHANNEL_NODE pNewNode = NULL; 2166 PCHANNEL_NODE pPrevNode = NULL; 2167 PCHANNEL_NODE pTempNode = NULL; 2168 NvBool bFoundOnce = NV_FALSE; 2169 NV_STATUS status = NV_OK; 2170 2171 if (!pKernelChannel) 2172 return NV_ERR_INVALID_ARGUMENT; 2173 2174 if (!pList) 2175 return NV_ERR_INVALID_ARGUMENT; 2176 2177 pNewNode = pList->pHead; 2178 pPrevNode = NULL; 2179 2180 while (pNewNode) 2181 { 2182 2183 if (pKernelChannel != pNewNode->pKernelChannel) 2184 { 2185 pPrevNode = pNewNode; 2186 pNewNode = pNewNode->pNext; 2187 continue; 2188 } 2189 2190 // Deleting first node 2191 if (pList->pHead == pNewNode) 2192 { 2193 pList->pHead = pNewNode->pNext; 2194 } 2195 2196 // Deleting tail node 2197 if (pList->pTail == pNewNode) 2198 { 2199 pList->pTail = pPrevNode; 2200 } 2201 2202 // First node does not have previous node. 2203 if (pPrevNode) 2204 { 2205 pPrevNode->pNext = pNewNode->pNext; 2206 } 2207 2208 pTempNode = pNewNode; 2209 pNewNode = pNewNode->pNext; 2210 portMemFree(pTempNode); 2211 2212 bFoundOnce = NV_TRUE; 2213 2214 if (0 == pKernelChannel->refCount) 2215 { 2216 NV_PRINTF(LEVEL_ERROR, "RefCount for channel is not right!!!\n"); 2217 DBG_BREAKPOINT(); 2218 status = NV_ERR_GENERIC; 2219 break; 2220 } 2221 2222 pKernelChannel->refCount--; 2223 } 2224 2225 2226 if (!bFoundOnce) 2227 { 2228 NV_PRINTF(LEVEL_INFO, 2229 "Can't find channel in channelGroupList (Normal during RC Recovery on " 2230 "GK110+ or if software scheduling is enabled).\n"); 2231 2232 status = NV_ERR_INVALID_CHANNEL; 2233 } 2234 2235 return status; 2236 } 2237 2238 /** 2239 * @brief Destroy channel list. 2240 * 2241 * @param pGpu 2242 * @param pKernelFifo 2243 * @param pList 2244 */ 2245 NV_STATUS 2246 kfifoChannelListDestroy_IMPL 2247 ( 2248 OBJGPU *pGpu, 2249 KernelFifo *pKernel, 2250 CHANNEL_LIST *pList 2251 ) 2252 { 2253 PCHANNEL_NODE pTempNode; 2254 2255 if (!pList) 2256 return NV_OK; 2257 2258 while (pList->pHead) 2259 { 2260 pTempNode = pList->pHead; 2261 2262 NV_ASSERT_OR_RETURN(pTempNode->pKernelChannel && pTempNode->pKernelChannel->refCount, NV_ERR_INVALID_STATE); 2263 2264 pTempNode->pKernelChannel->refCount--; 2265 2266 pList->pHead = pTempNode->pNext; 2267 portMemFree(pTempNode); 2268 } 2269 2270 portMemFree(pList); 2271 2272 return NV_OK; 2273 } 2274 2275 /*! 2276 * @brief Determines whether provided engines have any channels/contexts assigned 2277 * 2278 * @param[IN] pGpu OBJGPU 2279 * @param[IN] pKernelFifo KernelFifo 2280 * @param[IN] pEngines Which engines to check (RM_ENGINE_TYPE_***) 2281 * @param[IN] engineCount Number of engines to check 2282 * 2283 * @return Returns NV_TRUE if any provided engines are active 2284 */ 2285 NvBool 2286 kfifoEngineListHasChannel_IMPL 2287 ( 2288 OBJGPU *pGpu, 2289 KernelFifo *pKernelFifo, 2290 RM_ENGINE_TYPE *pEngines, 2291 NvU32 engineCount 2292 ) 2293 { 2294 KernelChannel *pKernelChannel; 2295 CHANNEL_ITERATOR it; 2296 NvU32 i; 2297 2298 NV_ASSERT_OR_RETURN((pEngines != NULL) && (engineCount > 0), NV_TRUE); 2299 2300 // Find any channels or contexts on passed engines 2301 kfifoGetChannelIterator(pGpu, pKernelFifo, &it); 2302 while (kchannelGetNextKernelChannel(pGpu, &it, &pKernelChannel) == NV_OK) 2303 { 2304 NV_ASSERT_OR_ELSE(pKernelChannel != NULL, continue); 2305 2306 // If the client supplied the engine type, directly check it 2307 if (RM_ENGINE_TYPE_IS_VALID(kchannelGetEngineType(pKernelChannel))) 2308 { 2309 for (i = 0; i < engineCount; ++i) 2310 { 2311 if (kchannelGetEngineType(pKernelChannel) == pEngines[i]) 2312 { 2313 NV_PRINTF(LEVEL_ERROR, 2314 "Found channel on engine 0x%x owned by 0x%x\n", 2315 kchannelGetEngineType(pKernelChannel), RES_GET_CLIENT_HANDLE(pKernelChannel)); 2316 2317 return NV_TRUE; 2318 } 2319 } 2320 } 2321 else 2322 { 2323 NvU32 runlistId; 2324 2325 // Ideally valid engine Id should always be set in channel if this property is enabled 2326 NV_ASSERT_OR_RETURN(!kfifoIsPerRunlistChramEnabled(pKernelFifo), NV_TRUE); 2327 2328 // 2329 // If runlist Id for channel is set then check if it matches with any of the engines 2330 // If channel is not associated with any engine then there is a chance 2331 // it can be created on one of the engines we care about. 2332 // 2333 if (kchannelIsRunlistSet(pGpu, pKernelChannel)) 2334 { 2335 for (i = 0; i < engineCount; ++i) 2336 { 2337 NV_ASSERT_OR_RETURN((kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo, 2338 ENGINE_INFO_TYPE_RM_ENGINE_TYPE, (NvU32)pEngines[i], 2339 ENGINE_INFO_TYPE_RUNLIST, &runlistId) == NV_OK), NV_TRUE); 2340 if (kchannelGetRunlistId(pKernelChannel) == runlistId) 2341 { 2342 NV_PRINTF(LEVEL_ERROR, 2343 "Found channel on runlistId 0x%x owned by 0x%x\n", 2344 kchannelGetRunlistId(pKernelChannel), RES_GET_CLIENT_HANDLE(pKernelChannel)); 2345 2346 return NV_TRUE; 2347 } 2348 } 2349 } 2350 else 2351 { 2352 NV_PRINTF(LEVEL_ERROR, 2353 "Found channel owned by 0x%x that can be associated to any engine\n", 2354 RES_GET_CLIENT_HANDLE(pKernelChannel)); 2355 2356 return NV_TRUE; 2357 } 2358 } 2359 } 2360 2361 return NV_FALSE; 2362 } 2363 2364 /** 2365 * @brief Maximum number of subcontexts for a non-legacy mode TSG 2366 */ 2367 CTX_BUF_POOL_INFO * 2368 kfifoGetRunlistBufPool_IMPL 2369 ( 2370 OBJGPU *pGpu, 2371 KernelFifo *pKernelFifo, 2372 RM_ENGINE_TYPE rmEngineType 2373 ) 2374 { 2375 return pKernelFifo->pRunlistBufPool[rmEngineType]; 2376 } 2377 2378 /** 2379 * @brief Get size and alignment requirements for runlist buffers 2380 * 2381 * @param[in] pGpu Pointer to OBJGPU 2382 * @param[in] pKernelFifo Pointer to KernelFifo 2383 * @param[in] runlistId Runlist ID 2384 * @param[in] bTsgSupported Is TSG supported 2385 * @param[in] maxRunlistEntries Max entries to be supported in a runlist 2386 * @param[out] pSize Size of runlist buffer 2387 * @param[out] pAlignment Alignment for runlist buffer 2388 */ 2389 NV_STATUS 2390 kfifoGetRunlistBufInfo_IMPL 2391 ( 2392 OBJGPU *pGpu, 2393 KernelFifo *pKernelFifo, 2394 NvU32 runlistId, 2395 NvBool bTsgSupported, 2396 NvU32 maxRunlistEntries, 2397 NvU64 *pSize, 2398 NvU64 *pAlignment 2399 ) 2400 { 2401 NvU32 runlistEntrySize = 0; 2402 NvU32 maxRunlistEntriesSupported = 0; 2403 CHID_MGR *pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, runlistId); 2404 2405 NV_ASSERT_OR_RETURN(pSize != NULL, NV_ERR_INVALID_ARGUMENT); 2406 NV_ASSERT_OR_RETURN(pAlignment != NULL, NV_ERR_INVALID_ARGUMENT); 2407 2408 if (kfifoIsPerRunlistChramEnabled(pKernelFifo)) 2409 { 2410 NV_ASSERT_OR_RETURN(pChidMgr != NULL, NV_ERR_INVALID_ARGUMENT); 2411 // 2412 // We assume worst case of one TSG wrapper per channel, and 2413 // the number of TSGs + number of channels is how we get 2414 // the 2 x number of fifos. 2415 // 2416 maxRunlistEntriesSupported = 2 * kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr); 2417 } 2418 else 2419 { 2420 maxRunlistEntriesSupported = kfifoGetMaxChannelsInSystem(pGpu, pKernelFifo); 2421 maxRunlistEntriesSupported += (bTsgSupported ? 2422 kfifoGetMaxChannelGroupsInSystem(pGpu, pKernelFifo) 2423 : 0); 2424 } 2425 2426 NV_ASSERT_OR_RETURN(maxRunlistEntries <= maxRunlistEntriesSupported, NV_ERR_INVALID_ARGUMENT); 2427 2428 if (maxRunlistEntries == 0) 2429 { 2430 maxRunlistEntries = maxRunlistEntriesSupported; 2431 } 2432 2433 runlistEntrySize = kfifoRunlistGetEntrySize_HAL(pKernelFifo); 2434 *pSize = (NvU64)runlistEntrySize * maxRunlistEntries; 2435 2436 *pAlignment = NVBIT64(kfifoRunlistGetBaseShift_HAL(pKernelFifo)); 2437 return NV_OK; 2438 } 2439 2440 /*! 2441 * @brief Gets total number of channels supported by the system 2442 */ 2443 NvU32 2444 kfifoGetMaxChannelsInSystem_IMPL 2445 ( 2446 OBJGPU *pGpu, 2447 KernelFifo *pKernelFifo 2448 ) 2449 { 2450 NvU32 numChannels = 0; 2451 NvU32 i; 2452 2453 for (i = 0; i < pKernelFifo->numChidMgrs; i++) 2454 { 2455 if (pKernelFifo->ppChidMgr[i] != NULL) 2456 { 2457 numChannels += kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pKernelFifo->ppChidMgr[i]); 2458 } 2459 } 2460 return numChannels; 2461 } 2462 2463 /*! 2464 * @brief Gets total number of channel groups supported by the system 2465 */ 2466 NvU32 2467 kfifoGetMaxChannelGroupsInSystem_IMPL 2468 ( 2469 OBJGPU *pGpu, 2470 KernelFifo *pKernelFifo 2471 ) 2472 { 2473 // Max channel groups is the same as max channels 2474 return kfifoGetMaxChannelsInSystem(pGpu, pKernelFifo); 2475 } 2476 2477 /*! 2478 * @brief Get runlist buffer allocation params 2479 * 2480 * @param[in] pGpu 2481 * @param[out] *pAperture Aperture to use for runlist buffer allocation 2482 * @param[out] *pAttr Attributes to use for runlits buffer allocation 2483 * @param[out] *pAllocFlags Allocation flags to use for runlist buffer allocation 2484 */ 2485 void 2486 kfifoRunlistGetBufAllocParams_IMPL 2487 ( 2488 OBJGPU *pGpu, 2489 NV_ADDRESS_SPACE *pAperture, 2490 NvU32 *pAttr, 2491 NvU64 *pAllocFlags 2492 ) 2493 { 2494 *pAperture = ADDR_FBMEM; 2495 *pAttr = NV_MEMORY_WRITECOMBINED; 2496 2497 memdescOverrideInstLoc(DRF_VAL(_REG_STR_RM, _INST_LOC, _RUNLIST, pGpu->instLocOverrides), 2498 "runlist", pAperture, pAttr); 2499 2500 *pAllocFlags = FLD_TEST_DRF(_REG_STR_RM, _INST_VPR, _RUNLIST, _TRUE, pGpu->instVprOverrides) 2501 ? MEMDESC_ALLOC_FLAGS_PROTECTED : MEMDESC_FLAGS_NONE; 2502 } 2503 2504 /*! 2505 * @brief Allocate Runlist buffers for a single runlistId 2506 * 2507 * @param[in] pGpu 2508 * @param[in] pKernelFifo 2509 * @param[in] bSupportTsg Will this runlist support TSGs? 2510 * @param[in] aperture NV_ADDRESS_SPACE requested 2511 * @param[in] runlistId runlistId to allocate buffer for 2512 * @param[in] attr CPU cacheability requested 2513 * @param[in] allocFlags MEMDESC_FLAGS_* 2514 * @param[in] maxRunlistEntries Can pass zero to determine in function 2515 * @param[in] bHWRL Is this runlist a HW runlist? (verif feature specific) 2516 * @param[out] ppMemDesc memdesc created/allocated by function 2517 */ 2518 NV_STATUS 2519 kfifoRunlistAllocBuffers_IMPL 2520 ( 2521 OBJGPU *pGpu, 2522 KernelFifo *pKernelFifo, 2523 NvBool bSupportTsg, 2524 NV_ADDRESS_SPACE aperture, 2525 NvU32 runlistId, 2526 NvU32 attr, 2527 NvU64 allocFlags, 2528 NvU64 maxRunlistEntries, 2529 NvBool bHWRL, 2530 MEMORY_DESCRIPTOR **ppMemDesc 2531 ) 2532 { 2533 NV_STATUS status = NV_OK; 2534 NvU64 runlistSz = 0; 2535 NvU64 runlistAlign = 0; 2536 NvU32 counter; 2537 2538 status = kfifoGetRunlistBufInfo(pGpu, pKernelFifo, runlistId, bSupportTsg, 2539 maxRunlistEntries, &runlistSz, &runlistAlign); 2540 if (status != NV_OK) 2541 { 2542 NV_PRINTF(LEVEL_ERROR, "failed to get runlist buffer info 0x%08x\n", 2543 status); 2544 DBG_BREAKPOINT(); 2545 goto failed; 2546 } 2547 2548 for (counter = 0; counter < NUM_BUFFERS_PER_RUNLIST; ++counter) 2549 { 2550 ppMemDesc[counter] = NULL; 2551 2552 status = memdescCreate(&ppMemDesc[counter], pGpu, runlistSz, runlistAlign, 2553 NV_TRUE, aperture, attr, allocFlags); 2554 if (status != NV_OK) 2555 { 2556 NV_PRINTF(LEVEL_ERROR, 2557 "Runlist buffer memdesc create failed 0x%08x\n", status); 2558 DBG_BREAKPOINT(); 2559 goto failed; 2560 } 2561 2562 // If flag is set then allocate runlist from ctx buf pool 2563 if (allocFlags & MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL) 2564 { 2565 RM_ENGINE_TYPE rmEngineType; 2566 CTX_BUF_POOL_INFO *pCtxBufPool = NULL; 2567 status = kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo, ENGINE_INFO_TYPE_RUNLIST, 2568 runlistId, ENGINE_INFO_TYPE_RM_ENGINE_TYPE, (NvU32 *)&rmEngineType); 2569 if (status != NV_OK) 2570 { 2571 NV_PRINTF(LEVEL_ERROR, 2572 "Failed to translate runlistId 0x%x to NV2080 engine type\n", runlistId); 2573 DBG_BREAKPOINT(); 2574 goto failed; 2575 } 2576 status = ctxBufPoolGetGlobalPool(pGpu, CTX_BUF_ID_RUNLIST, rmEngineType, &pCtxBufPool); 2577 if (status != NV_OK) 2578 { 2579 NV_PRINTF(LEVEL_ERROR, 2580 "Failed to get ctx buf pool for engine type 0x%x (0x%x)\n", 2581 gpuGetNv2080EngineType(rmEngineType), rmEngineType); 2582 DBG_BREAKPOINT(); 2583 goto failed; 2584 } 2585 status = memdescSetCtxBufPool(ppMemDesc[counter], pCtxBufPool); 2586 if (status != NV_OK) 2587 { 2588 NV_PRINTF(LEVEL_ERROR, 2589 "Failed to set ctx buf pool for runlistId 0x%x\n", runlistId); 2590 DBG_BREAKPOINT(); 2591 goto failed; 2592 } 2593 } 2594 2595 status = memdescAlloc(ppMemDesc[counter]); 2596 if (status != NV_OK) 2597 { 2598 NV_PRINTF(LEVEL_ERROR, "Runlist buffer mem alloc failed 0x%08x\n", 2599 status); 2600 DBG_BREAKPOINT(); 2601 goto failed; 2602 } 2603 } 2604 2605 return NV_OK; 2606 2607 failed: 2608 for (counter = 0; counter < NUM_BUFFERS_PER_RUNLIST; counter++) 2609 { 2610 if (ppMemDesc[counter]) 2611 { 2612 memdescFree(ppMemDesc[counter]); 2613 memdescDestroy(ppMemDesc[counter]); 2614 ppMemDesc[counter] = NULL; 2615 } 2616 } 2617 return status; 2618 } 2619 2620 NvU32 2621 kfifoGetMaxSubcontextFromGr_KERNEL 2622 ( 2623 OBJGPU *pGpu, 2624 KernelFifo *pKernelFifo 2625 ) 2626 { 2627 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 2628 2629 NV_ASSERT_OR_RETURN(pKernelGraphicsManager != NULL, 0); 2630 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.bInitialized, 0); 2631 NV_ASSERT_OR_RETURN(pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo != NULL, 0); 2632 2633 return pKernelGraphicsManager->legacyKgraphicsStaticInfo.pGrInfo->infoList[NV0080_CTRL_GR_INFO_INDEX_MAX_SUBCONTEXT_COUNT].data; 2634 } 2635 2636 NvU32 2637 kfifoReturnPushbufferCaps_IMPL 2638 ( 2639 OBJGPU *pGpu, 2640 KernelFifo *pKernelFifo 2641 ) 2642 { 2643 NvU32 kfifoBitMask = 0; 2644 2645 // PCI is always supported 2646 kfifoBitMask = PCI_PB_ALLOWED; 2647 2648 if (!gpuIsUnifiedMemorySpaceEnabled(pGpu)) 2649 { 2650 kfifoBitMask |= VID_PB_ALLOWED; 2651 } 2652 2653 return kfifoBitMask; 2654 } 2655 2656 /*! 2657 * @brief kfifoGetDeviceCaps 2658 * 2659 * This routine gets cap bits in unicast. If pbCapsInitialized is passed as 2660 * NV_FALSE, the caps will be copied into pKfifoCaps without OR/ANDING. 2661 * Otherwise, the caps bits for the current GPU will be ORed/ANDed together with 2662 * pKfifoCaps to create a single set of caps that accurately represents the 2663 * functionality of the device. 2664 */ 2665 void kfifoGetDeviceCaps_IMPL 2666 ( 2667 OBJGPU *pGpu, 2668 KernelFifo *pKernelFifo, 2669 NvU8 *pKfifoCaps, 2670 NvBool bCapsInitialized 2671 ) 2672 { 2673 NvU8 tempCaps[NV0080_CTRL_FIFO_CAPS_TBL_SIZE]; 2674 NvU8 temp; 2675 NvU32 kfifoBitMask; 2676 2677 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu)); 2678 2679 portMemSet(tempCaps, 0, NV0080_CTRL_FIFO_CAPS_TBL_SIZE); 2680 2681 kfifoBitMask = kfifoReturnPushbufferCaps(pGpu, pKernelFifo); 2682 2683 if (kfifoBitMask & PCI_PB_ALLOWED) 2684 RMCTRL_SET_CAP(tempCaps, NV0080_CTRL_FIFO_CAPS, _SUPPORT_PCI_PB); 2685 if (kfifoBitMask & VID_PB_ALLOWED) 2686 RMCTRL_SET_CAP(tempCaps, NV0080_CTRL_FIFO_CAPS, _SUPPORT_VID_PB); 2687 2688 if ((IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu)) && 2689 !gpuIsPipelinedPteMemEnabled(pGpu)) 2690 RMCTRL_SET_CAP(tempCaps, NV0080_CTRL_FIFO_CAPS, _NO_PIPELINED_PTE_BLIT); 2691 2692 if (kfifoIsUserdInSystemMemory(pKernelFifo)) 2693 RMCTRL_SET_CAP(tempCaps, NV0080_CTRL_FIFO_CAPS, _USERD_IN_SYSMEM); 2694 2695 if (kfifoIsUserdMapDmaSupported(pKernelFifo)) 2696 RMCTRL_SET_CAP(tempCaps, NV0080_CTRL_FIFO_CAPS, _GPU_MAP_CHANNEL); 2697 2698 if (kfifoHostHasLbOverflow(pKernelFifo)) 2699 RMCTRL_SET_CAP(tempCaps, NV0080_CTRL_FIFO_CAPS, _HAS_HOST_LB_OVERFLOW_BUG_1667921); 2700 2701 if (kfifoIsSubcontextSupported(pKernelFifo)) 2702 RMCTRL_SET_CAP(tempCaps, NV0080_CTRL_FIFO_CAPS, _MULTI_VAS_PER_CHANGRP); 2703 2704 if (kfifoIsWddmInterleavingPolicyEnabled(pKernelFifo)) 2705 RMCTRL_SET_CAP(tempCaps, NV0080_CTRL_FIFO_CAPS, _SUPPORT_WDDM_INTERLEAVING); 2706 2707 // if this is the first GPU in the device, then start with it's caps 2708 if (bCapsInitialized == NV_FALSE) 2709 { 2710 portMemCopy(pKfifoCaps, NV0080_CTRL_FIFO_CAPS_TBL_SIZE, 2711 tempCaps, NV0080_CTRL_FIFO_CAPS_TBL_SIZE); 2712 return; 2713 } 2714 2715 RMCTRL_AND_CAP(pKfifoCaps, tempCaps, temp, 2716 NV0080_CTRL_FIFO_CAPS, _SUPPORT_PCI_PB); 2717 RMCTRL_AND_CAP(pKfifoCaps, tempCaps, temp, 2718 NV0080_CTRL_FIFO_CAPS, _SUPPORT_VID_PB); 2719 RMCTRL_AND_CAP(pKfifoCaps, tempCaps, temp, 2720 NV0080_CTRL_FIFO_CAPS, _GPU_MAP_CHANNEL); 2721 2722 RMCTRL_OR_CAP(pKfifoCaps, tempCaps, temp, 2723 NV0080_CTRL_FIFO_CAPS, _MULTI_VAS_PER_CHANGRP); 2724 2725 RMCTRL_OR_CAP(pKfifoCaps, tempCaps, temp, 2726 NV0080_CTRL_FIFO_CAPS, _HAS_HOST_LB_OVERFLOW_BUG_1667921); 2727 2728 RMCTRL_OR_CAP(pKfifoCaps, tempCaps, temp, 2729 NV0080_CTRL_FIFO_CAPS, _SUPPORT_WDDM_INTERLEAVING); 2730 return; 2731 } 2732 2733 /*! 2734 * @brief Add handlers for scheduling enable and/or disable. 2735 * 2736 * @param[in] pGpu OBJGPU pointer 2737 * @param[in] pKernelFifo KernelFifo pointer 2738 * @param[in] pPostSchedulingEnableHandler No action if NULL 2739 * @param[in] pPostSchedulingEnableHandlerData Data to pass to 2740 * @p pPostSchedulingEnableHandler 2741 * @param[in] pPreSchedulingDisableHandler No action if NULL 2742 * @param[in] pPreSchedulingDisableHandlerData Data to pass to 2743 * @p pPreSchedulingDisableHandler 2744 * 2745 * @returns NV_OK if successfully processed both handlers 2746 * NV_WARN_NOTHING_TO_DO if: - Both handlers are NULL 2747 * - Both handlers are already installed 2748 * NV_ERR_INVALID_STATE if one handler is already installed, but not both 2749 */ 2750 NV_STATUS 2751 kfifoAddSchedulingHandler_IMPL 2752 ( 2753 OBJGPU *pGpu, 2754 KernelFifo *pKernelFifo, 2755 PFifoSchedulingHandler pPostSchedulingEnableHandler, 2756 void *pPostSchedulingEnableHandlerData, 2757 PFifoSchedulingHandler pPreSchedulingDisableHandler, 2758 void *pPreSchedulingDisableHandlerData 2759 ) 2760 { 2761 FifoSchedulingHandlerEntry *pEntry; 2762 NvBool bPostHandlerAlreadyPresent = NV_FALSE; 2763 NvBool bPreHandlerAlreadyPresent = NV_FALSE; 2764 FifoSchedulingHandlerEntry postEntry; 2765 FifoSchedulingHandlerEntry preEntry; 2766 2767 NV_CHECK_OR_RETURN(LEVEL_SILENT, 2768 (pPostSchedulingEnableHandler != NULL) || 2769 (pPreSchedulingDisableHandler != NULL), 2770 NV_WARN_NOTHING_TO_DO); 2771 2772 // Check for already installed handler if non-NULL 2773 if (pPostSchedulingEnableHandler != NULL) 2774 { 2775 for (pEntry = listHead(&pKernelFifo->postSchedulingEnableHandlerList); 2776 pEntry != NULL; 2777 pEntry = listNext(&pKernelFifo->postSchedulingEnableHandlerList, pEntry)) 2778 { 2779 if (pEntry->pCallback == pPostSchedulingEnableHandler && 2780 pEntry->pCallbackParam == pPostSchedulingEnableHandlerData) 2781 { 2782 bPostHandlerAlreadyPresent = NV_TRUE; 2783 break; 2784 } 2785 } 2786 } 2787 2788 // Check for already installed handler if non-NULL 2789 if (pPreSchedulingDisableHandler != NULL) 2790 { 2791 for (pEntry = listHead(&pKernelFifo->preSchedulingDisableHandlerList); 2792 pEntry != NULL; 2793 pEntry = listNext(&pKernelFifo->preSchedulingDisableHandlerList, pEntry)) 2794 { 2795 if (pEntry->pCallback == pPreSchedulingDisableHandler && 2796 pEntry->pCallbackParam == pPreSchedulingDisableHandlerData) 2797 { 2798 bPreHandlerAlreadyPresent = NV_TRUE; 2799 break; 2800 } 2801 } 2802 } 2803 2804 // 2805 // If we are installing both handlers, and one is already present, but not 2806 // the other, we will do nothing, so assert loudly in that case 2807 // 2808 if ((pPostSchedulingEnableHandler != NULL) && (pPreSchedulingDisableHandler != NULL)) 2809 { 2810 NV_ASSERT_OR_RETURN(!(bPostHandlerAlreadyPresent ^ bPreHandlerAlreadyPresent), 2811 NV_ERR_INVALID_STATE); 2812 } 2813 2814 // Return early unless all non-null handlers are not already installed 2815 NV_CHECK_OR_RETURN(LEVEL_SILENT, 2816 !bPostHandlerAlreadyPresent && !bPreHandlerAlreadyPresent, 2817 NV_WARN_NOTHING_TO_DO); 2818 2819 // Add handler entry to list unless NULL 2820 if (pPostSchedulingEnableHandler != NULL) 2821 { 2822 postEntry.pCallback = pPostSchedulingEnableHandler; 2823 postEntry.pCallbackParam = pPostSchedulingEnableHandlerData; 2824 postEntry.bHandled = NV_FALSE; 2825 NV_ASSERT_OR_RETURN(listPrependValue(&pKernelFifo->postSchedulingEnableHandlerList, &postEntry), 2826 NV_ERR_NO_MEMORY); 2827 } 2828 2829 // Add handler entry to list unless NULL 2830 if (pPreSchedulingDisableHandler != NULL) 2831 { 2832 preEntry.pCallback = pPreSchedulingDisableHandler; 2833 preEntry.pCallbackParam = pPreSchedulingDisableHandlerData; 2834 preEntry.bHandled = NV_FALSE; 2835 NV_ASSERT_OR_RETURN(listPrependValue(&pKernelFifo->preSchedulingDisableHandlerList, &preEntry), 2836 NV_ERR_NO_MEMORY); 2837 } 2838 2839 return NV_OK; 2840 } 2841 2842 /*! 2843 * @brief Remove handlers for scheduling enable and/or disable. 2844 * 2845 * @param[in] pGpu OBJGPU pointer 2846 * @param[in] pKernelFifo KernelFifo pointer 2847 * @param[in] pPostSchedulingEnableHandler No action if NULL 2848 * @param[in] pPostSchedulingEnableHandlerData Data argument set for the 2849 * handler. 2850 * @param[in] pPreSchedulingDisableHandler No action if NULL 2851 * @param[in] pPreSchedulingDisableHandlerData Data argument set for the 2852 * handler. 2853 */ 2854 void 2855 kfifoRemoveSchedulingHandler_IMPL 2856 ( 2857 OBJGPU *pGpu, 2858 KernelFifo *pKernelFifo, 2859 PFifoSchedulingHandler pPostSchedulingEnableHandler, 2860 void *pPostSchedulingEnableHandlerData, 2861 PFifoSchedulingHandler pPreSchedulingDisableHandler, 2862 void *pPreSchedulingDisableHandlerData 2863 ) 2864 { 2865 FifoSchedulingHandlerEntry *pEntry; 2866 FifoSchedulingHandlerEntry *pTemp; 2867 2868 // Search for the post handler in the post handler list and remove it if present 2869 pEntry = listHead(&pKernelFifo->postSchedulingEnableHandlerList); 2870 while (pEntry != NULL) 2871 { 2872 pTemp = listNext(&pKernelFifo->postSchedulingEnableHandlerList, pEntry); 2873 2874 if (pEntry->pCallback == pPostSchedulingEnableHandler && 2875 pEntry->pCallbackParam == pPostSchedulingEnableHandlerData) 2876 { 2877 listRemove(&pKernelFifo->postSchedulingEnableHandlerList, pEntry); 2878 } 2879 2880 pEntry = pTemp; 2881 } 2882 2883 // Search for the pre handler in the pre handler list and remove it if present 2884 pEntry = listHead(&pKernelFifo->preSchedulingDisableHandlerList); 2885 while (pEntry != NULL) 2886 { 2887 pTemp = listNext(&pKernelFifo->preSchedulingDisableHandlerList, pEntry); 2888 2889 if (pEntry->pCallback == pPreSchedulingDisableHandler && 2890 pEntry->pCallbackParam == pPreSchedulingDisableHandlerData) 2891 { 2892 listRemove(&pKernelFifo->preSchedulingDisableHandlerList, pEntry); 2893 } 2894 2895 pEntry = pTemp; 2896 } 2897 } 2898 2899 2900 /*! 2901 * @brief Notify handlers that scheduling has been enabled. 2902 * 2903 * @param[in] pGpu OBJGPU pointer 2904 * @param[in] pKernelFifo KernelFifo pointer 2905 * 2906 * @returns NV_STATUS 2907 */ 2908 NV_STATUS 2909 kfifoTriggerPostSchedulingEnableCallback_IMPL 2910 ( 2911 OBJGPU *pGpu, 2912 KernelFifo *pKernelFifo 2913 ) 2914 { 2915 NV_STATUS status = NV_OK; 2916 FifoSchedulingHandlerEntry *pEntry; 2917 NvBool bRetry = NV_FALSE; 2918 2919 for (pEntry = listHead(&pKernelFifo->postSchedulingEnableHandlerList); 2920 pEntry != NULL; 2921 pEntry = listNext(&pKernelFifo->postSchedulingEnableHandlerList, pEntry)) 2922 { 2923 NV_ASSERT_OR_ELSE(pEntry->pCallback != NULL, 2924 status = NV_ERR_INVALID_STATE; break;); 2925 2926 pEntry->bHandled = NV_FALSE; 2927 status = pEntry->pCallback(pGpu, pEntry->pCallbackParam); 2928 2929 // Retry mechanism: Some callbacks depend on other callbacks in this list. 2930 bRetry = bRetry || (status == NV_WARN_MORE_PROCESSING_REQUIRED); 2931 2932 if (status == NV_WARN_MORE_PROCESSING_REQUIRED) 2933 // Quash retry status 2934 status = NV_OK; 2935 else if (status == NV_OK) 2936 // Successfully handled, no need to retry 2937 pEntry->bHandled = NV_TRUE; 2938 else 2939 // Actual error, abort 2940 break; 2941 } 2942 2943 // If we hit an actual error or completed everything successfully, return early. 2944 if ((status != NV_OK) || !bRetry) 2945 return status; 2946 2947 // Second pass, retry anything that asked nicely to be deferred 2948 for (pEntry = listHead(&pKernelFifo->postSchedulingEnableHandlerList); 2949 pEntry != NULL; 2950 pEntry = listNext(&pKernelFifo->postSchedulingEnableHandlerList, pEntry)) 2951 { 2952 NV_ASSERT_OR_ELSE(pEntry->pCallback != NULL, 2953 status = NV_ERR_INVALID_STATE; break;); 2954 2955 // Skip anything that was completed successfully 2956 if (pEntry->bHandled) 2957 continue; 2958 2959 NV_CHECK_OK_OR_ELSE(status, LEVEL_ERROR, 2960 pEntry->pCallback(pGpu, pEntry->pCallbackParam), 2961 break; ); 2962 } 2963 2964 return status; 2965 } 2966 2967 2968 /*! 2969 * @brief Notify handlers that scheduling will soon be disabled. 2970 * 2971 * @param[in] pGpu OBJGPU pointer 2972 * @param[in] pKernelFifo KernelFifo pointer 2973 * 2974 * @returns NV_STATUS 2975 */ 2976 NV_STATUS 2977 kfifoTriggerPreSchedulingDisableCallback_IMPL 2978 ( 2979 OBJGPU *pGpu, 2980 KernelFifo *pKernelFifo 2981 ) 2982 { 2983 NV_STATUS status = NV_OK; 2984 FifoSchedulingHandlerEntry *pEntry; 2985 NvBool bRetry = NV_FALSE; 2986 2987 // First pass 2988 for (pEntry = listHead(&pKernelFifo->preSchedulingDisableHandlerList); 2989 pEntry != NULL; 2990 pEntry = listNext(&pKernelFifo->preSchedulingDisableHandlerList, pEntry)) 2991 { 2992 NV_ASSERT_OR_ELSE(pEntry->pCallback != NULL, 2993 status = NV_ERR_INVALID_STATE; break;); 2994 2995 pEntry->bHandled = NV_FALSE; 2996 status = pEntry->pCallback(pGpu, pEntry->pCallbackParam); 2997 2998 // Retry mechanism: Some callbacks depend on other callbacks in this list. 2999 bRetry = bRetry || (status == NV_WARN_MORE_PROCESSING_REQUIRED); 3000 3001 if (status == NV_WARN_MORE_PROCESSING_REQUIRED) 3002 // Quash retry status 3003 status = NV_OK; 3004 else if (status == NV_OK) 3005 // Successfully handled, no need to retry 3006 pEntry->bHandled = NV_TRUE; 3007 else 3008 // Actual error, abort 3009 break; 3010 } 3011 3012 // If we hit an actual error or completed everything successfully, return early. 3013 if ((status != NV_OK) || !bRetry) 3014 return status; 3015 3016 // Second pass, retry anything that asked nicely to be deferred 3017 for (pEntry = listHead(&pKernelFifo->preSchedulingDisableHandlerList); 3018 pEntry != NULL; 3019 pEntry = listNext(&pKernelFifo->preSchedulingDisableHandlerList, pEntry)) 3020 { 3021 NV_ASSERT_OR_ELSE(pEntry->pCallback != NULL, 3022 status = NV_ERR_INVALID_STATE; break;); 3023 3024 // Skip anything that was completed successfully 3025 if (pEntry->bHandled) 3026 continue; 3027 3028 NV_CHECK_OK_OR_ELSE(status, LEVEL_ERROR, 3029 pEntry->pCallback(pGpu, pEntry->pCallbackParam), 3030 break; ); 3031 } 3032 3033 return status; 3034 } 3035 3036 /** 3037 * @brief Gets vChid corresponding to a sChid 3038 */ 3039 NV_STATUS 3040 kfifoGetVChIdForSChId_FWCLIENT 3041 ( 3042 OBJGPU *pGpu, 3043 KernelFifo *pKernelFifo, 3044 NvU32 sChId, 3045 NvU32 gfid, 3046 NvU32 engineId, 3047 NvU32 *pVChid 3048 ) 3049 { 3050 KERNEL_HOST_VGPU_DEVICE *pKernelHostVgpuDevice = NULL; 3051 3052 NV_ASSERT_OR_RETURN(pVChid != NULL, NV_ERR_INVALID_ARGUMENT); 3053 *pVChid = sChId; 3054 3055 NV_CHECK_OR_RETURN(LEVEL_INFO, IS_GFID_VF(gfid), NV_OK); 3056 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextKernelHostVgpuDevice(pGpu, &pKernelHostVgpuDevice)); 3057 NV_ASSERT_OR_RETURN(pKernelHostVgpuDevice->gfid == gfid, NV_ERR_INVALID_ARGUMENT); 3058 NV_ASSERT_OR_RETURN(engineId < (sizeof(pKernelHostVgpuDevice->chidOffset) / sizeof(pKernelHostVgpuDevice->chidOffset[0])), 3059 NV_ERR_INVALID_STATE); 3060 NV_ASSERT_OR_RETURN(pKernelHostVgpuDevice->chidOffset[engineId] != 0, NV_ERR_INVALID_STATE); 3061 3062 NV_ASSERT_OR_RETURN(sChId >= pKernelHostVgpuDevice->chidOffset[engineId], NV_ERR_INVALID_ARGUMENT); 3063 3064 *pVChid = sChId - pKernelHostVgpuDevice->chidOffset[engineId]; 3065 3066 return NV_OK; 3067 } 3068 3069 /** 3070 * @brief cache vChid <-> sChid offset 3071 */ 3072 NV_STATUS 3073 kfifoSetChidOffset_IMPL 3074 ( 3075 OBJGPU *pGpu, 3076 KernelFifo *pKernelFifo, 3077 CHID_MGR *pChidMgr, 3078 NvU32 offset, 3079 NvU32 numChannels, 3080 NvU32 gfid, 3081 NvU32 *pChidOffset, 3082 NvU32 *pChannelCount, 3083 NvU32 hMigClient, 3084 NvU32 engineFifoListNumEntries, 3085 FIFO_ENGINE_LIST *pEngineFifoList 3086 ) 3087 { 3088 NV_STATUS status = NV_OK; 3089 RM_ENGINE_TYPE *pEngineIds = NULL; 3090 NvU32 maxEngines = kfifoGetNumEngines_HAL(pGpu, pKernelFifo); 3091 NvU32 numEngines, i; 3092 3093 NV_ASSERT_OR_RETURN(pChidMgr != NULL, NV_ERR_INVALID_ARGUMENT); 3094 NV_ASSERT_OR_RETURN(pChidOffset != NULL, NV_ERR_INVALID_ARGUMENT); 3095 3096 pEngineIds = portMemAllocNonPaged(sizeof(RM_ENGINE_TYPE) * maxEngines); 3097 NV_ASSERT_OR_RETURN((pEngineIds != NULL), NV_ERR_NO_MEMORY); 3098 portMemSet(pEngineIds, 0, sizeof(RM_ENGINE_TYPE) * maxEngines); 3099 3100 NV_ASSERT_OK_OR_GOTO(status, kfifoGetEngineListForRunlist(pGpu, pKernelFifo, pChidMgr->runlistId, pEngineIds, &numEngines), cleanup) 3101 3102 for (i = 0; i < numEngines; i++) 3103 { 3104 NV_ASSERT_OR_ELSE(NV2080_ENGINE_TYPE_IS_VALID(pEngineIds[i]), status = NV_ERR_INVALID_STATE; goto cleanup); 3105 pChidOffset[pEngineIds[i]] = offset; 3106 pChannelCount[pEngineIds[i]] = numChannels; 3107 } 3108 3109 NV_ASSERT_OK_OR_GOTO(status, kfifoProgramChIdTable_HAL(pGpu, pKernelFifo, pChidMgr, offset, numChannels, gfid, 3110 hMigClient, engineFifoListNumEntries, pEngineFifoList), cleanup); 3111 3112 cleanup: 3113 3114 portMemFree(pEngineIds); 3115 3116 return status; 3117 } 3118 3119 /* 3120 * @brief Gets a list of engine ids that use this runlist 3121 * 3122 * @param[in] runlistId Runlist id 3123 * @param[out] pOutEngineIds List of engineids 3124 * @param[out] pNumEngines # of entries in pOutEngines 3125 */ 3126 NV_STATUS 3127 kfifoGetEngineListForRunlist_IMPL 3128 ( 3129 OBJGPU *pGpu, 3130 KernelFifo *pKernelFifo, 3131 NvU32 runlistId, 3132 RM_ENGINE_TYPE *pOutEngineIds, 3133 NvU32 *pNumEngines 3134 ) 3135 { 3136 NV_STATUS status = NV_OK; 3137 NvU32 numEngines = kfifoGetNumEngines_HAL(pGpu, pKernelFifo); 3138 NvU32 i; 3139 3140 // Sanity check the input 3141 NV_CHECK_OR_RETURN(LEVEL_ERROR, pOutEngineIds != NULL, NV_ERR_INVALID_ARGUMENT); 3142 NV_CHECK_OR_RETURN(LEVEL_ERROR, pNumEngines != NULL, NV_ERR_INVALID_ARGUMENT); 3143 3144 *pNumEngines = 0; 3145 NV_PRINTF(LEVEL_INFO, "Engine list for runlistId 0x%x:\n", runlistId); 3146 3147 for (i = 0; i < numEngines; i++) 3148 { 3149 RM_ENGINE_TYPE rmEngineType; 3150 NvU32 thisRunlistId; 3151 3152 NV_ASSERT_OK_OR_GOTO(status, 3153 kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo, 3154 ENGINE_INFO_TYPE_INVALID, 3155 i, 3156 ENGINE_INFO_TYPE_RUNLIST, 3157 &thisRunlistId), done); 3158 if (runlistId == thisRunlistId) 3159 { 3160 NV_ASSERT_OK_OR_GOTO(status, 3161 kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo, 3162 ENGINE_INFO_TYPE_INVALID, 3163 i, 3164 ENGINE_INFO_TYPE_RM_ENGINE_TYPE, 3165 (NvU32 *)&rmEngineType), done); 3166 pOutEngineIds[(*pNumEngines)++] = rmEngineType; 3167 3168 NV_PRINTF(LEVEL_INFO, "Engine name: %s\n", 3169 kfifoGetEngineName_HAL(pKernelFifo, ENGINE_INFO_TYPE_RM_ENGINE_TYPE, 3170 (NvU32)rmEngineType)); 3171 } 3172 } 3173 done: 3174 if ((status != NV_OK) && (*pNumEngines != 0)) 3175 { 3176 portMemSet(pOutEngineIds, 0, sizeof(NvU32) * (*pNumEngines)); 3177 *pNumEngines = 0; 3178 } 3179 return status; 3180 } 3181 3182 /** 3183 * @brief Return bitmask of currently allocated channels for a given runlist 3184 * 3185 * @param[in] bitMaskSize @p pBitMask size in bytes 3186 * 3187 */ 3188 NV_STATUS 3189 kfifoGetAllocatedChannelMask_IMPL 3190 ( 3191 OBJGPU *pGpu, 3192 KernelFifo *pKernelFifo, 3193 NvU32 runlistId, 3194 NvU32 *pBitMask, 3195 NvLength bitMaskSize 3196 ) 3197 { 3198 CHID_MGR *pChidMgr; 3199 NvU32 chId; 3200 NvU32 numChannels; 3201 3202 NV_ASSERT(pBitMask != NULL); 3203 portMemSet(pBitMask, 0, bitMaskSize); 3204 3205 NV_ASSERT_OR_RETURN(bitMaskSize % sizeof(NvU32) == 0, 3206 NV_ERR_INVALID_ARGUMENT); 3207 3208 if (!kfifoIsPerRunlistChramEnabled(pKernelFifo)) 3209 { 3210 if (runlistId > 0) 3211 { 3212 return NV_ERR_OUT_OF_RANGE; 3213 } 3214 else 3215 { 3216 runlistId = CHIDMGR_RUNLIST_ID_LEGACY; 3217 } 3218 } 3219 else 3220 { 3221 if (!(runlistId < kfifoGetMaxNumRunlists_HAL(pGpu, pKernelFifo))) 3222 { 3223 return NV_ERR_OUT_OF_RANGE; 3224 } 3225 } 3226 3227 pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, runlistId); 3228 if (pChidMgr == NULL) 3229 { 3230 // 3231 // This runlist is not valid. This is not an error since it might be 3232 // possible for some runlists between [0, maxRunlists) to be invalid. 3233 // Simply return that it has no channels to simplify clients iterating 3234 // over all runlists. 3235 // 3236 numChannels = 0; 3237 } 3238 else 3239 { 3240 numChannels = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr); 3241 } 3242 3243 if (((numChannels + 7) / 8) > bitMaskSize) 3244 { 3245 return NV_ERR_BUFFER_TOO_SMALL; 3246 } 3247 3248 for (chId = 0; chId < numChannels; chId++) 3249 { 3250 KernelChannel *pKernelChannel; 3251 pKernelChannel = kfifoChidMgrGetKernelChannel(pGpu, pKernelFifo, 3252 pChidMgr, 3253 chId); 3254 if (pKernelChannel != NULL) 3255 { 3256 NV_BITMASK32_SET(pBitMask, chId); 3257 } 3258 } 3259 3260 return NV_OK; 3261 } 3262 3263 /*! 3264 * Get host channel class 3265 */ 3266 NvU32 3267 kfifoGetChannelClassId_IMPL 3268 ( 3269 OBJGPU *pGpu, 3270 KernelFifo *pKernelFifo 3271 ) 3272 { 3273 NvU32 numClasses; 3274 NvU32 *pClassList = NULL; 3275 CLI_CHANNEL_CLASS_INFO classInfo; 3276 NvU32 i; 3277 NvU32 class = 0; 3278 3279 NV_ASSERT_OR_RETURN(NV_OK == gpuGetClassList(pGpu, &numClasses, NULL, ENG_KERNEL_FIFO), 0); 3280 NV_ASSERT_OR_RETURN(numClasses > 0, 0); 3281 pClassList = portMemAllocNonPaged(sizeof(NvU32) * numClasses); 3282 NV_ASSERT_OR_RETURN((pClassList != NULL), 0); 3283 3284 if (NV_OK == gpuGetClassList(pGpu, &numClasses, pClassList, ENG_KERNEL_FIFO)) 3285 { 3286 for (i = 0; i < numClasses; i++) 3287 { 3288 if (pClassList[i] == PHYSICAL_CHANNEL_GPFIFO) 3289 { 3290 // Skip the physical channel class 3291 continue; 3292 } 3293 CliGetChannelClassInfo(pClassList[i], &classInfo); 3294 if (classInfo.classType == CHANNEL_CLASS_TYPE_GPFIFO) 3295 class = NV_MAX(class, pClassList[i]); 3296 } 3297 } 3298 3299 NV_ASSERT(class); 3300 portMemFree(pClassList); 3301 return class; 3302 } 3303 3304 /** 3305 * @brief Return the FIFO_GUEST_ENGINE_TABLE 3306 * 3307 * @param[out] pEngLookupTblSize To get the table size 3308 * 3309 * @return a pointer to FIFO_GUEST_ENGINE_TABLE 3310 * 3311 */ 3312 const FIFO_GUEST_ENGINE_TABLE * 3313 kfifoGetGuestEngineLookupTable_IMPL 3314 ( 3315 NvU32 *pEngLookupTblSize 3316 ) 3317 { 3318 // 3319 // This table is used for a guest RM to reconstruct the engine list data 3320 // received from the host RM. The host and guest RM can be running on 3321 // different versions. 3322 // 3323 // This table does not need to be HALified. It need to match NV2080_ENGINE_TYPE 3324 // defined in cl2080_notification.h 3325 // 3326 const static FIFO_GUEST_ENGINE_TABLE guestEngineLookupTable[] = 3327 { 3328 // nv2080EngineType mcIdx 3329 {NV2080_ENGINE_TYPE_GR0, MC_ENGINE_IDX_GR0}, 3330 {NV2080_ENGINE_TYPE_GR1, MC_ENGINE_IDX_GR1}, 3331 {NV2080_ENGINE_TYPE_GR2, MC_ENGINE_IDX_GR2}, 3332 {NV2080_ENGINE_TYPE_GR3, MC_ENGINE_IDX_GR3}, 3333 {NV2080_ENGINE_TYPE_GR4, MC_ENGINE_IDX_GR4}, 3334 {NV2080_ENGINE_TYPE_GR5, MC_ENGINE_IDX_GR5}, 3335 {NV2080_ENGINE_TYPE_GR6, MC_ENGINE_IDX_GR6}, 3336 {NV2080_ENGINE_TYPE_GR7, MC_ENGINE_IDX_GR7}, 3337 {NV2080_ENGINE_TYPE_COPY0, MC_ENGINE_IDX_CE0}, 3338 {NV2080_ENGINE_TYPE_COPY1, MC_ENGINE_IDX_CE1}, 3339 {NV2080_ENGINE_TYPE_COPY2, MC_ENGINE_IDX_CE2}, 3340 {NV2080_ENGINE_TYPE_COPY3, MC_ENGINE_IDX_CE3}, 3341 {NV2080_ENGINE_TYPE_COPY4, MC_ENGINE_IDX_CE4}, 3342 {NV2080_ENGINE_TYPE_COPY5, MC_ENGINE_IDX_CE5}, 3343 {NV2080_ENGINE_TYPE_COPY6, MC_ENGINE_IDX_CE6}, 3344 {NV2080_ENGINE_TYPE_COPY7, MC_ENGINE_IDX_CE7}, 3345 {NV2080_ENGINE_TYPE_COPY8, MC_ENGINE_IDX_CE8}, 3346 {NV2080_ENGINE_TYPE_COPY9, MC_ENGINE_IDX_CE9}, 3347 {NV2080_ENGINE_TYPE_NVENC0, MC_ENGINE_IDX_MSENC}, 3348 {NV2080_ENGINE_TYPE_NVENC1, MC_ENGINE_IDX_MSENC1}, 3349 {NV2080_ENGINE_TYPE_NVENC2, MC_ENGINE_IDX_MSENC2}, 3350 {NV2080_ENGINE_TYPE_NVDEC0, MC_ENGINE_IDX_NVDEC0}, 3351 {NV2080_ENGINE_TYPE_NVDEC1, MC_ENGINE_IDX_NVDEC1}, 3352 {NV2080_ENGINE_TYPE_NVDEC2, MC_ENGINE_IDX_NVDEC2}, 3353 {NV2080_ENGINE_TYPE_NVDEC3, MC_ENGINE_IDX_NVDEC3}, 3354 {NV2080_ENGINE_TYPE_NVDEC4, MC_ENGINE_IDX_NVDEC4}, 3355 {NV2080_ENGINE_TYPE_NVDEC5, MC_ENGINE_IDX_NVDEC5}, 3356 {NV2080_ENGINE_TYPE_NVDEC6, MC_ENGINE_IDX_NVDEC6}, 3357 {NV2080_ENGINE_TYPE_NVDEC7, MC_ENGINE_IDX_NVDEC7}, 3358 {NV2080_ENGINE_TYPE_SW, MC_ENGINE_IDX_NULL}, 3359 {NV2080_ENGINE_TYPE_SEC2, MC_ENGINE_IDX_SEC2}, 3360 {NV2080_ENGINE_TYPE_NVJPEG0, MC_ENGINE_IDX_NVJPEG0}, 3361 {NV2080_ENGINE_TYPE_NVJPEG1, MC_ENGINE_IDX_NVJPEG1}, 3362 {NV2080_ENGINE_TYPE_NVJPEG2, MC_ENGINE_IDX_NVJPEG2}, 3363 {NV2080_ENGINE_TYPE_NVJPEG3, MC_ENGINE_IDX_NVJPEG3}, 3364 {NV2080_ENGINE_TYPE_NVJPEG4, MC_ENGINE_IDX_NVJPEG4}, 3365 {NV2080_ENGINE_TYPE_NVJPEG5, MC_ENGINE_IDX_NVJPEG5}, 3366 {NV2080_ENGINE_TYPE_NVJPEG6, MC_ENGINE_IDX_NVJPEG6}, 3367 {NV2080_ENGINE_TYPE_NVJPEG7, MC_ENGINE_IDX_NVJPEG7}, 3368 {NV2080_ENGINE_TYPE_OFA, MC_ENGINE_IDX_OFA0}, 3369 }; 3370 3371 // 3372 // To trap NV2080_ENGINE_TYPE expansions. 3373 // Please update the table guestEngineLookupTable if this assertion is triggered. 3374 // 3375 ct_assert(NV2080_ENGINE_TYPE_LAST == 0x0000003e); 3376 3377 *pEngLookupTblSize = NV_ARRAY_ELEMENTS(guestEngineLookupTable); 3378 3379 return guestEngineLookupTable; 3380 }; 3381 3382