1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "kernel/gpu/fifo/kernel_fifo.h"
25 #include "kernel/gpu/fifo/kernel_channel.h"
26 #include "kernel/gpu/fifo/kernel_channel_group.h"
27 #include "kernel/gpu/fifo/kernel_channel_group_api.h"
28 #include "kernel/gpu/fifo/kernel_sched_mgr.h"
29 #include "virtualization/kernel_vgpu_mgr.h"
30 #include "rmapi/rs_utils.h"
31 #include "rmapi/client.h"
32 #include "gpu/device/device.h"
33 #include "gpu/subdevice/subdevice.h"
34
35 #include "kernel/core/locks.h"
36 #include "lib/base_utils.h"
37
38 #include "gpu/mmu/kern_gmmu.h"
39 #include "vgpu/rpc.h"
40 #include "vgpu/vgpu_events.h"
41 #include "nvRmReg.h"
42
43 #include "class/cl0080.h"
44 #include "class/cl2080.h"
45 #include "class/cl208f.h"
46 #include "class/clc572.h"
47
48 #include "ctrl/ctrl0080/ctrl0080fifo.h"
49
50 #define KFIFO_EHEAP_OWNER NvU32_BUILD('f','i','f','o')
51
52 static EHeapOwnershipComparator _kfifoUserdOwnerComparator;
53
54 static NV_STATUS _kfifoChidMgrAllocChidHeaps(OBJGPU *pGpu,
55 KernelFifo *pKernelFifo,
56 CHID_MGR *pChidMgr);
57
58 static NV_STATUS _kfifoChidMgrAllocVChidHeapPointers(OBJGPU *pGpu, CHID_MGR *pChidMgr);
59
60 static NV_STATUS _kfifoChidMgrInitChannelGroupMgr(OBJGPU *pGpu, CHID_MGR *pChidMgr);
61
62 static void _kfifoChidMgrDestroyChidHeaps(CHID_MGR *pChidMgr);
63
64 static void _kfifoChidMgrDestroyChannelGroupMgr(CHID_MGR *pChidMgr);
65
66 static NV_STATUS _kfifoChidMgrFreeIsolationId(CHID_MGR *pChidMgr, NvU32 ChID);
67
68 static NV_STATUS _kfifoChidMgrGetNextKernelChannel(OBJGPU *pGpu, KernelFifo *pKernelFifo,
69 CHID_MGR *pChidMgr, CHANNEL_ITERATOR *pIt,
70 KernelChannel **ppKernelChannel);
71
72
kfifoGetNumEschedDrivenEngines_IMPL(KernelFifo * pKernelFifo)73 NvU32 kfifoGetNumEschedDrivenEngines_IMPL
74 (
75 KernelFifo *pKernelFifo
76 )
77 {
78 const ENGINE_INFO *pEngineInfo;
79 NV_ASSERT(kfifoGetNumEngines_HAL(ENG_GET_GPU(pKernelFifo), pKernelFifo) >
80 0);
81 pEngineInfo = kfifoGetEngineInfo(pKernelFifo);
82 return pEngineInfo->numRunlists;
83 }
84
85
86 NV_STATUS
kfifoChidMgrConstruct_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo)87 kfifoChidMgrConstruct_IMPL
88 (
89 OBJGPU *pGpu,
90 KernelFifo *pKernelFifo
91 )
92 {
93 NV_STATUS status = NV_OK;
94 NvU32 i;
95 NvU32 numEngines;
96
97 //
98 // Allocate memory for the array of CHID_MGR pointers. Since this is an
99 // array, we allocate memory for pointers unto maxNumRunlists. We will only
100 // allocate objects for the valid ones.
101 //
102 if (kfifoIsPerRunlistChramEnabled(pKernelFifo))
103 {
104 //
105 // Construct the engine list if it isn't already constructed (internally
106 // checks if it was already constructed)
107 //
108 NV_ASSERT_OK_OR_RETURN(kfifoConstructEngineList_HAL(pGpu, pKernelFifo));
109 pKernelFifo->numChidMgrs = kfifoGetMaxNumRunlists_HAL(pGpu, pKernelFifo);
110 }
111 else
112 pKernelFifo->numChidMgrs = 1;
113
114 if (pKernelFifo->numChidMgrs > MAX_NUM_RUNLISTS)
115 {
116 //
117 // This only currently defines the size of our bitvector
118 // pKernelFifo->chidMgrValid. Catch this case if HW expands beyond this so we
119 // can increase the size allocated to the bitvector
120 //
121 NV_PRINTF(LEVEL_ERROR, "numChidMgrs 0x%x exceeds MAX_NUM_RUNLISTS\n",
122 pKernelFifo->numChidMgrs);
123 DBG_BREAKPOINT();
124 return NV_ERR_BUFFER_TOO_SMALL;
125 }
126
127 pKernelFifo->ppChidMgr = portMemAllocNonPaged(sizeof(CHID_MGR *) * pKernelFifo->numChidMgrs);
128 if (pKernelFifo->ppChidMgr == NULL)
129 {
130 status = NV_ERR_NO_MEMORY;
131 pKernelFifo->ppChidMgr = NULL;
132 NV_PRINTF(LEVEL_ERROR, "Failed to allocate pFifo->pChidMgr\n");
133 DBG_BREAKPOINT();
134 return status;
135 }
136 portMemSet(pKernelFifo->ppChidMgr, 0, sizeof(CHID_MGR *) * pKernelFifo->numChidMgrs);
137
138 // Initialize the valid mask
139 if (kfifoIsPerRunlistChramEnabled(pKernelFifo))
140 {
141 numEngines = kfifoGetNumEngines_HAL(pGpu, pKernelFifo);
142 for (i = 0; i < numEngines; i++)
143 {
144 NvU32 runlistId;
145 status = kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo,
146 ENGINE_INFO_TYPE_INVALID, i,
147 ENGINE_INFO_TYPE_RUNLIST, &runlistId);
148 if (status == NV_OK)
149 bitVectorSet(&pKernelFifo->chidMgrValid, runlistId);
150 else
151 {
152 NV_PRINTF(LEVEL_ERROR, "Translation to runlistId failed for engine %d\n", i);
153 DBG_BREAKPOINT();
154 goto fail;
155 }
156 }
157 }
158 else
159 {
160 bitVectorSet(&pKernelFifo->chidMgrValid, 0); // We only have 1 chidmgr
161 }
162
163 // Allocate memory for each CHID_MGR and its members (only the valid ones)
164 for (i = 0; i < pKernelFifo->numChidMgrs; i++)
165 {
166 if (!bitVectorTest(&pKernelFifo->chidMgrValid, i))
167 continue;
168
169 pKernelFifo->ppChidMgr[i] = portMemAllocNonPaged(sizeof(CHID_MGR));
170 if (pKernelFifo->ppChidMgr[i] == NULL)
171 {
172 status = NV_ERR_NO_MEMORY;
173 NV_PRINTF(LEVEL_ERROR, "Failed to allocate pFifo->pChidMgr[%d]\n", i);
174 DBG_BREAKPOINT();
175 goto fail;
176 }
177 portMemSet(pKernelFifo->ppChidMgr[i], 0, sizeof(CHID_MGR));
178
179 pKernelFifo->ppChidMgr[i]->runlistId = i;
180
181 pKernelFifo->ppChidMgr[i]->pChanGrpTree = portMemAllocNonPaged(sizeof(KernelChannelGroupMap));
182 mapInitIntrusive(pKernelFifo->ppChidMgr[i]->pChanGrpTree);
183
184 status = _kfifoChidMgrAllocChidHeaps(pGpu, pKernelFifo, pKernelFifo->ppChidMgr[i]);
185 if (status != NV_OK)
186 {
187 NV_PRINTF(LEVEL_ERROR, "Error allocating FifoDataHeap in "
188 "pChidMgr. Status = %s (0x%x)\n",
189 nvstatusToString(status), status);
190 DBG_BREAKPOINT();
191 goto fail;
192 }
193
194 status = _kfifoChidMgrInitChannelGroupMgr(pGpu, pKernelFifo->ppChidMgr[i]);
195 if (status != NV_OK)
196 goto fail;
197 }
198
199
200 return status;
201
202 fail:
203 kfifoChidMgrDestruct(pKernelFifo);
204 return status;
205 }
206
207 void
kfifoChidMgrDestruct_IMPL(KernelFifo * pKernelFifo)208 kfifoChidMgrDestruct_IMPL
209 (
210 KernelFifo *pKernelFifo
211 )
212 {
213 NvU32 i;
214
215 for (i = 0; i < pKernelFifo->numChidMgrs; i++)
216 {
217 if (pKernelFifo->ppChidMgr[i] != NULL)
218 {
219 mapDestroy(pKernelFifo->ppChidMgr[i]->pChanGrpTree);
220 portMemFree(pKernelFifo->ppChidMgr[i]->pChanGrpTree);
221 _kfifoChidMgrDestroyChidHeaps(pKernelFifo->ppChidMgr[i]);
222 _kfifoChidMgrDestroyChannelGroupMgr(pKernelFifo->ppChidMgr[i]);
223 portMemFree(pKernelFifo->ppChidMgr[i]);
224 pKernelFifo->ppChidMgr[i] = NULL;
225 }
226 }
227
228 portMemFree(pKernelFifo->ppChidMgr);
229 pKernelFifo->ppChidMgr = NULL;
230 bitVectorClrAll(&pKernelFifo->chidMgrValid);
231 pKernelFifo->numChidMgrs = 0;
232 }
233
234 /*
235 * @brief Allocate and initialize the virtual ChId heap pointers
236 */
237 static NV_STATUS
_kfifoChidMgrAllocVChidHeapPointers(OBJGPU * pGpu,CHID_MGR * pChidMgr)238 _kfifoChidMgrAllocVChidHeapPointers
239 (
240 OBJGPU *pGpu,
241 CHID_MGR *pChidMgr
242 )
243 {
244 NV_STATUS status = NV_OK;
245 NvU32 i;
246
247 if (IS_VIRTUAL(pGpu))
248 {
249 return NV_OK;
250 }
251
252 if (gpuIsSriovEnabled(pGpu))
253 {
254 //
255 // For Virtual Channel Heap
256 // Allocate Memory for Heap Object pointers
257 //
258 pChidMgr->ppVirtualChIDHeap = portMemAllocNonPaged(sizeof(OBJEHEAP *) * (VMMU_MAX_GFID));
259 if (pChidMgr->ppVirtualChIDHeap == NULL)
260 {
261 NV_PRINTF(LEVEL_ERROR,
262 "Error allocating memory for virtual channel heap pointers\n");
263 return NV_ERR_NO_MEMORY;
264 }
265
266 // initialize
267 for (i = 0; i < VMMU_MAX_GFID; i++)
268 {
269 pChidMgr->ppVirtualChIDHeap[i] = NULL;
270 }
271 }
272 return status;
273 }
274
275
276 /*
277 * @brief Allocates & initializes ChID heaps
278 */
279 static NV_STATUS
_kfifoChidMgrAllocChidHeaps(OBJGPU * pGpu,KernelFifo * pKernelFifo,CHID_MGR * pChidMgr)280 _kfifoChidMgrAllocChidHeaps
281 (
282 OBJGPU *pGpu,
283 KernelFifo *pKernelFifo,
284 CHID_MGR *pChidMgr
285 )
286 {
287 NV_STATUS status = NV_OK;
288
289 if (pChidMgr->numChannels == 0)
290 {
291 if (kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr) == 0)
292 {
293 NV_PRINTF(LEVEL_ERROR, "pChidMgr->numChannels is 0\n");
294 DBG_BREAKPOINT();
295 return NV_ERR_INVALID_STATE;
296 }
297 }
298
299 pChidMgr->pFifoDataHeap = portMemAllocNonPaged(sizeof(*pChidMgr->pFifoDataHeap));
300 if (pChidMgr->pFifoDataHeap == NULL)
301 {
302 status = NV_ERR_NO_MEMORY;
303 NV_PRINTF(LEVEL_ERROR,
304 "Error in Allocating memory for pFifoDataHeap! Status = %s (0x%x)\n",
305 nvstatusToString(status), status);
306 return status;
307 }
308 constructObjEHeap(pChidMgr->pFifoDataHeap, 0, pChidMgr->numChannels,
309 sizeof(KernelChannel *), 0);
310
311 if (kfifoIsChidHeapEnabled(pKernelFifo))
312 {
313 NvU32 userdBar1Size;
314 NvU32 numChannels = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr);
315 NvU32 subProcessIsolation = 1;
316
317 pChidMgr->pGlobalChIDHeap = portMemAllocNonPaged(sizeof(OBJEHEAP));
318 if (pChidMgr->pGlobalChIDHeap == NULL)
319 {
320 NV_PRINTF(LEVEL_ERROR,
321 "Error in Allocating memory for global ChID heap!\n");
322 return NV_ERR_NO_MEMORY;
323 }
324 constructObjEHeap(pChidMgr->pGlobalChIDHeap, 0, numChannels,
325 sizeof(PFIFO_ISOLATIONID), 0);
326
327 //
328 // Enable USERD allocation isolation. USERD allocated by different clients
329 // should not be in the same page
330 //
331 kfifoGetUserdSizeAlign_HAL(pKernelFifo, &userdBar1Size, NULL);
332 NvBool bIsolationEnabled = (pKernelFifo->bUsePerRunlistChram && pKernelFifo->bDisableChidIsolation) ? NV_FALSE : NV_TRUE;
333 pChidMgr->pGlobalChIDHeap->eheapSetOwnerIsolation(pChidMgr->pGlobalChIDHeap,
334 bIsolationEnabled,
335 RM_PAGE_SIZE / userdBar1Size);
336
337 // Disable USERD allocation isolation for guest if disabled from vmioplugin
338 if (IS_VIRTUAL(pGpu))
339 {
340 VGPU_STATIC_INFO *pVSI = GPU_GET_STATIC_INFO(pGpu);
341 if (pVSI != NULL)
342 {
343 subProcessIsolation = pVSI->subProcessIsolation;
344 }
345 }
346 else
347 {
348 // In this case subProcessIsolation is always 0
349 if (IS_GSP_CLIENT(pGpu))
350 {
351 subProcessIsolation = 0;
352 }
353 }
354 if (!subProcessIsolation)
355 {
356 pChidMgr->pGlobalChIDHeap->eheapSetOwnerIsolation(
357 pChidMgr->pGlobalChIDHeap,
358 NV_FALSE,
359 RM_PAGE_SIZE / userdBar1Size);
360 #if (defined(_WIN32) || defined(_WIN64) || defined(NV_UNIX)) && !RMCFG_FEATURE_MODS_FEATURES
361 NV_PRINTF(LEVEL_INFO,
362 "Sub Process channel isolation disabled by vGPU plugin\n");
363 #endif
364 }
365
366 status = _kfifoChidMgrAllocVChidHeapPointers(pGpu, pChidMgr);
367 }
368
369 return status;
370 }
371
372 static void
_kfifoChidMgrDestroyChidHeaps(CHID_MGR * pChidMgr)373 _kfifoChidMgrDestroyChidHeaps
374 (
375 CHID_MGR *pChidMgr
376 )
377 {
378 if (pChidMgr->pFifoDataHeap != NULL)
379 {
380 pChidMgr->pFifoDataHeap->eheapDestruct(pChidMgr->pFifoDataHeap);
381 portMemFree(pChidMgr->pFifoDataHeap);
382 pChidMgr->pFifoDataHeap = NULL;
383 }
384 if (pChidMgr->pGlobalChIDHeap != NULL)
385 {
386 pChidMgr->pGlobalChIDHeap->eheapDestruct(pChidMgr->pGlobalChIDHeap);
387 portMemFree(pChidMgr->pGlobalChIDHeap);
388 pChidMgr->pGlobalChIDHeap = NULL;
389 }
390
391 portMemFree(pChidMgr->ppVirtualChIDHeap);
392 pChidMgr->ppVirtualChIDHeap = NULL;
393 }
394
395
396 static NV_STATUS
_kfifoChidMgrInitChannelGroupMgr(OBJGPU * pGpu,CHID_MGR * pChidMgr)397 _kfifoChidMgrInitChannelGroupMgr
398 (
399 OBJGPU *pGpu,
400 CHID_MGR *pChidMgr
401 )
402 {
403 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu);
404 FIFO_HW_ID *pFifoHwID = &pChidMgr->channelGrpMgr;
405 NvU32 allocSize;
406 NvU32 numChannelGroups = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr);
407
408 if (numChannelGroups == 0)
409 {
410 return NV_OK;
411 }
412
413 // Rounds up to dword alignemnt, then converts bits to bytes.
414 allocSize = RM_ALIGN_UP(numChannelGroups, 32)/8;
415
416 pFifoHwID->pHwIdInUse = portMemAllocNonPaged(allocSize);
417 if (pFifoHwID->pHwIdInUse == NULL)
418 return NV_ERR_NO_MEMORY;
419
420 // bytes to NvU32[] elements
421 pFifoHwID->hwIdInUseSz = allocSize/4;
422
423 portMemSet(pFifoHwID->pHwIdInUse, 0, allocSize);
424
425 //
426 // If numChannelGroups isn't a multiple of 32 we need to set the bits > numChannelGroups to
427 // 1. Otherwise when we allocate IDs starting at the top we'll allocate
428 // ids >numChannelGroups.
429 //
430 if (numChannelGroups % 32 != 0)
431 {
432 pFifoHwID->pHwIdInUse[numChannelGroups/32] |= ~ ((1<<(numChannelGroups%32))-1);
433 }
434
435 return NV_OK;
436 }
437
438 static void
_kfifoChidMgrDestroyChannelGroupMgr(CHID_MGR * pChidMgr)439 _kfifoChidMgrDestroyChannelGroupMgr
440 (
441 CHID_MGR *pChidMgr
442 )
443 {
444 if (pChidMgr->channelGrpMgr.pHwIdInUse)
445 {
446 portMemFree(pChidMgr->channelGrpMgr.pHwIdInUse);
447 pChidMgr->channelGrpMgr.pHwIdInUse = NULL;
448 pChidMgr->channelGrpMgr.hwIdInUseSz = 0;
449 }
450 }
451
452 static NV_STATUS
_kfifoChidMgrFreeIsolationId(CHID_MGR * pChidMgr,NvU32 ChID)453 _kfifoChidMgrFreeIsolationId
454 (
455 CHID_MGR *pChidMgr,
456 NvU32 ChID
457 )
458 {
459 EMEMBLOCK *pIsolationIdBlock = pChidMgr->pGlobalChIDHeap->eheapGetBlock(
460 pChidMgr->pGlobalChIDHeap,
461 ChID,
462 NV_FALSE);
463
464 NV_ASSERT_OR_RETURN(pIsolationIdBlock, NV_ERR_OBJECT_NOT_FOUND);
465 NV_ASSERT(pIsolationIdBlock->refCount > 0);
466 NV_ASSERT(pIsolationIdBlock->pData != NULL);
467 portMemFree(pIsolationIdBlock->pData);
468
469 pIsolationIdBlock->pData = NULL;
470
471 return NV_OK;
472 }
473
474 /*!
475 * @breif Fifo defined call back comparator to compare eheap block ownership ID
476 *
477 * @param[in] pRequesterID Ownership ID constructed by caller
478 * @param[in] pIsolationID
479 *
480 * @return NV_TRUE if two ownership IDs belong to the same owner
481 */
482 static NvBool
_kfifoUserdOwnerComparator(void * pRequesterID,void * pIsolationID)483 _kfifoUserdOwnerComparator
484 (
485 void *pRequesterID,
486 void *pIsolationID
487 )
488 {
489 PFIFO_ISOLATIONID pAllocID = (PFIFO_ISOLATIONID)pRequesterID;
490 PFIFO_ISOLATIONID pBlockID = (PFIFO_ISOLATIONID)pIsolationID;
491
492 //
493 // The block's data will be NULL if the channel has been destroyed but there
494 // is still a refcount on the channel ID. In that case no work can be issued
495 // to that channel ID now or in the future, so we can act as though the
496 // channel does not exist.
497 //
498 if (!pBlockID)
499 return NV_TRUE;
500
501 if ((pAllocID->domain != pBlockID->domain) ||
502 (pAllocID->processID != pBlockID->processID) ||
503 (pAllocID->subProcessID != pBlockID->subProcessID))
504 {
505 return NV_FALSE;
506 }
507 else
508 {
509 return NV_TRUE;
510 }
511 }
512
513 /*
514 * @brief Returns the number of vGPU plugin channels.
515 *
516 * Depending on whether this code is executed on the CPU RM or the Physical RM,
517 * different structures are used to retrieve the number.
518 * On the CPU RM or the monolithic RM, KERNEL_HOST_VGPU_DEVICE::numPluginChannels is used,
519 * whereas on the physical RM it's HOST_VGPU_DEVICE::numPluginChannels.
520 */
521 static NV_STATUS
_kfifoGetVgpuPluginChannelsCount(OBJGPU * pGpu,NvU32 * pNumPluginChannels)522 _kfifoGetVgpuPluginChannelsCount
523 (
524 OBJGPU *pGpu,
525 NvU32 *pNumPluginChannels
526 )
527 {
528 NV_ASSERT_OR_RETURN(pNumPluginChannels != NULL, NV_ERR_INVALID_ARGUMENT);
529
530 if (!RMCFG_FEATURE_PLATFORM_GSP)
531 {
532 KERNEL_HOST_VGPU_DEVICE *pKernelHostVgpuDevice = NULL;
533
534 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextKernelHostVgpuDevice(pGpu, &pKernelHostVgpuDevice));
535 NV_ASSERT_OR_RETURN(pKernelHostVgpuDevice != NULL, NV_ERR_OBJECT_NOT_FOUND);
536
537 *pNumPluginChannels = pKernelHostVgpuDevice->numPluginChannels;
538 }
539
540 return NV_OK;
541 }
542
543 /*!
544 * @brief Allocates one Channel ID on heap
545 *
546 * @param[in] OBJGPU GPU Object
547 * @param[in] KernelFifo KernelFifo Object
548 * @param[in] CHID_MGR Channel ID manager
549 * @param[in] chFlag NVOS32_ALLOC_FLAGS_FORCE_MEM_GROWS_DOWN
550 * NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE
551 * NVOS32_ALLOC_FLAGS_FORCE_MEM_GROWS_UP
552 * @param[in] bForceInternalIdx true if requesting specific index within USERD
553 * page
554 * @param[in] internalIdx requested index within USERD page when
555 * bForceInternalIdx true
556 * @param[in] ChID ChID to assign in case of ADDRESS_ALLOCATE
557 * @param[in,out] pKernelChannel The previously allocated KernelChannel structure
558 *
559 * @return NV_OK if allocation is successful
560 * NV_ERR_NO_FREE_FIFOS: allocated channel ID exceeds MAX channels.
561 */
562 NV_STATUS
kfifoChidMgrAllocChid_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,CHID_MGR * pChidMgr,NvHandle hClient,CHANNEL_HW_ID_ALLOC_MODE chIdFlag,NvBool bForceInternalIdx,NvU32 internalIdx,NvBool bForceUserdPage,NvU32 userdPageIdx,NvU32 ChID,KernelChannel * pKernelChannel)563 kfifoChidMgrAllocChid_IMPL
564 (
565 OBJGPU *pGpu,
566 KernelFifo *pKernelFifo,
567 CHID_MGR *pChidMgr,
568 NvHandle hClient,
569 CHANNEL_HW_ID_ALLOC_MODE chIdFlag,
570 NvBool bForceInternalIdx,
571 NvU32 internalIdx,
572 NvBool bForceUserdPage,
573 NvU32 userdPageIdx,
574 NvU32 ChID,
575 KernelChannel *pKernelChannel
576 )
577 {
578 NvU64 chSize;
579 NvU32 chFlag = chIdFlag;
580 NvU64 ChID64 = 0;
581 NvU64 subProcessID = 0;
582 NvU64 processID = 0;
583 NvBool bIsSubProcessDisabled = NV_FALSE;
584 RmClient *pClient;
585 NvU32 offsetAlign = 1;
586 NvU32 gfid;
587 PFIFO_ISOLATIONID pIsolationID = NULL;
588 NV_STATUS status;
589 NvU32 numChannels;
590
591 NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_ARGUMENT);
592
593 numChannels = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr);
594
595 switch (chIdFlag)
596 {
597 case CHANNEL_HW_ID_ALLOC_MODE_GROW_DOWN:
598 chFlag = NVOS32_ALLOC_FLAGS_FORCE_MEM_GROWS_DOWN;
599 break;
600 case CHANNEL_HW_ID_ALLOC_MODE_GROW_UP:
601 chFlag = NVOS32_ALLOC_FLAGS_FORCE_MEM_GROWS_UP;
602 break;
603 case CHANNEL_HW_ID_ALLOC_MODE_PROVIDED:
604 ChID64 = ChID;
605 chFlag = NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE;
606 break;
607 default:
608 NV_PRINTF(LEVEL_ERROR, "Invalid channel ID alloc mode %d\n", chFlag);
609 DBG_BREAKPOINT();
610 return NV_ERR_INVALID_ARGUMENT;
611 }
612
613 // we are allocating only one Channel at a time
614 chSize = 1;
615
616 // Create unique isolation ID for each process
617 pClient = serverutilGetClientUnderLock(hClient);
618 if (pClient == NULL)
619 {
620 NV_PRINTF(LEVEL_ERROR, "Invalid client handle %ux\n", hClient);
621 DBG_BREAKPOINT();
622 return NV_ERR_INVALID_CLIENT;
623 }
624
625 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &gfid));
626
627 // SRIOV: In guest plugin context allocate the chid.
628 // In guest RM context allocate the same chid as guest
629 if (IS_GFID_VF(gfid))
630 {
631 NvU32 numPluginChannels;
632 NvU64 rangeLo, rangeHi, base, size;
633
634 NV_ASSERT_OR_RETURN(pChidMgr->ppVirtualChIDHeap[gfid],
635 NV_ERR_INVALID_STATE);
636
637 NV_ASSERT_OK_OR_RETURN(_kfifoGetVgpuPluginChannelsCount(pGpu, &numPluginChannels));
638
639 pChidMgr->ppVirtualChIDHeap[gfid]->eheapGetBase(
640 pChidMgr->ppVirtualChIDHeap[gfid],
641 &base);
642 pChidMgr->ppVirtualChIDHeap[gfid]->eheapGetSize(
643 pChidMgr->ppVirtualChIDHeap[gfid],
644 &size);
645
646 rangeLo = base;
647 rangeHi = base + size - 1;
648
649 // Route plugin channels to be allocated at the top
650 NV_ASSERT_OR_RETURN(numPluginChannels < size,
651 NV_ERR_INVALID_PARAMETER);
652 if (pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->bIsCallingContextVgpuPlugin &&
653 numPluginChannels > 0)
654 {
655 rangeLo = rangeHi - numPluginChannels + 1;
656 }
657 else
658 {
659 rangeHi = rangeHi - numPluginChannels;
660 }
661
662 status = pChidMgr->ppVirtualChIDHeap[gfid]->eheapSetAllocRange(
663 pChidMgr->ppVirtualChIDHeap[gfid],
664 rangeLo,
665 rangeHi);
666
667 NV_ASSERT_OK_OR_RETURN(status);
668
669 if (bForceUserdPage)
670 {
671 NV_ASSERT_OR_RETURN(!bForceInternalIdx, NV_ERR_INVALID_STATE);
672 ChID64 = ((NvU64)userdPageIdx) *
673 pChidMgr->ppVirtualChIDHeap[gfid]->ownerGranularity +
674 internalIdx;
675 chFlag |= NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE;
676 }
677 else if (bForceInternalIdx)
678 {
679 chFlag |= NVOS32_ALLOC_FLAGS_FORCE_INTERNAL_INDEX;
680 offsetAlign = internalIdx;
681 }
682
683 if (chFlag & NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE)
684 NV_ASSERT_OR_RETURN((ChID64 <= rangeHi) && (ChID64 >= rangeLo),
685 NV_ERR_INVALID_PARAMETER);
686
687 // We'll allocate from the VirtualChIdHeap for the guest
688 status = pChidMgr->ppVirtualChIDHeap[gfid]->eheapAlloc(
689 pChidMgr->ppVirtualChIDHeap[gfid], // This Heap
690 KFIFO_EHEAP_OWNER, // owner
691 &chFlag, // Alloc Flags
692 &ChID64, // Alloc Offset
693 &chSize, // Size
694 offsetAlign, // offsetAlign
695 1, // sizeAlign
696 NULL, // Allocated mem block
697 NULL, // Isolation ID
698 NULL // Fifo defined ownership comparator
699 );
700 if (status != NV_OK)
701 {
702 NV_PRINTF(LEVEL_ERROR,
703 "Failed to allocate Channel ID 0x%llx %d on heap \n",
704 ChID64,
705 chIdFlag);
706 DBG_BREAKPOINT();
707 goto fail;
708 }
709 }
710 else
711 {
712 //
713 // Legacy / SRIOV vGPU Host, SRIOV guest, baremetal CPU RM, GSP FW, GSP
714 // client allocate from global heap
715 //
716 pIsolationID = portMemAllocNonPaged(sizeof(FIFO_ISOLATIONID));
717 NV_ASSERT_OR_RETURN((pIsolationID != NULL), NV_ERR_NO_MEMORY);
718 portMemSet(pIsolationID, 0, sizeof(FIFO_ISOLATIONID));
719
720 //
721 // Check if the allocation request is from the guest RM or host RM
722 //
723 processID = pClient->ProcID;
724 subProcessID = pClient->SubProcessID;
725 bIsSubProcessDisabled = pClient->bIsSubProcessDisabled;
726
727 if (RMCFG_FEATURE_PLATFORM_GSP || kchannelCheckIsKernel(pKernelChannel))
728 {
729 //
730 // If not GSPFW: Allocation request is from host RM kernel
731 // If GSPFW: ChID has already been chosen by CPU-RM, but pClient
732 // doesn't have the true processID, so just allow the whole pool.
733 //
734 pIsolationID->domain = HOST_KERNEL;
735 processID = KERNEL_PID;
736 }
737 else
738 {
739 if (0x0 != subProcessID)
740 {
741 //
742 // Allocation request is from the guest RM
743 //
744 if (KERNEL_PID == subProcessID)
745 {
746 pIsolationID->domain = GUEST_KERNEL;
747 }
748 else
749 {
750 pIsolationID->domain = GUEST_USER;
751 }
752 }
753 else
754 {
755 pIsolationID->domain = HOST_USER;
756 }
757 }
758
759 pIsolationID->processID = processID;
760 pIsolationID->subProcessID = subProcessID;
761
762 //
763 // Overwrite isolation ID if guest USERD isolation is disabled
764 //
765 if ((subProcessID != 0x0) && (bIsSubProcessDisabled))
766 {
767 pIsolationID->domain = GUEST_INSECURE;
768 pIsolationID->subProcessID = KERNEL_PID;
769 }
770
771 /* Channel USERD manipuliation only supported without GFID */
772 if (bForceUserdPage)
773 {
774 NV_ASSERT_OR_RETURN(!bForceInternalIdx, NV_ERR_INVALID_STATE);
775 ChID64 = ((NvU64)userdPageIdx) *
776 pChidMgr->pGlobalChIDHeap->ownerGranularity +
777 internalIdx;
778 chFlag |= NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE;
779 }
780 else if (bForceInternalIdx)
781 {
782 chFlag |= NVOS32_ALLOC_FLAGS_FORCE_INTERNAL_INDEX;
783 offsetAlign = internalIdx;
784 }
785
786 status = pChidMgr->pGlobalChIDHeap->eheapAlloc(
787 pChidMgr->pGlobalChIDHeap, // This Heap
788 KFIFO_EHEAP_OWNER, // owner
789 &chFlag, // Alloc Flags
790 &ChID64, // Alloc Offset
791 &chSize, // Size
792 offsetAlign, // offsetAlign
793 1, // sizeAlign
794 NULL, // Allocated mem block
795 pIsolationID, // Isolation ID
796 _kfifoUserdOwnerComparator // Fifo defined ownership comparator
797 );
798
799 if (status != NV_OK)
800 {
801 NV_PRINTF(LEVEL_ERROR, "Failed to allocate Channel ID on heap\n");
802 DBG_BREAKPOINT();
803 goto fail;
804 }
805 }
806
807 //
808 // Now allocate at a fixed offset from the pFifoDataHeap once the previous
809 // ID allocation told us which ID to use.
810 //
811 chFlag = NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE;
812 status = pChidMgr->pFifoDataHeap->eheapAlloc(
813 pChidMgr->pFifoDataHeap, // This Heap
814 KFIFO_EHEAP_OWNER, // owner
815 &chFlag, // Alloc Flags
816 &ChID64, // Alloc Offset
817 &chSize, // Size
818 1, // offsetAlign
819 1, // sizeAlign
820 NULL, // Allocated mem block
821 NULL, // Isolation ID
822 NULL // ownership comparator
823 );
824
825 if (status != NV_OK)
826 {
827 //
828 // Should never happen since we're mirroring the global chid heap, or
829 // pre-reserving space on the global chid heap for SR-IOV capable
830 // systems.
831 //
832 NV_PRINTF(LEVEL_ERROR, "Failed to allocate Channel on fifo data heap\n");
833 goto fail;
834 }
835
836 ChID = NvU64_LO32(ChID64);
837
838 if (ChID < numChannels)
839 {
840 EMEMBLOCK *pFifoDataBlock = pChidMgr->pFifoDataHeap->eheapGetBlock(
841 pChidMgr->pFifoDataHeap,
842 ChID,
843 NV_FALSE);
844 EMEMBLOCK *pIsolationIdBlock = pChidMgr->pGlobalChIDHeap->eheapGetBlock(
845 pChidMgr->pGlobalChIDHeap,
846 ChID,
847 NV_FALSE);
848
849 if (IS_GFID_PF(gfid))
850 pIsolationIdBlock->pData = pIsolationID;
851
852 pFifoDataBlock->pData = pKernelChannel;
853 pKernelChannel->ChID = ChID;
854 }
855 else
856 {
857 NV_PRINTF(LEVEL_WARNING, "No allocatable FIFO available.\n");
858 status = NV_ERR_NO_FREE_FIFOS;
859 goto fail;
860 }
861 return NV_OK;
862
863 fail:
864 // We already know that pIsolationID is non-NULL here.
865 portMemFree(pIsolationID);
866 return status;
867 }
868
869 /*
870 * Retain a channel ID which has already been allocated by
871 * kfifoChidMgrAllocChid. Until released, the HW channel ID will not be
872 * allocated by any new channels even after kfifoChidMgrFreeChid has been
873 * called.
874 */
875 NV_STATUS
kfifoChidMgrRetainChid_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,CHID_MGR * pChidMgr,NvU32 ChID)876 kfifoChidMgrRetainChid_IMPL
877 (
878 OBJGPU *pGpu,
879 KernelFifo *pKernelFifo,
880 CHID_MGR *pChidMgr,
881 NvU32 ChID
882 )
883 {
884 NvU32 gfid;
885 EMEMBLOCK *pFifoDataBlock = NULL;
886
887 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &gfid));
888
889 if (IS_GFID_VF(gfid))
890 {
891 NV_ASSERT_OR_RETURN(pChidMgr->ppVirtualChIDHeap[gfid] != NULL,
892 NV_ERR_INVALID_STATE);
893 EMEMBLOCK *pVirtChIdBlock = pChidMgr->ppVirtualChIDHeap[gfid]->eheapGetBlock(
894 pChidMgr->ppVirtualChIDHeap[gfid],
895 ChID,
896 NV_FALSE);
897 NV_ASSERT_OR_RETURN(pVirtChIdBlock != NULL, NV_ERR_OBJECT_NOT_FOUND);
898 NV_ASSERT(pVirtChIdBlock->refCount > 0);
899 ++pVirtChIdBlock->refCount;
900 }
901 else
902 {
903 NV_ASSERT_OR_RETURN(pChidMgr->pGlobalChIDHeap != NULL, NV_ERR_INVALID_STATE);
904 EMEMBLOCK *pChIdBlock = pChidMgr->pGlobalChIDHeap->eheapGetBlock(
905 pChidMgr->pGlobalChIDHeap,
906 ChID,
907 NV_FALSE);
908 NV_ASSERT_OR_RETURN(pChIdBlock != NULL, NV_ERR_OBJECT_NOT_FOUND);
909 NV_ASSERT(pChIdBlock->refCount > 0);
910 ++pChIdBlock->refCount;
911 }
912
913 NV_ASSERT_OR_RETURN(pChidMgr->pFifoDataHeap != NULL, NV_ERR_INVALID_STATE);
914 pFifoDataBlock = pChidMgr->pFifoDataHeap->eheapGetBlock(
915 pChidMgr->pFifoDataHeap,
916 ChID,
917 NV_FALSE);
918 NV_ASSERT_OR_RETURN(pFifoDataBlock != NULL, NV_ERR_OBJECT_NOT_FOUND);
919 NV_ASSERT(pFifoDataBlock->refCount > 0);
920 ++pFifoDataBlock->refCount;
921
922 return NV_OK;
923 }
924
925 /*
926 * Drop the refcount on the given channel (ID), removing it from pFifo's heap if
927 * its refcount reaches 0.
928 */
929 NV_STATUS
kfifoChidMgrReleaseChid_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,CHID_MGR * pChidMgr,NvU32 ChID)930 kfifoChidMgrReleaseChid_IMPL
931 (
932 OBJGPU *pGpu,
933 KernelFifo *pKernelFifo,
934 CHID_MGR *pChidMgr,
935 NvU32 ChID
936 )
937 {
938 NvU32 gfid;
939
940 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &gfid));
941
942 if (IS_GFID_VF(gfid))
943 {
944 NV_ASSERT_OR_RETURN(pChidMgr->ppVirtualChIDHeap[gfid] != NULL, NV_ERR_INVALID_STATE);
945 NV_ASSERT_OK(pChidMgr->ppVirtualChIDHeap[gfid]->eheapFree(pChidMgr->ppVirtualChIDHeap[gfid], ChID));
946 }
947 else
948 {
949 NV_ASSERT_OR_RETURN(pChidMgr->pGlobalChIDHeap != NULL, NV_ERR_INVALID_STATE);
950 NV_ASSERT_OK(pChidMgr->pGlobalChIDHeap->eheapFree(pChidMgr->pGlobalChIDHeap, ChID));
951 }
952
953 NV_ASSERT_OR_RETURN(pChidMgr->pFifoDataHeap != NULL, NV_ERR_INVALID_STATE);
954 NV_ASSERT_OK_OR_RETURN(pChidMgr->pFifoDataHeap->eheapFree(pChidMgr->pFifoDataHeap, ChID));
955
956 return NV_OK;
957 }
958
959 /*
960 * Removes the association between pKernelChannel and its channel ID. Note that this
961 * will not remove the channel ID itself from pFifo's heap if
962 * fifoHeapRetainChannelId has been called.
963 */
964 NV_STATUS
kfifoChidMgrFreeChid_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,CHID_MGR * pChidMgr,NvU32 ChID)965 kfifoChidMgrFreeChid_IMPL
966 (
967 OBJGPU *pGpu,
968 KernelFifo *pKernelFifo,
969 CHID_MGR *pChidMgr,
970 NvU32 ChID
971 )
972 {
973 EMEMBLOCK *pFifoDataBlock;
974 NV_STATUS status;
975 NvU32 gfid;
976
977 //
978 // This channel is going away, so clear its pointer from the channel ID's heap
979 // block.
980 //
981 pFifoDataBlock = pChidMgr->pFifoDataHeap->eheapGetBlock(
982 pChidMgr->pFifoDataHeap,
983 ChID,
984 NV_FALSE);
985 NV_ASSERT_OR_RETURN(pFifoDataBlock != NULL, NV_ERR_OBJECT_NOT_FOUND);
986 NV_ASSERT(pFifoDataBlock->refCount > 0);
987 pFifoDataBlock->pData = NULL;
988
989 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &gfid));
990
991 if (IS_GFID_PF(gfid))
992 {
993 //
994 // This marks the channel ID as orphaned and causes it to be ignored for
995 // isolation purposes. This only matters if there will still be a reference
996 // on the ID after we release ours below.
997 //
998 status = _kfifoChidMgrFreeIsolationId(pChidMgr, ChID);
999 if(status != NV_OK)
1000 {
1001 NV_PRINTF(LEVEL_ERROR,
1002 "Failed to free IsolationId. Status = 0x%x\n", status);
1003 DBG_BREAKPOINT();
1004 return status;
1005 }
1006 }
1007
1008 return kfifoChidMgrReleaseChid(pGpu, pKernelFifo, pChidMgr, ChID);
1009 }
1010
1011 /**
1012 * @brief Reserve a contiguous set of SCHIDs from the end of our CHID heap for
1013 * the given GFID
1014 *
1015
1016 * @param[in] pChidMgr CHID_MGR pointer
1017 * @param[in] numChannels Number of SCHIDs to reserve
1018 * @param[in] pHostVgpuDevice HOST_VGPU_DEVICE
1019 *
1020 * @return NV_OK if success
1021 */
1022 NV_STATUS
kfifoChidMgrReserveSystemChids_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,CHID_MGR * pChidMgr,NvU32 numChannels,NvU32 flags,NvU32 gfid,NvU32 * pChidOffset,NvU64 offset,NvU32 * pChannelCount,Device * pMigDevice,NvU32 engineFifoListNumEntries,FIFO_ENGINE_LIST * pEngineFifoList)1023 kfifoChidMgrReserveSystemChids_IMPL
1024 (
1025 OBJGPU *pGpu,
1026 KernelFifo *pKernelFifo,
1027 CHID_MGR *pChidMgr,
1028 NvU32 numChannels,
1029 NvU32 flags,
1030 NvU32 gfid,
1031 NvU32 *pChidOffset,
1032 NvU64 offset,
1033 NvU32 *pChannelCount,
1034 Device *pMigDevice,
1035 NvU32 engineFifoListNumEntries,
1036 FIFO_ENGINE_LIST *pEngineFifoList
1037 )
1038 {
1039 NV_STATUS status = NV_OK;
1040 NvU64 chSize;
1041 PFIFO_ISOLATIONID pIsolationID = NULL;
1042 EMEMBLOCK *pIsolationIdBlock;
1043 NvU32 userdBar1Size;
1044
1045 if (IS_VIRTUAL(pGpu))
1046 {
1047 // Not supported on guest or when SRIOV is disabled
1048 return NV_ERR_NOT_SUPPORTED;
1049 }
1050
1051 pIsolationID = portMemAllocNonPaged(sizeof(FIFO_ISOLATIONID));
1052 NV_ASSERT_OR_RETURN((pIsolationID != NULL), NV_ERR_NO_MEMORY);
1053 portMemSet(pIsolationID, 0, sizeof(FIFO_ISOLATIONID));
1054
1055 chSize = numChannels;
1056
1057 status = pChidMgr->pGlobalChIDHeap->eheapAlloc(
1058 pChidMgr->pGlobalChIDHeap, // This Heap
1059 KFIFO_EHEAP_OWNER, // owner
1060 &flags, // Alloc Flags
1061 &offset, // Alloc Offset
1062 &chSize, // Size
1063 1, // offsetAlign
1064 1, // sizeAlign
1065 NULL, // Allocated mem block
1066 pIsolationID, // IsolationID
1067 _kfifoUserdOwnerComparator // Fifo defined ownership comparator
1068 );
1069
1070 if(status != NV_OK)
1071 {
1072 NV_PRINTF(LEVEL_ERROR, "Failed to reserve channel IDs. Status = 0x%x\n", status);
1073 DBG_BREAKPOINT();
1074
1075 //
1076 // Free the allocated memory and return early. After this, all failure
1077 // points can goto the common cleanup label
1078 //
1079 portMemFree(pIsolationID);
1080 return status;
1081 }
1082
1083 pIsolationIdBlock = pChidMgr->pGlobalChIDHeap->eheapGetBlock(
1084 pChidMgr->pGlobalChIDHeap,
1085 offset,
1086 NV_FALSE);
1087 if (pIsolationIdBlock == NULL)
1088 {
1089 // Something bad happened. This should not fail if allocation succeeded
1090 NV_PRINTF(LEVEL_ERROR, "Could not fetch block from eheap\n");
1091 DBG_BREAKPOINT();
1092 goto cleanup;
1093 }
1094 pIsolationIdBlock->pData = pIsolationID;
1095
1096 status = kfifoSetChidOffset(pGpu, pKernelFifo, pChidMgr, (NvU32)offset,
1097 numChannels, gfid, pChidOffset, pChannelCount,
1098 pMigDevice, engineFifoListNumEntries, pEngineFifoList);
1099
1100 if (status != NV_OK)
1101 {
1102 NV_PRINTF(LEVEL_ERROR,
1103 "Failed to program the CHID table\n");
1104 goto cleanup;
1105 }
1106
1107 pChidMgr->ppVirtualChIDHeap[gfid] = portMemAllocNonPaged(sizeof(OBJEHEAP));
1108 if (pChidMgr->ppVirtualChIDHeap[gfid] == NULL)
1109 {
1110 status = NV_ERR_NO_MEMORY;
1111 NV_PRINTF(LEVEL_ERROR, "Error allocating memory for virtual "
1112 "channel ID heap\n");
1113 goto cleanup;
1114 }
1115 portMemSet(pChidMgr->ppVirtualChIDHeap[gfid], 0, sizeof(OBJEHEAP));
1116
1117 //
1118 // Construct heap using low as offset and size of numChannels. This heap
1119 // will be used for guest channel ID allocations, but will be in the
1120 // system channel ID space, hence it only manages IDs from offset to
1121 // (offset + numChannels).
1122 //
1123 constructObjEHeap(pChidMgr->ppVirtualChIDHeap[gfid], offset,
1124 (offset + numChannels), 0, 0);
1125
1126 kfifoGetUserdSizeAlign_HAL(pKernelFifo, &userdBar1Size, NULL);
1127 pChidMgr->ppVirtualChIDHeap[gfid]->eheapSetOwnerIsolation(
1128 pChidMgr->ppVirtualChIDHeap[gfid],
1129 NV_FALSE,
1130 RM_PAGE_SIZE/userdBar1Size);
1131
1132 return status;
1133
1134 cleanup:
1135 portMemFree(pChidMgr->ppVirtualChIDHeap[gfid]);
1136 NV_ASSERT(kfifoSetChidOffset(pGpu, pKernelFifo, pChidMgr, 0, 0,
1137 gfid, pChidOffset, pChannelCount, pMigDevice,
1138 engineFifoListNumEntries, pEngineFifoList) == NV_OK);
1139 NV_ASSERT(pChidMgr->pGlobalChIDHeap->eheapFree(pChidMgr->pGlobalChIDHeap, offset) == NV_OK);
1140 portMemFree(pIsolationID);
1141 return status;
1142 }
1143
1144 /*! Frees a block of contiguous SCHIDs previously reserved for the given GFID */
1145 NV_STATUS
kfifoChidMgrFreeSystemChids_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,CHID_MGR * pChidMgr,NvU32 gfid,NvU32 * pChidOffset,NvU32 * pChannelCount,Device * pMigDevice,NvU32 engineFifoListNumEntries,FIFO_ENGINE_LIST * pEngineFifoList)1146 kfifoChidMgrFreeSystemChids_IMPL
1147 (
1148 OBJGPU *pGpu,
1149 KernelFifo *pKernelFifo,
1150 CHID_MGR *pChidMgr,
1151 NvU32 gfid,
1152 NvU32 *pChidOffset,
1153 NvU32 *pChannelCount,
1154 Device *pMigDevice,
1155 NvU32 engineFifoListNumEntries,
1156 FIFO_ENGINE_LIST *pEngineFifoList
1157 )
1158 {
1159 NV_STATUS status, tmpStatus;
1160 NvU64 chId;
1161
1162 if (IS_VIRTUAL(pGpu))
1163 {
1164 // Not supported on guest or when SRIOV is disabled
1165 return NV_ERR_NOT_SUPPORTED;
1166 }
1167
1168 // Get the schid base
1169 pChidMgr->ppVirtualChIDHeap[gfid]->eheapGetBase(
1170 pChidMgr->ppVirtualChIDHeap[gfid],
1171 &chId);
1172
1173 status = _kfifoChidMgrFreeIsolationId(pChidMgr, (NvU32)chId);
1174 if(status != NV_OK)
1175 {
1176 NV_PRINTF(LEVEL_ERROR,
1177 "Failed to free IsolationId. Status = 0x%x\n",
1178 status);
1179 DBG_BREAKPOINT();
1180 return status;
1181 }
1182
1183 status = pChidMgr->pGlobalChIDHeap->eheapFree(pChidMgr->pGlobalChIDHeap, chId);
1184 if(status != NV_OK)
1185 {
1186 NV_PRINTF(LEVEL_ERROR,
1187 "Failed to free channel IDs. Status = 0x%x\n",
1188 status);
1189 DBG_BREAKPOINT();
1190
1191 //
1192 // March on anyway to program the ChId table. We'll return an error
1193 // if we get here though.
1194 //
1195 }
1196
1197 tmpStatus = kfifoSetChidOffset(pGpu, pKernelFifo, pChidMgr, 0, 0,
1198 gfid, pChidOffset, pChannelCount, pMigDevice,
1199 engineFifoListNumEntries, pEngineFifoList);
1200 if (tmpStatus != NV_OK)
1201 {
1202 NV_PRINTF(LEVEL_ERROR,
1203 "Failed to program the CHID table\n");
1204 DBG_BREAKPOINT();
1205 return tmpStatus;
1206 }
1207
1208 pChidMgr->ppVirtualChIDHeap[gfid]->eheapDestruct(pChidMgr->ppVirtualChIDHeap[gfid]);
1209 portMemFree(pChidMgr->ppVirtualChIDHeap[gfid]);
1210 pChidMgr->ppVirtualChIDHeap[gfid] = NULL;
1211
1212 return status;
1213 }
1214
1215 NvU32
kfifoChidMgrGetNumChannels_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,CHID_MGR * pChidMgr)1216 kfifoChidMgrGetNumChannels_IMPL
1217 (
1218 OBJGPU *pGpu,
1219 KernelFifo *pKernelFifo,
1220 CHID_MGR *pChidMgr
1221 )
1222 {
1223 // Cache ChidMgr's numChannels if not set
1224 if (pChidMgr->numChannels == 0)
1225 {
1226 NvU32 numChannels = kfifoRunlistQueryNumChannels_HAL(pGpu, pKernelFifo,
1227 pChidMgr->runlistId);
1228
1229 if (pKernelFifo->bNumChannelsOverride)
1230 {
1231 pChidMgr->numChannels = NV_MIN(pKernelFifo->numChannelsOverride, numChannels);
1232 }
1233 else
1234 {
1235 pChidMgr->numChannels = numChannels;
1236 }
1237
1238 // Once we have set calculated value disable any overrides.
1239 pKernelFifo->bNumChannelsOverride = 0;
1240 }
1241
1242 return pChidMgr->numChannels;
1243 }
1244
1245 NvU32
kfifoRunlistQueryNumChannels_KERNEL(OBJGPU * pGpu,KernelFifo * pKernelFifo,NvU32 runlistId)1246 kfifoRunlistQueryNumChannels_KERNEL
1247 (
1248 OBJGPU *pGpu,
1249 KernelFifo *pKernelFifo,
1250 NvU32 runlistId
1251 )
1252 {
1253 NvU32 numChannels = 0;
1254 NvU32 status;
1255
1256 // For vgpu, read numChannels from VGPU_STATIC_INFO
1257 if (IS_VIRTUAL(pGpu))
1258 {
1259 VGPU_STATIC_INFO *pVSI;
1260 pVSI = GPU_GET_STATIC_INFO(pGpu);
1261
1262 if (pVSI)
1263 {
1264 numChannels = pVSI->vgpuStaticProperties.channelCount;
1265 }
1266 else
1267 {
1268 DBG_BREAKPOINT();
1269 return 0;
1270 }
1271 }
1272
1273 // Do internal control call and set numChannels
1274 if (IS_GSP_CLIENT(pGpu))
1275 {
1276 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
1277 NV2080_CTRL_INTERNAL_FIFO_GET_NUM_CHANNELS_PARAMS numChannelsParams = {0};
1278
1279 numChannelsParams.runlistId = runlistId;
1280
1281 status = pRmApi->Control(pRmApi,
1282 pGpu->hInternalClient,
1283 pGpu->hInternalSubdevice,
1284 NV2080_CTRL_CMD_INTERNAL_FIFO_GET_NUM_CHANNELS,
1285 &numChannelsParams,
1286 sizeof(NV2080_CTRL_INTERNAL_FIFO_GET_NUM_CHANNELS_PARAMS));
1287 if (status != NV_OK)
1288 {
1289 DBG_BREAKPOINT();
1290 return 0;
1291 }
1292
1293 numChannels = numChannelsParams.numChannels;
1294 }
1295
1296 NV_ASSERT(numChannels > 0);
1297
1298 return numChannels;
1299 }
1300
1301 /**
1302 * @brief reserves a hardware channel slot for a channel group
1303 *
1304 * Only responsible for indicating a hardware channel is in use, does not set
1305 * any other software state.
1306 *
1307 * This function is not called in broadcast mode
1308 *
1309 * @param pGpu
1310 * @param pKernelFifo
1311 * @param pChidMgr
1312 * @param[out] grpID
1313 */
1314 NV_STATUS
kfifoChidMgrAllocChannelGroupHwID_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,CHID_MGR * pChidMgr,NvU32 * pChGrpID)1315 kfifoChidMgrAllocChannelGroupHwID_IMPL
1316 (
1317 OBJGPU *pGpu,
1318 KernelFifo *pKernelFifo,
1319 CHID_MGR *pChidMgr,
1320 NvU32 *pChGrpID
1321 )
1322 {
1323 NvU32 maxChannelGroups;
1324
1325 if (pChGrpID == NULL)
1326 return NV_ERR_INVALID_ARGUMENT;
1327
1328 maxChannelGroups = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr);
1329 if (maxChannelGroups == 0)
1330 {
1331 NV_PRINTF(LEVEL_ERROR, "Zero max channel groups!!!\n");
1332 return NV_ERR_INVALID_ARGUMENT;
1333 }
1334
1335 // Find the least unused grpID
1336 *pChGrpID = nvBitFieldLSZero(pChidMgr->channelGrpMgr.pHwIdInUse,
1337 pChidMgr->channelGrpMgr.hwIdInUseSz);
1338
1339 if (*pChGrpID < maxChannelGroups)
1340 {
1341 nvBitFieldSet(pChidMgr->channelGrpMgr.pHwIdInUse,
1342 pChidMgr->channelGrpMgr.hwIdInUseSz, *pChGrpID, NV_TRUE);
1343 }
1344 else
1345 {
1346 *pChGrpID = maxChannelGroups;
1347 NV_PRINTF(LEVEL_ERROR, "No allocatable FIFO available.\n");
1348 return NV_ERR_NO_FREE_FIFOS;
1349 }
1350 return NV_OK;
1351 }
1352
1353
1354 /**
1355 * @brief Releases a hardware channel group ID.
1356 *
1357 * Not responsible for freeing any software state beyond that which indicates a
1358 * hardware channel is in use.
1359 *
1360 * This function is not called in broadcast mode
1361 *
1362 * @param pGpu
1363 * @param pFifo
1364 * @param pChidMgr
1365 * @param chGrpID
1366 */
1367 NV_STATUS
kfifoChidMgrFreeChannelGroupHwID_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,CHID_MGR * pChidMgr,NvU32 chGrpID)1368 kfifoChidMgrFreeChannelGroupHwID_IMPL
1369 (
1370 OBJGPU *pGpu,
1371 KernelFifo *pKernelFifo,
1372 CHID_MGR *pChidMgr,
1373 NvU32 chGrpID
1374 )
1375 {
1376 NvU32 maxChannelGroups;
1377
1378 maxChannelGroups = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr);
1379 if (maxChannelGroups == 0)
1380 {
1381 NV_PRINTF(LEVEL_ERROR, "Zero max channel groups!!!\n");
1382 return NV_ERR_INVALID_ARGUMENT;
1383 }
1384 NV_ASSERT_OR_RETURN(chGrpID < maxChannelGroups, NV_ERR_INVALID_ARGUMENT);
1385
1386 //
1387 // Look for the channel group, check to make sure it's InUse bit is set
1388 // and then clear it to indicate the grpID is no longer in use
1389 //
1390 NV_ASSERT(nvBitFieldTest(pChidMgr->channelGrpMgr.pHwIdInUse,
1391 pChidMgr->channelGrpMgr.hwIdInUseSz, chGrpID));
1392 nvBitFieldSet(pChidMgr->channelGrpMgr.pHwIdInUse,
1393 pChidMgr->channelGrpMgr.hwIdInUseSz, chGrpID, NV_FALSE);
1394
1395 return NV_OK;
1396 }
1397
1398 CHID_MGR *
kfifoGetChidMgr_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,NvU32 runlistId)1399 kfifoGetChidMgr_IMPL
1400 (
1401 OBJGPU *pGpu,
1402 KernelFifo *pKernelFifo,
1403 NvU32 runlistId
1404 )
1405 {
1406 if (!kfifoIsPerRunlistChramEnabled(pKernelFifo))
1407 {
1408 // We only have 1 chidmgr when we don't have a per-runlist channel RAM
1409 if ((pKernelFifo->numChidMgrs != 1) ||
1410 (pKernelFifo->ppChidMgr == NULL) ||
1411 !bitVectorTest(&pKernelFifo->chidMgrValid, 0))
1412 {
1413 return NULL;
1414 }
1415 return pKernelFifo->ppChidMgr[0];
1416 }
1417 else
1418 {
1419 if (runlistId >= pKernelFifo->numChidMgrs)
1420 {
1421 return NULL;
1422 }
1423 //
1424 // It is valid to return a NULL value as long as runlistId is less than
1425 // maxNumRunlists since it is possible that not everything in the range
1426 // [0, numChidMgrs) represents a valid runlistId. The onus is on the
1427 // caller to check for NULL and only then use the CHIDMGR pointer
1428 //
1429 return pKernelFifo->ppChidMgr[runlistId];
1430 }
1431 }
1432
1433 /*! Gets associated CHIDMGR object for given FIFO_ENGINE_INFO_TYPE and value */
1434 NV_STATUS
kfifoGetChidMgrFromType_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,NvU32 engineInfoType,NvU32 val,CHID_MGR ** ppChidMgr)1435 kfifoGetChidMgrFromType_IMPL
1436 (
1437 OBJGPU *pGpu,
1438 KernelFifo *pKernelFifo,
1439 NvU32 engineInfoType,
1440 NvU32 val,
1441 CHID_MGR **ppChidMgr
1442 )
1443 {
1444 NV_STATUS status = NV_OK;
1445 NvU32 runlistId;
1446
1447 NV_CHECK_OR_RETURN(LEVEL_INFO, ppChidMgr != NULL, NV_ERR_INVALID_ARGUMENT);
1448
1449 // Initialize the pointer to NULL, in case we fail and return early
1450 *ppChidMgr = NULL;
1451
1452 status = kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo,
1453 engineInfoType, val,
1454 ENGINE_INFO_TYPE_RUNLIST, &runlistId);
1455 NV_CHECK_OR_RETURN(LEVEL_INFO, NV_OK == status, status);
1456
1457 *ppChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, runlistId);
1458
1459 return NV_OK;
1460 }
1461
1462 /*!
1463 * @brief Fetch pKernelChannel based on chidmgr and chid.
1464 *
1465 * This look-up uses the chid heap. It should find the first allocation of the channel,
1466 * which is useful if the handle is duped to another client.
1467 *
1468 * @param[in] pGpu
1469 * @param[in] pKernelFifo
1470 * @param[in] pChidMgr the ChIDMgr (per-runlist)
1471 * @param[in] ChID the ChID
1472 *
1473 * @return the KernelChannel * or NULL
1474 */
1475 KernelChannel *
kfifoChidMgrGetKernelChannel_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,CHID_MGR * pChidMgr,NvU32 ChID)1476 kfifoChidMgrGetKernelChannel_IMPL
1477 (
1478 OBJGPU *pGpu,
1479 KernelFifo *pKernelFifo,
1480 CHID_MGR *pChidMgr,
1481 NvU32 ChID
1482 )
1483 {
1484 EMEMBLOCK *pFifoDataBlock;
1485 NvU32 numChannels;
1486
1487 NV_ASSERT_OR_RETURN(pChidMgr != NULL, NULL);
1488 // Lite mode channels don't have KernelChannel yet
1489 NV_ASSERT_OR_RETURN(!kfifoIsLiteModeEnabled_HAL(pGpu, pKernelFifo), NULL);
1490
1491 numChannels = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr);
1492 if (ChID >= numChannels)
1493 {
1494 return NULL;
1495 }
1496
1497 pFifoDataBlock = pChidMgr->pFifoDataHeap->eheapGetBlock(
1498 pChidMgr->pFifoDataHeap,
1499 ChID,
1500 NV_FALSE);
1501 if (pFifoDataBlock != NULL)
1502 {
1503 return (KernelChannel *)pFifoDataBlock->pData;
1504 }
1505
1506 return NULL;
1507 }
1508
1509 /*! Gets channel group data corresponding to grpID */
1510 KernelChannelGroup *
kfifoChidMgrGetKernelChannelGroup_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,CHID_MGR * pChidMgr,NvU32 grpID)1511 kfifoChidMgrGetKernelChannelGroup_IMPL
1512 (
1513 OBJGPU *pGpu,
1514 KernelFifo *pKernelFifo,
1515 CHID_MGR *pChidMgr,
1516 NvU32 grpID
1517 )
1518 {
1519 KernelChannelGroup *pKernelChannelGroup = NULL;
1520
1521 pKernelChannelGroup = mapFind(pChidMgr->pChanGrpTree, grpID);
1522 if (pKernelChannelGroup == NULL)
1523 {
1524 NV_PRINTF(LEVEL_INFO, "Can't find channel group %d\n", grpID);
1525 }
1526
1527 return pKernelChannelGroup;
1528 }
1529
1530 /*!
1531 * @brief Gets channel group data corresponding to grpID
1532 *
1533 * This function is not called in broadcast mode
1534 *
1535 * @param pGpu
1536 * @param pFifo
1537 * @param[in] grpID
1538 * @param[in] runlistID pass CHIDMGR_RUNLIST_ID_LEGACY if not known
1539 *
1540 * @returns KernelChannelGroup * on success
1541 * NULL if channel group is not found
1542 */
1543 KernelChannelGroup *
kfifoGetChannelGroup_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,NvU32 grpID,NvU32 runlistID)1544 kfifoGetChannelGroup_IMPL
1545 (
1546 OBJGPU *pGpu,
1547 KernelFifo *pKernelFifo,
1548 NvU32 grpID,
1549 NvU32 runlistID
1550 )
1551 {
1552 CHID_MGR *pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, runlistID);
1553
1554 return kfifoChidMgrGetKernelChannelGroup(pGpu, pKernelFifo, pChidMgr, grpID);
1555 }
1556
1557 /*! Gets total number of channel groups in use */
1558 NvU32
kfifoGetChannelGroupsInUse_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo)1559 kfifoGetChannelGroupsInUse_IMPL
1560 (
1561 OBJGPU *pGpu,
1562 KernelFifo *pKernelFifo
1563 )
1564 {
1565 NvU32 numChannelGroups = 0;
1566 NvU32 numChannelGroupsInUse = 0;
1567 NvU32 chGrpID, i;
1568
1569 for (i = 0; i < pKernelFifo->numChidMgrs; i++)
1570 {
1571 if (pKernelFifo->ppChidMgr[i] != NULL)
1572 {
1573 numChannelGroups = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo,
1574 pKernelFifo->ppChidMgr[i]);
1575
1576 for (chGrpID = 0; chGrpID < numChannelGroups; chGrpID++)
1577 {
1578 if (nvBitFieldTest(pKernelFifo->ppChidMgr[i]->channelGrpMgr.pHwIdInUse,
1579 pKernelFifo->ppChidMgr[i]->channelGrpMgr.hwIdInUseSz,
1580 chGrpID))
1581 {
1582 numChannelGroupsInUse++;
1583 }
1584 }
1585 }
1586 }
1587 return numChannelGroupsInUse;
1588 }
1589
1590 /*! Gets total number of channel groups in use per engine */
1591 NvU32
kfifoGetRunlistChannelGroupsInUse_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,NvU32 runlistId)1592 kfifoGetRunlistChannelGroupsInUse_IMPL
1593 (
1594 OBJGPU *pGpu,
1595 KernelFifo *pKernelFifo,
1596 NvU32 runlistId
1597 )
1598 {
1599 NvU32 numChannelGroups = 0;
1600 NvU32 numChannelGroupsInUse = 0;
1601 NvU32 chGrpID;
1602 CHID_MGR *pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, runlistId);
1603
1604 numChannelGroups = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr);
1605 for (chGrpID = 0; chGrpID < numChannelGroups; chGrpID++)
1606 {
1607 if (nvBitFieldTest(pChidMgr->channelGrpMgr.pHwIdInUse,
1608 pChidMgr->channelGrpMgr.hwIdInUseSz,
1609 chGrpID))
1610 {
1611 numChannelGroupsInUse++;
1612 }
1613 }
1614 return numChannelGroupsInUse;
1615 }
1616
1617 /**
1618 * @brief Sets the timeslice for the specified channel group.
1619 *
1620 * @returns NV_OK if success, appropriate error otherwise
1621 */
1622 NV_STATUS
kfifoChannelGroupSetTimeslice_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,KernelChannelGroup * pKernelChannelGroup,NvU64 timesliceUs,NvBool bSkipSubmit)1623 kfifoChannelGroupSetTimeslice_IMPL
1624 (
1625 OBJGPU *pGpu,
1626 KernelFifo *pKernelFifo,
1627 KernelChannelGroup *pKernelChannelGroup,
1628 NvU64 timesliceUs,
1629 NvBool bSkipSubmit
1630 )
1631 {
1632 NV_STATUS status = NV_OK;
1633
1634 NV_PRINTF(LEVEL_INFO, "Setting TSG %d Timeslice to %lldus\n",
1635 pKernelChannelGroup->grpID, timesliceUs);
1636
1637 if (timesliceUs < kfifoRunlistGetMinTimeSlice_HAL(pKernelFifo))
1638 {
1639 NV_PRINTF(LEVEL_ERROR,
1640 "Setting Timeslice to %lldus not allowed. Min value is %lldus\n",
1641 timesliceUs, kfifoRunlistGetMinTimeSlice_HAL(pKernelFifo));
1642 return NV_ERR_NOT_SUPPORTED;
1643 }
1644
1645 pKernelChannelGroup->timesliceUs = timesliceUs;
1646
1647 NV_ASSERT_OK_OR_RETURN(kfifoChannelGroupSetTimesliceSched(pGpu,
1648 pKernelFifo,
1649 pKernelChannelGroup,
1650 timesliceUs,
1651 bSkipSubmit));
1652
1653 return status;
1654 }
1655
1656 void
kfifoFillMemInfo_IMPL(KernelFifo * pKernelFifo,MEMORY_DESCRIPTOR * pMemDesc,NV2080_CTRL_FIFO_MEM_INFO * pMemory)1657 kfifoFillMemInfo_IMPL
1658 (
1659 KernelFifo *pKernelFifo,
1660 MEMORY_DESCRIPTOR *pMemDesc,
1661 NV2080_CTRL_FIFO_MEM_INFO *pMemory
1662 )
1663 {
1664 if (pMemDesc == NULL)
1665 {
1666 pMemory->aperture = NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_APERTURE_INVALID;
1667 NV_PRINTF(LEVEL_ERROR, "kfifoFillMemInfo: pMemDesc = NULL\n");
1668 }
1669 else
1670 {
1671 if (memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM)
1672 {
1673 pMemory->aperture = NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_APERTURE_VIDMEM;
1674 }
1675 else if (memdescGetAddressSpace(pMemDesc) == ADDR_SYSMEM)
1676 {
1677 if (memdescGetCpuCacheAttrib(pMemDesc) == NV_MEMORY_CACHED)
1678 {
1679 pMemory->aperture = NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_APERTURE_SYSMEM_COH;
1680 }
1681 else if (memdescGetCpuCacheAttrib(pMemDesc) == NV_MEMORY_UNCACHED)
1682 {
1683 pMemory->aperture = NV2080_CTRL_CMD_FIFO_GET_CHANNEL_MEM_APERTURE_SYSMEM_NCOH;
1684 }
1685 else
1686 {
1687 NV_PRINTF(LEVEL_ERROR,
1688 "kfifoFillMemInfo: Unknown cache attribute for sysmem aperture\n");
1689 NV_ASSERT(NV_FALSE);
1690 }
1691 }
1692 pMemory->base = memdescGetPhysAddr(pMemDesc, AT_GPU, 0);
1693 pMemory->size = pMemDesc->Size;
1694 }
1695 }
1696
1697 /*
1698 * Initializes an iterator to iterate through all channels of a runlist
1699 * If runlistId is INVALID_RUNLIST_ID then it iterates channels for all runlists
1700 */
1701 void
kfifoGetChannelIterator_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,CHANNEL_ITERATOR * pIt,NvU32 runlistId)1702 kfifoGetChannelIterator_IMPL
1703 (
1704 OBJGPU *pGpu,
1705 KernelFifo *pKernelFifo,
1706 CHANNEL_ITERATOR *pIt,
1707 NvU32 runlistId
1708 )
1709 {
1710 portMemSet(pIt, 0, sizeof(*pIt));
1711 pIt->physicalChannelID = 0;
1712 pIt->pFifoDataBlock = NULL;
1713 pIt->runlistId = 0;
1714 pIt->numRunlists = 1;
1715
1716 // Do we want to ierate all runlist channels
1717 if (runlistId == INVALID_RUNLIST_ID)
1718 {
1719 if (kfifoIsPerRunlistChramEnabled(pKernelFifo))
1720 {
1721 pIt->numRunlists = kfifoGetMaxNumRunlists_HAL(pGpu, pKernelFifo);
1722 }
1723 }
1724 else
1725 {
1726 pIt->runlistId = runlistId;
1727 }
1728 }
1729
1730 // return next channel for a specific chidMgr
1731 static NV_STATUS
_kfifoChidMgrGetNextKernelChannel(OBJGPU * pGpu,KernelFifo * pKernelFifo,CHID_MGR * pChidMgr,CHANNEL_ITERATOR * pIt,KernelChannel ** ppKernelChannel)1732 _kfifoChidMgrGetNextKernelChannel
1733 (
1734 OBJGPU *pGpu,
1735 KernelFifo *pKernelFifo,
1736 CHID_MGR *pChidMgr,
1737 CHANNEL_ITERATOR *pIt,
1738 KernelChannel **ppKernelChannel
1739 )
1740 {
1741 KernelChannel *pKernelChannel = NULL;
1742 pIt->numChannels = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr);
1743
1744 if (pIt->pFifoDataBlock == NULL)
1745 {
1746 pIt->pFifoDataBlock = pChidMgr->pFifoDataHeap->eheapGetBlock(
1747 pChidMgr->pFifoDataHeap,
1748 pIt->physicalChannelID,
1749 NV_TRUE);
1750 }
1751
1752 while (pIt->physicalChannelID < pIt->numChannels)
1753 {
1754 if (pIt->pFifoDataBlock->owner == NVOS32_BLOCK_TYPE_FREE)
1755 {
1756 pIt->physicalChannelID = pIt->pFifoDataBlock->end + 1;
1757 }
1758 else
1759 {
1760 pIt->physicalChannelID++;
1761 pKernelChannel = (KernelChannel *)pIt->pFifoDataBlock->pData;
1762
1763 //
1764 // This iterator can be used during an interrupt, when a KernelChannel may
1765 // be in the process of being destroyed. If a KernelChannel expects a pChannel
1766 // but does not have one, it means it's being destroyed and we don't want to
1767 // return it.
1768 //
1769 if (pKernelChannel != NULL && kchannelIsValid_HAL(pKernelChannel))
1770 {
1771 // Prepare iterator to check next block in pChidMgr->pFifoDataHeap
1772 pIt->pFifoDataBlock = pIt->pFifoDataBlock->next;
1773 *ppKernelChannel = pKernelChannel;
1774 return NV_OK;
1775 }
1776 }
1777
1778 // Check next block in pChidMgr->pFifoDataHeap
1779 pIt->pFifoDataBlock = pIt->pFifoDataBlock->next;
1780 }
1781 return NV_ERR_OBJECT_NOT_FOUND;
1782 }
1783
1784 /**
1785 * @brief Returns the next KernelChannel from the iterator.
1786 *
1787 * Iterates over runlist IDs and ChIDs and returns the next KernelChannel found
1788 * on the heap, if any.
1789 *
1790 * (error guaranteed if pointer is NULL; non-NULL pointer guaranteed if NV_OK)
1791 *
1792 * @param[in] pGpu
1793 * @param[in] pKernelFifo
1794 * @param[in] pIt the channel iterator
1795 * @param[out] ppKernelChannel returns a KernelChannel *
1796 *
1797 * @return NV_OK if the returned pointer is valid or error
1798 */
kfifoGetNextKernelChannel_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,CHANNEL_ITERATOR * pIt,KernelChannel ** ppKernelChannel)1799 NV_STATUS kfifoGetNextKernelChannel_IMPL
1800 (
1801 OBJGPU *pGpu,
1802 KernelFifo *pKernelFifo,
1803 CHANNEL_ITERATOR *pIt,
1804 KernelChannel **ppKernelChannel
1805 )
1806 {
1807 if (ppKernelChannel == NULL)
1808 return NV_ERR_INVALID_ARGUMENT;
1809
1810 *ppKernelChannel = NULL;
1811
1812 if (pIt->numRunlists == 1)
1813 {
1814 CHID_MGR *pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, pIt->runlistId);
1815 NV_ASSERT_OR_RETURN(pChidMgr != NULL, NV_ERR_INVALID_ARGUMENT);
1816 return _kfifoChidMgrGetNextKernelChannel(pGpu, pKernelFifo,
1817 pChidMgr, pIt, ppKernelChannel);
1818 }
1819 while (pIt->runlistId < pIt->numRunlists)
1820 {
1821 CHID_MGR *pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, pIt->runlistId);
1822
1823 if (pChidMgr == NULL)
1824 {
1825 pIt->runlistId++;
1826 continue;
1827 }
1828
1829 if (_kfifoChidMgrGetNextKernelChannel(pGpu, pKernelFifo, pChidMgr,
1830 pIt, ppKernelChannel) == NV_OK)
1831 {
1832 return NV_OK;
1833 }
1834 else
1835 {
1836 pIt->runlistId++;
1837 // Reset iterator for next runlist
1838 pIt->physicalChannelID = 0;
1839 pIt->pFifoDataBlock = NULL;
1840 }
1841 }
1842
1843 return NV_ERR_OBJECT_NOT_FOUND;
1844 }
1845
1846 /*!
1847 * @brief Localize the engine info received from the host
1848 *
1849 * The host and the guest can run in different version of drivers, the guest driver
1850 * can not directly use the raw MC_ENGINE_IDX values from the host.
1851 * RM does not guarantee those values are consistent cross branches.
1852 *
1853 * To keep the compatibility between different version of branches, this function reconstructs
1854 * of RM_ENGINE_TYPE, MC_ENGINE_IDX values based on NV2080_ENGINE_TYPE
1855 *
1856 * @param[in] pGpu OBJGPU pointer
1857 * @param[in] pFifo KernelFifo pointer
1858 * @param[in/out] pEngine Pointer to engine info table to update
1859 *
1860 */
1861 static void
_kfifoLocalizeGuestEngineData(OBJGPU * pGpu,KernelFifo * pKernelFifo,ENGINE_INFO * pEngineInfo)1862 _kfifoLocalizeGuestEngineData
1863 (
1864 OBJGPU *pGpu,
1865 KernelFifo *pKernelFifo,
1866 ENGINE_INFO *pEngineInfo
1867 )
1868 {
1869 const FIFO_GUEST_ENGINE_TABLE *guestEngineTable;
1870 NvU32 guestEngineTableSz;
1871 NvU32 nv2080EngineType;
1872 NvU32 engineIdx;
1873 NvU32 newEngineIdx;
1874 NvU32 guestTableIdx;
1875
1876 // This function should only be called in a vgpu guest RM
1877 NV_ASSERT_OR_RETURN_VOID(IS_VIRTUAL(pGpu));
1878
1879 guestEngineTable = kfifoGetGuestEngineLookupTable(&guestEngineTableSz);
1880
1881 newEngineIdx = 0;
1882
1883 for (engineIdx = 0; engineIdx < pEngineInfo->engineInfoListSize; engineIdx++)
1884 {
1885 FIFO_ENGINE_LIST *pEngine = &pEngineInfo->engineInfoList[engineIdx];
1886
1887 // The actual data in engineData[ENGINE_INFO_TYPE_RM_ENGINE_TYPE] is NV2080 ENGINE TYPE.
1888 nv2080EngineType = pEngine->engineData[ENGINE_INFO_TYPE_RM_ENGINE_TYPE];
1889
1890 for (guestTableIdx = 0; guestTableIdx < guestEngineTableSz; guestTableIdx++)
1891 {
1892 // Find the engine type supported by the guest
1893 if (guestEngineTable[guestTableIdx].nv2080EngineType == nv2080EngineType)
1894 break;
1895 }
1896
1897 if (guestTableIdx < guestEngineTableSz)
1898 {
1899 // Update the MC for the guest
1900 pEngine->engineData[ENGINE_INFO_TYPE_MC] = guestEngineTable[guestTableIdx].mcIdx;
1901
1902 // Update it with correct engine type
1903 pEngine->engineData[ENGINE_INFO_TYPE_RM_ENGINE_TYPE] = gpuGetRmEngineType(nv2080EngineType);
1904
1905 if (newEngineIdx != engineIdx)
1906 {
1907 //
1908 // Move the engine info up to make sure the engine info table only contains data for
1909 // guest supported engine types.
1910 //
1911 portMemCopy(&pEngineInfo->engineInfoList[newEngineIdx], sizeof(FIFO_ENGINE_LIST),
1912 &pEngineInfo->engineInfoList[engineIdx], sizeof(FIFO_ENGINE_LIST));
1913 }
1914
1915 newEngineIdx++;
1916 }
1917 }
1918
1919 pEngineInfo->engineInfoListSize = newEngineIdx;
1920 }
1921
1922 /*!
1923 * @brief Performs an RPC into Host RM to read its device info table.
1924 *
1925 * This is necessary because in virtual environments, we cannot directly read
1926 * the device info table, and do not have the physical GPU partitioning
1927 * information to determine which engines belong to this guest, so we have Host
1928 * RM do the filtering and send us the filtered table.
1929 *
1930 * @param[in] pGpu
1931 * @param[in] pKernelFifo
1932 *
1933 * @return NV_OK if succcessful,
1934 * NV_ERR_NOT_SUPPORTED if Host RM calls this interface
1935 * NV_ERR_INVALID_STATE if host supplied invalid data
1936 * NV_STATUS supplied by RPC response from Host
1937 */
1938
1939 NV_STATUS
kfifoGetHostDeviceInfoTable_KERNEL(OBJGPU * pGpu,KernelFifo * pKernelFifo,ENGINE_INFO * pEngineInfo,Device * pMigDevice)1940 kfifoGetHostDeviceInfoTable_KERNEL
1941 (
1942 OBJGPU *pGpu,
1943 KernelFifo *pKernelFifo,
1944 ENGINE_INFO *pEngineInfo,
1945 Device *pMigDevice
1946 )
1947 {
1948 NV_STATUS status = NV_OK;
1949 NvHandle hClient = NV01_NULL_OBJECT;
1950 NvHandle hObject = NV01_NULL_OBJECT;
1951 NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_PARAMS *pParams;
1952 NV2080_CTRL_FIFO_DEVICE_ENTRY *pFetchedTable;
1953 NvU32 numEntries;
1954 NvU32 device;
1955 NvU32 entry;
1956 NvU32 numRunlists;
1957 NvU32 maxRunlistId;
1958 NvU32 maxPbdmaId;
1959 NvU32 i;
1960 struct {
1961 NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_PARAMS params;
1962 NV2080_CTRL_FIFO_DEVICE_ENTRY entries[NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_MAX_DEVICES];
1963 } *pLocals;
1964
1965
1966 NV_ASSERT_OR_RETURN(IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu),
1967 NV_ERR_NOT_SUPPORTED);
1968
1969 // RPC call for GSP will throw INVALID_CLIENT error with NULL handles
1970 if (IS_GSP_CLIENT(pGpu))
1971 {
1972 if (!IS_MIG_IN_USE(pGpu))
1973 {
1974 hClient = pGpu->hInternalClient;
1975 hObject = pGpu->hInternalSubdevice;
1976 }
1977 else
1978 {
1979 RsClient *pClient = RES_GET_CLIENT(pMigDevice);
1980 Subdevice *pSubdevice;
1981
1982 NV_ASSERT_OK_OR_RETURN(
1983 subdeviceGetByInstance(pClient, RES_GET_HANDLE(pMigDevice), 0, &pSubdevice));
1984
1985 GPU_RES_SET_THREAD_BC_STATE(pSubdevice);
1986
1987 hClient = pClient->hClient;
1988 hObject = RES_GET_HANDLE(pSubdevice);
1989 }
1990 }
1991
1992 // Allocate pFetchedTable and params on the heap to avoid stack overflow
1993 pLocals = portMemAllocNonPaged(sizeof(*pLocals));
1994 NV_ASSERT_OR_RETURN((pLocals != NULL), NV_ERR_NO_MEMORY);
1995
1996 pParams = &pLocals->params;
1997 pFetchedTable = pLocals->entries;
1998
1999 //
2000 // Read device info table entries from Host RM until Host indicates that
2001 // there are no more valid entries in the table (by setting bMore flag)
2002 //
2003 numEntries = 0;
2004 for (device = 0;
2005 device < NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_MAX_DEVICES;
2006 device += NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_MAX_ENTRIES)
2007 {
2008 portMemSet(pParams, 0x0, sizeof(*pParams));
2009 pParams->baseIndex = device;
2010
2011 NV_RM_RPC_CONTROL(pGpu,
2012 hClient,
2013 hObject,
2014 NV2080_CTRL_CMD_FIFO_GET_DEVICE_INFO_TABLE,
2015 pParams,
2016 sizeof(*pParams),
2017 status);
2018
2019 if (status != NV_OK)
2020 goto cleanup;
2021
2022 // Assert that host RM didn't tell us an invalid number of entries
2023 if (pParams->numEntries >
2024 NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_MAX_ENTRIES)
2025 {
2026 DBG_BREAKPOINT();
2027 status = NV_ERR_INVALID_STATE;
2028 goto cleanup;
2029 }
2030
2031 portMemCopy(&pFetchedTable[device],
2032 pParams->numEntries * (sizeof *pFetchedTable),
2033 pParams->entries,
2034 pParams->numEntries * (sizeof *pParams->entries));
2035
2036 numEntries += pParams->numEntries;
2037
2038 if (!pParams->bMore)
2039 {
2040 break;
2041 }
2042 }
2043
2044 pEngineInfo->engineInfoListSize = numEntries;
2045 pEngineInfo->engineInfoList = portMemAllocNonPaged(sizeof(*pEngineInfo->engineInfoList) *
2046 pEngineInfo->engineInfoListSize);
2047 if (pEngineInfo->engineInfoList == NULL)
2048 {
2049 NV_CHECK(LEVEL_ERROR, pEngineInfo->engineInfoList != NULL);
2050 status = NV_ERR_NO_MEMORY;
2051 goto cleanup;
2052 }
2053
2054 // Copy each entry from the table
2055 numRunlists = 0;
2056 maxRunlistId = 0;
2057 maxPbdmaId = 0;
2058 for (entry = 0; entry < numEntries; ++entry)
2059 {
2060 FIFO_ENGINE_LIST *pLocalEntry = &pEngineInfo->engineInfoList[entry];
2061 NV2080_CTRL_FIFO_DEVICE_ENTRY *pFetchedEntry = &pFetchedTable[entry];
2062
2063 ct_assert(sizeof pLocalEntry->engineData <=
2064 sizeof pFetchedEntry->engineData);
2065 portMemCopy(pLocalEntry->engineData,
2066 sizeof pLocalEntry->engineData,
2067 pFetchedEntry->engineData,
2068 NV_ARRAY_ELEMENTS(pLocalEntry->engineData) *
2069 (sizeof *pFetchedEntry->engineData));
2070
2071 pLocalEntry->numPbdmas = pFetchedEntry->numPbdmas;
2072 NV_ASSERT_TRUE_OR_GOTO(status,
2073 pLocalEntry->numPbdmas <=
2074 NV_ARRAY_ELEMENTS(pLocalEntry->pbdmaIds) &&
2075 pLocalEntry->numPbdmas <=
2076 NV_ARRAY_ELEMENTS(pLocalEntry->pbdmaFaultIds),
2077 NV_ERR_INVALID_STATE,
2078 cleanup);
2079 portMemCopy(
2080 pLocalEntry->pbdmaIds,
2081 pLocalEntry->numPbdmas * (sizeof *(pLocalEntry->pbdmaIds)),
2082 pFetchedEntry->pbdmaIds,
2083 pLocalEntry->numPbdmas * (sizeof *(pFetchedEntry->pbdmaIds)));
2084 portMemCopy(
2085 pLocalEntry->pbdmaFaultIds,
2086 pLocalEntry->numPbdmas * (sizeof *(pLocalEntry->pbdmaFaultIds)),
2087 pFetchedEntry->pbdmaFaultIds,
2088 pLocalEntry->numPbdmas * (sizeof *(pFetchedEntry->pbdmaFaultIds)));
2089
2090 portStringCopy(pLocalEntry->engineName,
2091 sizeof pLocalEntry->engineName,
2092 pFetchedEntry->engineName,
2093 sizeof pFetchedEntry->engineName);
2094
2095 if (pLocalEntry->engineData[ENGINE_INFO_TYPE_IS_HOST_DRIVEN_ENGINE] != 0)
2096 {
2097 numRunlists++;
2098 }
2099 maxRunlistId = NV_MAX(maxRunlistId,
2100 pFetchedEntry->engineData[ENGINE_INFO_TYPE_RUNLIST]);
2101
2102 for (i = 0; i < pLocalEntry->numPbdmas; i++)
2103 {
2104 maxPbdmaId = NV_MAX(maxPbdmaId, pLocalEntry->pbdmaIds[i]);
2105
2106 //
2107 // SW engine while being constructed does not populate any PBDMA Fault IDs.
2108 // Hence, skipping it.
2109 //
2110 if (pLocalEntry->engineData[ENGINE_INFO_TYPE_ENG_DESC] != ENG_SW)
2111 {
2112 bitVectorSet(&pEngineInfo->validEngineIdsForPbdmas, pLocalEntry->pbdmaFaultIds[i]);
2113 }
2114 }
2115 }
2116
2117 NV_ASSERT_OK_OR_GOTO(status,
2118 kfifoReservePbdmaFaultIds_HAL(pGpu, pKernelFifo,
2119 pEngineInfo->engineInfoList,
2120 pEngineInfo->engineInfoListSize),
2121 cleanup);
2122
2123 if (IS_VIRTUAL(pGpu))
2124 {
2125 _kfifoLocalizeGuestEngineData(pGpu, pKernelFifo, pEngineInfo);
2126 }
2127
2128 pEngineInfo->numRunlists = numRunlists;
2129 pEngineInfo->maxNumRunlists = maxRunlistId + 1;
2130 pEngineInfo->maxNumPbdmas = maxPbdmaId + 1;
2131
2132 cleanup:
2133 portMemFree(pLocals);
2134
2135 return status;
2136 }
2137
2138 /*!
2139 * @brief Constructs EngineInfo List
2140 *
2141 * @param[in] pGpu
2142 * @param[in] pKernelFifo
2143 *
2144 * @return NV_OK if succcessful,
2145 * NV_STATUS supplied by HALs called
2146 */
2147 NV_STATUS
kfifoConstructEngineList_KERNEL(OBJGPU * pGpu,KernelFifo * pKernelFifo)2148 kfifoConstructEngineList_KERNEL
2149 (
2150 OBJGPU *pGpu,
2151 KernelFifo *pKernelFifo
2152 )
2153 {
2154 ENGINE_INFO *pEngineInfo = &pKernelFifo->engineInfo;
2155
2156 // Return early if EngineList is already constructed
2157 if (pEngineInfo->engineInfoList != NULL)
2158 return NV_OK;
2159
2160 if (IS_GSP_CLIENT(pGpu))
2161 {
2162 NV_ASSERT_OK_OR_RETURN(gpuConstructDeviceInfoTable_HAL(pGpu));
2163 }
2164
2165 NV_ASSERT_OK_OR_RETURN(kfifoGetHostDeviceInfoTable_HAL(pGpu, pKernelFifo, pEngineInfo, 0));
2166
2167 return NV_OK;
2168 }
2169
2170 /**
2171 * @brief Create a list of channels.
2172 *
2173 * @param pGpu
2174 * @param pKernelFifo
2175 * @param pList
2176 */
2177 NV_STATUS
kfifoChannelListCreate_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,CHANNEL_LIST ** ppList)2178 kfifoChannelListCreate_IMPL
2179 (
2180 OBJGPU *pGpu,
2181 KernelFifo *pKernelFifo,
2182 CHANNEL_LIST **ppList
2183 )
2184 {
2185 if (!ppList)
2186 return NV_ERR_INVALID_ARGUMENT;
2187
2188 *ppList = portMemAllocNonPaged(sizeof(CHANNEL_LIST));
2189 NV_ASSERT_OR_RETURN((*ppList != NULL), NV_ERR_NO_MEMORY);
2190
2191 (*ppList)->pHead = NULL;
2192 (*ppList)->pTail = NULL;
2193
2194 return NV_OK;
2195 }
2196
2197 /**
2198 * @brief Append a channel to a channel list.
2199 *
2200 * @param pGpu
2201 * @param pKernelFifo
2202 * @param pKernelChannel
2203 * @param pList
2204 */
2205
2206 NV_STATUS
kfifoChannelListAppend_IMPL(OBJGPU * pGpu,KernelFifo * pKernel,KernelChannel * pKernelChannel,CHANNEL_LIST * pList)2207 kfifoChannelListAppend_IMPL
2208 (
2209 OBJGPU *pGpu,
2210 KernelFifo *pKernel,
2211 KernelChannel *pKernelChannel,
2212 CHANNEL_LIST *pList
2213 )
2214 {
2215 PCHANNEL_NODE pNewNode = NULL;
2216
2217 if (!pKernelChannel || !pList)
2218 return NV_ERR_INVALID_ARGUMENT;
2219
2220 pNewNode = portMemAllocNonPaged(sizeof(CHANNEL_NODE));
2221 NV_ASSERT_OR_RETURN((pNewNode != NULL), NV_ERR_NO_MEMORY);
2222
2223 pNewNode->pKernelChannel = pKernelChannel;
2224 pKernelChannel->refCount++;
2225
2226 pNewNode->pNext = NULL;
2227
2228 // Searching based on the ChID
2229 if (pList->pTail)
2230 {
2231 pList->pTail->pNext = pNewNode;
2232 pList->pTail = pNewNode;
2233 }
2234 else
2235 {
2236 pList->pHead = pNewNode;
2237 pList->pTail = pNewNode;
2238 }
2239
2240 return NV_OK;
2241 }
2242
2243 /**
2244 * @brief remove channel from the given channel list
2245 * look for duplicates.
2246 *
2247 * @param pGpu
2248 * @param pKernelFifo
2249 * @param pKernelChannel
2250 * @param pList
2251 */
2252 NV_STATUS
kfifoChannelListRemove_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,KernelChannel * pKernelChannel,CHANNEL_LIST * pList)2253 kfifoChannelListRemove_IMPL
2254 (
2255 OBJGPU *pGpu,
2256 KernelFifo *pKernelFifo,
2257 KernelChannel *pKernelChannel,
2258 CHANNEL_LIST *pList
2259 )
2260 {
2261 PCHANNEL_NODE pNewNode = NULL;
2262 PCHANNEL_NODE pPrevNode = NULL;
2263 PCHANNEL_NODE pTempNode = NULL;
2264 NvBool bFoundOnce = NV_FALSE;
2265 NV_STATUS status = NV_OK;
2266
2267 if (!pKernelChannel)
2268 return NV_ERR_INVALID_ARGUMENT;
2269
2270 if (!pList)
2271 return NV_ERR_INVALID_ARGUMENT;
2272
2273 pNewNode = pList->pHead;
2274 pPrevNode = NULL;
2275
2276 while (pNewNode)
2277 {
2278
2279 if (pKernelChannel != pNewNode->pKernelChannel)
2280 {
2281 pPrevNode = pNewNode;
2282 pNewNode = pNewNode->pNext;
2283 continue;
2284 }
2285
2286 // Deleting first node
2287 if (pList->pHead == pNewNode)
2288 {
2289 pList->pHead = pNewNode->pNext;
2290 }
2291
2292 // Deleting tail node
2293 if (pList->pTail == pNewNode)
2294 {
2295 pList->pTail = pPrevNode;
2296 }
2297
2298 // First node does not have previous node.
2299 if (pPrevNode)
2300 {
2301 pPrevNode->pNext = pNewNode->pNext;
2302 }
2303
2304 pTempNode = pNewNode;
2305 pNewNode = pNewNode->pNext;
2306 portMemFree(pTempNode);
2307
2308 bFoundOnce = NV_TRUE;
2309
2310 if (0 == pKernelChannel->refCount)
2311 {
2312 NV_PRINTF(LEVEL_ERROR, "RefCount for channel is not right!!!\n");
2313 DBG_BREAKPOINT();
2314 status = NV_ERR_GENERIC;
2315 break;
2316 }
2317
2318 pKernelChannel->refCount--;
2319 }
2320
2321
2322 if (!bFoundOnce)
2323 {
2324 NV_PRINTF(LEVEL_INFO,
2325 "Can't find channel in channelGroupList (Normal during RC Recovery on "
2326 "GK110+ or if software scheduling is enabled).\n");
2327
2328 status = NV_ERR_INVALID_CHANNEL;
2329 }
2330
2331 return status;
2332 }
2333
2334 /**
2335 * @brief Destroy channel list.
2336 *
2337 * @param pGpu
2338 * @param pKernelFifo
2339 * @param pList
2340 */
2341 NV_STATUS
kfifoChannelListDestroy_IMPL(OBJGPU * pGpu,KernelFifo * pKernel,CHANNEL_LIST * pList)2342 kfifoChannelListDestroy_IMPL
2343 (
2344 OBJGPU *pGpu,
2345 KernelFifo *pKernel,
2346 CHANNEL_LIST *pList
2347 )
2348 {
2349 PCHANNEL_NODE pTempNode;
2350
2351 if (!pList)
2352 return NV_OK;
2353
2354 while (pList->pHead)
2355 {
2356 pTempNode = pList->pHead;
2357
2358 NV_ASSERT_OR_RETURN(pTempNode->pKernelChannel && pTempNode->pKernelChannel->refCount, NV_ERR_INVALID_STATE);
2359
2360 pTempNode->pKernelChannel->refCount--;
2361
2362 pList->pHead = pTempNode->pNext;
2363 portMemFree(pTempNode);
2364 }
2365
2366 portMemFree(pList);
2367
2368 return NV_OK;
2369 }
2370
2371 /*!
2372 * @brief Determines whether provided engines have any channels/contexts assigned
2373 *
2374 * @param[IN] pGpu OBJGPU
2375 * @param[IN] pKernelFifo KernelFifo
2376 * @param[IN] pEngines Which engines to check (RM_ENGINE_TYPE_***)
2377 * @param[IN] engineCount Number of engines to check
2378 *
2379 * @return Returns NV_TRUE if any provided engines are active
2380 */
2381 NvBool
kfifoEngineListHasChannel_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,RM_ENGINE_TYPE * pEngines,NvU32 engineCount)2382 kfifoEngineListHasChannel_IMPL
2383 (
2384 OBJGPU *pGpu,
2385 KernelFifo *pKernelFifo,
2386 RM_ENGINE_TYPE *pEngines,
2387 NvU32 engineCount
2388 )
2389 {
2390 KernelChannel *pKernelChannel;
2391 CHANNEL_ITERATOR it;
2392 NvU32 i;
2393
2394 NV_ASSERT_OR_RETURN((pEngines != NULL) && (engineCount > 0), NV_TRUE);
2395
2396 // Find any channels or contexts on passed engines
2397 kfifoGetChannelIterator(pGpu, pKernelFifo, &it, INVALID_RUNLIST_ID);
2398 while (kchannelGetNextKernelChannel(pGpu, &it, &pKernelChannel) == NV_OK)
2399 {
2400 NV_ASSERT_OR_ELSE(pKernelChannel != NULL, continue);
2401
2402 // If the client supplied the engine type, directly check it
2403 if (RM_ENGINE_TYPE_IS_VALID(kchannelGetEngineType(pKernelChannel)))
2404 {
2405 for (i = 0; i < engineCount; ++i)
2406 {
2407 if (kchannelGetEngineType(pKernelChannel) == pEngines[i])
2408 {
2409 NV_PRINTF(LEVEL_ERROR,
2410 "Found channel on engine 0x%x owned by 0x%x\n",
2411 kchannelGetEngineType(pKernelChannel), RES_GET_CLIENT_HANDLE(pKernelChannel));
2412
2413 return NV_TRUE;
2414 }
2415 }
2416 }
2417 else
2418 {
2419 NvU32 runlistId;
2420
2421 // Ideally valid engine Id should always be set in channel if this property is enabled
2422 NV_ASSERT_OR_RETURN(!kfifoIsPerRunlistChramEnabled(pKernelFifo), NV_TRUE);
2423
2424 //
2425 // If runlist Id for channel is set then check if it matches with any of the engines
2426 // If channel is not associated with any engine then there is a chance
2427 // it can be created on one of the engines we care about.
2428 //
2429 if (kchannelIsRunlistSet(pGpu, pKernelChannel))
2430 {
2431 for (i = 0; i < engineCount; ++i)
2432 {
2433 NV_ASSERT_OR_RETURN((kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo,
2434 ENGINE_INFO_TYPE_RM_ENGINE_TYPE, (NvU32)pEngines[i],
2435 ENGINE_INFO_TYPE_RUNLIST, &runlistId) == NV_OK), NV_TRUE);
2436 if (kchannelGetRunlistId(pKernelChannel) == runlistId)
2437 {
2438 NV_PRINTF(LEVEL_ERROR,
2439 "Found channel on runlistId 0x%x owned by 0x%x\n",
2440 kchannelGetRunlistId(pKernelChannel), RES_GET_CLIENT_HANDLE(pKernelChannel));
2441
2442 return NV_TRUE;
2443 }
2444 }
2445 }
2446 else
2447 {
2448 NV_PRINTF(LEVEL_ERROR,
2449 "Found channel owned by 0x%x that can be associated to any engine\n",
2450 RES_GET_CLIENT_HANDLE(pKernelChannel));
2451
2452 return NV_TRUE;
2453 }
2454 }
2455 }
2456
2457 return NV_FALSE;
2458 }
2459
2460 /**
2461 * @brief Maximum number of subcontexts for a non-legacy mode TSG
2462 */
2463 CTX_BUF_POOL_INFO *
kfifoGetRunlistBufPool_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,RM_ENGINE_TYPE rmEngineType)2464 kfifoGetRunlistBufPool_IMPL
2465 (
2466 OBJGPU *pGpu,
2467 KernelFifo *pKernelFifo,
2468 RM_ENGINE_TYPE rmEngineType
2469 )
2470 {
2471 return pKernelFifo->pRunlistBufPool[rmEngineType];
2472 }
2473
2474 /**
2475 * @brief Get size and alignment requirements for runlist buffers
2476 *
2477 * @param[in] pGpu Pointer to OBJGPU
2478 * @param[in] pKernelFifo Pointer to KernelFifo
2479 * @param[in] runlistId Runlist ID
2480 * @param[in] bTsgSupported Is TSG supported
2481 * @param[in] maxRunlistEntries Max entries to be supported in a runlist
2482 * @param[out] pSize Size of runlist buffer
2483 * @param[out] pAlignment Alignment for runlist buffer
2484 */
2485 NV_STATUS
kfifoGetRunlistBufInfo_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,NvU32 runlistId,NvBool bTsgSupported,NvU32 maxRunlistEntries,NvU64 * pSize,NvU64 * pAlignment)2486 kfifoGetRunlistBufInfo_IMPL
2487 (
2488 OBJGPU *pGpu,
2489 KernelFifo *pKernelFifo,
2490 NvU32 runlistId,
2491 NvBool bTsgSupported,
2492 NvU32 maxRunlistEntries,
2493 NvU64 *pSize,
2494 NvU64 *pAlignment
2495 )
2496 {
2497 NvU32 runlistEntrySize = 0;
2498 NvU32 maxRunlistEntriesSupported = 0;
2499 CHID_MGR *pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, runlistId);
2500
2501 NV_ASSERT_OR_RETURN(pSize != NULL, NV_ERR_INVALID_ARGUMENT);
2502 NV_ASSERT_OR_RETURN(pAlignment != NULL, NV_ERR_INVALID_ARGUMENT);
2503
2504 if (kfifoIsPerRunlistChramEnabled(pKernelFifo))
2505 {
2506 NV_ASSERT_OR_RETURN(pChidMgr != NULL, NV_ERR_INVALID_ARGUMENT);
2507 //
2508 // We assume worst case of one TSG wrapper per channel, and
2509 // the number of TSGs + number of channels is how we get
2510 // the 2 x number of fifos.
2511 //
2512 maxRunlistEntriesSupported = 2 * kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr);
2513 }
2514 else
2515 {
2516 maxRunlistEntriesSupported = kfifoGetMaxChannelsInSystem(pGpu, pKernelFifo);
2517 maxRunlistEntriesSupported += (bTsgSupported ?
2518 kfifoGetMaxChannelGroupsInSystem(pGpu, pKernelFifo)
2519 : 0);
2520 }
2521
2522 NV_ASSERT_OR_RETURN(maxRunlistEntries <= maxRunlistEntriesSupported, NV_ERR_INVALID_ARGUMENT);
2523
2524 if (maxRunlistEntries == 0)
2525 {
2526 maxRunlistEntries = maxRunlistEntriesSupported;
2527 }
2528
2529 runlistEntrySize = kfifoRunlistGetEntrySize_HAL(pKernelFifo);
2530 *pSize = (NvU64)runlistEntrySize * maxRunlistEntries;
2531
2532 *pAlignment = NVBIT64(kfifoRunlistGetBaseShift_HAL(pKernelFifo));
2533 return NV_OK;
2534 }
2535
2536 /*!
2537 * @brief Gets total number of channels supported by the system
2538 */
2539 NvU32
kfifoGetMaxChannelsInSystem_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo)2540 kfifoGetMaxChannelsInSystem_IMPL
2541 (
2542 OBJGPU *pGpu,
2543 KernelFifo *pKernelFifo
2544 )
2545 {
2546 NvU32 numChannels = 0;
2547 NvU32 i;
2548
2549 for (i = 0; i < pKernelFifo->numChidMgrs; i++)
2550 {
2551 if (pKernelFifo->ppChidMgr[i] != NULL)
2552 {
2553 numChannels += kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pKernelFifo->ppChidMgr[i]);
2554 }
2555 }
2556 return numChannels;
2557 }
2558
2559 /*!
2560 * @brief Gets total number of channel groups supported by the system
2561 */
2562 NvU32
kfifoGetMaxChannelGroupsInSystem_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo)2563 kfifoGetMaxChannelGroupsInSystem_IMPL
2564 (
2565 OBJGPU *pGpu,
2566 KernelFifo *pKernelFifo
2567 )
2568 {
2569 // Max channel groups is the same as max channels
2570 return kfifoGetMaxChannelsInSystem(pGpu, pKernelFifo);
2571 }
2572
2573 /*!
2574 * @brief Get runlist buffer allocation params
2575 *
2576 * @param[in] pGpu
2577 * @param[out] *pAperture Aperture to use for runlist buffer allocation
2578 * @param[out] *pAttr Attributes to use for runlits buffer allocation
2579 * @param[out] *pAllocFlags Allocation flags to use for runlist buffer allocation
2580 */
2581 void
kfifoRunlistGetBufAllocParams_IMPL(OBJGPU * pGpu,NV_ADDRESS_SPACE * pAperture,NvU32 * pAttr,NvU64 * pAllocFlags)2582 kfifoRunlistGetBufAllocParams_IMPL
2583 (
2584 OBJGPU *pGpu,
2585 NV_ADDRESS_SPACE *pAperture,
2586 NvU32 *pAttr,
2587 NvU64 *pAllocFlags
2588 )
2589 {
2590 *pAperture = ADDR_FBMEM;
2591 *pAttr = NV_MEMORY_WRITECOMBINED;
2592
2593 memdescOverrideInstLoc(DRF_VAL(_REG_STR_RM, _INST_LOC, _RUNLIST, pGpu->instLocOverrides),
2594 "runlist", pAperture, pAttr);
2595
2596 *pAllocFlags = FLD_TEST_DRF(_REG_STR_RM, _INST_VPR, _RUNLIST, _TRUE, pGpu->instVprOverrides)
2597 ? MEMDESC_ALLOC_FLAGS_PROTECTED : MEMDESC_FLAGS_NONE;
2598 }
2599
2600 /*!
2601 * @brief Allocate Runlist buffers for a single runlistId
2602 *
2603 * @param[in] pGpu
2604 * @param[in] pKernelFifo
2605 * @param[in] bSupportTsg Will this runlist support TSGs?
2606 * @param[in] aperture NV_ADDRESS_SPACE requested
2607 * @param[in] runlistId runlistId to allocate buffer for
2608 * @param[in] attr CPU cacheability requested
2609 * @param[in] allocFlags MEMDESC_FLAGS_*
2610 * @param[in] maxRunlistEntries Can pass zero to determine in function
2611 * @param[in] bHWRL Is this runlist a HW runlist? (verif feature specific)
2612 * @param[out] ppMemDesc memdesc created/allocated by function
2613 */
2614 NV_STATUS
kfifoRunlistAllocBuffers_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,NvBool bSupportTsg,NV_ADDRESS_SPACE aperture,NvU32 runlistId,NvU32 attr,NvU64 allocFlags,NvU64 maxRunlistEntries,NvBool bHWRL,MEMORY_DESCRIPTOR ** ppMemDesc)2615 kfifoRunlistAllocBuffers_IMPL
2616 (
2617 OBJGPU *pGpu,
2618 KernelFifo *pKernelFifo,
2619 NvBool bSupportTsg,
2620 NV_ADDRESS_SPACE aperture,
2621 NvU32 runlistId,
2622 NvU32 attr,
2623 NvU64 allocFlags,
2624 NvU64 maxRunlistEntries,
2625 NvBool bHWRL,
2626 MEMORY_DESCRIPTOR **ppMemDesc
2627 )
2628 {
2629 NV_STATUS status = NV_OK;
2630 NvU64 runlistSz = 0;
2631 NvU64 runlistAlign = 0;
2632 NvU32 counter;
2633
2634 status = kfifoGetRunlistBufInfo(pGpu, pKernelFifo, runlistId, bSupportTsg,
2635 maxRunlistEntries, &runlistSz, &runlistAlign);
2636 if (status != NV_OK)
2637 {
2638 NV_PRINTF(LEVEL_ERROR, "failed to get runlist buffer info 0x%08x\n",
2639 status);
2640 DBG_BREAKPOINT();
2641 goto failed;
2642 }
2643
2644 for (counter = 0; counter < NUM_BUFFERS_PER_RUNLIST; ++counter)
2645 {
2646 ppMemDesc[counter] = NULL;
2647
2648 status = memdescCreate(&ppMemDesc[counter], pGpu, runlistSz, runlistAlign,
2649 NV_TRUE, aperture, attr, allocFlags);
2650 if (status != NV_OK)
2651 {
2652 NV_PRINTF(LEVEL_ERROR,
2653 "Runlist buffer memdesc create failed 0x%08x\n", status);
2654 DBG_BREAKPOINT();
2655 goto failed;
2656 }
2657
2658 // If flag is set then allocate runlist from ctx buf pool
2659 if (allocFlags & MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL)
2660 {
2661 RM_ENGINE_TYPE rmEngineType;
2662 CTX_BUF_POOL_INFO *pCtxBufPool = NULL;
2663 status = kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo, ENGINE_INFO_TYPE_RUNLIST,
2664 runlistId, ENGINE_INFO_TYPE_RM_ENGINE_TYPE, (NvU32 *)&rmEngineType);
2665 if (status != NV_OK)
2666 {
2667 NV_PRINTF(LEVEL_ERROR,
2668 "Failed to translate runlistId 0x%x to NV2080 engine type\n", runlistId);
2669 DBG_BREAKPOINT();
2670 goto failed;
2671 }
2672 status = ctxBufPoolGetGlobalPool(pGpu, CTX_BUF_ID_RUNLIST, rmEngineType, &pCtxBufPool);
2673 if (status != NV_OK)
2674 {
2675 NV_PRINTF(LEVEL_ERROR,
2676 "Failed to get ctx buf pool for engine type 0x%x (0x%x)\n",
2677 gpuGetNv2080EngineType(rmEngineType), rmEngineType);
2678 DBG_BREAKPOINT();
2679 goto failed;
2680 }
2681 status = memdescSetCtxBufPool(ppMemDesc[counter], pCtxBufPool);
2682 if (status != NV_OK)
2683 {
2684 NV_PRINTF(LEVEL_ERROR,
2685 "Failed to set ctx buf pool for runlistId 0x%x\n", runlistId);
2686 DBG_BREAKPOINT();
2687 goto failed;
2688 }
2689 }
2690
2691 memdescTagAlloc(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_101,
2692 ppMemDesc[counter]);
2693 if (status != NV_OK)
2694 {
2695 NV_PRINTF(LEVEL_ERROR, "Runlist buffer mem alloc failed 0x%08x\n",
2696 status);
2697 DBG_BREAKPOINT();
2698 goto failed;
2699 }
2700 }
2701
2702 return NV_OK;
2703
2704 failed:
2705 for (counter = 0; counter < NUM_BUFFERS_PER_RUNLIST; counter++)
2706 {
2707 if (ppMemDesc[counter])
2708 {
2709 memdescFree(ppMemDesc[counter]);
2710 memdescDestroy(ppMemDesc[counter]);
2711 ppMemDesc[counter] = NULL;
2712 }
2713 }
2714 return status;
2715 }
2716
2717 NvU32
kfifoGetMaxSubcontextFromGr_KERNEL(OBJGPU * pGpu,KernelFifo * pKernelFifo)2718 kfifoGetMaxSubcontextFromGr_KERNEL
2719 (
2720 OBJGPU *pGpu,
2721 KernelFifo *pKernelFifo
2722 )
2723 {
2724 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu);
2725
2726 NV_ASSERT_OR_RETURN(pKernelGraphicsManager != NULL, 0);
2727 NV_ASSERT_OR_RETURN(kgrmgrGetLegacyKGraphicsStaticInfo(pGpu, pKernelGraphicsManager)->bInitialized, 0);
2728 NV_ASSERT_OR_RETURN(kgrmgrGetLegacyKGraphicsStaticInfo(pGpu, pKernelGraphicsManager)->pGrInfo != NULL, 0);
2729
2730 return kgrmgrGetLegacyKGraphicsStaticInfo(pGpu, pKernelGraphicsManager)->pGrInfo->infoList[NV0080_CTRL_GR_INFO_INDEX_MAX_SUBCONTEXT_COUNT].data;
2731 }
2732
2733 NvU32
kfifoReturnPushbufferCaps_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo)2734 kfifoReturnPushbufferCaps_IMPL
2735 (
2736 OBJGPU *pGpu,
2737 KernelFifo *pKernelFifo
2738 )
2739 {
2740 NvU32 kfifoBitMask = 0;
2741
2742 // PCI is always supported
2743 kfifoBitMask = PCI_PB_ALLOWED;
2744
2745 if (!gpuIsUnifiedMemorySpaceEnabled(pGpu))
2746 {
2747 kfifoBitMask |= VID_PB_ALLOWED;
2748 }
2749
2750 return kfifoBitMask;
2751 }
2752
2753 /*!
2754 * @brief kfifoGetDeviceCaps
2755 *
2756 * This routine gets cap bits in unicast. If pbCapsInitialized is passed as
2757 * NV_FALSE, the caps will be copied into pKfifoCaps without OR/ANDING.
2758 * Otherwise, the caps bits for the current GPU will be ORed/ANDed together with
2759 * pKfifoCaps to create a single set of caps that accurately represents the
2760 * functionality of the device.
2761 */
kfifoGetDeviceCaps_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,NvU8 * pKfifoCaps,NvBool bCapsInitialized)2762 void kfifoGetDeviceCaps_IMPL
2763 (
2764 OBJGPU *pGpu,
2765 KernelFifo *pKernelFifo,
2766 NvU8 *pKfifoCaps,
2767 NvBool bCapsInitialized
2768 )
2769 {
2770 NvU8 tempCaps[NV0080_CTRL_FIFO_CAPS_TBL_SIZE];
2771 NvU8 temp;
2772 NvU32 kfifoBitMask;
2773
2774 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu));
2775
2776 portMemSet(tempCaps, 0, NV0080_CTRL_FIFO_CAPS_TBL_SIZE);
2777
2778 kfifoBitMask = kfifoReturnPushbufferCaps(pGpu, pKernelFifo);
2779
2780 if (kfifoBitMask & PCI_PB_ALLOWED)
2781 RMCTRL_SET_CAP(tempCaps, NV0080_CTRL_FIFO_CAPS, _SUPPORT_PCI_PB);
2782 if (kfifoBitMask & VID_PB_ALLOWED)
2783 RMCTRL_SET_CAP(tempCaps, NV0080_CTRL_FIFO_CAPS, _SUPPORT_VID_PB);
2784
2785 if ((IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu)) &&
2786 !gpuIsPipelinedPteMemEnabled(pGpu))
2787 RMCTRL_SET_CAP(tempCaps, NV0080_CTRL_FIFO_CAPS, _NO_PIPELINED_PTE_BLIT);
2788
2789 if (kfifoIsUserdInSystemMemory(pKernelFifo))
2790 RMCTRL_SET_CAP(tempCaps, NV0080_CTRL_FIFO_CAPS, _USERD_IN_SYSMEM);
2791
2792 if (kfifoIsUserdMapDmaSupported(pKernelFifo))
2793 RMCTRL_SET_CAP(tempCaps, NV0080_CTRL_FIFO_CAPS, _GPU_MAP_CHANNEL);
2794
2795 if (kfifoHostHasLbOverflow(pKernelFifo))
2796 RMCTRL_SET_CAP(tempCaps, NV0080_CTRL_FIFO_CAPS, _HAS_HOST_LB_OVERFLOW_BUG_1667921);
2797
2798 if (kfifoIsSubcontextSupported(pKernelFifo))
2799 RMCTRL_SET_CAP(tempCaps, NV0080_CTRL_FIFO_CAPS, _MULTI_VAS_PER_CHANGRP);
2800
2801 if (kfifoIsWddmInterleavingPolicyEnabled(pKernelFifo))
2802 RMCTRL_SET_CAP(tempCaps, NV0080_CTRL_FIFO_CAPS, _SUPPORT_WDDM_INTERLEAVING);
2803
2804 // if this is the first GPU in the device, then start with it's caps
2805 if (bCapsInitialized == NV_FALSE)
2806 {
2807 portMemCopy(pKfifoCaps, NV0080_CTRL_FIFO_CAPS_TBL_SIZE,
2808 tempCaps, NV0080_CTRL_FIFO_CAPS_TBL_SIZE);
2809 return;
2810 }
2811
2812 RMCTRL_AND_CAP(pKfifoCaps, tempCaps, temp,
2813 NV0080_CTRL_FIFO_CAPS, _SUPPORT_PCI_PB);
2814 RMCTRL_AND_CAP(pKfifoCaps, tempCaps, temp,
2815 NV0080_CTRL_FIFO_CAPS, _SUPPORT_VID_PB);
2816 RMCTRL_AND_CAP(pKfifoCaps, tempCaps, temp,
2817 NV0080_CTRL_FIFO_CAPS, _GPU_MAP_CHANNEL);
2818
2819 RMCTRL_OR_CAP(pKfifoCaps, tempCaps, temp,
2820 NV0080_CTRL_FIFO_CAPS, _MULTI_VAS_PER_CHANGRP);
2821
2822 RMCTRL_OR_CAP(pKfifoCaps, tempCaps, temp,
2823 NV0080_CTRL_FIFO_CAPS, _HAS_HOST_LB_OVERFLOW_BUG_1667921);
2824
2825 RMCTRL_OR_CAP(pKfifoCaps, tempCaps, temp,
2826 NV0080_CTRL_FIFO_CAPS, _SUPPORT_WDDM_INTERLEAVING);
2827 return;
2828 }
2829
2830 /**
2831 * @brief Get the start offset of USERD BAR1 map region
2832 */
2833 NvU64
kfifoGetUserdBar1MapStartOffset_VF(OBJGPU * pGpu,KernelFifo * pKernelFifo)2834 kfifoGetUserdBar1MapStartOffset_VF(OBJGPU *pGpu, KernelFifo *pKernelFifo)
2835 {
2836
2837 return 0;
2838 }
2839
2840 /*!
2841 * @brief Add handlers for scheduling enable and/or disable.
2842 *
2843 * @param[in] pGpu OBJGPU pointer
2844 * @param[in] pKernelFifo KernelFifo pointer
2845 * @param[in] pPostSchedulingEnableHandler No action if NULL
2846 * @param[in] pPostSchedulingEnableHandlerData Data to pass to
2847 * @p pPostSchedulingEnableHandler
2848 * @param[in] pPreSchedulingDisableHandler No action if NULL
2849 * @param[in] pPreSchedulingDisableHandlerData Data to pass to
2850 * @p pPreSchedulingDisableHandler
2851 *
2852 * @returns NV_OK if successfully processed both handlers
2853 * NV_WARN_NOTHING_TO_DO if: - Both handlers are NULL
2854 * - Both handlers are already installed
2855 * NV_ERR_INVALID_STATE if one handler is already installed, but not both
2856 */
2857 NV_STATUS
kfifoAddSchedulingHandler_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,PFifoSchedulingHandler pPostSchedulingEnableHandler,void * pPostSchedulingEnableHandlerData,PFifoSchedulingHandler pPreSchedulingDisableHandler,void * pPreSchedulingDisableHandlerData)2858 kfifoAddSchedulingHandler_IMPL
2859 (
2860 OBJGPU *pGpu,
2861 KernelFifo *pKernelFifo,
2862 PFifoSchedulingHandler pPostSchedulingEnableHandler,
2863 void *pPostSchedulingEnableHandlerData,
2864 PFifoSchedulingHandler pPreSchedulingDisableHandler,
2865 void *pPreSchedulingDisableHandlerData
2866 )
2867 {
2868 FifoSchedulingHandlerEntry *pEntry;
2869 NvBool bPostHandlerAlreadyPresent = NV_FALSE;
2870 NvBool bPreHandlerAlreadyPresent = NV_FALSE;
2871 FifoSchedulingHandlerEntry postEntry;
2872 FifoSchedulingHandlerEntry preEntry;
2873
2874 NV_CHECK_OR_RETURN(LEVEL_SILENT,
2875 (pPostSchedulingEnableHandler != NULL) ||
2876 (pPreSchedulingDisableHandler != NULL),
2877 NV_WARN_NOTHING_TO_DO);
2878
2879 // Check for already installed handler if non-NULL
2880 if (pPostSchedulingEnableHandler != NULL)
2881 {
2882 for (pEntry = listHead(&pKernelFifo->postSchedulingEnableHandlerList);
2883 pEntry != NULL;
2884 pEntry = listNext(&pKernelFifo->postSchedulingEnableHandlerList, pEntry))
2885 {
2886 if (pEntry->pCallback == pPostSchedulingEnableHandler &&
2887 pEntry->pCallbackParam == pPostSchedulingEnableHandlerData)
2888 {
2889 bPostHandlerAlreadyPresent = NV_TRUE;
2890 break;
2891 }
2892 }
2893 }
2894
2895 // Check for already installed handler if non-NULL
2896 if (pPreSchedulingDisableHandler != NULL)
2897 {
2898 for (pEntry = listHead(&pKernelFifo->preSchedulingDisableHandlerList);
2899 pEntry != NULL;
2900 pEntry = listNext(&pKernelFifo->preSchedulingDisableHandlerList, pEntry))
2901 {
2902 if (pEntry->pCallback == pPreSchedulingDisableHandler &&
2903 pEntry->pCallbackParam == pPreSchedulingDisableHandlerData)
2904 {
2905 bPreHandlerAlreadyPresent = NV_TRUE;
2906 break;
2907 }
2908 }
2909 }
2910
2911 //
2912 // If we are installing both handlers, and one is already present, but not
2913 // the other, we will do nothing, so assert loudly in that case
2914 //
2915 if ((pPostSchedulingEnableHandler != NULL) && (pPreSchedulingDisableHandler != NULL))
2916 {
2917 NV_ASSERT_OR_RETURN(!(bPostHandlerAlreadyPresent ^ bPreHandlerAlreadyPresent),
2918 NV_ERR_INVALID_STATE);
2919 }
2920
2921 // Return early unless all non-null handlers are not already installed
2922 NV_CHECK_OR_RETURN(LEVEL_SILENT,
2923 !bPostHandlerAlreadyPresent && !bPreHandlerAlreadyPresent,
2924 NV_WARN_NOTHING_TO_DO);
2925
2926 // Add handler entry to list unless NULL
2927 if (pPostSchedulingEnableHandler != NULL)
2928 {
2929 postEntry.pCallback = pPostSchedulingEnableHandler;
2930 postEntry.pCallbackParam = pPostSchedulingEnableHandlerData;
2931 postEntry.bHandled = NV_FALSE;
2932 NV_ASSERT_OR_RETURN(listPrependValue(&pKernelFifo->postSchedulingEnableHandlerList, &postEntry),
2933 NV_ERR_NO_MEMORY);
2934 }
2935
2936 // Add handler entry to list unless NULL
2937 if (pPreSchedulingDisableHandler != NULL)
2938 {
2939 preEntry.pCallback = pPreSchedulingDisableHandler;
2940 preEntry.pCallbackParam = pPreSchedulingDisableHandlerData;
2941 preEntry.bHandled = NV_FALSE;
2942 NV_ASSERT_OR_RETURN(listPrependValue(&pKernelFifo->preSchedulingDisableHandlerList, &preEntry),
2943 NV_ERR_NO_MEMORY);
2944 }
2945
2946 return NV_OK;
2947 }
2948
2949 /*!
2950 * @brief Remove handlers for scheduling enable and/or disable.
2951 *
2952 * @param[in] pGpu OBJGPU pointer
2953 * @param[in] pKernelFifo KernelFifo pointer
2954 * @param[in] pPostSchedulingEnableHandler No action if NULL
2955 * @param[in] pPostSchedulingEnableHandlerData Data argument set for the
2956 * handler.
2957 * @param[in] pPreSchedulingDisableHandler No action if NULL
2958 * @param[in] pPreSchedulingDisableHandlerData Data argument set for the
2959 * handler.
2960 */
2961 void
kfifoRemoveSchedulingHandler_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,PFifoSchedulingHandler pPostSchedulingEnableHandler,void * pPostSchedulingEnableHandlerData,PFifoSchedulingHandler pPreSchedulingDisableHandler,void * pPreSchedulingDisableHandlerData)2962 kfifoRemoveSchedulingHandler_IMPL
2963 (
2964 OBJGPU *pGpu,
2965 KernelFifo *pKernelFifo,
2966 PFifoSchedulingHandler pPostSchedulingEnableHandler,
2967 void *pPostSchedulingEnableHandlerData,
2968 PFifoSchedulingHandler pPreSchedulingDisableHandler,
2969 void *pPreSchedulingDisableHandlerData
2970 )
2971 {
2972 FifoSchedulingHandlerEntry *pEntry;
2973 FifoSchedulingHandlerEntry *pTemp;
2974
2975 // Search for the post handler in the post handler list and remove it if present
2976 pEntry = listHead(&pKernelFifo->postSchedulingEnableHandlerList);
2977 while (pEntry != NULL)
2978 {
2979 pTemp = listNext(&pKernelFifo->postSchedulingEnableHandlerList, pEntry);
2980
2981 if (pEntry->pCallback == pPostSchedulingEnableHandler &&
2982 pEntry->pCallbackParam == pPostSchedulingEnableHandlerData)
2983 {
2984 listRemove(&pKernelFifo->postSchedulingEnableHandlerList, pEntry);
2985 }
2986
2987 pEntry = pTemp;
2988 }
2989
2990 // Search for the pre handler in the pre handler list and remove it if present
2991 pEntry = listHead(&pKernelFifo->preSchedulingDisableHandlerList);
2992 while (pEntry != NULL)
2993 {
2994 pTemp = listNext(&pKernelFifo->preSchedulingDisableHandlerList, pEntry);
2995
2996 if (pEntry->pCallback == pPreSchedulingDisableHandler &&
2997 pEntry->pCallbackParam == pPreSchedulingDisableHandlerData)
2998 {
2999 listRemove(&pKernelFifo->preSchedulingDisableHandlerList, pEntry);
3000 }
3001
3002 pEntry = pTemp;
3003 }
3004 }
3005
3006
3007 /*!
3008 * @brief Notify handlers that scheduling has been enabled.
3009 *
3010 * @param[in] pGpu OBJGPU pointer
3011 * @param[in] pKernelFifo KernelFifo pointer
3012 *
3013 * @returns NV_STATUS
3014 */
3015 NV_STATUS
kfifoTriggerPostSchedulingEnableCallback_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo)3016 kfifoTriggerPostSchedulingEnableCallback_IMPL
3017 (
3018 OBJGPU *pGpu,
3019 KernelFifo *pKernelFifo
3020 )
3021 {
3022 NV_STATUS status = NV_OK;
3023 FifoSchedulingHandlerEntry *pEntry;
3024 NvBool bFirstPass = NV_TRUE;
3025 NvBool bRetry;
3026
3027 do
3028 {
3029 NvBool bMadeProgress = NV_FALSE;
3030
3031 bRetry = NV_FALSE;
3032
3033 for (pEntry = listHead(&pKernelFifo->postSchedulingEnableHandlerList);
3034 pEntry != NULL;
3035 pEntry = listNext(&pKernelFifo->postSchedulingEnableHandlerList, pEntry))
3036 {
3037 NV_ASSERT_OR_ELSE(pEntry->pCallback != NULL,
3038 status = NV_ERR_INVALID_STATE; break;);
3039
3040 if (bFirstPass)
3041 {
3042 // Reset bHandled set by previous call (fore example, for dor suspend-resume)
3043 pEntry->bHandled = NV_FALSE;
3044 }
3045 else if (pEntry->bHandled)
3046 {
3047 continue;
3048 }
3049
3050 status = pEntry->pCallback(pGpu, pEntry->pCallbackParam);
3051
3052 if (status == NV_WARN_MORE_PROCESSING_REQUIRED)
3053 {
3054 // Retry mechanism: Some callbacks depend on other callbacks in this list.
3055 bRetry = NV_TRUE;
3056 // Quash retry status
3057 status = NV_OK;
3058 }
3059 else if (status == NV_OK)
3060 {
3061 // Successfully handled, no need to retry
3062 pEntry->bHandled = NV_TRUE;
3063 bMadeProgress = NV_TRUE;
3064 }
3065 else
3066 {
3067 // Actual error, abort
3068 NV_ASSERT(0);
3069 break;
3070 }
3071 }
3072
3073 // We are stuck in a loop, and all remaining callbacks are returning NV_WARN_MORE_PROCESSING_REQUIRED
3074 NV_ASSERT_OR_RETURN(bMadeProgress || status != NV_OK, NV_ERR_INVALID_STATE);
3075
3076 bFirstPass = NV_FALSE;
3077 } while (bRetry && status == NV_OK);
3078
3079 return status;
3080 }
3081
3082 /*!
3083 * @brief Notify handlers that scheduling will soon be disabled.
3084 *
3085 * @param[in] pGpu OBJGPU pointer
3086 * @param[in] pKernelFifo KernelFifo pointer
3087 *
3088 * @returns NV_STATUS
3089 */
3090 NV_STATUS
kfifoTriggerPreSchedulingDisableCallback_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo)3091 kfifoTriggerPreSchedulingDisableCallback_IMPL
3092 (
3093 OBJGPU *pGpu,
3094 KernelFifo *pKernelFifo
3095 )
3096 {
3097 NV_STATUS status = NV_OK;
3098 FifoSchedulingHandlerEntry *pEntry;
3099 NvBool bFirstPass = NV_TRUE;
3100 NvBool bRetry;
3101
3102 do
3103 {
3104 NvBool bMadeProgress = NV_FALSE;
3105
3106 bRetry = NV_FALSE;
3107
3108 for (pEntry = listHead(&pKernelFifo->preSchedulingDisableHandlerList);
3109 pEntry != NULL;
3110 pEntry = listNext(&pKernelFifo->preSchedulingDisableHandlerList, pEntry))
3111 {
3112 NV_ASSERT_OR_ELSE(pEntry->pCallback != NULL,
3113 status = NV_ERR_INVALID_STATE; break;);
3114
3115 if (bFirstPass)
3116 {
3117 // Reset bHandled set by previous call (fore example, for dor suspend-resume)
3118 pEntry->bHandled = NV_FALSE;
3119 }
3120 else if (pEntry->bHandled)
3121 {
3122 continue;
3123 }
3124
3125 status = pEntry->pCallback(pGpu, pEntry->pCallbackParam);
3126
3127 if (status == NV_WARN_MORE_PROCESSING_REQUIRED)
3128 {
3129 // Retry mechanism: Some callbacks depend on other callbacks in this list.
3130 bRetry = NV_TRUE;
3131 // Quash retry status
3132 status = NV_OK;
3133 }
3134 else if (status == NV_OK)
3135 {
3136 // Successfully handled, no need to retry
3137 pEntry->bHandled = NV_TRUE;
3138 bMadeProgress = NV_TRUE;
3139 }
3140 else
3141 {
3142 // Actual error, abort
3143 NV_ASSERT(0);
3144 break;
3145 }
3146 }
3147
3148 // We are stuck in a loop, and all remaining callbacks are returning NV_WARN_MORE_PROCESSING_REQUIRED
3149 NV_ASSERT_OR_RETURN(bMadeProgress || status != NV_OK, NV_ERR_INVALID_STATE);
3150
3151 bFirstPass = NV_FALSE;
3152 } while (bRetry && status == NV_OK);
3153
3154 return status;
3155 }
3156
3157 /**
3158 * @brief Gets vChid corresponding to a sChid
3159 */
3160 NV_STATUS
kfifoGetVChIdForSChId_FWCLIENT(OBJGPU * pGpu,KernelFifo * pKernelFifo,NvU32 sChId,NvU32 gfid,NvU32 engineId,NvU32 * pVChid)3161 kfifoGetVChIdForSChId_FWCLIENT
3162 (
3163 OBJGPU *pGpu,
3164 KernelFifo *pKernelFifo,
3165 NvU32 sChId,
3166 NvU32 gfid,
3167 NvU32 engineId,
3168 NvU32 *pVChid
3169 )
3170 {
3171 KERNEL_HOST_VGPU_DEVICE *pKernelHostVgpuDevice = NULL;
3172
3173 NV_ASSERT_OR_RETURN(pVChid != NULL, NV_ERR_INVALID_ARGUMENT);
3174 *pVChid = sChId;
3175
3176 NV_CHECK_OR_RETURN(LEVEL_INFO, IS_GFID_VF(gfid), NV_OK);
3177 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextKernelHostVgpuDevice(pGpu, &pKernelHostVgpuDevice));
3178 NV_ASSERT_OR_RETURN(pKernelHostVgpuDevice->gfid == gfid, NV_ERR_INVALID_ARGUMENT);
3179 NV_ASSERT_OR_RETURN(engineId < (sizeof(pKernelHostVgpuDevice->chidOffset) / sizeof(pKernelHostVgpuDevice->chidOffset[0])),
3180 NV_ERR_INVALID_STATE);
3181 NV_ASSERT_OR_RETURN(pKernelHostVgpuDevice->chidOffset[engineId] != 0, NV_ERR_INVALID_STATE);
3182
3183 NV_ASSERT_OR_RETURN(sChId >= pKernelHostVgpuDevice->chidOffset[engineId], NV_ERR_INVALID_ARGUMENT);
3184
3185 *pVChid = sChId - pKernelHostVgpuDevice->chidOffset[engineId];
3186
3187 return NV_OK;
3188 }
3189
3190 /**
3191 * @brief cache vChid <-> sChid offset
3192 */
3193 NV_STATUS
kfifoSetChidOffset_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,CHID_MGR * pChidMgr,NvU32 offset,NvU32 numChannels,NvU32 gfid,NvU32 * pChidOffset,NvU32 * pChannelCount,Device * pMigDevice,NvU32 engineFifoListNumEntries,FIFO_ENGINE_LIST * pEngineFifoList)3194 kfifoSetChidOffset_IMPL
3195 (
3196 OBJGPU *pGpu,
3197 KernelFifo *pKernelFifo,
3198 CHID_MGR *pChidMgr,
3199 NvU32 offset,
3200 NvU32 numChannels,
3201 NvU32 gfid,
3202 NvU32 *pChidOffset,
3203 NvU32 *pChannelCount,
3204 Device *pMigDevice,
3205 NvU32 engineFifoListNumEntries,
3206 FIFO_ENGINE_LIST *pEngineFifoList
3207 )
3208 {
3209 NV_STATUS status = NV_OK;
3210 RM_ENGINE_TYPE *pEngineIds = NULL;
3211 NvU32 maxEngines = kfifoGetNumEngines_HAL(pGpu, pKernelFifo);
3212 NvU32 numEngines, i;
3213
3214 NV_ASSERT_OR_RETURN(pChidMgr != NULL, NV_ERR_INVALID_ARGUMENT);
3215 NV_ASSERT_OR_RETURN(pChidOffset != NULL, NV_ERR_INVALID_ARGUMENT);
3216
3217 pEngineIds = portMemAllocNonPaged(sizeof(RM_ENGINE_TYPE) * maxEngines);
3218 NV_ASSERT_OR_RETURN((pEngineIds != NULL), NV_ERR_NO_MEMORY);
3219 portMemSet(pEngineIds, 0, sizeof(RM_ENGINE_TYPE) * maxEngines);
3220
3221 NV_ASSERT_OK_OR_GOTO(status, kfifoGetEngineListForRunlist(pGpu, pKernelFifo, pChidMgr->runlistId, pEngineIds, &numEngines), cleanup)
3222
3223 for (i = 0; i < numEngines; i++)
3224 {
3225 NV_ASSERT_OR_ELSE(NV2080_ENGINE_TYPE_IS_VALID(pEngineIds[i]), status = NV_ERR_INVALID_STATE; goto cleanup);
3226 pChidOffset[pEngineIds[i]] = offset;
3227 pChannelCount[pEngineIds[i]] = numChannels;
3228 }
3229
3230 NV_ASSERT_OK_OR_GOTO(status, kfifoProgramChIdTable_HAL(pGpu, pKernelFifo, pChidMgr, offset, numChannels, gfid,
3231 pMigDevice, engineFifoListNumEntries, pEngineFifoList), cleanup);
3232
3233 cleanup:
3234
3235 portMemFree(pEngineIds);
3236
3237 return status;
3238 }
3239
3240 /*
3241 * @brief Gets a list of engine ids that use this runlist
3242 *
3243 * @param[in] runlistId Runlist id
3244 * @param[out] pOutEngineIds List of engineids
3245 * @param[out] pNumEngines # of entries in pOutEngines
3246 */
3247 NV_STATUS
kfifoGetEngineListForRunlist_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,NvU32 runlistId,RM_ENGINE_TYPE * pOutEngineIds,NvU32 * pNumEngines)3248 kfifoGetEngineListForRunlist_IMPL
3249 (
3250 OBJGPU *pGpu,
3251 KernelFifo *pKernelFifo,
3252 NvU32 runlistId,
3253 RM_ENGINE_TYPE *pOutEngineIds,
3254 NvU32 *pNumEngines
3255 )
3256 {
3257 NV_STATUS status = NV_OK;
3258 NvU32 numEngines = kfifoGetNumEngines_HAL(pGpu, pKernelFifo);
3259 NvU32 i;
3260
3261 // Sanity check the input
3262 NV_CHECK_OR_RETURN(LEVEL_ERROR, pOutEngineIds != NULL, NV_ERR_INVALID_ARGUMENT);
3263 NV_CHECK_OR_RETURN(LEVEL_ERROR, pNumEngines != NULL, NV_ERR_INVALID_ARGUMENT);
3264
3265 *pNumEngines = 0;
3266 NV_PRINTF(LEVEL_INFO, "Engine list for runlistId 0x%x:\n", runlistId);
3267
3268 for (i = 0; i < numEngines; i++)
3269 {
3270 RM_ENGINE_TYPE rmEngineType;
3271 NvU32 thisRunlistId;
3272
3273 NV_ASSERT_OK_OR_GOTO(status,
3274 kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo,
3275 ENGINE_INFO_TYPE_INVALID,
3276 i,
3277 ENGINE_INFO_TYPE_RUNLIST,
3278 &thisRunlistId), done);
3279 if (runlistId == thisRunlistId)
3280 {
3281 NV_ASSERT_OK_OR_GOTO(status,
3282 kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo,
3283 ENGINE_INFO_TYPE_INVALID,
3284 i,
3285 ENGINE_INFO_TYPE_RM_ENGINE_TYPE,
3286 (NvU32 *)&rmEngineType), done);
3287 pOutEngineIds[(*pNumEngines)++] = rmEngineType;
3288
3289 NV_PRINTF(LEVEL_INFO, "Engine name: %s\n",
3290 kfifoGetEngineName_HAL(pKernelFifo, ENGINE_INFO_TYPE_RM_ENGINE_TYPE,
3291 (NvU32)rmEngineType));
3292 }
3293 }
3294 done:
3295 if ((status != NV_OK) && (*pNumEngines != 0))
3296 {
3297 portMemSet(pOutEngineIds, 0, sizeof(NvU32) * (*pNumEngines));
3298 *pNumEngines = 0;
3299 }
3300 return status;
3301 }
3302
3303 /**
3304 * @brief Return bitmask of currently allocated channels for a given runlist
3305 *
3306 * @param[in] bitMaskSize @p pBitMask size in bytes
3307 *
3308 */
3309 NV_STATUS
kfifoGetAllocatedChannelMask_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,NvU32 runlistId,NvU32 * pBitMask,NvLength bitMaskSize)3310 kfifoGetAllocatedChannelMask_IMPL
3311 (
3312 OBJGPU *pGpu,
3313 KernelFifo *pKernelFifo,
3314 NvU32 runlistId,
3315 NvU32 *pBitMask,
3316 NvLength bitMaskSize
3317 )
3318 {
3319 CHID_MGR *pChidMgr;
3320 NvU32 chId;
3321 NvU32 numChannels;
3322
3323 NV_ASSERT(pBitMask != NULL);
3324 portMemSet(pBitMask, 0, bitMaskSize);
3325
3326 NV_ASSERT_OR_RETURN(bitMaskSize % sizeof(NvU32) == 0,
3327 NV_ERR_INVALID_ARGUMENT);
3328
3329 if (!kfifoIsPerRunlistChramEnabled(pKernelFifo))
3330 {
3331 if (runlistId > 0)
3332 {
3333 return NV_ERR_OUT_OF_RANGE;
3334 }
3335 else
3336 {
3337 runlistId = CHIDMGR_RUNLIST_ID_LEGACY;
3338 }
3339 }
3340 else
3341 {
3342 if (!(runlistId < kfifoGetMaxNumRunlists_HAL(pGpu, pKernelFifo)))
3343 {
3344 return NV_ERR_OUT_OF_RANGE;
3345 }
3346 }
3347
3348 pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, runlistId);
3349 if (pChidMgr == NULL)
3350 {
3351 //
3352 // This runlist is not valid. This is not an error since it might be
3353 // possible for some runlists between [0, maxRunlists) to be invalid.
3354 // Simply return that it has no channels to simplify clients iterating
3355 // over all runlists.
3356 //
3357 numChannels = 0;
3358 }
3359 else
3360 {
3361 numChannels = kfifoChidMgrGetNumChannels(pGpu, pKernelFifo, pChidMgr);
3362 }
3363
3364 if (((numChannels + 7) / 8) > bitMaskSize)
3365 {
3366 return NV_ERR_BUFFER_TOO_SMALL;
3367 }
3368
3369 for (chId = 0; chId < numChannels; chId++)
3370 {
3371 KernelChannel *pKernelChannel;
3372 pKernelChannel = kfifoChidMgrGetKernelChannel(pGpu, pKernelFifo,
3373 pChidMgr,
3374 chId);
3375 if (pKernelChannel != NULL)
3376 {
3377 NV_BITMASK32_SET(pBitMask, chId);
3378 }
3379 }
3380
3381 return NV_OK;
3382 }
3383
3384 /*!
3385 * Get host channel class
3386 */
3387 NvU32
kfifoGetChannelClassId_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo)3388 kfifoGetChannelClassId_IMPL
3389 (
3390 OBJGPU *pGpu,
3391 KernelFifo *pKernelFifo
3392 )
3393 {
3394 NvU32 numClasses;
3395 NvU32 *pClassList = NULL;
3396 CLI_CHANNEL_CLASS_INFO classInfo;
3397 NvU32 i;
3398 NvU32 class = 0;
3399
3400 NV_ASSERT_OR_RETURN(NV_OK == gpuGetClassList(pGpu, &numClasses, NULL, ENG_KERNEL_FIFO), 0);
3401 NV_ASSERT_OR_RETURN(numClasses > 0, 0);
3402 pClassList = portMemAllocNonPaged(sizeof(NvU32) * numClasses);
3403 NV_ASSERT_OR_RETURN((pClassList != NULL), 0);
3404
3405 if (NV_OK == gpuGetClassList(pGpu, &numClasses, pClassList, ENG_KERNEL_FIFO))
3406 {
3407 for (i = 0; i < numClasses; i++)
3408 {
3409 if (pClassList[i] == PHYSICAL_CHANNEL_GPFIFO)
3410 {
3411 // Skip the physical channel class
3412 continue;
3413 }
3414 CliGetChannelClassInfo(pClassList[i], &classInfo);
3415 if (classInfo.classType == CHANNEL_CLASS_TYPE_GPFIFO)
3416 class = NV_MAX(class, pClassList[i]);
3417 }
3418 }
3419
3420 NV_ASSERT(class);
3421 portMemFree(pClassList);
3422 return class;
3423 }
3424
3425 /**
3426 * @brief Return the FIFO_GUEST_ENGINE_TABLE
3427 *
3428 * @param[out] pEngLookupTblSize To get the table size
3429 *
3430 * @return a pointer to FIFO_GUEST_ENGINE_TABLE
3431 *
3432 */
3433 const FIFO_GUEST_ENGINE_TABLE *
kfifoGetGuestEngineLookupTable_IMPL(NvU32 * pEngLookupTblSize)3434 kfifoGetGuestEngineLookupTable_IMPL
3435 (
3436 NvU32 *pEngLookupTblSize
3437 )
3438 {
3439 //
3440 // This table is used for a guest RM to reconstruct the engine list data
3441 // received from the host RM. The host and guest RM can be running on
3442 // different versions.
3443 //
3444 // This table does not need to be HALified. It need to match NV2080_ENGINE_TYPE
3445 // defined in cl2080_notification.h
3446 //
3447 const static FIFO_GUEST_ENGINE_TABLE guestEngineLookupTable[] =
3448 {
3449 // nv2080EngineType mcIdx
3450 {NV2080_ENGINE_TYPE_GR0, MC_ENGINE_IDX_GR0},
3451 {NV2080_ENGINE_TYPE_GR1, MC_ENGINE_IDX_GR1},
3452 {NV2080_ENGINE_TYPE_GR2, MC_ENGINE_IDX_GR2},
3453 {NV2080_ENGINE_TYPE_GR3, MC_ENGINE_IDX_GR3},
3454 {NV2080_ENGINE_TYPE_GR4, MC_ENGINE_IDX_GR4},
3455 {NV2080_ENGINE_TYPE_GR5, MC_ENGINE_IDX_GR5},
3456 {NV2080_ENGINE_TYPE_GR6, MC_ENGINE_IDX_GR6},
3457 {NV2080_ENGINE_TYPE_GR7, MC_ENGINE_IDX_GR7},
3458 {NV2080_ENGINE_TYPE_COPY0, MC_ENGINE_IDX_CE0},
3459 {NV2080_ENGINE_TYPE_COPY1, MC_ENGINE_IDX_CE1},
3460 {NV2080_ENGINE_TYPE_COPY2, MC_ENGINE_IDX_CE2},
3461 {NV2080_ENGINE_TYPE_COPY3, MC_ENGINE_IDX_CE3},
3462 {NV2080_ENGINE_TYPE_COPY4, MC_ENGINE_IDX_CE4},
3463 {NV2080_ENGINE_TYPE_COPY5, MC_ENGINE_IDX_CE5},
3464 {NV2080_ENGINE_TYPE_COPY6, MC_ENGINE_IDX_CE6},
3465 {NV2080_ENGINE_TYPE_COPY7, MC_ENGINE_IDX_CE7},
3466 {NV2080_ENGINE_TYPE_COPY8, MC_ENGINE_IDX_CE8},
3467 {NV2080_ENGINE_TYPE_COPY9, MC_ENGINE_IDX_CE9},
3468 {NV2080_ENGINE_TYPE_NVENC0, MC_ENGINE_IDX_NVENC},
3469 {NV2080_ENGINE_TYPE_NVENC1, MC_ENGINE_IDX_NVENC1},
3470 {NV2080_ENGINE_TYPE_NVENC2, MC_ENGINE_IDX_NVENC2},
3471 {NV2080_ENGINE_TYPE_NVDEC0, MC_ENGINE_IDX_NVDEC0},
3472 {NV2080_ENGINE_TYPE_NVDEC1, MC_ENGINE_IDX_NVDEC1},
3473 {NV2080_ENGINE_TYPE_NVDEC2, MC_ENGINE_IDX_NVDEC2},
3474 {NV2080_ENGINE_TYPE_NVDEC3, MC_ENGINE_IDX_NVDEC3},
3475 {NV2080_ENGINE_TYPE_NVDEC4, MC_ENGINE_IDX_NVDEC4},
3476 {NV2080_ENGINE_TYPE_NVDEC5, MC_ENGINE_IDX_NVDEC5},
3477 {NV2080_ENGINE_TYPE_NVDEC6, MC_ENGINE_IDX_NVDEC6},
3478 {NV2080_ENGINE_TYPE_NVDEC7, MC_ENGINE_IDX_NVDEC7},
3479 {NV2080_ENGINE_TYPE_SW, MC_ENGINE_IDX_NULL},
3480 {NV2080_ENGINE_TYPE_SEC2, MC_ENGINE_IDX_SEC2},
3481 {NV2080_ENGINE_TYPE_NVJPEG0, MC_ENGINE_IDX_NVJPEG0},
3482 {NV2080_ENGINE_TYPE_NVJPEG1, MC_ENGINE_IDX_NVJPEG1},
3483 {NV2080_ENGINE_TYPE_NVJPEG2, MC_ENGINE_IDX_NVJPEG2},
3484 {NV2080_ENGINE_TYPE_NVJPEG3, MC_ENGINE_IDX_NVJPEG3},
3485 {NV2080_ENGINE_TYPE_NVJPEG4, MC_ENGINE_IDX_NVJPEG4},
3486 {NV2080_ENGINE_TYPE_NVJPEG5, MC_ENGINE_IDX_NVJPEG5},
3487 {NV2080_ENGINE_TYPE_NVJPEG6, MC_ENGINE_IDX_NVJPEG6},
3488 {NV2080_ENGINE_TYPE_NVJPEG7, MC_ENGINE_IDX_NVJPEG7},
3489 {NV2080_ENGINE_TYPE_OFA0, MC_ENGINE_IDX_OFA0},
3490 };
3491
3492 //
3493 // To trap NV2080_ENGINE_TYPE expansions.
3494 // Please update the table guestEngineLookupTable if this assertion is triggered.
3495 //
3496 ct_assert(NV2080_ENGINE_TYPE_LAST == 0x00000040);
3497
3498 *pEngLookupTblSize = NV_ARRAY_ELEMENTS(guestEngineLookupTable);
3499
3500 return guestEngineLookupTable;
3501 };
3502
3503 /**
3504 * @brief Fetch the maximum number of secure channels supported by SEC2
3505 * and secure CEs when confidential compute is enabled
3506 *
3507 */
3508 NV_STATUS
kfifoGetMaxSecureChannels_KERNEL(OBJGPU * pGpu,KernelFifo * pKernelFifo)3509 kfifoGetMaxSecureChannels_KERNEL
3510 (
3511 OBJGPU *pGpu,
3512 KernelFifo *pKernelFifo
3513 )
3514 {
3515 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
3516 NV2080_CTRL_INTERNAL_FIFO_GET_NUM_SECURE_CHANNELS_PARAMS numSecureChannelsParams = {0};
3517
3518 if (gpuIsCCFeatureEnabled(pGpu))
3519 {
3520 NV_ASSERT_OK_OR_RETURN(
3521 pRmApi->Control(pRmApi,
3522 pGpu->hInternalClient,
3523 pGpu->hInternalSubdevice,
3524 NV2080_CTRL_CMD_INTERNAL_FIFO_GET_NUM_SECURE_CHANNELS,
3525 &numSecureChannelsParams,
3526 sizeof(numSecureChannelsParams)));
3527 }
3528
3529 pKernelFifo->maxSec2SecureChannels = numSecureChannelsParams.maxSec2SecureChannels;
3530 pKernelFifo->maxCeSecureChannels = numSecureChannelsParams.maxCeSecureChannels;
3531
3532 return NV_OK;
3533 }
3534
3535 /**
3536 * @brief Checking if the engine ID belongs to a PBDMA or not
3537 * @param[in] pGpu
3538 * @param[in] pKernelFifo
3539 * @param[in] engineId
3540 *
3541 * @return TRUE if engine ID belongs to a PBDMA
3542 */
3543 NvBool
kfifoIsMmuFaultEngineIdPbdma_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,NvU32 engineId)3544 kfifoIsMmuFaultEngineIdPbdma_IMPL
3545 (
3546 OBJGPU *pGpu,
3547 KernelFifo *pKernelFifo,
3548 NvU32 engineId
3549 )
3550 {
3551 const ENGINE_INFO *pEngineInfo = kfifoGetEngineInfo(pKernelFifo);
3552
3553 NV_ASSERT_OR_RETURN(pEngineInfo != NULL, NV_FALSE);
3554 return bitVectorTest(&pEngineInfo->validEngineIdsForPbdmas, engineId);
3555 }
3556
3557 /**
3558 * @brief Function to get PBDMA ID from the given MMU falut ID
3559 *
3560 * @param[in] pGpu
3561 * @param[in] pKernelFifo
3562 * @param[in] mmuFaultId
3563 * @param[out] pPbdmaId
3564 */
3565 NV_STATUS
kfifoGetPbdmaIdFromMmuFaultId_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,NvU32 mmuFaultId,NvU32 * pPbdmaId)3566 kfifoGetPbdmaIdFromMmuFaultId_IMPL
3567 (
3568 OBJGPU *pGpu,
3569 KernelFifo *pKernelFifo,
3570 NvU32 mmuFaultId,
3571 NvU32 *pPbdmaId
3572 )
3573 {
3574 const ENGINE_INFO *pEngineInfo = kfifoGetEngineInfo(pKernelFifo);
3575 NvU32 pbdmaFaultIdStart;
3576
3577 NV_ASSERT_OR_RETURN(pEngineInfo != NULL, NV_ERR_INVALID_STATE);
3578
3579 //
3580 // HW guarantees mmu fault engine ids used for PBDMAs will be assigned in sequential order
3581 // "PBDMA of MMU_ENGINE_ID = MMU_ENGINE_ID - BASE_PBDMA_FAULT_ID" relation holds for all assignments
3582 // This is helping SW to derive the pbdma id using base pbdma fault id and mmu fault engine id
3583 //
3584 pbdmaFaultIdStart = bitVectorCountTrailingZeros(&pEngineInfo->validEngineIdsForPbdmas);
3585 *pPbdmaId = mmuFaultId - pbdmaFaultIdStart;
3586
3587 return NV_OK;
3588 }
3589
3590 /*!
3591 * @brief Function to get RM engine type from the given pbdma falut id
3592 *
3593 * @param[in] pGpu
3594 * @param[in] pKernelFifo
3595 * @param[in] pbdmaFaultId
3596 * @param[out] pRmEngineType
3597 *
3598 * @returns NV_OK when engine type found for given pbdma falut id
3599 */
3600 NV_STATUS
kfifoGetEngineTypeFromPbdmaFaultId_IMPL(OBJGPU * pGpu,KernelFifo * pKernelFifo,NvU32 pbdmaFaultId,RM_ENGINE_TYPE * pRmEngineType)3601 kfifoGetEngineTypeFromPbdmaFaultId_IMPL
3602 (
3603 OBJGPU *pGpu,
3604 KernelFifo *pKernelFifo,
3605 NvU32 pbdmaFaultId,
3606 RM_ENGINE_TYPE *pRmEngineType
3607 )
3608 {
3609 const ENGINE_INFO *pEngineInfo = kfifoGetEngineInfo(pKernelFifo);
3610 NvU32 i, j;
3611
3612 for (i = 0; i < pEngineInfo->engineInfoListSize; i++)
3613 {
3614 for (j = 0; j < pEngineInfo->engineInfoList[i].numPbdmas; j++)
3615 {
3616 if (pbdmaFaultId == pEngineInfo->engineInfoList[i].pbdmaFaultIds[j])
3617 {
3618 *pRmEngineType = pEngineInfo->engineInfoList[i].engineData[ENGINE_INFO_TYPE_RM_ENGINE_TYPE];
3619 return NV_OK;
3620 }
3621 }
3622 }
3623
3624 *pRmEngineType = RM_ENGINE_TYPE_NULL;
3625 return NV_ERR_OBJECT_NOT_FOUND;
3626 }
3627