1 /*
2  * SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3  * SPDX-License-Identifier: MIT
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21  * DEALINGS IN THE SOFTWARE.
22  */
23 
24 #include "kernel/gpu/fifo/kernel_channel.h"
25 #include "kernel/gpu/fifo/kernel_channel_group.h"
26 #include "kernel/gpu/fifo/kernel_channel_group_api.h"
27 #include "kernel/mem_mgr/mem.h"
28 #include "kernel/gpu/mem_sys/kern_mem_sys.h"
29 #include "gpu/mem_mgr/mem_mgr.h"
30 #include "gpu/mem_mgr/mem_desc.h"
31 #include "platform/sli/sli.h"
32 
33 #include "class/cl906f.h"
34 
35 #include "published/maxwell/gm107/dev_ram.h"
36 #include "published/maxwell/gm107/dev_mmu.h"
37 
38 static NV_STATUS _kchannelCreateRMUserdMemDesc(OBJGPU *pGpu, KernelChannel *pKernelChannel);
39 
40 static NV_STATUS _kchannelDestroyRMUserdMemDesc(OBJGPU *pGpu, KernelChannel *pKernelChannel);
41 
42 /*!
43  * The reason this is a hal method is because it primarily concerns with returning EngineID
44  * and unfortunately, the defines for these are not standard across chips.
45  *
46  * Reason we have a classEngineID concat is to present an opaque handle that clients can do
47  * setobject with directly. Some of them may also require to know the class, engine IDs.
48  */
49 NV_STATUS
kchannelGetClassEngineID_GM107(OBJGPU * pGpu,KernelChannel * pKernelChannel,NvHandle handle,NvU32 * pClassEngineID,NvU32 * pClassID,RM_ENGINE_TYPE * pRmEngineID)50 kchannelGetClassEngineID_GM107
51 (
52     OBJGPU         *pGpu,
53     KernelChannel  *pKernelChannel,
54     NvHandle        handle,
55     NvU32          *pClassEngineID,
56     NvU32          *pClassID,
57     RM_ENGINE_TYPE *pRmEngineID
58 )
59 {
60     NV_STATUS          status = NV_OK;
61     NvU32              halEngineTag;
62     NvU32              classID;
63     ChannelDescendant *pObject = NULL;
64 
65     NV_CHECK_OK_OR_RETURN(LEVEL_INFO,
66         kchannelFindChildByHandle(pKernelChannel, handle, &pObject));
67     NV_ASSERT_OR_RETURN(pObject != NULL, NV_ERR_OBJECT_NOT_FOUND);
68 
69     *pClassID = classID = RES_GET_EXT_CLASS_ID(pObject);
70     halEngineTag = pObject->resourceDesc.engDesc;
71 
72     if (halEngineTag == ENG_SW)
73     {
74         classID = pObject->classID;
75     }
76 
77     status = gpuXlateEngDescToClientEngineId(pGpu, halEngineTag, pRmEngineID);
78 
79     if (status == NV_OK)
80     {
81         *pClassEngineID = DRF_NUM(906F, _SET_OBJECT, _NVCLASS, classID);
82     }
83 
84     NV_PRINTF(LEVEL_INFO,
85               "class ID: 0x%08x classEngine ID: 0x%08x\n",
86               classID, *pClassEngineID);
87 
88     return status;
89 }
90 
91 NV_STATUS
kchannelEnableVirtualContext_GM107(KernelChannel * pKernelChannel)92 kchannelEnableVirtualContext_GM107
93 (
94     KernelChannel *pKernelChannel
95 )
96 {
97     pKernelChannel->bSkipCtxBufferAlloc = NV_TRUE;
98 
99     return NV_OK;
100 }
101 
102 /*!
103  * Create the sub memory descriptor from preallocated USERD memory
104  * allocated by RM for a channel
105  */
106 static NV_STATUS
_kchannelCreateRMUserdMemDesc(OBJGPU * pGpu,KernelChannel * pKernelChannel)107 _kchannelCreateRMUserdMemDesc
108 (
109     OBJGPU        *pGpu,
110     KernelChannel *pKernelChannel
111 )
112 {
113     NV_STATUS            status = NV_OK;
114     NvU32                userdSize;
115     KernelFifo          *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu);
116     const PREALLOCATED_USERD_INFO *pUserdInfo = kfifoGetPreallocatedUserdInfo(pKernelFifo);
117     NvU32                subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu);
118     MEMORY_DESCRIPTOR  **ppUserdSubdevMemDesc =
119                          &pKernelChannel->pUserdSubDeviceMemDesc[subdevInst];
120 
121     kfifoGetUserdSizeAlign_HAL(pKernelFifo, &userdSize, NULL);
122 
123     status = memdescCreateSubMem(ppUserdSubdevMemDesc,
124                                  pUserdInfo->userdPhysDesc[subdevInst],
125                                  pGpu,
126                                  pKernelChannel->ChID * userdSize,
127                                  userdSize );
128     return status;
129 }
130 
131 /*! Delete prealloc userd submemdesc for the channel */
132 static NV_STATUS
_kchannelDestroyRMUserdMemDesc(OBJGPU * pGpu,KernelChannel * pKernelChannel)133 _kchannelDestroyRMUserdMemDesc
134 (
135     OBJGPU        *pGpu,
136     KernelChannel *pKernelChannel
137 )
138 {
139     MEMORY_DESCRIPTOR  **ppUserdSubdevMemDesc =
140                          &pKernelChannel->pUserdSubDeviceMemDesc[gpumgrGetSubDeviceInstanceFromGpu(pGpu)];
141     if ((ppUserdSubdevMemDesc != NULL) && (*ppUserdSubdevMemDesc != NULL))
142     {
143         memdescFree(*ppUserdSubdevMemDesc);
144         memdescDestroy(*ppUserdSubdevMemDesc);
145         *ppUserdSubdevMemDesc = NULL;
146     }
147 
148     return NV_OK;
149 }
150 
151 /*!
152  * @brief Create and alloc channel instance mem,
153  *        ramfc and userd subdevice memdescs.
154  *
155  * @param pGpu[in]           OBJGPU pointer
156  * @param pKernelChannel[in] KernelChannel pointer
157  * @param flags[in]          Flags
158  * @param verifFlags[in]     verifFlags
159  *
160  * @returns NV_STATUS
161  */
kchannelAllocMem_GM107(OBJGPU * pGpu,KernelChannel * pKernelChannel,NvU32 Flags,NvU32 verifFlags)162 NV_STATUS kchannelAllocMem_GM107
163 (
164     OBJGPU             *pGpu,
165     KernelChannel      *pKernelChannel,
166     NvU32               Flags,
167     NvU32               verifFlags
168 )
169 {
170     KernelFifo             *pKernelFifo     = GPU_GET_KERNEL_FIFO(pGpu);
171     NV_STATUS               status          = NV_OK;
172     NvU32                   CpuCacheAttrib;
173     FIFO_INSTANCE_BLOCK    *pInstanceBlock  = NULL;
174     NvU32                   userdSize;
175     NvU64                   instMemSize;
176     NvU64                   instMemAlign;
177     NvBool                  bInstProtectedMem;
178     const NV_ADDRESS_SPACE *pInstAllocList;
179     CTX_BUF_POOL_INFO      *pChannelBufPool = NULL;
180     NvU64                   allocFlags      = MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE;
181     NvU32                   scgType;
182     NvU32                   runqueue;
183     KernelChannelGroup     *pKernelChannelGroup;
184     NvU32                   subdevInst;
185 
186     NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_ARGUMENT);
187     pKernelChannelGroup = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup;
188 
189     scgType     = DRF_VAL(OS04, _FLAGS, _GROUP_CHANNEL_THREAD, Flags);
190     runqueue    = DRF_VAL(OS04, _FLAGS, _GROUP_CHANNEL_RUNQUEUE, Flags);
191 
192     if (!kfifoValidateSCGTypeAndRunqueue_HAL(pKernelFifo, scgType, runqueue))
193         return NV_ERR_INVALID_ARGUMENT;
194 
195     kfifoGetUserdSizeAlign_HAL(pKernelFifo, &userdSize, NULL);
196 
197     status = kfifoGetInstMemInfo_HAL(pKernelFifo, &instMemSize, &instMemAlign,
198                                     &bInstProtectedMem, &CpuCacheAttrib, &pInstAllocList);
199     if (status != NV_OK)
200     {
201         NV_PRINTF(LEVEL_ERROR, "Unable to get instance memory info!\n");
202         goto fail;
203     }
204 
205     ///  Alloc Instance block
206     if (IsSLIEnabled(pGpu) || IS_GSP_CLIENT(pGpu))
207     {
208         pInstAllocList = ADDRLIST_FBMEM_ONLY;
209         CpuCacheAttrib = NV_MEMORY_UNCACHED;
210     }
211 
212     // check for allocating VPR memory
213     if (bInstProtectedMem)
214         allocFlags |= MEMDESC_ALLOC_FLAGS_PROTECTED;
215 
216     pChannelBufPool = pKernelChannelGroup->pChannelBufPool;
217     if (pChannelBufPool != NULL)
218         allocFlags |= MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL;
219 
220     SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY)
221 
222     subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu);
223     pInstanceBlock = pKernelChannel->pFifoHalData[subdevInst];
224 
225     if (pInstanceBlock == NULL)
226     {
227         NV_PRINTF(LEVEL_ERROR,
228                   "Instance block is NULL for hClient 0x%x Channel 0x%x!\n",
229                   RES_GET_CLIENT_HANDLE(pKernelChannel),
230                   RES_GET_HANDLE(pKernelChannel));
231         SLI_LOOP_BREAK;
232     }
233 
234     status = memdescCreate(&pInstanceBlock->pInstanceBlockDesc, pGpu,
235                            instMemSize, instMemAlign, NV_TRUE,
236                            ADDR_UNKNOWN, CpuCacheAttrib, allocFlags);
237     if (status != NV_OK)
238     {
239         NV_PRINTF(LEVEL_ERROR, "Unable to allocate instance memory descriptor!\n");
240         SLI_LOOP_BREAK;
241     }
242 
243     if ((memdescGetAddressSpace(pInstanceBlock->pInstanceBlockDesc) == ADDR_SYSMEM) &&
244         (gpuIsInstanceMemoryAlwaysCached(pGpu)))
245     {
246         memdescSetGpuCacheAttrib(pInstanceBlock->pInstanceBlockDesc, NV_MEMORY_CACHED);
247     }
248 
249     if (pChannelBufPool != NULL)
250     {
251         status = memdescSetCtxBufPool(pInstanceBlock->pInstanceBlockDesc, pChannelBufPool);
252         if (status != NV_OK)
253         {
254             NV_ASSERT(status == NV_OK);
255             SLI_LOOP_BREAK;
256         }
257     }
258 
259     memdescTagAllocList(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_116,
260                         pInstanceBlock->pInstanceBlockDesc, pInstAllocList);
261     if (status == NV_OK)
262     {
263         MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
264 
265         // Initialize the instance block of the channel with zeros
266         status = memmgrMemDescMemSet(pMemoryManager,
267                                      pInstanceBlock->pInstanceBlockDesc,
268                                      0,
269                                      TRANSFER_FLAGS_NONE);
270         if (status != NV_OK)
271         {
272             NV_ASSERT(status == NV_OK);
273             SLI_LOOP_BREAK;
274         }
275 
276         memdescSetName(pGpu, pInstanceBlock->pInstanceBlockDesc, NV_RM_SURF_NAME_INSTANCE_BLOCK, NULL);
277     }
278     else
279     {
280         NV_PRINTF(LEVEL_ERROR,
281                   "Instance block allocation for hClient 0x%x hChannel 0x%x failed\n",
282                   RES_GET_CLIENT_HANDLE(pKernelChannel), RES_GET_HANDLE(pKernelChannel));
283         SLI_LOOP_BREAK;
284     }
285 
286     /// Alloc RAMFC Desc
287     status = memdescCreateSubMem(&pInstanceBlock->pRamfcDesc,
288                                  pInstanceBlock->pInstanceBlockDesc,
289                                  pGpu, 0, DRF_SIZE( NV_RAMIN_RAMFC ) / 8);
290     if (status != NV_OK)
291     {
292         NV_PRINTF(LEVEL_ERROR, "Could not allocate memdesc for RAMFC\n");
293         SLI_LOOP_BREAK;
294     }
295 
296     // TODO: Move this elsewhere.
297     if (!pKernelChannel->bClientAllocatedUserD)
298     {
299         NV_ASSERT(pKernelChannel->pUserdSubDeviceMemDesc[subdevInst] == NULL);
300 
301         status = _kchannelCreateRMUserdMemDesc(pGpu, pKernelChannel);
302         if (status  != NV_OK)
303         {
304             NV_PRINTF(LEVEL_ERROR, "Could not allocate sub memdesc for USERD\n"); // TODO SLI BREAK
305             SLI_LOOP_BREAK;
306         }
307     }
308 
309     status = kchannelCreateUserMemDesc_HAL(pGpu, pKernelChannel);
310     if (status != NV_OK)
311     {
312         NV_PRINTF(LEVEL_ERROR, "kchannelCreateUserMemDesc failed \n");
313         SLI_LOOP_BREAK;
314     }
315 
316     NV_PRINTF(LEVEL_INFO,
317               "hChannel 0x%x hClient 0x%x, Class ID 0x%x "
318               "Instance Block @ 0x%llx (%s %x) "
319               "USERD @ 0x%llx "
320               "for subdevice %d\n",
321               RES_GET_HANDLE(pKernelChannel), RES_GET_CLIENT_HANDLE(pKernelChannel), RES_GET_EXT_CLASS_ID(pKernelChannel),
322               memdescGetPhysAddr(pInstanceBlock->pInstanceBlockDesc, AT_GPU, 0),
323               memdescGetApertureString(memdescGetAddressSpace(pInstanceBlock->pInstanceBlockDesc)),
324               (NvU32)(memdescGetAddressSpace(pInstanceBlock->pInstanceBlockDesc)),
325               (pKernelChannel->pUserdSubDeviceMemDesc[subdevInst] == NULL) ? 0x0LL :
326               memdescGetPhysAddr(pKernelChannel->pUserdSubDeviceMemDesc[subdevInst], AT_GPU, 0LL),
327               subdevInst);
328 
329     SLI_LOOP_END
330 
331 fail:
332     // Just a note about our failure path, null and unallocated
333     // memdescFrees are allowed so this is not a bug.
334     if (status != NV_OK)
335     {
336         NV_PRINTF(LEVEL_ERROR, "Could not create Channel\n");
337         DBG_BREAKPOINT();
338     }
339 
340     return status;
341 }
342 
343 /*!
344  * @brief Free and destroy channel memdescs
345  *        created during channel alloc mem
346  *
347  * @param pGpu[in]            OBJGPU pointer
348  * @param pKernelChannel[in]  KernelChannel pointer
349  *
350  * @return void
351  */
352 void
kchannelDestroyMem_GM107(OBJGPU * pGpu,KernelChannel * pKernelChannel)353 kchannelDestroyMem_GM107
354 (
355     OBJGPU        *pGpu,
356     KernelChannel *pKernelChannel
357 )
358 {
359     FIFO_INSTANCE_BLOCK *pInstanceBlock  = NULL;
360     NvU32                subdevInst;
361 
362     NV_ASSERT_OR_RETURN_VOID(pKernelChannel != NULL);
363 
364     SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY)
365 
366     subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu);
367 
368     if (!pKernelChannel->bClientAllocatedUserD)
369     {
370         _kchannelDestroyRMUserdMemDesc(pGpu, pKernelChannel);
371     }
372 
373     pInstanceBlock = pKernelChannel->pFifoHalData[subdevInst];
374     if (pInstanceBlock != NULL)
375     {
376         // Release RAMFC sub memdesc
377         if (pInstanceBlock->pRamfcDesc != NULL)
378         {
379             memdescFree(pInstanceBlock->pRamfcDesc);
380             memdescDestroy(pInstanceBlock->pRamfcDesc);
381             pInstanceBlock->pRamfcDesc = NULL;
382         }
383 
384         // Release Inst block Desc
385         if (pInstanceBlock->pInstanceBlockDesc != NULL)
386         {
387             memdescFree(pInstanceBlock->pInstanceBlockDesc);
388             memdescDestroy(pInstanceBlock->pInstanceBlockDesc);
389             pInstanceBlock->pInstanceBlockDesc = NULL;
390         }
391     }
392 
393     // Remove USERD memDescs
394     memdescDestroy(pKernelChannel->pInstSubDeviceMemDesc[subdevInst]);
395     pKernelChannel->pInstSubDeviceMemDesc[subdevInst] = NULL;
396 
397     SLI_LOOP_END
398 
399     return;
400 }
401 
402 /**
403  * @brief reserves a hardware channel slot
404  *
405  * Only responsible for indicating a hardware channel is in use, does not set
406  * any other software state.
407  *
408  * @param     pGpu
409  * @param[in] pKernelChannel  the pre-allocated KernelChannel
410  * @param[in] hClient
411  * @param[in] allocMode CHANNEL_HW_ID_ALLC_MODE_*
412  * @param[in] ChID
413  * @param[in] bForceInternalIdx true if requesting specific index within USERD page
414  * @param[in] internalIdx requested index within USERD page when bForceInternalIdx
415  *                        true
416  */
417 NV_STATUS
kchannelAllocHwID_GM107(OBJGPU * pGpu,KernelChannel * pKernelChannel,NvHandle hClient,NvU32 Flags,NvU32 verifFlags2,NvU32 ChID)418 kchannelAllocHwID_GM107
419 (
420     OBJGPU   *pGpu,
421     KernelChannel *pKernelChannel,
422     NvHandle  hClient,
423     NvU32     Flags,
424     NvU32     verifFlags2,
425     NvU32     ChID
426 )
427 {
428     NV_STATUS status;
429     KernelFifo              *pKernelFifo       = GPU_GET_KERNEL_FIFO(pGpu);
430     CHID_MGR                *pChidMgr          = NULL;
431     NvU32                    internalIdx       = 0;
432     NvU32                    userdPageIdx      = 0;
433     NvBool                   bForceInternalIdx = NV_FALSE;
434     NvBool                   bForceUserdPage   = NV_FALSE;
435     CHANNEL_HW_ID_ALLOC_MODE allocMode         = CHANNEL_HW_ID_ALLOC_MODE_GROW_UP;
436 
437     if (IS_VIRTUAL_WITHOUT_SRIOV(pGpu))
438     {
439         // per runlist channel heap is supported for sriov only
440         NV_CHECK_OR_RETURN(LEVEL_ERROR,
441                            !kfifoIsPerRunlistChramEnabled(pKernelFifo),
442                            NV_ERR_INVALID_STATE);
443 
444         pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo,
445                                 CHIDMGR_RUNLIST_ID_LEGACY);
446 
447         NV_CHECK_OR_RETURN(LEVEL_INFO,
448                            kfifoChidMgrGetKernelChannel(pGpu, pKernelFifo,
449                                                         pChidMgr, ChID) == NULL,
450                            NV_OK);
451         allocMode = CHANNEL_HW_ID_ALLOC_MODE_PROVIDED;
452     }
453     else
454     {
455         if (FLD_TEST_DRF(OS04, _FLAGS, _CHANNEL_USERD_INDEX_PAGE_FIXED, _TRUE, Flags))
456         {
457             bForceUserdPage = NV_TRUE;
458             userdPageIdx = DRF_VAL(OS04, _FLAGS, _CHANNEL_USERD_INDEX_PAGE_VALUE, Flags);
459             internalIdx = DRF_VAL(OS04, _FLAGS, _CHANNEL_USERD_INDEX_VALUE, Flags);
460 
461             NV_ASSERT_OR_RETURN(FLD_TEST_DRF(OS04,
462                                              _FLAGS,
463                                              _CHANNEL_USERD_INDEX_FIXED,
464                                              _FALSE,
465                                              Flags),
466                                 NV_ERR_INVALID_STATE);
467         }
468 
469         if (FLD_TEST_DRF(OS04, _FLAGS, _CHANNEL_USERD_INDEX_FIXED, _TRUE, Flags))
470         {
471             bForceInternalIdx = NV_TRUE;
472             internalIdx = DRF_VAL(OS04, _FLAGS, _CHANNEL_USERD_INDEX_VALUE, Flags);
473         }
474     }
475 
476     pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, kchannelGetRunlistId(pKernelChannel));
477 
478     status = kfifoChidMgrAllocChid(pGpu,
479                                    pKernelFifo,
480                                    pChidMgr,
481                                    hClient,
482                                    allocMode,
483                                    bForceInternalIdx,
484                                    internalIdx,
485                                    bForceUserdPage,
486                                    userdPageIdx,
487                                    ChID,
488                                    pKernelChannel);
489 
490     return status;
491 }
492 
493 /**
494  * @brief Releases a hardware channel ID.
495  *
496  * Not responsible for freeing any software state beyond that which indicates a
497  * hardware channel is in use.
498  *
499  * @param pGpu
500  * @param pKernelChannel
501  */
502 NV_STATUS
kchannelFreeHwID_GM107(OBJGPU * pGpu,KernelChannel * pKernelChannel)503 kchannelFreeHwID_GM107
504 (
505     OBJGPU        *pGpu,
506     KernelChannel *pKernelChannel
507 )
508 {
509     NV_STATUS   status;
510     KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu);
511     CHID_MGR   *pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo,
512                                            kchannelGetRunlistId(pKernelChannel));
513     EMEMBLOCK  *pFifoDataBlock;
514 
515     NV_ASSERT_OR_RETURN(pChidMgr != NULL, NV_ERR_OBJECT_NOT_FOUND);
516     pFifoDataBlock = pChidMgr->pFifoDataHeap->eheapGetBlock(
517         pChidMgr->pFifoDataHeap,
518         pKernelChannel->ChID,
519         NV_FALSE);
520     NV_ASSERT_OR_RETURN(pFifoDataBlock != NULL, NV_ERR_OBJECT_NOT_FOUND);
521     NV_ASSERT(pFifoDataBlock->pData == pKernelChannel);
522 
523     status = kfifoChidMgrFreeChid(pGpu, pKernelFifo, pChidMgr, pKernelChannel->ChID);
524     if (status != NV_OK)
525     {
526         NV_PRINTF(LEVEL_ERROR,
527                   "Unable to Free Channel From Heap: %d\n",
528                   pKernelChannel->ChID);
529         DBG_BREAKPOINT();
530     }
531 
532     return status;
533 }
534 
535 NV_STATUS
kchannelGetUserdInfo_GM107(OBJGPU * pGpu,KernelChannel * pKernelChannel,NvU64 * userBase,NvU64 * offset,NvU64 * length)536 kchannelGetUserdInfo_GM107
537 (
538     OBJGPU         *pGpu,
539     KernelChannel  *pKernelChannel,
540     NvU64          *userBase,
541     NvU64          *offset,
542     NvU64          *length
543 )
544 {
545     NV_STATUS              status;
546     NvU64                  bar1MapOffset;
547     NvU32                  bar1MapSize;
548     CLI_CHANNEL_CLASS_INFO classInfo;
549     KernelMemorySystem     *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu);
550 
551     NvBool bCoherentCpuMapping = pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING);
552 
553     CliGetChannelClassInfo(RES_GET_EXT_CLASS_ID(pKernelChannel), &classInfo);
554 
555     switch (classInfo.classType)
556     {
557         case CHANNEL_CLASS_TYPE_GPFIFO:
558             NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_ARGUMENT);
559 
560             // USERD is not pre-allocated in BAR1 so there is no offset/userBase
561             NV_ASSERT_OR_RETURN(!pKernelChannel->bClientAllocatedUserD,
562                                 NV_ERR_INVALID_REQUEST);
563 
564             status = kchannelGetUserdBar1MapOffset_HAL(pGpu,
565                                                        pKernelChannel,
566                                                        &bar1MapOffset,
567                                                        &bar1MapSize);
568             if (status == NV_OK)
569             {
570                 *offset = bar1MapOffset;
571                 *length = bar1MapSize;
572 
573                 if (userBase)
574                 {
575                     if (bCoherentCpuMapping)
576                     {
577                         NV_ASSERT(pGpu->getProperty(pGpu, PDB_PROP_GPU_ATS_SUPPORTED));
578                         *userBase = pKernelMemorySystem->coherentCpuFbBase;
579                     }
580                     else
581                     {
582                         *userBase = gpumgrGetGpuPhysFbAddr(pGpu);
583                     }
584                 }
585             }
586             break;
587 
588         default:
589             NV_PRINTF(LEVEL_ERROR,
590                       "class = %x not supported for user base mapping\n",
591                       RES_GET_EXT_CLASS_ID(pKernelChannel));
592             status = NV_ERR_GENERIC;
593             break;
594     }
595     return status;
596 }
597 
598 //
599 // Takes as input a Channel * and returns the BAR1 offset that this channel's
600 // USERD has been mapped to. Also returns the size of the BAR1 mapping that
601 // pertains to this channel. The BAR1 map of all USERDs should have already
602 // been setup before the first channel was created.
603 //
604 // For example, USERD of 40 channels have been mapped at BAR1 offset 0x100.
605 // USERD of one channel is of size 4k. In which case this function will return
606 // ( 0x100 + ( 0x1000 * 0xa ) ) if the input ChID = 0xa.
607 //
608 NV_STATUS
kchannelGetUserdBar1MapOffset_GM107(OBJGPU * pGpu,KernelChannel * pKernelChannel,NvU64 * bar1MapOffset,NvU32 * bar1MapSize)609 kchannelGetUserdBar1MapOffset_GM107
610 (
611     OBJGPU        *pGpu,
612     KernelChannel *pKernelChannel,
613     NvU64         *bar1MapOffset,
614     NvU32         *bar1MapSize
615 )
616 {
617     KernelFifo        *pKernelFifo     = GPU_GET_KERNEL_FIFO(pGpu);
618     const PREALLOCATED_USERD_INFO *pUserdInfo = kfifoGetPreallocatedUserdInfo(pKernelFifo);
619 
620     NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_ARGUMENT);
621 
622     //
623     // only supported when bUsePerRunlistChannelRam is disabled.
624     // We don't pre-allocate userd for all channels across all runlists; we expect
625     // clients to have moved to client allocated userd.
626     //
627     NV_ASSERT_OR_RETURN(!kfifoIsPerRunlistChramEnabled(pKernelFifo),
628                         NV_ERR_NOT_SUPPORTED);
629 
630     if (pUserdInfo->userdBar1MapSize == 0)
631     {
632         NV_PRINTF(LEVEL_ERROR,
633                   "fifoGetUserdBar1Offset_GF100: BAR1 map of USERD has not "
634                   "been setup yet\n");
635         NV_ASSERT(0);
636         return NV_ERR_GENERIC;
637     }
638 
639     kfifoGetUserdSizeAlign_HAL(pKernelFifo, bar1MapSize, NULL);
640 
641     *bar1MapOffset = pKernelChannel->ChID * *bar1MapSize +
642                      pUserdInfo->userdBar1MapStartOffset;
643 
644     NV_ASSERT((*bar1MapOffset + *bar1MapSize) <=
645               (pUserdInfo->userdBar1MapStartOffset +
646                pUserdInfo->userdBar1MapSize));
647 
648     return NV_OK;
649 }
650 
651 /*!
652  * @brief Creates a memory descriptor to be used for creating a GPU mapped MMIO
653  *        region for a given channel.
654  */
655 NV_STATUS
kchannelCreateUserMemDesc_GM107(OBJGPU * pGpu,KernelChannel * pKernelChannel)656 kchannelCreateUserMemDesc_GM107
657 (
658     OBJGPU        *pGpu,
659     KernelChannel *pKernelChannel
660 )
661 {
662     MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
663     KernelFifo    *pKernelFifo    = GPU_GET_KERNEL_FIFO(pGpu);
664     NV_STATUS status;
665     MEMORY_DESCRIPTOR *pSubDevInstMemDesc = NULL;
666     MEMORY_DESCRIPTOR **ppMemDesc =
667          &pKernelChannel->pInstSubDeviceMemDesc[gpumgrGetSubDeviceInstanceFromGpu(pGpu)];
668 
669     NV_ASSERT_OR_RETURN(!gpumgrGetBcEnabledStatus(pGpu), NV_ERR_INVALID_STATE);
670 
671     status = kfifoChannelGetFifoContextMemDesc_HAL(pGpu, pKernelFifo, pKernelChannel,
672                                                    FIFO_CTX_INST_BLOCK, &pSubDevInstMemDesc);
673 
674     if (status != NV_OK)
675         return status;
676 
677     NV_ASSERT_OR_RETURN(pSubDevInstMemDesc, NV_ERR_OBJECT_NOT_FOUND);
678 
679     status = memdescCreate(ppMemDesc, pGpu, RM_PAGE_SIZE, 0,
680                            memdescGetContiguity(pSubDevInstMemDesc, AT_GPU),
681                            memdescGetAddressSpace(pSubDevInstMemDesc),
682                            memdescGetCpuCacheAttrib(pSubDevInstMemDesc),
683                            MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE);
684 
685     if (status != NV_OK)
686         return status;
687 
688     NV_ASSERT(*ppMemDesc);
689 
690     memdescDescribe(*ppMemDesc, memdescGetAddressSpace(pSubDevInstMemDesc),
691                     memdescGetPhysAddr(pSubDevInstMemDesc, AT_GPU, 0), RM_PAGE_SIZE);
692     memmgrSetMemDescPageSize_HAL(pGpu, pMemoryManager, *ppMemDesc, AT_GPU, RM_ATTR_PAGE_SIZE_4KB);
693 
694     memdescSetPteKind(*ppMemDesc, NV_MMU_PTE_KIND_SMHOST_MESSAGE);
695 
696     return NV_OK;
697 }
698 
699 /**
700  * @brief Retrieves the engine ID (NV_PFIFO_ENGINE_*) a given channel is operating on.
701  *
702  * This value will not be valid for a channel that has not been scheduled.
703  *
704  * @param       pGpu
705  * @param       pKernelChannel
706  * @param[out]  pEngDesc
707  */
708 NV_STATUS
kchannelGetEngine_GM107(OBJGPU * pGpu,KernelChannel * pKernelChannel,NvU32 * pEngDesc)709 kchannelGetEngine_GM107
710 (
711     OBJGPU        *pGpu,
712     KernelChannel *pKernelChannel,
713     NvU32         *pEngDesc
714 )
715 {
716     KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu);
717 
718     if (pEngDesc == NULL)
719         return NV_ERR_INVALID_ARGUMENT;
720 
721     NV_PRINTF(LEVEL_INFO, "0x%x\n", kchannelGetDebugTag(pKernelChannel));
722 
723     *pEngDesc = kchannelGetRunlistId(pKernelChannel);
724 
725     // This will pick the first engine on this runlist (may not be the only one).
726     NV_ASSERT_OK_OR_RETURN(kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo,
727                 ENGINE_INFO_TYPE_RUNLIST, *pEngDesc,
728                 ENGINE_INFO_TYPE_ENG_DESC, pEngDesc));
729 
730     return NV_OK;
731 }
732