1 /*
2  * SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3  * SPDX-License-Identifier: MIT
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21  * DEALINGS IN THE SOFTWARE.
22  */
23 
24 #include "kernel/gpu/fifo/kernel_channel.h"
25 #include "kernel/gpu/fifo/kernel_channel_group.h"
26 #include "kernel/gpu/fifo/kernel_channel_group_api.h"
27 #include "kernel/mem_mgr/mem.h"
28 #include "kernel/gpu/mem_sys/kern_mem_sys.h"
29 #include "gpu/mem_mgr/mem_mgr.h"
30 #include "gpu/mem_mgr/mem_desc.h"
31 #include "platform/sli/sli.h"
32 
33 #include "class/cl906f.h"
34 
35 #include "published/maxwell/gm107/dev_ram.h"
36 #include "published/maxwell/gm107/dev_mmu.h"
37 
38 static NV_STATUS _kchannelCreateRMUserdMemDesc(OBJGPU *pGpu, KernelChannel *pKernelChannel);
39 
40 static NV_STATUS _kchannelDestroyRMUserdMemDesc(OBJGPU *pGpu, KernelChannel *pKernelChannel);
41 
42 /*!
43  * The reason this is a hal method is because it primarily concerns with returning EngineID
44  * and unfortunately, the defines for these are not standard across chips.
45  *
46  * Reason we have a classEngineID concat is to present an opaque handle that clients can do
47  * setobject with directly. Some of them may also require to know the class, engine IDs.
48  */
49 NV_STATUS
50 kchannelGetClassEngineID_GM107
51 (
52     OBJGPU         *pGpu,
53     KernelChannel  *pKernelChannel,
54     NvHandle        handle,
55     NvU32          *pClassEngineID,
56     NvU32          *pClassID,
57     RM_ENGINE_TYPE *pRmEngineID
58 )
59 {
60     NV_STATUS          status = NV_OK;
61     NvU32              halEngineTag;
62     NvU32              classID;
63     ChannelDescendant *pObject = NULL;
64 
65     NV_CHECK_OK_OR_RETURN(LEVEL_INFO,
66         kchannelFindChildByHandle(pKernelChannel, handle, &pObject));
67     NV_ASSERT_OR_RETURN(pObject != NULL, NV_ERR_OBJECT_NOT_FOUND);
68 
69     *pClassID = classID = RES_GET_EXT_CLASS_ID(pObject);
70     halEngineTag = pObject->resourceDesc.engDesc;
71 
72     if (halEngineTag == ENG_SW)
73     {
74         classID = pObject->classID;
75     }
76 
77     status = gpuXlateEngDescToClientEngineId(pGpu, halEngineTag, pRmEngineID);
78 
79     if (status == NV_OK)
80     {
81         *pClassEngineID = DRF_NUM(906F, _SET_OBJECT, _NVCLASS, classID);
82     }
83 
84     NV_PRINTF(LEVEL_INFO,
85               "class ID: 0x%08x classEngine ID: 0x%08x\n",
86               classID, *pClassEngineID);
87 
88     return status;
89 }
90 
91 NV_STATUS
92 kchannelEnableVirtualContext_GM107
93 (
94     KernelChannel *pKernelChannel
95 )
96 {
97     pKernelChannel->bSkipCtxBufferAlloc = NV_TRUE;
98 
99     return NV_OK;
100 }
101 
102 /*!
103  * Create the sub memory descriptor from preallocated USERD memory
104  * allocated by RM for a channel
105  */
106 static NV_STATUS
107 _kchannelCreateRMUserdMemDesc
108 (
109     OBJGPU        *pGpu,
110     KernelChannel *pKernelChannel
111 )
112 {
113     NV_STATUS            status = NV_OK;
114     NvU32                userdSize;
115     KernelFifo          *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu);
116     const PREALLOCATED_USERD_INFO *pUserdInfo = kfifoGetPreallocatedUserdInfo(pKernelFifo);
117     NvU32                subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu);
118     MEMORY_DESCRIPTOR  **ppUserdSubdevMemDesc =
119                          &pKernelChannel->pUserdSubDeviceMemDesc[subdevInst];
120 
121     kfifoGetUserdSizeAlign_HAL(pKernelFifo, &userdSize, NULL);
122 
123     status = memdescCreateSubMem(ppUserdSubdevMemDesc,
124                                  pUserdInfo->userdPhysDesc[subdevInst],
125                                  pGpu,
126                                  pKernelChannel->ChID * userdSize,
127                                  userdSize );
128     return status;
129 }
130 
131 /*! Delete prealloc userd submemdesc for the channel */
132 static NV_STATUS
133 _kchannelDestroyRMUserdMemDesc
134 (
135     OBJGPU        *pGpu,
136     KernelChannel *pKernelChannel
137 )
138 {
139     MEMORY_DESCRIPTOR  **ppUserdSubdevMemDesc =
140                          &pKernelChannel->pUserdSubDeviceMemDesc[gpumgrGetSubDeviceInstanceFromGpu(pGpu)];
141     if ((ppUserdSubdevMemDesc != NULL) && (*ppUserdSubdevMemDesc != NULL))
142     {
143         memdescFree(*ppUserdSubdevMemDesc);
144         memdescDestroy(*ppUserdSubdevMemDesc);
145         *ppUserdSubdevMemDesc = NULL;
146     }
147 
148     return NV_OK;
149 }
150 
151 /*!
152  * @brief Create and alloc channel instance mem,
153  *        ramfc and userd subdevice memdescs.
154  *
155  * @param pGpu[in]           OBJGPU pointer
156  * @param pKernelChannel[in] KernelChannel pointer
157  * @param flags[in]          Flags
158  * @param verifFlags[in]     verifFlags
159  *
160  * @returns NV_STATUS
161  */
162 NV_STATUS kchannelAllocMem_GM107
163 (
164     OBJGPU             *pGpu,
165     KernelChannel      *pKernelChannel,
166     NvU32               Flags,
167     NvU32               verifFlags
168 )
169 {
170     KernelFifo             *pKernelFifo     = GPU_GET_KERNEL_FIFO(pGpu);
171     NV_STATUS               status          = NV_OK;
172     NvU32                   CpuCacheAttrib;
173     FIFO_INSTANCE_BLOCK    *pInstanceBlock  = NULL;
174     NvU32                   userdSize;
175     NvU64                   instMemSize;
176     NvU64                   instMemAlign;
177     NvBool                  bInstProtectedMem;
178     const NV_ADDRESS_SPACE *pInstAllocList;
179     CTX_BUF_POOL_INFO      *pChannelBufPool = NULL;
180     NvU64                   allocFlags      = MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE;
181     NvU32                   scgType;
182     NvU32                   runqueue;
183     KernelChannelGroup     *pKernelChannelGroup;
184     NvU32                   subdevInst;
185 
186     NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_ARGUMENT);
187     pKernelChannelGroup = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup;
188 
189     scgType     = DRF_VAL(OS04, _FLAGS, _GROUP_CHANNEL_THREAD, Flags);
190     runqueue    = DRF_VAL(OS04, _FLAGS, _GROUP_CHANNEL_RUNQUEUE, Flags);
191 
192     if (!kfifoValidateSCGTypeAndRunqueue_HAL(pKernelFifo, scgType, runqueue))
193         return NV_ERR_INVALID_ARGUMENT;
194 
195     kfifoGetUserdSizeAlign_HAL(pKernelFifo, &userdSize, NULL);
196 
197     status = kfifoGetInstMemInfo_HAL(pKernelFifo, &instMemSize, &instMemAlign,
198                                     &bInstProtectedMem, &CpuCacheAttrib, &pInstAllocList);
199     if (status != NV_OK)
200     {
201         NV_PRINTF(LEVEL_ERROR, "Unable to get instance memory info!\n");
202         goto fail;
203     }
204 
205     ///  Alloc Instance block
206     if (IsSLIEnabled(pGpu) || IS_GSP_CLIENT(pGpu))
207     {
208         pInstAllocList = ADDRLIST_FBMEM_ONLY;
209         CpuCacheAttrib = NV_MEMORY_UNCACHED;
210     }
211 
212     // check for allocating VPR memory
213     if (bInstProtectedMem)
214         allocFlags |= MEMDESC_ALLOC_FLAGS_PROTECTED;
215 
216     pChannelBufPool = pKernelChannelGroup->pChannelBufPool;
217     if (pChannelBufPool != NULL)
218         allocFlags |= MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL;
219 
220     SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY)
221 
222     subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu);
223     pInstanceBlock = pKernelChannel->pFifoHalData[subdevInst];
224 
225     if (pInstanceBlock == NULL)
226     {
227         NV_PRINTF(LEVEL_ERROR,
228                   "Instance block is NULL for hClient 0x%x Channel 0x%x!\n",
229                   RES_GET_CLIENT_HANDLE(pKernelChannel),
230                   RES_GET_HANDLE(pKernelChannel));
231         SLI_LOOP_BREAK;
232     }
233 
234     status = memdescCreate(&pInstanceBlock->pInstanceBlockDesc, pGpu,
235                            instMemSize, instMemAlign, NV_TRUE,
236                            ADDR_UNKNOWN, CpuCacheAttrib, allocFlags);
237     if (status != NV_OK)
238     {
239         NV_PRINTF(LEVEL_ERROR, "Unable to allocate instance memory descriptor!\n");
240         SLI_LOOP_BREAK;
241     }
242 
243     if ((memdescGetAddressSpace(pInstanceBlock->pInstanceBlockDesc) == ADDR_SYSMEM) &&
244         (gpuIsInstanceMemoryAlwaysCached(pGpu)))
245     {
246         memdescSetGpuCacheAttrib(pInstanceBlock->pInstanceBlockDesc, NV_MEMORY_CACHED);
247     }
248 
249     if (pChannelBufPool != NULL)
250     {
251         status = memdescSetCtxBufPool(pInstanceBlock->pInstanceBlockDesc, pChannelBufPool);
252         if (status != NV_OK)
253         {
254             NV_ASSERT(status == NV_OK);
255             SLI_LOOP_BREAK;
256         }
257     }
258 
259     memdescTagAllocList(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_116,
260                         pInstanceBlock->pInstanceBlockDesc, pInstAllocList);
261     if (status == NV_OK)
262     {
263         MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
264 
265         // Initialize the instance block of the channel with zeros
266         status = memmgrMemDescMemSet(pMemoryManager,
267                                      pInstanceBlock->pInstanceBlockDesc,
268                                      0,
269                                      TRANSFER_FLAGS_NONE);
270         if (status != NV_OK)
271         {
272             NV_ASSERT(status == NV_OK);
273             SLI_LOOP_BREAK;
274         }
275 
276         memdescSetName(pGpu, pInstanceBlock->pInstanceBlockDesc, NV_RM_SURF_NAME_INSTANCE_BLOCK, NULL);
277     }
278     else
279     {
280         NV_PRINTF(LEVEL_ERROR,
281                   "Instance block allocation for hClient 0x%x hChannel 0x%x failed\n",
282                   RES_GET_CLIENT_HANDLE(pKernelChannel), RES_GET_HANDLE(pKernelChannel));
283         SLI_LOOP_BREAK;
284     }
285 
286     /// Alloc RAMFC Desc
287     status = memdescCreateSubMem(&pInstanceBlock->pRamfcDesc,
288                                  pInstanceBlock->pInstanceBlockDesc,
289                                  pGpu, 0, DRF_SIZE( NV_RAMIN_RAMFC ) / 8);
290     if (status != NV_OK)
291     {
292         NV_PRINTF(LEVEL_ERROR, "Could not allocate memdesc for RAMFC\n");
293         SLI_LOOP_BREAK;
294     }
295 
296     // TODO: Move this elsewhere.
297     if (!pKernelChannel->bClientAllocatedUserD)
298     {
299         NV_ASSERT(pKernelChannel->pUserdSubDeviceMemDesc[subdevInst] == NULL);
300 
301         status = _kchannelCreateRMUserdMemDesc(pGpu, pKernelChannel);
302         if (status  != NV_OK)
303         {
304             NV_PRINTF(LEVEL_ERROR, "Could not allocate sub memdesc for USERD\n"); // TODO SLI BREAK
305             SLI_LOOP_BREAK;
306         }
307     }
308 
309     status = kchannelCreateUserMemDesc_HAL(pGpu, pKernelChannel);
310     if (status != NV_OK)
311     {
312         NV_PRINTF(LEVEL_ERROR, "kchannelCreateUserMemDesc failed \n");
313         SLI_LOOP_BREAK;
314     }
315 
316     NV_PRINTF(LEVEL_INFO,
317               "hChannel 0x%x hClient 0x%x, Class ID 0x%x "
318               "Instance Block @ 0x%llx (%s %x) "
319               "USERD @ 0x%llx "
320               "for subdevice %d\n",
321               RES_GET_HANDLE(pKernelChannel), RES_GET_CLIENT_HANDLE(pKernelChannel), RES_GET_EXT_CLASS_ID(pKernelChannel),
322               memdescGetPhysAddr(pInstanceBlock->pInstanceBlockDesc, AT_GPU, 0),
323               memdescGetApertureString(memdescGetAddressSpace(pInstanceBlock->pInstanceBlockDesc)),
324               (NvU32)(memdescGetAddressSpace(pInstanceBlock->pInstanceBlockDesc)),
325               (pKernelChannel->pUserdSubDeviceMemDesc[subdevInst] == NULL) ? 0x0LL :
326               memdescGetPhysAddr(pKernelChannel->pUserdSubDeviceMemDesc[subdevInst], AT_GPU, 0LL),
327               subdevInst);
328 
329     SLI_LOOP_END
330 
331 fail:
332     // Just a note about our failure path, null and unallocated
333     // memdescFrees are allowed so this is not a bug.
334     if (status != NV_OK)
335     {
336         NV_PRINTF(LEVEL_ERROR, "Could not create Channel\n");
337         DBG_BREAKPOINT();
338     }
339 
340     return status;
341 }
342 
343 /*!
344  * @brief Free and destroy channel memdescs
345  *        created during channel alloc mem
346  *
347  * @param pGpu[in]            OBJGPU pointer
348  * @param pKernelChannel[in]  KernelChannel pointer
349  *
350  * @return void
351  */
352 void
353 kchannelDestroyMem_GM107
354 (
355     OBJGPU        *pGpu,
356     KernelChannel *pKernelChannel
357 )
358 {
359     FIFO_INSTANCE_BLOCK *pInstanceBlock  = NULL;
360     NvU32                subdevInst;
361 
362     NV_ASSERT_OR_RETURN_VOID(pKernelChannel != NULL);
363 
364     SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY)
365 
366     subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu);
367 
368     if (!pKernelChannel->bClientAllocatedUserD)
369     {
370         _kchannelDestroyRMUserdMemDesc(pGpu, pKernelChannel);
371     }
372 
373     pInstanceBlock = pKernelChannel->pFifoHalData[subdevInst];
374     if (pInstanceBlock != NULL)
375     {
376         // Release RAMFC sub memdesc
377         if (pInstanceBlock->pRamfcDesc != NULL)
378         {
379             memdescFree(pInstanceBlock->pRamfcDesc);
380             memdescDestroy(pInstanceBlock->pRamfcDesc);
381             pInstanceBlock->pRamfcDesc = NULL;
382         }
383 
384         // Release Inst block Desc
385         if (pInstanceBlock->pInstanceBlockDesc != NULL)
386         {
387             memdescFree(pInstanceBlock->pInstanceBlockDesc);
388             memdescDestroy(pInstanceBlock->pInstanceBlockDesc);
389             pInstanceBlock->pInstanceBlockDesc = NULL;
390         }
391     }
392 
393     // Remove USERD memDescs
394     memdescDestroy(pKernelChannel->pInstSubDeviceMemDesc[subdevInst]);
395     pKernelChannel->pInstSubDeviceMemDesc[subdevInst] = NULL;
396 
397     SLI_LOOP_END
398 
399     return;
400 }
401 
402 /**
403  * @brief reserves a hardware channel slot
404  *
405  * Only responsible for indicating a hardware channel is in use, does not set
406  * any other software state.
407  *
408  * @param     pGpu
409  * @param[in] pKernelChannel  the pre-allocated KernelChannel
410  * @param[in] hClient
411  * @param[in] allocMode CHANNEL_HW_ID_ALLC_MODE_*
412  * @param[in] ChID
413  * @param[in] bForceInternalIdx true if requesting specific index within USERD page
414  * @param[in] internalIdx requested index within USERD page when bForceInternalIdx
415  *                        true
416  */
417 NV_STATUS
418 kchannelAllocHwID_GM107
419 (
420     OBJGPU   *pGpu,
421     KernelChannel *pKernelChannel,
422     NvHandle  hClient,
423     NvU32     Flags,
424     NvU32     verifFlags2,
425     NvU32     ChID
426 )
427 {
428     NV_STATUS status;
429     KernelFifo              *pKernelFifo       = GPU_GET_KERNEL_FIFO(pGpu);
430     CHID_MGR                *pChidMgr          = NULL;
431     NvU32                    internalIdx       = 0;
432     NvU32                    userdPageIdx      = 0;
433     NvBool                   bForceInternalIdx = NV_FALSE;
434     NvBool                   bForceUserdPage   = NV_FALSE;
435     CHANNEL_HW_ID_ALLOC_MODE allocMode         = CHANNEL_HW_ID_ALLOC_MODE_GROW_UP;
436 
437     if (IS_VIRTUAL_WITHOUT_SRIOV(pGpu))
438     {
439         // per runlist channel heap is supported for sriov only
440         NV_CHECK_OR_RETURN(LEVEL_ERROR,
441                            !kfifoIsPerRunlistChramEnabled(pKernelFifo),
442                            NV_ERR_INVALID_STATE);
443 
444         pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo,
445                                 CHIDMGR_RUNLIST_ID_LEGACY);
446 
447         NV_CHECK_OR_RETURN(LEVEL_INFO,
448                            kfifoChidMgrGetKernelChannel(pGpu, pKernelFifo,
449                                                         pChidMgr, ChID) == NULL,
450                            NV_OK);
451         allocMode = CHANNEL_HW_ID_ALLOC_MODE_PROVIDED;
452     }
453     else
454     {
455         if (FLD_TEST_DRF(OS04, _FLAGS, _CHANNEL_USERD_INDEX_PAGE_FIXED, _TRUE, Flags))
456         {
457             bForceUserdPage = NV_TRUE;
458             userdPageIdx = DRF_VAL(OS04, _FLAGS, _CHANNEL_USERD_INDEX_PAGE_VALUE, Flags);
459             internalIdx = DRF_VAL(OS04, _FLAGS, _CHANNEL_USERD_INDEX_VALUE, Flags);
460 
461             NV_ASSERT_OR_RETURN(FLD_TEST_DRF(OS04,
462                                              _FLAGS,
463                                              _CHANNEL_USERD_INDEX_FIXED,
464                                              _FALSE,
465                                              Flags),
466                                 NV_ERR_INVALID_STATE);
467         }
468 
469         if (FLD_TEST_DRF(OS04, _FLAGS, _CHANNEL_USERD_INDEX_FIXED, _TRUE, Flags))
470         {
471             bForceInternalIdx = NV_TRUE;
472             internalIdx = DRF_VAL(OS04, _FLAGS, _CHANNEL_USERD_INDEX_VALUE, Flags);
473         }
474     }
475 
476     pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, kchannelGetRunlistId(pKernelChannel));
477 
478     status = kfifoChidMgrAllocChid(pGpu,
479                                    pKernelFifo,
480                                    pChidMgr,
481                                    hClient,
482                                    allocMode,
483                                    bForceInternalIdx,
484                                    internalIdx,
485                                    bForceUserdPage,
486                                    userdPageIdx,
487                                    ChID,
488                                    pKernelChannel);
489 
490     return status;
491 }
492 
493 /**
494  * @brief Releases a hardware channel ID.
495  *
496  * Not responsible for freeing any software state beyond that which indicates a
497  * hardware channel is in use.
498  *
499  * @param pGpu
500  * @param pKernelChannel
501  */
502 NV_STATUS
503 kchannelFreeHwID_GM107
504 (
505     OBJGPU        *pGpu,
506     KernelChannel *pKernelChannel
507 )
508 {
509     NV_STATUS   status;
510     KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu);
511     CHID_MGR   *pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo,
512                                            kchannelGetRunlistId(pKernelChannel));
513     EMEMBLOCK  *pFifoDataBlock;
514 
515     pFifoDataBlock = pChidMgr->pFifoDataHeap->eheapGetBlock(
516         pChidMgr->pFifoDataHeap,
517         pKernelChannel->ChID,
518         NV_FALSE);
519     NV_ASSERT_OR_RETURN(pFifoDataBlock, NV_ERR_OBJECT_NOT_FOUND);
520     NV_ASSERT(pFifoDataBlock->pData == pKernelChannel);
521 
522     status = kfifoChidMgrFreeChid(pGpu, pKernelFifo, pChidMgr, pKernelChannel->ChID);
523     if (status != NV_OK)
524     {
525         NV_PRINTF(LEVEL_ERROR,
526                   "Unable to Free Channel From Heap: %d\n",
527                   pKernelChannel->ChID);
528         DBG_BREAKPOINT();
529     }
530 
531     return status;
532 }
533 
534 NV_STATUS
535 kchannelGetUserdInfo_GM107
536 (
537     OBJGPU         *pGpu,
538     KernelChannel  *pKernelChannel,
539     NvU64          *userBase,
540     NvU64          *offset,
541     NvU64          *length
542 )
543 {
544     NV_STATUS              status;
545     NvU64                  bar1MapOffset;
546     NvU32                  bar1MapSize;
547     CLI_CHANNEL_CLASS_INFO classInfo;
548     KernelMemorySystem     *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu);
549 
550     NvBool bCoherentCpuMapping = pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING);
551 
552     CliGetChannelClassInfo(RES_GET_EXT_CLASS_ID(pKernelChannel), &classInfo);
553 
554     switch (classInfo.classType)
555     {
556         case CHANNEL_CLASS_TYPE_GPFIFO:
557             NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_ARGUMENT);
558 
559             // USERD is not pre-allocated in BAR1 so there is no offset/userBase
560             NV_ASSERT_OR_RETURN(!pKernelChannel->bClientAllocatedUserD,
561                                 NV_ERR_INVALID_REQUEST);
562 
563             status = kchannelGetUserdBar1MapOffset_HAL(pGpu,
564                                                        pKernelChannel,
565                                                        &bar1MapOffset,
566                                                        &bar1MapSize);
567             if (status == NV_OK)
568             {
569                 *offset = bar1MapOffset;
570                 *length = bar1MapSize;
571 
572                 if (userBase)
573                 {
574                     if (bCoherentCpuMapping)
575                     {
576                         NV_ASSERT(pGpu->getProperty(pGpu, PDB_PROP_GPU_ATS_SUPPORTED));
577                         *userBase = pKernelMemorySystem->coherentCpuFbBase;
578                     }
579                     else
580                     {
581                         *userBase = gpumgrGetGpuPhysFbAddr(pGpu);
582                     }
583                 }
584             }
585             break;
586 
587         default:
588             NV_PRINTF(LEVEL_ERROR,
589                       "class = %x not supported for user base mapping\n",
590                       RES_GET_EXT_CLASS_ID(pKernelChannel));
591             status = NV_ERR_GENERIC;
592             break;
593     }
594     return status;
595 }
596 
597 //
598 // Takes as input a Channel * and returns the BAR1 offset that this channel's
599 // USERD has been mapped to. Also returns the size of the BAR1 mapping that
600 // pertains to this channel. The BAR1 map of all USERDs should have already
601 // been setup before the first channel was created.
602 //
603 // For example, USERD of 40 channels have been mapped at BAR1 offset 0x100.
604 // USERD of one channel is of size 4k. In which case this function will return
605 // ( 0x100 + ( 0x1000 * 0xa ) ) if the input ChID = 0xa.
606 //
607 NV_STATUS
608 kchannelGetUserdBar1MapOffset_GM107
609 (
610     OBJGPU        *pGpu,
611     KernelChannel *pKernelChannel,
612     NvU64         *bar1MapOffset,
613     NvU32         *bar1MapSize
614 )
615 {
616     KernelFifo        *pKernelFifo     = GPU_GET_KERNEL_FIFO(pGpu);
617     const PREALLOCATED_USERD_INFO *pUserdInfo = kfifoGetPreallocatedUserdInfo(pKernelFifo);
618 
619     NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_ARGUMENT);
620 
621     //
622     // only supported when bUsePerRunlistChannelRam is disabled.
623     // We don't pre-allocate userd for all channels across all runlists; we expect
624     // clients to have moved to client allocated userd.
625     //
626     NV_ASSERT_OR_RETURN(!kfifoIsPerRunlistChramEnabled(pKernelFifo),
627                         NV_ERR_NOT_SUPPORTED);
628 
629     if (pUserdInfo->userdBar1MapSize == 0)
630     {
631         NV_PRINTF(LEVEL_ERROR,
632                   "fifoGetUserdBar1Offset_GF100: BAR1 map of USERD has not "
633                   "been setup yet\n");
634         NV_ASSERT(0);
635         return NV_ERR_GENERIC;
636     }
637 
638     kfifoGetUserdSizeAlign_HAL(pKernelFifo, bar1MapSize, NULL);
639 
640     *bar1MapOffset = pKernelChannel->ChID * *bar1MapSize +
641                      pUserdInfo->userdBar1MapStartOffset;
642 
643     NV_ASSERT((*bar1MapOffset + *bar1MapSize) <=
644               (pUserdInfo->userdBar1MapStartOffset +
645                pUserdInfo->userdBar1MapSize));
646 
647     return NV_OK;
648 }
649 
650 /*!
651  * @brief Creates a memory descriptor to be used for creating a GPU mapped MMIO
652  *        region for a given channel.
653  */
654 NV_STATUS
655 kchannelCreateUserMemDesc_GM107
656 (
657     OBJGPU        *pGpu,
658     KernelChannel *pKernelChannel
659 )
660 {
661     MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
662     KernelFifo    *pKernelFifo    = GPU_GET_KERNEL_FIFO(pGpu);
663     NV_STATUS status;
664     MEMORY_DESCRIPTOR *pSubDevInstMemDesc = NULL;
665     MEMORY_DESCRIPTOR **ppMemDesc =
666          &pKernelChannel->pInstSubDeviceMemDesc[gpumgrGetSubDeviceInstanceFromGpu(pGpu)];
667 
668     NV_ASSERT_OR_RETURN(!gpumgrGetBcEnabledStatus(pGpu), NV_ERR_INVALID_STATE);
669 
670     status = kfifoChannelGetFifoContextMemDesc_HAL(pGpu, pKernelFifo, pKernelChannel,
671                                                    FIFO_CTX_INST_BLOCK, &pSubDevInstMemDesc);
672 
673     if (status != NV_OK)
674         return status;
675 
676     NV_ASSERT_OR_RETURN(pSubDevInstMemDesc, NV_ERR_OBJECT_NOT_FOUND);
677 
678     status = memdescCreate(ppMemDesc, pGpu, RM_PAGE_SIZE, 0,
679                            memdescGetContiguity(pSubDevInstMemDesc, AT_GPU),
680                            memdescGetAddressSpace(pSubDevInstMemDesc),
681                            memdescGetCpuCacheAttrib(pSubDevInstMemDesc),
682                            MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE);
683 
684     if (status != NV_OK)
685         return status;
686 
687     NV_ASSERT(*ppMemDesc);
688 
689     memdescDescribe(*ppMemDesc, memdescGetAddressSpace(pSubDevInstMemDesc),
690                     memdescGetPhysAddr(pSubDevInstMemDesc, AT_GPU, 0), RM_PAGE_SIZE);
691     memmgrSetMemDescPageSize_HAL(pGpu, pMemoryManager, *ppMemDesc, AT_GPU, RM_ATTR_PAGE_SIZE_4KB);
692 
693     memdescSetPteKind(*ppMemDesc, NV_MMU_PTE_KIND_SMHOST_MESSAGE);
694 
695     return NV_OK;
696 }
697 
698 /**
699  * @brief Retrieves the engine ID (NV_PFIFO_ENGINE_*) a given channel is operating on.
700  *
701  * This value will not be valid for a channel that has not been scheduled.
702  *
703  * @param       pGpu
704  * @param       pKernelChannel
705  * @param[out]  pEngDesc
706  */
707 NV_STATUS
708 kchannelGetEngine_GM107
709 (
710     OBJGPU        *pGpu,
711     KernelChannel *pKernelChannel,
712     NvU32         *pEngDesc
713 )
714 {
715     KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu);
716 
717     if (pEngDesc == NULL)
718         return NV_ERR_INVALID_ARGUMENT;
719 
720     NV_PRINTF(LEVEL_INFO, "0x%x\n", kchannelGetDebugTag(pKernelChannel));
721 
722     *pEngDesc = kchannelGetRunlistId(pKernelChannel);
723 
724     // This will pick the first engine on this runlist (may not be the only one).
725     NV_ASSERT_OK_OR_RETURN(kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo,
726                 ENGINE_INFO_TYPE_RUNLIST, *pEngDesc,
727                 ENGINE_INFO_TYPE_ENG_DESC, pEngDesc));
728 
729     return NV_OK;
730 }
731