1 /*
2  * SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3  * SPDX-License-Identifier: MIT
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21  * DEALINGS IN THE SOFTWARE.
22  */
23 
24 #include "kernel/gpu/fifo/kernel_channel.h"
25 #include "kernel/gpu/fifo/kernel_channel_group.h"
26 #include "kernel/gpu/fifo/kernel_channel_group_api.h"
27 #include "kernel/mem_mgr/mem.h"
28 #include "kernel/gpu/mem_sys/kern_mem_sys.h"
29 #include "gpu/mem_mgr/mem_mgr.h"
30 #include "gpu/mem_mgr/mem_desc.h"
31 
32 #include "class/cl906f.h"
33 
34 #include "published/maxwell/gm107/dev_ram.h"
35 #include "published/maxwell/gm107/dev_mmu.h"
36 
37 static NV_STATUS _kchannelCreateRMUserdMemDesc(OBJGPU *pGpu, KernelChannel *pKernelChannel);
38 
39 static NV_STATUS _kchannelDestroyRMUserdMemDesc(OBJGPU *pGpu, KernelChannel *pKernelChannel);
40 
41 /*!
42  * The reason this is a hal method is because it primarily concerns with returning EngineID
43  * and unfortunately, the defines for these are not standard across chips.
44  *
45  * Reason we have a classEngineID concat is to present an opaque handle that clients can do
46  * setobject with directly. Some of them may also require to know the class, engine IDs.
47  */
48 NV_STATUS
49 kchannelGetClassEngineID_GM107
50 (
51     OBJGPU         *pGpu,
52     KernelChannel  *pKernelChannel,
53     NvHandle        handle,
54     NvU32          *pClassEngineID,
55     NvU32          *pClassID,
56     RM_ENGINE_TYPE *pRmEngineID
57 )
58 {
59     NV_STATUS          status = NV_OK;
60     NvU32              halEngineTag;
61     NvU32              classID;
62     ChannelDescendant *pObject = NULL;
63 
64     NV_CHECK_OK_OR_RETURN(LEVEL_INFO,
65         kchannelFindChildByHandle(pKernelChannel, handle, &pObject));
66     NV_ASSERT_OR_RETURN(pObject != NULL, NV_ERR_OBJECT_NOT_FOUND);
67 
68     *pClassID = classID = RES_GET_EXT_CLASS_ID(pObject);
69     halEngineTag = pObject->resourceDesc.engDesc;
70 
71     if (halEngineTag == ENG_SW)
72     {
73         classID = pObject->classID;
74     }
75 
76     status = gpuXlateEngDescToClientEngineId(pGpu, halEngineTag, pRmEngineID);
77 
78     if (status == NV_OK)
79     {
80         *pClassEngineID = DRF_NUM(906F, _SET_OBJECT, _NVCLASS, classID);
81     }
82 
83     NV_PRINTF(LEVEL_INFO,
84               "class ID: 0x%08x classEngine ID: 0x%08x\n",
85               classID, *pClassEngineID);
86 
87     return status;
88 }
89 
90 NV_STATUS
91 kchannelEnableVirtualContext_GM107
92 (
93     KernelChannel *pKernelChannel
94 )
95 {
96     pKernelChannel->bSkipCtxBufferAlloc = NV_TRUE;
97 
98     return NV_OK;
99 }
100 
101 /*!
102  * Create the sub memory descriptor from preallocated USERD memory
103  * allocated by RM for a channel
104  */
105 static NV_STATUS
106 _kchannelCreateRMUserdMemDesc
107 (
108     OBJGPU        *pGpu,
109     KernelChannel *pKernelChannel
110 )
111 {
112     NV_STATUS            status = NV_OK;
113     NvU32                userdSize;
114     KernelFifo          *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu);
115     const PREALLOCATED_USERD_INFO *pUserdInfo = kfifoGetPreallocatedUserdInfo(pKernelFifo);
116     NvU32                subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu);
117     MEMORY_DESCRIPTOR  **ppUserdSubdevMemDesc =
118                          &pKernelChannel->pUserdSubDeviceMemDesc[subdevInst];
119 
120     kfifoGetUserdSizeAlign_HAL(pKernelFifo, &userdSize, NULL);
121 
122     status = memdescCreateSubMem(ppUserdSubdevMemDesc,
123                                  pUserdInfo->userdPhysDesc[subdevInst],
124                                  pGpu,
125                                  pKernelChannel->ChID * userdSize,
126                                  userdSize );
127     return status;
128 }
129 
130 /*! Delete prealloc userd submemdesc for the channel */
131 static NV_STATUS
132 _kchannelDestroyRMUserdMemDesc
133 (
134     OBJGPU        *pGpu,
135     KernelChannel *pKernelChannel
136 )
137 {
138     MEMORY_DESCRIPTOR  **ppUserdSubdevMemDesc =
139                          &pKernelChannel->pUserdSubDeviceMemDesc[gpumgrGetSubDeviceInstanceFromGpu(pGpu)];
140     if ((ppUserdSubdevMemDesc != NULL) && (*ppUserdSubdevMemDesc != NULL))
141     {
142         memdescFree(*ppUserdSubdevMemDesc);
143         memdescDestroy(*ppUserdSubdevMemDesc);
144         *ppUserdSubdevMemDesc = NULL;
145     }
146 
147     return NV_OK;
148 }
149 
150 /*!
151  * @brief Create and alloc channel instance mem,
152  *        ramfc and userd subdevice memdescs.
153  *
154  * @param pGpu[in]           OBJGPU pointer
155  * @param pKernelChannel[in] KernelChannel pointer
156  * @param flags[in]          Flags
157  * @param verifFlags[in]     verifFlags
158  *
159  * @returns NV_STATUS
160  */
161 NV_STATUS kchannelAllocMem_GM107
162 (
163     OBJGPU             *pGpu,
164     KernelChannel      *pKernelChannel,
165     NvU32               Flags,
166     NvU32               verifFlags
167 )
168 {
169     KernelFifo             *pKernelFifo     = GPU_GET_KERNEL_FIFO(pGpu);
170     NV_STATUS               status          = NV_OK;
171     NvU32                   CpuCacheAttrib;
172     FIFO_INSTANCE_BLOCK    *pInstanceBlock  = NULL;
173     NvU32                   userdSize;
174     NvU64                   instMemSize;
175     NvU64                   instMemAlign;
176     NvBool                  bInstProtectedMem;
177     const NV_ADDRESS_SPACE *pInstAllocList;
178     CTX_BUF_POOL_INFO      *pChannelBufPool = NULL;
179     NvU64                   allocFlags      = MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE;
180     NvU32                   scgType;
181     NvU32                   runqueue;
182     KernelChannelGroup     *pKernelChannelGroup;
183     NvU32                   subdevInst;
184 
185     NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_ARGUMENT);
186     pKernelChannelGroup = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup;
187 
188     scgType     = DRF_VAL(OS04, _FLAGS, _GROUP_CHANNEL_THREAD, Flags);
189     runqueue    = DRF_VAL(OS04, _FLAGS, _GROUP_CHANNEL_RUNQUEUE, Flags);
190 
191     if (!kfifoValidateSCGTypeAndRunqueue_HAL(pKernelFifo, scgType, runqueue))
192         return NV_ERR_INVALID_ARGUMENT;
193 
194     kfifoGetUserdSizeAlign_HAL(pKernelFifo, &userdSize, NULL);
195 
196     status = kfifoGetInstMemInfo_HAL(pKernelFifo, &instMemSize, &instMemAlign,
197                                     &bInstProtectedMem, &CpuCacheAttrib, &pInstAllocList);
198     if (status != NV_OK)
199     {
200         NV_PRINTF(LEVEL_ERROR, "Unable to get instance memory info!\n");
201         goto fail;
202     }
203 
204     ///  Alloc Instance block
205     if (IsSLIEnabled(pGpu) || IS_GSP_CLIENT(pGpu))
206     {
207         pInstAllocList = ADDRLIST_FBMEM_ONLY;
208         CpuCacheAttrib = NV_MEMORY_UNCACHED;
209     }
210 
211     // check for allocating VPR memory
212     if (bInstProtectedMem)
213         allocFlags |= MEMDESC_ALLOC_FLAGS_PROTECTED;
214 
215     pChannelBufPool = pKernelChannelGroup->pChannelBufPool;
216     if (pChannelBufPool != NULL)
217         allocFlags |= MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL;
218 
219     SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY)
220 
221     subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu);
222     pInstanceBlock = pKernelChannel->pFifoHalData[subdevInst];
223 
224     if (pInstanceBlock == NULL)
225     {
226         NV_PRINTF(LEVEL_ERROR,
227                   "Instance block is NULL for hClient 0x%x Channel 0x%x!\n",
228                   RES_GET_CLIENT_HANDLE(pKernelChannel),
229                   RES_GET_HANDLE(pKernelChannel));
230         SLI_LOOP_BREAK;
231     }
232 
233     status = memdescCreate(&pInstanceBlock->pInstanceBlockDesc, pGpu,
234                            instMemSize, instMemAlign, NV_TRUE,
235                            ADDR_UNKNOWN, CpuCacheAttrib, allocFlags);
236     if (status != NV_OK)
237     {
238         NV_PRINTF(LEVEL_ERROR, "Unable to allocate instance memory descriptor!\n");
239         SLI_LOOP_BREAK;
240     }
241 
242     if ((memdescGetAddressSpace(pInstanceBlock->pInstanceBlockDesc) == ADDR_SYSMEM) &&
243         (gpuIsInstanceMemoryAlwaysCached(pGpu)))
244     {
245         memdescSetGpuCacheAttrib(pInstanceBlock->pInstanceBlockDesc, NV_MEMORY_CACHED);
246     }
247 
248     if (pChannelBufPool != NULL)
249     {
250         status = memdescSetCtxBufPool(pInstanceBlock->pInstanceBlockDesc, pChannelBufPool);
251         if (status != NV_OK)
252         {
253             NV_ASSERT(status == NV_OK);
254             SLI_LOOP_BREAK;
255         }
256     }
257 
258     memdescTagAllocList(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_116,
259                         pInstanceBlock->pInstanceBlockDesc, pInstAllocList);
260     if (status == NV_OK)
261     {
262         MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
263 
264         // Initialize the instance block of the channel with zeros
265         status = memmgrMemDescMemSet(pMemoryManager,
266                                      pInstanceBlock->pInstanceBlockDesc,
267                                      0,
268                                      TRANSFER_FLAGS_NONE);
269         if (status != NV_OK)
270         {
271             NV_ASSERT(status == NV_OK);
272             SLI_LOOP_BREAK;
273         }
274 
275         memdescSetName(pGpu, pInstanceBlock->pInstanceBlockDesc, NV_RM_SURF_NAME_INSTANCE_BLOCK, NULL);
276     }
277     else
278     {
279         NV_PRINTF(LEVEL_ERROR,
280                   "Instance block allocation for hClient 0x%x hChannel 0x%x failed\n",
281                   RES_GET_CLIENT_HANDLE(pKernelChannel), RES_GET_HANDLE(pKernelChannel));
282         SLI_LOOP_BREAK;
283     }
284 
285     /// Alloc RAMFC Desc
286     status = memdescCreateSubMem(&pInstanceBlock->pRamfcDesc,
287                                  pInstanceBlock->pInstanceBlockDesc,
288                                  pGpu, 0, DRF_SIZE( NV_RAMIN_RAMFC ) / 8);
289     if (status != NV_OK)
290     {
291         NV_PRINTF(LEVEL_ERROR, "Could not allocate memdesc for RAMFC\n");
292         SLI_LOOP_BREAK;
293     }
294 
295     // TODO: Move this elsewhere.
296     if (!pKernelChannel->bClientAllocatedUserD)
297     {
298         NV_ASSERT(pKernelChannel->pUserdSubDeviceMemDesc[subdevInst] == NULL);
299 
300         status = _kchannelCreateRMUserdMemDesc(pGpu, pKernelChannel);
301         if (status  != NV_OK)
302         {
303             NV_PRINTF(LEVEL_ERROR, "Could not allocate sub memdesc for USERD\n"); // TODO SLI BREAK
304             SLI_LOOP_BREAK;
305         }
306     }
307 
308     status = kchannelCreateUserMemDesc_HAL(pGpu, pKernelChannel);
309     if (status != NV_OK)
310     {
311         NV_PRINTF(LEVEL_ERROR, "kchannelCreateUserMemDesc failed \n");
312         SLI_LOOP_BREAK;
313     }
314 
315     NV_PRINTF(LEVEL_INFO,
316               "hChannel 0x%x hClient 0x%x, Class ID 0x%x "
317               "Instance Block @ 0x%llx (%s %x) "
318               "USERD @ 0x%llx "
319               "for subdevice %d\n",
320               RES_GET_HANDLE(pKernelChannel), RES_GET_CLIENT_HANDLE(pKernelChannel), RES_GET_EXT_CLASS_ID(pKernelChannel),
321               memdescGetPhysAddr(pInstanceBlock->pInstanceBlockDesc, AT_GPU, 0),
322               memdescGetApertureString(memdescGetAddressSpace(pInstanceBlock->pInstanceBlockDesc)),
323               (NvU32)(memdescGetAddressSpace(pInstanceBlock->pInstanceBlockDesc)),
324               (pKernelChannel->pUserdSubDeviceMemDesc[subdevInst] == NULL) ? 0x0LL :
325               memdescGetPhysAddr(pKernelChannel->pUserdSubDeviceMemDesc[subdevInst], AT_GPU, 0LL),
326               subdevInst);
327 
328     SLI_LOOP_END
329 
330 fail:
331     // Just a note about our failure path, null and unallocated
332     // memdescFrees are allowed so this is not a bug.
333     if (status != NV_OK)
334     {
335         NV_PRINTF(LEVEL_ERROR, "Could not create Channel\n");
336         DBG_BREAKPOINT();
337     }
338 
339     return status;
340 }
341 
342 /*!
343  * @brief Free and destroy channel memdescs
344  *        created during channel alloc mem
345  *
346  * @param pGpu[in]            OBJGPU pointer
347  * @param pKernelChannel[in]  KernelChannel pointer
348  *
349  * @return void
350  */
351 void
352 kchannelDestroyMem_GM107
353 (
354     OBJGPU        *pGpu,
355     KernelChannel *pKernelChannel
356 )
357 {
358     FIFO_INSTANCE_BLOCK *pInstanceBlock  = NULL;
359     NvU32                subdevInst;
360 
361     NV_ASSERT_OR_RETURN_VOID(pKernelChannel != NULL);
362 
363     SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY)
364 
365     subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu);
366 
367     if (!pKernelChannel->bClientAllocatedUserD)
368     {
369         _kchannelDestroyRMUserdMemDesc(pGpu, pKernelChannel);
370     }
371 
372     pInstanceBlock = pKernelChannel->pFifoHalData[subdevInst];
373     if (pInstanceBlock != NULL)
374     {
375         // Release RAMFC sub memdesc
376         if (pInstanceBlock->pRamfcDesc != NULL)
377         {
378             memdescFree(pInstanceBlock->pRamfcDesc);
379             memdescDestroy(pInstanceBlock->pRamfcDesc);
380             pInstanceBlock->pRamfcDesc = NULL;
381         }
382 
383         // Release Inst block Desc
384         if (pInstanceBlock->pInstanceBlockDesc != NULL)
385         {
386             memdescFree(pInstanceBlock->pInstanceBlockDesc);
387             memdescDestroy(pInstanceBlock->pInstanceBlockDesc);
388             pInstanceBlock->pInstanceBlockDesc = NULL;
389         }
390     }
391 
392     // Remove USERD memDescs
393     memdescDestroy(pKernelChannel->pInstSubDeviceMemDesc[subdevInst]);
394     pKernelChannel->pInstSubDeviceMemDesc[subdevInst] = NULL;
395 
396     SLI_LOOP_END
397 
398     return;
399 }
400 
401 /**
402  * @brief reserves a hardware channel slot
403  *
404  * Only responsible for indicating a hardware channel is in use, does not set
405  * any other software state.
406  *
407  * @param     pGpu
408  * @param[in] pKernelChannel  the pre-allocated KernelChannel
409  * @param[in] hClient
410  * @param[in] allocMode CHANNEL_HW_ID_ALLC_MODE_*
411  * @param[in] ChID
412  * @param[in] bForceInternalIdx true if requesting specific index within USERD page
413  * @param[in] internalIdx requested index within USERD page when bForceInternalIdx
414  *                        true
415  */
416 NV_STATUS
417 kchannelAllocHwID_GM107
418 (
419     OBJGPU   *pGpu,
420     KernelChannel *pKernelChannel,
421     NvHandle  hClient,
422     NvU32     Flags,
423     NvU32     verifFlags2,
424     NvU32     ChID
425 )
426 {
427     NV_STATUS status;
428     KernelFifo              *pKernelFifo       = GPU_GET_KERNEL_FIFO(pGpu);
429     CHID_MGR                *pChidMgr          = NULL;
430     NvU32                    internalIdx       = 0;
431     NvU32                    userdPageIdx      = 0;
432     NvBool                   bForceInternalIdx = NV_FALSE;
433     NvBool                   bForceUserdPage   = NV_FALSE;
434     CHANNEL_HW_ID_ALLOC_MODE allocMode         = CHANNEL_HW_ID_ALLOC_MODE_GROW_UP;
435 
436     if (IS_VIRTUAL_WITHOUT_SRIOV(pGpu))
437     {
438         // per runlist channel heap is supported for sriov only
439         NV_CHECK_OR_RETURN(LEVEL_ERROR,
440                            !kfifoIsPerRunlistChramEnabled(pKernelFifo),
441                            NV_ERR_INVALID_STATE);
442 
443         pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo,
444                                 CHIDMGR_RUNLIST_ID_LEGACY);
445 
446         NV_CHECK_OR_RETURN(LEVEL_INFO,
447                            kfifoChidMgrGetKernelChannel(pGpu, pKernelFifo,
448                                                         pChidMgr, ChID) == NULL,
449                            NV_OK);
450         allocMode = CHANNEL_HW_ID_ALLOC_MODE_PROVIDED;
451     }
452     else
453     {
454         if (FLD_TEST_DRF(OS04, _FLAGS, _CHANNEL_USERD_INDEX_PAGE_FIXED, _TRUE, Flags))
455         {
456             bForceUserdPage = NV_TRUE;
457             userdPageIdx = DRF_VAL(OS04, _FLAGS, _CHANNEL_USERD_INDEX_PAGE_VALUE, Flags);
458             internalIdx = DRF_VAL(OS04, _FLAGS, _CHANNEL_USERD_INDEX_VALUE, Flags);
459 
460             NV_ASSERT_OR_RETURN(FLD_TEST_DRF(OS04,
461                                              _FLAGS,
462                                              _CHANNEL_USERD_INDEX_FIXED,
463                                              _FALSE,
464                                              Flags),
465                                 NV_ERR_INVALID_STATE);
466         }
467 
468         if (FLD_TEST_DRF(OS04, _FLAGS, _CHANNEL_USERD_INDEX_FIXED, _TRUE, Flags))
469         {
470             bForceInternalIdx = NV_TRUE;
471             internalIdx = DRF_VAL(OS04, _FLAGS, _CHANNEL_USERD_INDEX_VALUE, Flags);
472         }
473     }
474 
475     pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, kchannelGetRunlistId(pKernelChannel));
476 
477     status = kfifoChidMgrAllocChid(pGpu,
478                                    pKernelFifo,
479                                    pChidMgr,
480                                    hClient,
481                                    allocMode,
482                                    bForceInternalIdx,
483                                    internalIdx,
484                                    bForceUserdPage,
485                                    userdPageIdx,
486                                    ChID,
487                                    pKernelChannel);
488 
489     return status;
490 }
491 
492 /**
493  * @brief Releases a hardware channel ID.
494  *
495  * Not responsible for freeing any software state beyond that which indicates a
496  * hardware channel is in use.
497  *
498  * @param pGpu
499  * @param pKernelChannel
500  */
501 NV_STATUS
502 kchannelFreeHwID_GM107
503 (
504     OBJGPU        *pGpu,
505     KernelChannel *pKernelChannel
506 )
507 {
508     NV_STATUS   status;
509     KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu);
510     CHID_MGR   *pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo,
511                                            kchannelGetRunlistId(pKernelChannel));
512     EMEMBLOCK  *pFifoDataBlock;
513 
514     pFifoDataBlock = pChidMgr->pFifoDataHeap->eheapGetBlock(
515         pChidMgr->pFifoDataHeap,
516         pKernelChannel->ChID,
517         NV_FALSE);
518     NV_ASSERT_OR_RETURN(pFifoDataBlock, NV_ERR_OBJECT_NOT_FOUND);
519     NV_ASSERT(pFifoDataBlock->pData == pKernelChannel);
520 
521     status = kfifoChidMgrFreeChid(pGpu, pKernelFifo, pChidMgr, pKernelChannel->ChID);
522     if (status != NV_OK)
523     {
524         NV_PRINTF(LEVEL_ERROR,
525                   "Unable to Free Channel From Heap: %d\n",
526                   pKernelChannel->ChID);
527         DBG_BREAKPOINT();
528     }
529 
530     return status;
531 }
532 
533 NV_STATUS
534 kchannelGetUserdInfo_GM107
535 (
536     OBJGPU         *pGpu,
537     KernelChannel  *pKernelChannel,
538     NvU64          *userBase,
539     NvU64          *offset,
540     NvU64          *length
541 )
542 {
543     NV_STATUS              status;
544     NvU64                  bar1MapOffset;
545     NvU32                  bar1MapSize;
546     CLI_CHANNEL_CLASS_INFO classInfo;
547     KernelMemorySystem     *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu);
548 
549     NvBool bCoherentCpuMapping = pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING);
550 
551     CliGetChannelClassInfo(RES_GET_EXT_CLASS_ID(pKernelChannel), &classInfo);
552 
553     switch (classInfo.classType)
554     {
555         case CHANNEL_CLASS_TYPE_GPFIFO:
556             NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_ARGUMENT);
557 
558             // USERD is not pre-allocated in BAR1 so there is no offset/userBase
559             NV_ASSERT_OR_RETURN(!pKernelChannel->bClientAllocatedUserD,
560                                 NV_ERR_INVALID_REQUEST);
561 
562             status = kchannelGetUserdBar1MapOffset_HAL(pGpu,
563                                                        pKernelChannel,
564                                                        &bar1MapOffset,
565                                                        &bar1MapSize);
566             if (status == NV_OK)
567             {
568                 *offset = bar1MapOffset;
569                 *length = bar1MapSize;
570 
571                 if (userBase)
572                 {
573                     if (bCoherentCpuMapping)
574                     {
575                         NV_ASSERT(pGpu->getProperty(pGpu, PDB_PROP_GPU_ATS_SUPPORTED));
576                         *userBase = pKernelMemorySystem->coherentCpuFbBase;
577                     }
578                     else
579                     {
580                         *userBase = gpumgrGetGpuPhysFbAddr(pGpu);
581                     }
582                 }
583             }
584             break;
585 
586         default:
587             NV_PRINTF(LEVEL_ERROR,
588                       "class = %x not supported for user base mapping\n",
589                       RES_GET_EXT_CLASS_ID(pKernelChannel));
590             status = NV_ERR_GENERIC;
591             break;
592     }
593     return status;
594 }
595 
596 //
597 // Takes as input a Channel * and returns the BAR1 offset that this channel's
598 // USERD has been mapped to. Also returns the size of the BAR1 mapping that
599 // pertains to this channel. The BAR1 map of all USERDs should have already
600 // been setup before the first channel was created.
601 //
602 // For example, USERD of 40 channels have been mapped at BAR1 offset 0x100.
603 // USERD of one channel is of size 4k. In which case this function will return
604 // ( 0x100 + ( 0x1000 * 0xa ) ) if the input ChID = 0xa.
605 //
606 NV_STATUS
607 kchannelGetUserdBar1MapOffset_GM107
608 (
609     OBJGPU        *pGpu,
610     KernelChannel *pKernelChannel,
611     NvU64         *bar1MapOffset,
612     NvU32         *bar1MapSize
613 )
614 {
615     KernelFifo        *pKernelFifo     = GPU_GET_KERNEL_FIFO(pGpu);
616     const PREALLOCATED_USERD_INFO *pUserdInfo = kfifoGetPreallocatedUserdInfo(pKernelFifo);
617 
618     NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_ARGUMENT);
619 
620     //
621     // only supported when bUsePerRunlistChannelRam is disabled.
622     // We don't pre-allocate userd for all channels across all runlists; we expect
623     // clients to have moved to client allocated userd.
624     //
625     NV_ASSERT_OR_RETURN(!kfifoIsPerRunlistChramEnabled(pKernelFifo),
626                         NV_ERR_NOT_SUPPORTED);
627 
628     if (pUserdInfo->userdBar1MapSize == 0)
629     {
630         NV_PRINTF(LEVEL_ERROR,
631                   "fifoGetUserdBar1Offset_GF100: BAR1 map of USERD has not "
632                   "been setup yet\n");
633         NV_ASSERT(0);
634         return NV_ERR_GENERIC;
635     }
636 
637     kfifoGetUserdSizeAlign_HAL(pKernelFifo, bar1MapSize, NULL);
638 
639     *bar1MapOffset = pKernelChannel->ChID * *bar1MapSize +
640                      pUserdInfo->userdBar1MapStartOffset;
641 
642     NV_ASSERT((*bar1MapOffset + *bar1MapSize) <=
643               (pUserdInfo->userdBar1MapStartOffset +
644                pUserdInfo->userdBar1MapSize));
645 
646     return NV_OK;
647 }
648 
649 /*!
650  * @brief Creates a memory descriptor to be used for creating a GPU mapped MMIO
651  *        region for a given channel.
652  */
653 NV_STATUS
654 kchannelCreateUserMemDesc_GM107
655 (
656     OBJGPU        *pGpu,
657     KernelChannel *pKernelChannel
658 )
659 {
660     MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
661     KernelFifo    *pKernelFifo    = GPU_GET_KERNEL_FIFO(pGpu);
662     NV_STATUS status;
663     MEMORY_DESCRIPTOR *pSubDevInstMemDesc = NULL;
664     MEMORY_DESCRIPTOR **ppMemDesc =
665          &pKernelChannel->pInstSubDeviceMemDesc[gpumgrGetSubDeviceInstanceFromGpu(pGpu)];
666 
667     NV_ASSERT_OR_RETURN(!gpumgrGetBcEnabledStatus(pGpu), NV_ERR_INVALID_STATE);
668 
669     status = kfifoChannelGetFifoContextMemDesc_HAL(pGpu, pKernelFifo, pKernelChannel,
670                                                    FIFO_CTX_INST_BLOCK, &pSubDevInstMemDesc);
671 
672     if (status != NV_OK)
673         return status;
674 
675     NV_ASSERT_OR_RETURN(pSubDevInstMemDesc, NV_ERR_OBJECT_NOT_FOUND);
676 
677     status = memdescCreate(ppMemDesc, pGpu, RM_PAGE_SIZE, 0,
678                            memdescGetContiguity(pSubDevInstMemDesc, AT_GPU),
679                            memdescGetAddressSpace(pSubDevInstMemDesc),
680                            memdescGetCpuCacheAttrib(pSubDevInstMemDesc),
681                            MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE);
682 
683     if (status != NV_OK)
684         return status;
685 
686     NV_ASSERT(*ppMemDesc);
687 
688     memdescDescribe(*ppMemDesc, memdescGetAddressSpace(pSubDevInstMemDesc),
689                     memdescGetPhysAddr(pSubDevInstMemDesc, AT_GPU, 0), RM_PAGE_SIZE);
690     memmgrSetMemDescPageSize_HAL(pGpu, pMemoryManager, *ppMemDesc, AT_GPU, RM_ATTR_PAGE_SIZE_4KB);
691 
692     memdescSetPteKind(*ppMemDesc, NV_MMU_PTE_KIND_SMHOST_MESSAGE);
693 
694     return NV_OK;
695 }
696 
697 /**
698  * @brief Retrieves the engine ID (NV_PFIFO_ENGINE_*) a given channel is operating on.
699  *
700  * This value will not be valid for a channel that has not been scheduled.
701  *
702  * @param       pGpu
703  * @param       pKernelChannel
704  * @param[out]  pEngDesc
705  */
706 NV_STATUS
707 kchannelGetEngine_GM107
708 (
709     OBJGPU        *pGpu,
710     KernelChannel *pKernelChannel,
711     NvU32         *pEngDesc
712 )
713 {
714     KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu);
715 
716     if (pEngDesc == NULL)
717         return NV_ERR_INVALID_ARGUMENT;
718 
719     NV_PRINTF(LEVEL_INFO, "0x%x\n", kchannelGetDebugTag(pKernelChannel));
720 
721     *pEngDesc = kchannelGetRunlistId(pKernelChannel);
722 
723     // This will pick the first engine on this runlist (may not be the only one).
724     NV_ASSERT_OK_OR_RETURN(kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo,
725                 ENGINE_INFO_TYPE_RUNLIST, *pEngDesc,
726                 ENGINE_INFO_TYPE_ENG_DESC, pEngDesc));
727 
728     return NV_OK;
729 }
730