1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "kernel/gpu/fifo/kernel_channel.h"
25 #include "kernel/gpu/fifo/kernel_channel_group.h"
26 #include "kernel/gpu/fifo/kernel_channel_group_api.h"
27 #include "kernel/mem_mgr/mem.h"
28 #include "kernel/gpu/mem_sys/kern_mem_sys.h"
29 #include "gpu/mem_mgr/mem_mgr.h"
30 #include "gpu/mem_mgr/mem_desc.h"
31 #include "platform/sli/sli.h"
32
33 #include "class/cl906f.h"
34
35 #include "published/maxwell/gm107/dev_ram.h"
36 #include "published/maxwell/gm107/dev_mmu.h"
37
38 static NV_STATUS _kchannelCreateRMUserdMemDesc(OBJGPU *pGpu, KernelChannel *pKernelChannel);
39
40 static NV_STATUS _kchannelDestroyRMUserdMemDesc(OBJGPU *pGpu, KernelChannel *pKernelChannel);
41
42 /*!
43 * The reason this is a hal method is because it primarily concerns with returning EngineID
44 * and unfortunately, the defines for these are not standard across chips.
45 *
46 * Reason we have a classEngineID concat is to present an opaque handle that clients can do
47 * setobject with directly. Some of them may also require to know the class, engine IDs.
48 */
49 NV_STATUS
kchannelGetClassEngineID_GM107(OBJGPU * pGpu,KernelChannel * pKernelChannel,NvHandle handle,NvU32 * pClassEngineID,NvU32 * pClassID,RM_ENGINE_TYPE * pRmEngineID)50 kchannelGetClassEngineID_GM107
51 (
52 OBJGPU *pGpu,
53 KernelChannel *pKernelChannel,
54 NvHandle handle,
55 NvU32 *pClassEngineID,
56 NvU32 *pClassID,
57 RM_ENGINE_TYPE *pRmEngineID
58 )
59 {
60 NV_STATUS status = NV_OK;
61 NvU32 halEngineTag;
62 NvU32 classID;
63 ChannelDescendant *pObject = NULL;
64
65 NV_CHECK_OK_OR_RETURN(LEVEL_INFO,
66 kchannelFindChildByHandle(pKernelChannel, handle, &pObject));
67 NV_ASSERT_OR_RETURN(pObject != NULL, NV_ERR_OBJECT_NOT_FOUND);
68
69 *pClassID = classID = RES_GET_EXT_CLASS_ID(pObject);
70 halEngineTag = pObject->resourceDesc.engDesc;
71
72 if (halEngineTag == ENG_SW)
73 {
74 classID = pObject->classID;
75 }
76
77 status = gpuXlateEngDescToClientEngineId(pGpu, halEngineTag, pRmEngineID);
78
79 if (status == NV_OK)
80 {
81 *pClassEngineID = DRF_NUM(906F, _SET_OBJECT, _NVCLASS, classID);
82 }
83
84 NV_PRINTF(LEVEL_INFO,
85 "class ID: 0x%08x classEngine ID: 0x%08x\n",
86 classID, *pClassEngineID);
87
88 return status;
89 }
90
91 NV_STATUS
kchannelEnableVirtualContext_GM107(KernelChannel * pKernelChannel)92 kchannelEnableVirtualContext_GM107
93 (
94 KernelChannel *pKernelChannel
95 )
96 {
97 pKernelChannel->bSkipCtxBufferAlloc = NV_TRUE;
98
99 return NV_OK;
100 }
101
102 /*!
103 * Create the sub memory descriptor from preallocated USERD memory
104 * allocated by RM for a channel
105 */
106 static NV_STATUS
_kchannelCreateRMUserdMemDesc(OBJGPU * pGpu,KernelChannel * pKernelChannel)107 _kchannelCreateRMUserdMemDesc
108 (
109 OBJGPU *pGpu,
110 KernelChannel *pKernelChannel
111 )
112 {
113 NV_STATUS status = NV_OK;
114 NvU32 userdSize;
115 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu);
116 const PREALLOCATED_USERD_INFO *pUserdInfo = kfifoGetPreallocatedUserdInfo(pKernelFifo);
117 NvU32 subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu);
118 MEMORY_DESCRIPTOR **ppUserdSubdevMemDesc =
119 &pKernelChannel->pUserdSubDeviceMemDesc[subdevInst];
120
121 kfifoGetUserdSizeAlign_HAL(pKernelFifo, &userdSize, NULL);
122
123 status = memdescCreateSubMem(ppUserdSubdevMemDesc,
124 pUserdInfo->userdPhysDesc[subdevInst],
125 pGpu,
126 pKernelChannel->ChID * userdSize,
127 userdSize );
128 return status;
129 }
130
131 /*! Delete prealloc userd submemdesc for the channel */
132 static NV_STATUS
_kchannelDestroyRMUserdMemDesc(OBJGPU * pGpu,KernelChannel * pKernelChannel)133 _kchannelDestroyRMUserdMemDesc
134 (
135 OBJGPU *pGpu,
136 KernelChannel *pKernelChannel
137 )
138 {
139 MEMORY_DESCRIPTOR **ppUserdSubdevMemDesc =
140 &pKernelChannel->pUserdSubDeviceMemDesc[gpumgrGetSubDeviceInstanceFromGpu(pGpu)];
141 if ((ppUserdSubdevMemDesc != NULL) && (*ppUserdSubdevMemDesc != NULL))
142 {
143 memdescFree(*ppUserdSubdevMemDesc);
144 memdescDestroy(*ppUserdSubdevMemDesc);
145 *ppUserdSubdevMemDesc = NULL;
146 }
147
148 return NV_OK;
149 }
150
151 /*!
152 * @brief Create and alloc channel instance mem,
153 * ramfc and userd subdevice memdescs.
154 *
155 * @param pGpu[in] OBJGPU pointer
156 * @param pKernelChannel[in] KernelChannel pointer
157 * @param flags[in] Flags
158 * @param verifFlags[in] verifFlags
159 *
160 * @returns NV_STATUS
161 */
kchannelAllocMem_GM107(OBJGPU * pGpu,KernelChannel * pKernelChannel,NvU32 Flags,NvU32 verifFlags)162 NV_STATUS kchannelAllocMem_GM107
163 (
164 OBJGPU *pGpu,
165 KernelChannel *pKernelChannel,
166 NvU32 Flags,
167 NvU32 verifFlags
168 )
169 {
170 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu);
171 NV_STATUS status = NV_OK;
172 NvU32 CpuCacheAttrib;
173 FIFO_INSTANCE_BLOCK *pInstanceBlock = NULL;
174 NvU32 userdSize;
175 NvU64 instMemSize;
176 NvU64 instMemAlign;
177 NvBool bInstProtectedMem;
178 const NV_ADDRESS_SPACE *pInstAllocList;
179 CTX_BUF_POOL_INFO *pChannelBufPool = NULL;
180 NvU64 allocFlags = MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE;
181 NvU32 scgType;
182 NvU32 runqueue;
183 KernelChannelGroup *pKernelChannelGroup;
184 NvU32 subdevInst;
185
186 NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_ARGUMENT);
187 pKernelChannelGroup = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup;
188
189 scgType = DRF_VAL(OS04, _FLAGS, _GROUP_CHANNEL_THREAD, Flags);
190 runqueue = DRF_VAL(OS04, _FLAGS, _GROUP_CHANNEL_RUNQUEUE, Flags);
191
192 if (!kfifoValidateSCGTypeAndRunqueue_HAL(pKernelFifo, scgType, runqueue))
193 return NV_ERR_INVALID_ARGUMENT;
194
195 kfifoGetUserdSizeAlign_HAL(pKernelFifo, &userdSize, NULL);
196
197 status = kfifoGetInstMemInfo_HAL(pKernelFifo, &instMemSize, &instMemAlign,
198 &bInstProtectedMem, &CpuCacheAttrib, &pInstAllocList);
199 if (status != NV_OK)
200 {
201 NV_PRINTF(LEVEL_ERROR, "Unable to get instance memory info!\n");
202 goto fail;
203 }
204
205 /// Alloc Instance block
206 if (IsSLIEnabled(pGpu) || IS_GSP_CLIENT(pGpu))
207 {
208 pInstAllocList = ADDRLIST_FBMEM_ONLY;
209 CpuCacheAttrib = NV_MEMORY_UNCACHED;
210 }
211
212 // check for allocating VPR memory
213 if (bInstProtectedMem)
214 allocFlags |= MEMDESC_ALLOC_FLAGS_PROTECTED;
215
216 pChannelBufPool = pKernelChannelGroup->pChannelBufPool;
217 if (pChannelBufPool != NULL)
218 allocFlags |= MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL;
219
220 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY)
221
222 subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu);
223 pInstanceBlock = pKernelChannel->pFifoHalData[subdevInst];
224
225 if (pInstanceBlock == NULL)
226 {
227 NV_PRINTF(LEVEL_ERROR,
228 "Instance block is NULL for hClient 0x%x Channel 0x%x!\n",
229 RES_GET_CLIENT_HANDLE(pKernelChannel),
230 RES_GET_HANDLE(pKernelChannel));
231 SLI_LOOP_BREAK;
232 }
233
234 status = memdescCreate(&pInstanceBlock->pInstanceBlockDesc, pGpu,
235 instMemSize, instMemAlign, NV_TRUE,
236 ADDR_UNKNOWN, CpuCacheAttrib, allocFlags);
237 if (status != NV_OK)
238 {
239 NV_PRINTF(LEVEL_ERROR, "Unable to allocate instance memory descriptor!\n");
240 SLI_LOOP_BREAK;
241 }
242
243 if ((memdescGetAddressSpace(pInstanceBlock->pInstanceBlockDesc) == ADDR_SYSMEM) &&
244 (gpuIsInstanceMemoryAlwaysCached(pGpu)))
245 {
246 memdescSetGpuCacheAttrib(pInstanceBlock->pInstanceBlockDesc, NV_MEMORY_CACHED);
247 }
248
249 if (pChannelBufPool != NULL)
250 {
251 status = memdescSetCtxBufPool(pInstanceBlock->pInstanceBlockDesc, pChannelBufPool);
252 if (status != NV_OK)
253 {
254 NV_ASSERT(status == NV_OK);
255 SLI_LOOP_BREAK;
256 }
257 }
258
259 memdescTagAllocList(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_116,
260 pInstanceBlock->pInstanceBlockDesc, pInstAllocList);
261 if (status == NV_OK)
262 {
263 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
264
265 // Initialize the instance block of the channel with zeros
266 status = memmgrMemDescMemSet(pMemoryManager,
267 pInstanceBlock->pInstanceBlockDesc,
268 0,
269 TRANSFER_FLAGS_NONE);
270 if (status != NV_OK)
271 {
272 NV_ASSERT(status == NV_OK);
273 SLI_LOOP_BREAK;
274 }
275
276 memdescSetName(pGpu, pInstanceBlock->pInstanceBlockDesc, NV_RM_SURF_NAME_INSTANCE_BLOCK, NULL);
277 }
278 else
279 {
280 NV_PRINTF(LEVEL_ERROR,
281 "Instance block allocation for hClient 0x%x hChannel 0x%x failed\n",
282 RES_GET_CLIENT_HANDLE(pKernelChannel), RES_GET_HANDLE(pKernelChannel));
283 SLI_LOOP_BREAK;
284 }
285
286 /// Alloc RAMFC Desc
287 status = memdescCreateSubMem(&pInstanceBlock->pRamfcDesc,
288 pInstanceBlock->pInstanceBlockDesc,
289 pGpu, 0, DRF_SIZE( NV_RAMIN_RAMFC ) / 8);
290 if (status != NV_OK)
291 {
292 NV_PRINTF(LEVEL_ERROR, "Could not allocate memdesc for RAMFC\n");
293 SLI_LOOP_BREAK;
294 }
295
296 // TODO: Move this elsewhere.
297 if (!pKernelChannel->bClientAllocatedUserD)
298 {
299 NV_ASSERT(pKernelChannel->pUserdSubDeviceMemDesc[subdevInst] == NULL);
300
301 status = _kchannelCreateRMUserdMemDesc(pGpu, pKernelChannel);
302 if (status != NV_OK)
303 {
304 NV_PRINTF(LEVEL_ERROR, "Could not allocate sub memdesc for USERD\n"); // TODO SLI BREAK
305 SLI_LOOP_BREAK;
306 }
307 }
308
309 status = kchannelCreateUserMemDesc_HAL(pGpu, pKernelChannel);
310 if (status != NV_OK)
311 {
312 NV_PRINTF(LEVEL_ERROR, "kchannelCreateUserMemDesc failed \n");
313 SLI_LOOP_BREAK;
314 }
315
316 NV_PRINTF(LEVEL_INFO,
317 "hChannel 0x%x hClient 0x%x, Class ID 0x%x "
318 "Instance Block @ 0x%llx (%s %x) "
319 "USERD @ 0x%llx "
320 "for subdevice %d\n",
321 RES_GET_HANDLE(pKernelChannel), RES_GET_CLIENT_HANDLE(pKernelChannel), RES_GET_EXT_CLASS_ID(pKernelChannel),
322 memdescGetPhysAddr(pInstanceBlock->pInstanceBlockDesc, AT_GPU, 0),
323 memdescGetApertureString(memdescGetAddressSpace(pInstanceBlock->pInstanceBlockDesc)),
324 (NvU32)(memdescGetAddressSpace(pInstanceBlock->pInstanceBlockDesc)),
325 (pKernelChannel->pUserdSubDeviceMemDesc[subdevInst] == NULL) ? 0x0LL :
326 memdescGetPhysAddr(pKernelChannel->pUserdSubDeviceMemDesc[subdevInst], AT_GPU, 0LL),
327 subdevInst);
328
329 SLI_LOOP_END
330
331 fail:
332 // Just a note about our failure path, null and unallocated
333 // memdescFrees are allowed so this is not a bug.
334 if (status != NV_OK)
335 {
336 NV_PRINTF(LEVEL_ERROR, "Could not create Channel\n");
337 DBG_BREAKPOINT();
338 }
339
340 return status;
341 }
342
343 /*!
344 * @brief Free and destroy channel memdescs
345 * created during channel alloc mem
346 *
347 * @param pGpu[in] OBJGPU pointer
348 * @param pKernelChannel[in] KernelChannel pointer
349 *
350 * @return void
351 */
352 void
kchannelDestroyMem_GM107(OBJGPU * pGpu,KernelChannel * pKernelChannel)353 kchannelDestroyMem_GM107
354 (
355 OBJGPU *pGpu,
356 KernelChannel *pKernelChannel
357 )
358 {
359 FIFO_INSTANCE_BLOCK *pInstanceBlock = NULL;
360 NvU32 subdevInst;
361
362 NV_ASSERT_OR_RETURN_VOID(pKernelChannel != NULL);
363
364 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY)
365
366 subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu);
367
368 if (!pKernelChannel->bClientAllocatedUserD)
369 {
370 _kchannelDestroyRMUserdMemDesc(pGpu, pKernelChannel);
371 }
372
373 pInstanceBlock = pKernelChannel->pFifoHalData[subdevInst];
374 if (pInstanceBlock != NULL)
375 {
376 // Release RAMFC sub memdesc
377 if (pInstanceBlock->pRamfcDesc != NULL)
378 {
379 memdescFree(pInstanceBlock->pRamfcDesc);
380 memdescDestroy(pInstanceBlock->pRamfcDesc);
381 pInstanceBlock->pRamfcDesc = NULL;
382 }
383
384 // Release Inst block Desc
385 if (pInstanceBlock->pInstanceBlockDesc != NULL)
386 {
387 memdescFree(pInstanceBlock->pInstanceBlockDesc);
388 memdescDestroy(pInstanceBlock->pInstanceBlockDesc);
389 pInstanceBlock->pInstanceBlockDesc = NULL;
390 }
391 }
392
393 // Remove USERD memDescs
394 memdescDestroy(pKernelChannel->pInstSubDeviceMemDesc[subdevInst]);
395 pKernelChannel->pInstSubDeviceMemDesc[subdevInst] = NULL;
396
397 SLI_LOOP_END
398
399 return;
400 }
401
402 /**
403 * @brief reserves a hardware channel slot
404 *
405 * Only responsible for indicating a hardware channel is in use, does not set
406 * any other software state.
407 *
408 * @param pGpu
409 * @param[in] pKernelChannel the pre-allocated KernelChannel
410 * @param[in] hClient
411 * @param[in] allocMode CHANNEL_HW_ID_ALLC_MODE_*
412 * @param[in] ChID
413 * @param[in] bForceInternalIdx true if requesting specific index within USERD page
414 * @param[in] internalIdx requested index within USERD page when bForceInternalIdx
415 * true
416 */
417 NV_STATUS
kchannelAllocHwID_GM107(OBJGPU * pGpu,KernelChannel * pKernelChannel,NvHandle hClient,NvU32 Flags,NvU32 verifFlags2,NvU32 ChID)418 kchannelAllocHwID_GM107
419 (
420 OBJGPU *pGpu,
421 KernelChannel *pKernelChannel,
422 NvHandle hClient,
423 NvU32 Flags,
424 NvU32 verifFlags2,
425 NvU32 ChID
426 )
427 {
428 NV_STATUS status;
429 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu);
430 CHID_MGR *pChidMgr = NULL;
431 NvU32 internalIdx = 0;
432 NvU32 userdPageIdx = 0;
433 NvBool bForceInternalIdx = NV_FALSE;
434 NvBool bForceUserdPage = NV_FALSE;
435 CHANNEL_HW_ID_ALLOC_MODE allocMode = CHANNEL_HW_ID_ALLOC_MODE_GROW_UP;
436
437 if (IS_VIRTUAL_WITHOUT_SRIOV(pGpu))
438 {
439 // per runlist channel heap is supported for sriov only
440 NV_CHECK_OR_RETURN(LEVEL_ERROR,
441 !kfifoIsPerRunlistChramEnabled(pKernelFifo),
442 NV_ERR_INVALID_STATE);
443
444 pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo,
445 CHIDMGR_RUNLIST_ID_LEGACY);
446
447 NV_CHECK_OR_RETURN(LEVEL_INFO,
448 kfifoChidMgrGetKernelChannel(pGpu, pKernelFifo,
449 pChidMgr, ChID) == NULL,
450 NV_OK);
451 allocMode = CHANNEL_HW_ID_ALLOC_MODE_PROVIDED;
452 }
453 else
454 {
455 if (FLD_TEST_DRF(OS04, _FLAGS, _CHANNEL_USERD_INDEX_PAGE_FIXED, _TRUE, Flags))
456 {
457 bForceUserdPage = NV_TRUE;
458 userdPageIdx = DRF_VAL(OS04, _FLAGS, _CHANNEL_USERD_INDEX_PAGE_VALUE, Flags);
459 internalIdx = DRF_VAL(OS04, _FLAGS, _CHANNEL_USERD_INDEX_VALUE, Flags);
460
461 NV_ASSERT_OR_RETURN(FLD_TEST_DRF(OS04,
462 _FLAGS,
463 _CHANNEL_USERD_INDEX_FIXED,
464 _FALSE,
465 Flags),
466 NV_ERR_INVALID_STATE);
467 }
468
469 if (FLD_TEST_DRF(OS04, _FLAGS, _CHANNEL_USERD_INDEX_FIXED, _TRUE, Flags))
470 {
471 bForceInternalIdx = NV_TRUE;
472 internalIdx = DRF_VAL(OS04, _FLAGS, _CHANNEL_USERD_INDEX_VALUE, Flags);
473 }
474 }
475
476 pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, kchannelGetRunlistId(pKernelChannel));
477
478 status = kfifoChidMgrAllocChid(pGpu,
479 pKernelFifo,
480 pChidMgr,
481 hClient,
482 allocMode,
483 bForceInternalIdx,
484 internalIdx,
485 bForceUserdPage,
486 userdPageIdx,
487 ChID,
488 pKernelChannel);
489
490 return status;
491 }
492
493 /**
494 * @brief Releases a hardware channel ID.
495 *
496 * Not responsible for freeing any software state beyond that which indicates a
497 * hardware channel is in use.
498 *
499 * @param pGpu
500 * @param pKernelChannel
501 */
502 NV_STATUS
kchannelFreeHwID_GM107(OBJGPU * pGpu,KernelChannel * pKernelChannel)503 kchannelFreeHwID_GM107
504 (
505 OBJGPU *pGpu,
506 KernelChannel *pKernelChannel
507 )
508 {
509 NV_STATUS status;
510 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu);
511 CHID_MGR *pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo,
512 kchannelGetRunlistId(pKernelChannel));
513 EMEMBLOCK *pFifoDataBlock;
514
515 NV_ASSERT_OR_RETURN(pChidMgr != NULL, NV_ERR_OBJECT_NOT_FOUND);
516 pFifoDataBlock = pChidMgr->pFifoDataHeap->eheapGetBlock(
517 pChidMgr->pFifoDataHeap,
518 pKernelChannel->ChID,
519 NV_FALSE);
520 NV_ASSERT_OR_RETURN(pFifoDataBlock != NULL, NV_ERR_OBJECT_NOT_FOUND);
521 NV_ASSERT(pFifoDataBlock->pData == pKernelChannel);
522
523 status = kfifoChidMgrFreeChid(pGpu, pKernelFifo, pChidMgr, pKernelChannel->ChID);
524 if (status != NV_OK)
525 {
526 NV_PRINTF(LEVEL_ERROR,
527 "Unable to Free Channel From Heap: %d\n",
528 pKernelChannel->ChID);
529 DBG_BREAKPOINT();
530 }
531
532 return status;
533 }
534
535 NV_STATUS
kchannelGetUserdInfo_GM107(OBJGPU * pGpu,KernelChannel * pKernelChannel,NvU64 * userBase,NvU64 * offset,NvU64 * length)536 kchannelGetUserdInfo_GM107
537 (
538 OBJGPU *pGpu,
539 KernelChannel *pKernelChannel,
540 NvU64 *userBase,
541 NvU64 *offset,
542 NvU64 *length
543 )
544 {
545 NV_STATUS status;
546 NvU64 bar1MapOffset;
547 NvU32 bar1MapSize;
548 CLI_CHANNEL_CLASS_INFO classInfo;
549 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu);
550
551 NvBool bCoherentCpuMapping = pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING);
552
553 CliGetChannelClassInfo(RES_GET_EXT_CLASS_ID(pKernelChannel), &classInfo);
554
555 switch (classInfo.classType)
556 {
557 case CHANNEL_CLASS_TYPE_GPFIFO:
558 NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_ARGUMENT);
559
560 // USERD is not pre-allocated in BAR1 so there is no offset/userBase
561 NV_ASSERT_OR_RETURN(!pKernelChannel->bClientAllocatedUserD,
562 NV_ERR_INVALID_REQUEST);
563
564 status = kchannelGetUserdBar1MapOffset_HAL(pGpu,
565 pKernelChannel,
566 &bar1MapOffset,
567 &bar1MapSize);
568 if (status == NV_OK)
569 {
570 *offset = bar1MapOffset;
571 *length = bar1MapSize;
572
573 if (userBase)
574 {
575 if (bCoherentCpuMapping)
576 {
577 NV_ASSERT(pGpu->getProperty(pGpu, PDB_PROP_GPU_ATS_SUPPORTED));
578 *userBase = pKernelMemorySystem->coherentCpuFbBase;
579 }
580 else
581 {
582 *userBase = gpumgrGetGpuPhysFbAddr(pGpu);
583 }
584 }
585 }
586 break;
587
588 default:
589 NV_PRINTF(LEVEL_ERROR,
590 "class = %x not supported for user base mapping\n",
591 RES_GET_EXT_CLASS_ID(pKernelChannel));
592 status = NV_ERR_GENERIC;
593 break;
594 }
595 return status;
596 }
597
598 //
599 // Takes as input a Channel * and returns the BAR1 offset that this channel's
600 // USERD has been mapped to. Also returns the size of the BAR1 mapping that
601 // pertains to this channel. The BAR1 map of all USERDs should have already
602 // been setup before the first channel was created.
603 //
604 // For example, USERD of 40 channels have been mapped at BAR1 offset 0x100.
605 // USERD of one channel is of size 4k. In which case this function will return
606 // ( 0x100 + ( 0x1000 * 0xa ) ) if the input ChID = 0xa.
607 //
608 NV_STATUS
kchannelGetUserdBar1MapOffset_GM107(OBJGPU * pGpu,KernelChannel * pKernelChannel,NvU64 * bar1MapOffset,NvU32 * bar1MapSize)609 kchannelGetUserdBar1MapOffset_GM107
610 (
611 OBJGPU *pGpu,
612 KernelChannel *pKernelChannel,
613 NvU64 *bar1MapOffset,
614 NvU32 *bar1MapSize
615 )
616 {
617 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu);
618 const PREALLOCATED_USERD_INFO *pUserdInfo = kfifoGetPreallocatedUserdInfo(pKernelFifo);
619
620 NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_ARGUMENT);
621
622 //
623 // only supported when bUsePerRunlistChannelRam is disabled.
624 // We don't pre-allocate userd for all channels across all runlists; we expect
625 // clients to have moved to client allocated userd.
626 //
627 NV_ASSERT_OR_RETURN(!kfifoIsPerRunlistChramEnabled(pKernelFifo),
628 NV_ERR_NOT_SUPPORTED);
629
630 if (pUserdInfo->userdBar1MapSize == 0)
631 {
632 NV_PRINTF(LEVEL_ERROR,
633 "fifoGetUserdBar1Offset_GF100: BAR1 map of USERD has not "
634 "been setup yet\n");
635 NV_ASSERT(0);
636 return NV_ERR_GENERIC;
637 }
638
639 kfifoGetUserdSizeAlign_HAL(pKernelFifo, bar1MapSize, NULL);
640
641 *bar1MapOffset = pKernelChannel->ChID * *bar1MapSize +
642 pUserdInfo->userdBar1MapStartOffset;
643
644 NV_ASSERT((*bar1MapOffset + *bar1MapSize) <=
645 (pUserdInfo->userdBar1MapStartOffset +
646 pUserdInfo->userdBar1MapSize));
647
648 return NV_OK;
649 }
650
651 /*!
652 * @brief Creates a memory descriptor to be used for creating a GPU mapped MMIO
653 * region for a given channel.
654 */
655 NV_STATUS
kchannelCreateUserMemDesc_GM107(OBJGPU * pGpu,KernelChannel * pKernelChannel)656 kchannelCreateUserMemDesc_GM107
657 (
658 OBJGPU *pGpu,
659 KernelChannel *pKernelChannel
660 )
661 {
662 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
663 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu);
664 NV_STATUS status;
665 MEMORY_DESCRIPTOR *pSubDevInstMemDesc = NULL;
666 MEMORY_DESCRIPTOR **ppMemDesc =
667 &pKernelChannel->pInstSubDeviceMemDesc[gpumgrGetSubDeviceInstanceFromGpu(pGpu)];
668
669 NV_ASSERT_OR_RETURN(!gpumgrGetBcEnabledStatus(pGpu), NV_ERR_INVALID_STATE);
670
671 status = kfifoChannelGetFifoContextMemDesc_HAL(pGpu, pKernelFifo, pKernelChannel,
672 FIFO_CTX_INST_BLOCK, &pSubDevInstMemDesc);
673
674 if (status != NV_OK)
675 return status;
676
677 NV_ASSERT_OR_RETURN(pSubDevInstMemDesc, NV_ERR_OBJECT_NOT_FOUND);
678
679 status = memdescCreate(ppMemDesc, pGpu, RM_PAGE_SIZE, 0,
680 memdescGetContiguity(pSubDevInstMemDesc, AT_GPU),
681 memdescGetAddressSpace(pSubDevInstMemDesc),
682 memdescGetCpuCacheAttrib(pSubDevInstMemDesc),
683 MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE);
684
685 if (status != NV_OK)
686 return status;
687
688 NV_ASSERT(*ppMemDesc);
689
690 memdescDescribe(*ppMemDesc, memdescGetAddressSpace(pSubDevInstMemDesc),
691 memdescGetPhysAddr(pSubDevInstMemDesc, AT_GPU, 0), RM_PAGE_SIZE);
692 memmgrSetMemDescPageSize_HAL(pGpu, pMemoryManager, *ppMemDesc, AT_GPU, RM_ATTR_PAGE_SIZE_4KB);
693
694 memdescSetPteKind(*ppMemDesc, NV_MMU_PTE_KIND_SMHOST_MESSAGE);
695
696 return NV_OK;
697 }
698
699 /**
700 * @brief Retrieves the engine ID (NV_PFIFO_ENGINE_*) a given channel is operating on.
701 *
702 * This value will not be valid for a channel that has not been scheduled.
703 *
704 * @param pGpu
705 * @param pKernelChannel
706 * @param[out] pEngDesc
707 */
708 NV_STATUS
kchannelGetEngine_GM107(OBJGPU * pGpu,KernelChannel * pKernelChannel,NvU32 * pEngDesc)709 kchannelGetEngine_GM107
710 (
711 OBJGPU *pGpu,
712 KernelChannel *pKernelChannel,
713 NvU32 *pEngDesc
714 )
715 {
716 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu);
717
718 if (pEngDesc == NULL)
719 return NV_ERR_INVALID_ARGUMENT;
720
721 NV_PRINTF(LEVEL_INFO, "0x%x\n", kchannelGetDebugTag(pKernelChannel));
722
723 *pEngDesc = kchannelGetRunlistId(pKernelChannel);
724
725 // This will pick the first engine on this runlist (may not be the only one).
726 NV_ASSERT_OK_OR_RETURN(kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo,
727 ENGINE_INFO_TYPE_RUNLIST, *pEngDesc,
728 ENGINE_INFO_TYPE_ENG_DESC, pEngDesc));
729
730 return NV_OK;
731 }
732