1 /*
2  * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3  * SPDX-License-Identifier: MIT
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21  * DEALINGS IN THE SOFTWARE.
22  */
23 #include "gpu/falcon/kernel_falcon.h"
24 #include "gpu/sec2/kernel_sec2.h"
25 #include "gpu/gsp/kernel_gsp.h"
26 
27 #include "gpu/fifo/kernel_fifo.h"
28 #include "gpu/fifo/kernel_channel.h"
29 #include "gpu/fifo/kernel_channel_group.h"
30 #include "gpu/fifo/kernel_channel_group_api.h"
31 #include "gpu/intr/intr.h"
32 #include "gpu/subdevice/subdevice.h"
33 #include "gpu/mem_mgr/mem_desc.h"
34 #include "mem_mgr/gpu_vaspace.h"
35 #include "mem_mgr/ctx_buf_pool.h"
36 #include "rmapi/rmapi.h"
37 
38 
39 void kflcnConfigureEngine_IMPL(OBJGPU *pGpu, KernelFalcon *pKernelFalcon, KernelFalconEngineConfig *pFalconConfig)
40 {
41     pKernelFalcon->registerBase       = pFalconConfig->registerBase;
42     pKernelFalcon->riscvRegisterBase  = pFalconConfig->riscvRegisterBase;
43     pKernelFalcon->fbifBase           = pFalconConfig->fbifBase;
44     pKernelFalcon->bBootFromHs        = pFalconConfig->bBootFromHs;
45     pKernelFalcon->pmcEnableMask      = pFalconConfig->pmcEnableMask;
46     pKernelFalcon->bIsPmcDeviceEngine = pFalconConfig->bIsPmcDeviceEngine;
47     pKernelFalcon->physEngDesc        = pFalconConfig->physEngDesc;
48     pKernelFalcon->ctxAttr            = pFalconConfig->ctxAttr;
49     pKernelFalcon->ctxBufferSize      = pFalconConfig->ctxBufferSize;
50     pKernelFalcon->addrSpaceList      = pFalconConfig->addrSpaceList;
51 
52     NV_PRINTF(LEVEL_INFO, "for physEngDesc 0x%x\n", pKernelFalcon->physEngDesc);
53 }
54 
55 KernelFalcon *kflcnGetKernelFalconForEngine_IMPL(OBJGPU *pGpu, ENGDESCRIPTOR physEngDesc)
56 {
57     //
58     // Check for any special objects that are instantiated as GPU children.
59     // Otherwise, OBJGPU keeps track of all falcons as reported by GSP
60     //
61     switch (physEngDesc)
62     {
63         // this list is mirrored in subdeviceCtrlCmdInternalGetConstructedFalconInfo_IMPL
64         case ENG_SEC2:     return staticCast(GPU_GET_KERNEL_SEC2(pGpu), KernelFalcon);
65         case ENG_GSP:      return staticCast(GPU_GET_KERNEL_GSP(pGpu), KernelFalcon);
66         default:
67             return staticCast(gpuGetGenericKernelFalconForEngine(pGpu, physEngDesc), KernelFalcon);
68     }
69 }
70 
71 
72 static NvBool _kflcnNeedToAllocContext(OBJGPU *pGpu, KernelChannel *pKernelChannel)
73 {
74     NvU32 gfid = kchannelGetGfid(pKernelChannel);
75 
76     //
77     // In case of vGPU, when client allocated ctx buffer feature enabled, vGPU guest
78     // RM will alloc all FLCN context buffers for VF channels.
79     // But, for PF channels (IS_GFID_PF(gfid) is TRUE), host RM needs to allocate the
80     // FLCN buffers.
81     //
82     if (!gpuIsClientRmAllocatedCtxBufferEnabled(pGpu) || IS_GFID_VF(gfid))
83         return NV_FALSE;
84 
85     return NV_TRUE;
86 }
87 
88 static NV_STATUS _kflcnAllocAndMapCtxBuffer
89 (
90     OBJGPU *pGpu,
91     KernelFalcon *pKernelFalcon,
92     KernelChannel *pKernelChannel
93 )
94 {
95     MEMORY_DESCRIPTOR  *pCtxMemDesc = NULL;
96     CTX_BUF_POOL_INFO  *pCtxBufPool = NULL;
97     KernelChannelGroup *pKernelChannelGroup = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup;
98     OBJGVASPACE        *pGVAS = dynamicCast(pKernelChannel->pVAS, OBJGVASPACE);
99     NvU8               *pInstMem;
100     NV_STATUS           status = NV_OK;
101     NvU64               flags = MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE;
102 
103     if (kchannelIsCtxBufferAllocSkipped(pKernelChannel))
104         return NV_OK;
105 
106     kchangrpGetEngineContextMemDesc(pGpu, pKernelChannelGroup, &pCtxMemDesc);
107     if (pCtxMemDesc != NULL)
108     {
109         NV_PRINTF(LEVEL_ERROR, "This channel already has a falcon engine instance on engine %d:%d\n",
110                   ENGDESC_FIELD(pKernelFalcon->physEngDesc, _CLASS),
111                   ENGDESC_FIELD(pKernelFalcon->physEngDesc, _INST));
112         return NV_OK;
113     }
114 
115     if (ctxBufPoolIsSupported(pGpu) && pKernelChannelGroup->pCtxBufPool != NULL)
116     {
117         flags |= MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL;
118         pCtxBufPool = pKernelChannelGroup->pCtxBufPool;
119     }
120 
121     //
122     // Setup an engine context and initialize.
123     //
124     NV_ASSERT_OK_OR_RETURN(memdescCreate(&pCtxMemDesc, pGpu,
125                pKernelFalcon->ctxBufferSize,
126                FLCN_BLK_ALIGNMENT,
127                NV_TRUE,
128                ADDR_UNKNOWN,
129                pKernelFalcon->ctxAttr,
130                flags));
131     NV_ASSERT_OK_OR_GOTO(status,
132         memdescSetCtxBufPool(pCtxMemDesc, pCtxBufPool),
133         done);
134     NV_ASSERT_OK_OR_GOTO(status,
135         memdescAllocList(pCtxMemDesc, memdescU32ToAddrSpaceList(pKernelFalcon->addrSpaceList)),
136         done);
137 
138     pInstMem = memdescMapInternal(pGpu, pCtxMemDesc, 0);
139     if (pInstMem != NULL)
140     {
141         // Clear the engine context buffer
142         NvU32 i;
143         for (i = 0; i < pKernelFalcon->ctxBufferSize; i += 4)
144         {
145             MEM_WR32(pInstMem + i, 0);
146         }
147         memdescUnmapInternal(pGpu, pCtxMemDesc, 0);
148     }
149     else
150     {
151         status = NV_ERR_INSUFFICIENT_RESOURCES;
152         goto done;
153     }
154 
155     NV_ASSERT_OK_OR_GOTO(status,
156         kchannelSetEngineContextMemDesc(pGpu, pKernelChannel, pKernelFalcon->physEngDesc, pCtxMemDesc),
157         done);
158 
159     if (!gvaspaceIsExternallyOwned(pGVAS))
160     {
161         NV_ASSERT_OK_OR_GOTO(status,
162             kchannelMapEngineCtxBuf(pGpu, pKernelChannel, pKernelFalcon->physEngDesc),
163             done);
164     }
165 
166 done:
167     if (status != NV_OK)
168     {
169         memdescFree(pCtxMemDesc);
170         memdescDestroy(pCtxMemDesc);
171     }
172 
173     return status;
174 }
175 
176 static NV_STATUS _kflcnPromoteContext
177 (
178     OBJGPU *pGpu,
179     KernelFalcon *pKernelFalcon,
180     KernelChannel *pKernelChannel
181 )
182 {
183     RM_API                *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
184     RsClient              *pClient = RES_GET_CLIENT(pKernelChannel);
185     Subdevice             *pSubdevice;
186     NvU64                  addr;
187     RM_ENGINE_TYPE         rmEngineType;
188     ENGINE_CTX_DESCRIPTOR *pEngCtx;
189     NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS rmCtrlParams = {0};
190 
191     NV_ASSERT_OK_OR_RETURN(subdeviceGetByGpu(pClient, pGpu, &pSubdevice));
192     NV_ASSERT_OR_RETURN(gpumgrGetSubDeviceInstanceFromGpu(pGpu) == 0, NV_ERR_INVALID_STATE);
193 
194     pEngCtx = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->ppEngCtxDesc[0];
195     NV_ASSERT_OR_RETURN(pEngCtx != NULL, NV_ERR_INVALID_ARGUMENT);
196     NV_ASSERT_OK_OR_RETURN(vaListFindVa(&pEngCtx->vaList, pKernelChannel->pVAS, &addr));
197 
198     NV_ASSERT_OK_OR_RETURN(kfifoEngineInfoXlate_HAL(pGpu, GPU_GET_KERNEL_FIFO(pGpu),
199                             ENGINE_INFO_TYPE_ENG_DESC, pKernelFalcon->physEngDesc,
200                             ENGINE_INFO_TYPE_RM_ENGINE_TYPE, (NvU32 *)&rmEngineType));
201 
202     rmCtrlParams.hClient     = pClient->hClient;
203     rmCtrlParams.hObject     = RES_GET_HANDLE(pKernelChannel);
204     rmCtrlParams.hChanClient = pClient->hClient;
205     rmCtrlParams.virtAddress = addr;
206     rmCtrlParams.size        = pKernelFalcon->ctxBufferSize;
207     rmCtrlParams.engineType  = gpuGetNv2080EngineType(rmEngineType);
208     rmCtrlParams.ChID        = pKernelChannel->ChID;
209 
210     NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi, pClient->hClient, RES_GET_HANDLE(pSubdevice),
211         NV2080_CTRL_CMD_GPU_PROMOTE_CTX, &rmCtrlParams, sizeof(rmCtrlParams)));
212 
213     return NV_OK;
214 }
215 
216 
217 NV_STATUS kflcnAllocContext_IMPL
218 (
219     OBJGPU        *pGpu,
220     KernelFalcon  *pKernelFalcon,
221     KernelChannel *pKernelChannel,
222     NvU32          classNum
223 )
224 {
225     NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_CHANNEL);
226 
227     if (!_kflcnNeedToAllocContext(pGpu, pKernelChannel))
228         return NV_OK;
229 
230     NV_ASSERT_OR_RETURN(gpuIsClassSupported(pGpu, classNum), NV_ERR_INVALID_OBJECT);
231 
232     NV_ASSERT_OK_OR_RETURN(_kflcnAllocAndMapCtxBuffer(pGpu, pKernelFalcon, pKernelChannel));
233 
234     return _kflcnPromoteContext(pGpu, pKernelFalcon, pKernelChannel);
235 }
236 
237 NV_STATUS kflcnFreeContext_IMPL
238 (
239     OBJGPU        *pGpu,
240     KernelFalcon  *pKernelFalcon,
241     KernelChannel *pKernelChannel,
242     NvU32          classNum
243 )
244 {
245     MEMORY_DESCRIPTOR *pCtxMemDesc = NULL;
246     NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_CHANNEL);
247 
248     if (!_kflcnNeedToAllocContext(pGpu, pKernelChannel))
249         return NV_OK;
250 
251     if (kchannelIsCtxBufferAllocSkipped(pKernelChannel))
252         return NV_OK;
253 
254     kchangrpGetEngineContextMemDesc(pGpu,
255         pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup,
256         &pCtxMemDesc);
257 
258     if (pCtxMemDesc == NULL)
259     {
260         NV_PRINTF(LEVEL_WARNING,
261                   "The channel 0x%x does not have a falcon engine instance for engDesc=0x%x\n",
262                   kchannelGetDebugTag(pKernelChannel), pKernelFalcon->physEngDesc);
263         return NV_OK;
264     }
265 
266     kchannelUnmapEngineCtxBuf(pGpu, pKernelChannel, pKernelFalcon->physEngDesc);
267     kchannelSetEngineContextMemDesc(pGpu, pKernelChannel, pKernelFalcon->physEngDesc, NULL);
268     memdescFree(pCtxMemDesc);
269     memdescDestroy(pCtxMemDesc);
270 
271     return NV_OK;
272 }
273 
274 NV_STATUS gkflcnConstruct_IMPL
275 (
276     GenericKernelFalcon *pGenericKernelFalcon,
277     OBJGPU *pGpu,
278     KernelFalconEngineConfig *pFalconConfig
279 )
280 {
281     KernelFalcon *pKernelFalcon = staticCast(pGenericKernelFalcon, KernelFalcon);
282     if (pFalconConfig != NULL)
283     {
284         kflcnConfigureEngine(pGpu, pKernelFalcon, pFalconConfig);
285     }
286     return NV_OK;
287 }
288 
289 NV_STATUS gkflcnResetHw_IMPL(OBJGPU *pGpu, GenericKernelFalcon *pGenKernFlcn)
290 {
291     NV_ASSERT_FAILED("This should only be called on full KernelFalcon implementations");
292     return NV_ERR_NOT_SUPPORTED;
293 }
294 
295 void gkflcnRegisterIntrService_IMPL(OBJGPU *pGpu, GenericKernelFalcon *pGenericKernelFalcon, IntrServiceRecord pRecords[MC_ENGINE_IDX_MAX])
296 {
297     KernelFalcon *pKernelFalcon = staticCast(pGenericKernelFalcon, KernelFalcon);
298     NV_ASSERT_OR_RETURN_VOID(pKernelFalcon);
299 
300     NV_PRINTF(LEVEL_INFO, "physEngDesc 0x%x\n", pKernelFalcon->physEngDesc);
301 
302     if (!IS_NVDEC(pKernelFalcon->physEngDesc) &&
303         pKernelFalcon->physEngDesc != ENG_OFA &&
304         !IS_NVJPEG(pKernelFalcon->physEngDesc) &&
305         !IS_MSENC(pKernelFalcon->physEngDesc))
306         return;
307 
308     // Register to handle nonstalling interrupts of the corresponding physical falcon in kernel rm
309     if (pKernelFalcon->physEngDesc != ENG_INVALID)
310     {
311         NvU32 mcIdx = MC_ENGINE_IDX_NULL;
312 
313         NV_STATUS status = kfifoEngineInfoXlate_HAL(pGpu, GPU_GET_KERNEL_FIFO(pGpu),
314             ENGINE_INFO_TYPE_ENG_DESC, pKernelFalcon->physEngDesc,
315             ENGINE_INFO_TYPE_MC, &mcIdx);
316 
317         NV_ASSERT_OR_RETURN_VOID(status == NV_OK);
318 
319         NV_PRINTF(LEVEL_INFO, "Registering 0x%x/0x%x to handle nonstall intr\n", pKernelFalcon->physEngDesc, mcIdx);
320 
321         NV_ASSERT(pRecords[mcIdx].pNotificationService == NULL);
322         pRecords[mcIdx].bFifoWaiveNotify = NV_FALSE;
323         pRecords[mcIdx].pNotificationService = staticCast(pGenericKernelFalcon, IntrService);
324     }
325 }
326 
327 NV_STATUS gkflcnServiceNotificationInterrupt_IMPL(OBJGPU *pGpu, GenericKernelFalcon *pGenericKernelFalcon, IntrServiceServiceNotificationInterruptArguments *pParams)
328 {
329     NvU32 idxMc = pParams->engineIdx;
330     RM_ENGINE_TYPE rmEngineType = RM_ENGINE_TYPE_NULL;
331 
332     NV_PRINTF(LEVEL_INFO, "nonstall intr for MC 0x%x\n", idxMc);
333 
334     if (MC_ENGINE_IDX_NVDECn(0) <= idxMc &&
335         idxMc < MC_ENGINE_IDX_NVDECn(RM_ENGINE_TYPE_NVDEC_SIZE))
336     {
337         NvU32 nvdecIdx = idxMc - MC_ENGINE_IDX_NVDECn(0);
338         rmEngineType = RM_ENGINE_TYPE_NVDEC(nvdecIdx);
339     }
340     else if (idxMc == MC_ENGINE_IDX_OFA0)
341     {
342         rmEngineType = RM_ENGINE_TYPE_OFA;
343     }
344     else if (MC_ENGINE_IDX_NVJPEGn(0) <= idxMc &&
345              idxMc < MC_ENGINE_IDX_NVJPEGn(RM_ENGINE_TYPE_NVJPEG_SIZE))
346     {
347         NvU32 nvjpgIdx = idxMc - MC_ENGINE_IDX_NVJPEGn(0);
348         rmEngineType = RM_ENGINE_TYPE_NVJPEG(nvjpgIdx);
349     }
350     else if (MC_ENGINE_IDX_MSENCn(0) <= idxMc &&
351              idxMc < MC_ENGINE_IDX_MSENCn(RM_ENGINE_TYPE_NVENC_SIZE))
352     {
353         NvU32 msencIdx = idxMc - MC_ENGINE_IDX_MSENCn(0);
354         rmEngineType = RM_ENGINE_TYPE_NVENC(msencIdx);
355     }
356 
357     NV_ASSERT_OR_RETURN(rmEngineType != RM_ENGINE_TYPE_NULL, NV_ERR_INVALID_STATE);
358 
359     // Wake up channels waiting on this event
360     engineNonStallIntrNotify(pGpu, rmEngineType);
361 
362     return NV_OK;
363 }
364