1 /*
2  * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3  * SPDX-License-Identifier: MIT
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21  * DEALINGS IN THE SOFTWARE.
22  */
23 
24 #include "gpu/gpu.h"
25 #include "nvoc/prelude.h"
26 #include "nvstatuscodes.h"
27 #include "rmapi/rs_utils.h"
28 #include "gpu/hwpm/profiler_v2.h"
29 #include "ctrl/ctrlb0cc/ctrlb0ccinternal.h"
30 #include "ctrl/ctrlb0cc/ctrlb0ccprofiler.h"
31 #include "mem_mgr/mem.h"
32 #include "vgpu/rpc.h"
33 
34 static NV_STATUS _issueRpcToHost(OBJGPU *pGpu)
35 {
36     NV_STATUS status = NV_OK;
37     CALL_CONTEXT *pCallContext = resservGetTlsCallContext();
38     RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams;
39 
40     NV_RM_RPC_CONTROL(pGpu,
41                       pRmCtrlParams->hClient,
42                       pRmCtrlParams->hObject,
43                       pRmCtrlParams->cmd,
44                       pRmCtrlParams->pParams,
45                       pRmCtrlParams->paramsSize,
46                       status);
47 
48     return status;
49 }
50 
51 NV_STATUS
52 profilerBaseCtrlCmdFreePmaStream_IMPL
53 (
54     ProfilerBase *pProfiler,
55     NVB0CC_CTRL_FREE_PMA_STREAM_PARAMS *pParams
56 )
57 {
58     RM_API            *pRmApi     = GPU_GET_PHYSICAL_RMAPI(GPU_RES_GET_GPU(pProfiler));
59     NVB0CC_CTRL_INTERNAL_FREE_PMA_STREAM_PARAMS internalParams;
60 
61     portMemSet(&internalParams, 0, sizeof(NVB0CC_CTRL_INTERNAL_FREE_PMA_STREAM_PARAMS));
62     internalParams.pmaChannelIdx = pParams->pmaChannelIdx;
63 
64     //
65     // The following security checks are valid only if MemPmaBuffer and MemPmaBytesAvailable
66     // buffers are accessible and are set during PMA alloc
67     // Those buffers are not accessible on vGpu host (accessible on vGPU Guest and baremetal)
68     //
69     if (pProfiler->ppBytesAvailable != NULL && pProfiler->ppStreamBuffers != NULL)
70     {
71         RsResourceRef     *pCountRef = NULL;
72         RsResourceRef     *pBufferRef = NULL;
73 
74         if (pProfiler->maxPmaChannels <= pParams->pmaChannelIdx)
75         {
76             goto err;
77         }
78 
79         pCountRef = pProfiler->ppBytesAvailable[pParams->pmaChannelIdx];
80         pProfiler->ppBytesAvailable[pParams->pmaChannelIdx] = NULL;
81         pBufferRef = pProfiler->ppStreamBuffers[pParams->pmaChannelIdx];
82         pProfiler->ppStreamBuffers[pParams->pmaChannelIdx] = NULL;
83 
84         if(pProfiler->pBoundCntBuf == pCountRef && pProfiler->pBoundPmaBuf == pBufferRef)
85         {
86             Memory *pCntMem = dynamicCast(pCountRef->pResource, Memory);
87             Memory *pBufMem = dynamicCast(pBufferRef->pResource, Memory);
88             pProfiler->pBoundCntBuf = NULL;
89             pProfiler->pBoundPmaBuf = NULL;
90             pCntMem->pMemDesc->bRmExclusiveUse = NV_FALSE;
91             pBufMem->pMemDesc->bRmExclusiveUse = NV_FALSE;
92 
93         }
94         if (pCountRef != NULL)
95         {
96             refRemoveDependant(pCountRef, RES_GET_REF(pProfiler));
97         }
98         if (pBufferRef != NULL)
99         {
100             refRemoveDependant(pBufferRef, RES_GET_REF(pProfiler));
101         }
102     }
103 err:
104 
105     return pRmApi->Control(pRmApi,
106                            RES_GET_CLIENT_HANDLE(pProfiler),
107                            RES_GET_HANDLE(pProfiler),
108                            NVB0CC_CTRL_CMD_INTERNAL_FREE_PMA_STREAM,
109                            &internalParams, sizeof(internalParams));
110 }
111 
112 NV_STATUS
113 profilerBaseCtrlCmdBindPmResources_IMPL
114 (
115     ProfilerBase *pProfiler
116 )
117 {
118     OBJGPU        *pGpu                       = GPU_RES_GET_GPU(pProfiler);
119     RM_API        *pRmApi                     = GPU_GET_PHYSICAL_RMAPI(pGpu);
120     NvHandle       hClient                    = RES_GET_CLIENT_HANDLE(pProfiler);
121     NvHandle       hObject                    = RES_GET_HANDLE(pProfiler);
122     NV_STATUS      status                     = NV_OK;
123     Memory        *pCntMem                    = NULL;
124     Memory        *pBufMem                    = NULL;
125 
126     //
127     // The following security checks are valid only if MemPmaBuffer and MemPmaBytesAvailable
128     // buffers are accessible and are set during PMA alloc
129     // Those buffers are not accessible on vGpu host (accessible on vGPU Guest and baremetal)
130     //
131     if (pProfiler->ppBytesAvailable != NULL && pProfiler->ppStreamBuffers != NULL)
132     {
133         RsResourceRef *pCntRef                    = NULL;
134         RsResourceRef *pBufRef                    = NULL;
135 
136         NV_CHECK_OR_GOTO(LEVEL_INFO,
137             !pProfiler->bLegacyHwpm && pProfiler->maxPmaChannels != 0, physical_control);
138 
139         if (pProfiler->maxPmaChannels <= pProfiler->pmaVchIdx)
140         {
141             return NV_ERR_INVALID_ARGUMENT;
142         }
143 
144         pCntRef = pProfiler->ppBytesAvailable[pProfiler->pmaVchIdx];
145         pBufRef = pProfiler->ppStreamBuffers[pProfiler->pmaVchIdx];
146 
147         NV_CHECK_OR_GOTO(LEVEL_INFO,
148             pCntRef != NULL && pBufRef != NULL, physical_control);
149 
150         pCntMem = dynamicCast(pCntRef->pResource, Memory);
151         pBufMem = dynamicCast(pBufRef->pResource, Memory);
152 
153         NV_ASSERT_OR_RETURN(pCntMem != NULL && pBufMem != NULL, NV_ERR_INVALID_STATE);
154 
155         if (!memdescAcquireRmExclusiveUse(pCntMem->pMemDesc) ||
156             !memdescAcquireRmExclusiveUse(pBufMem->pMemDesc))
157         {
158             pCntMem->pMemDesc->bRmExclusiveUse = NV_FALSE;
159             pBufMem->pMemDesc->bRmExclusiveUse = NV_FALSE;
160             return NV_ERR_INVALID_ARGUMENT;
161         }
162 
163         pProfiler->pBoundCntBuf = pCntRef;
164         pProfiler->pBoundPmaBuf = pBufRef;
165     }
166 
167 physical_control:
168 
169     if (IS_VIRTUAL(pGpu))
170     {
171         return _issueRpcToHost(pGpu);
172     }
173 
174     status = pRmApi->Control(pRmApi, hClient, hObject,
175                              NVB0CC_CTRL_CMD_INTERNAL_BIND_PM_RESOURCES,
176                              NULL, 0);
177     if (status != NV_OK && pCntMem != NULL && pBufMem != NULL)
178     {
179         pCntMem->pMemDesc->bRmExclusiveUse = NV_FALSE;
180         pBufMem->pMemDesc->bRmExclusiveUse = NV_FALSE;
181         pProfiler->pBoundCntBuf = NULL;
182         pProfiler->pBoundPmaBuf = NULL;
183     }
184     return status;
185 }
186 
187 NV_STATUS
188 profilerBaseCtrlCmdUnbindPmResources_IMPL
189 (
190     ProfilerBase *pProfiler
191 )
192 {
193     OBJGPU   *pGpu                            = GPU_RES_GET_GPU(pProfiler);
194     RM_API   *pRmApi                          = GPU_GET_PHYSICAL_RMAPI(pGpu);
195     NvHandle  hClient                         = RES_GET_CLIENT_HANDLE(pProfiler);
196     NvHandle  hObject                         = RES_GET_HANDLE(pProfiler);
197     RsResourceRef *pCntRef                    = NULL;
198     RsResourceRef *pBufRef                    = NULL;
199 
200     pCntRef = pProfiler->pBoundCntBuf;
201     pBufRef = pProfiler->pBoundPmaBuf;
202 
203     if (pCntRef != NULL)
204     {
205         Memory *pCntMem = dynamicCast(pCntRef->pResource, Memory);
206         if (pCntMem != NULL)
207         {
208             pCntMem->pMemDesc->bRmExclusiveUse = NV_FALSE;
209         }
210         pProfiler->pBoundCntBuf = NULL;
211     }
212 
213     if (pBufRef != NULL)
214     {
215         Memory *pBufMem = dynamicCast(pBufRef->pResource, Memory);
216         if (pBufMem != NULL)
217         {
218             pBufMem->pMemDesc->bRmExclusiveUse = NV_FALSE;
219         }
220         pProfiler->pBoundPmaBuf = NULL;
221     }
222 
223     if (IS_VIRTUAL(pGpu))
224     {
225         return _issueRpcToHost(pGpu);
226     }
227 
228     return pRmApi->Control(pRmApi, hClient, hObject,
229                            NVB0CC_CTRL_CMD_INTERNAL_UNBIND_PM_RESOURCES,
230                            NULL, 0);
231 }
232 
233 NV_STATUS
234 profilerBaseCtrlCmdReserveHwpmLegacy_IMPL
235 (
236     ProfilerBase *pProfiler,
237     NVB0CC_CTRL_RESERVE_HWPM_LEGACY_PARAMS *pParams
238 )
239 {
240     OBJGPU   *pGpu                            = GPU_RES_GET_GPU(pProfiler);
241     RM_API   *pRmApi                          = GPU_GET_PHYSICAL_RMAPI(pGpu);
242     NvHandle  hClient                         = RES_GET_CLIENT_HANDLE(pProfiler);
243     NvHandle  hObject                         = RES_GET_HANDLE(pProfiler);
244 
245     pProfiler->bLegacyHwpm = NV_TRUE;
246     return pRmApi->Control(pRmApi, hClient, hObject,
247                            NVB0CC_CTRL_CMD_INTERNAL_RESERVE_HWPM_LEGACY,
248                            pParams, sizeof(*pParams));
249 }
250 
251 NV_STATUS
252 profilerBaseCtrlCmdAllocPmaStream_IMPL
253 (
254     ProfilerBase *pProfiler,
255     NVB0CC_CTRL_ALLOC_PMA_STREAM_PARAMS *pParams
256 )
257 {
258     NV_STATUS status                          = NV_OK;
259     OBJGPU   *pGpu                            = GPU_RES_GET_GPU(pProfiler);
260     RM_API   *pRmApi                          = GPU_GET_PHYSICAL_RMAPI(pGpu);
261     NvHandle  hClient                         = RES_GET_CLIENT_HANDLE(pProfiler);
262     NvHandle  hParent                         = RES_GET_PARENT_HANDLE(pProfiler);
263     NvHandle  hObject                         = RES_GET_HANDLE(pProfiler);
264     NvBool    bMemPmaBufferRegistered         = NV_FALSE;
265     NvBool    bMemPmaBytesAvailableRegistered = NV_FALSE;
266     NVB0CC_CTRL_INTERNAL_ALLOC_PMA_STREAM_PARAMS internalParams;
267     RsResourceRef     *pMemoryRef = NULL;
268     //
269     // REGISTER  MEMDESCs TO GSP
270     // These are no-op with BareMetal/No GSP
271     //
272     NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR,
273                         memdescRegisterToGSP(pGpu, hClient, hParent, pParams->hMemPmaBuffer),
274                         fail);
275     bMemPmaBufferRegistered = NV_TRUE;
276 
277     NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR,
278                         memdescRegisterToGSP(pGpu, hClient, hParent, pParams->hMemPmaBytesAvailable),
279                         fail);
280     bMemPmaBytesAvailableRegistered = NV_TRUE;
281 
282     portMemSet(&internalParams, 0, sizeof(NVB0CC_CTRL_INTERNAL_ALLOC_PMA_STREAM_PARAMS));
283     internalParams.hMemPmaBuffer = pParams->hMemPmaBuffer;
284     internalParams.pmaBufferOffset = pParams->pmaBufferOffset;
285     internalParams.pmaBufferSize = pParams->pmaBufferSize;
286     internalParams.hMemPmaBytesAvailable = pParams->hMemPmaBytesAvailable;
287     internalParams.pmaBytesAvailableOffset = pParams->pmaBytesAvailableOffset;
288     internalParams.ctxsw = pParams->ctxsw;
289     internalParams.pmaChannelIdx = pParams->pmaChannelIdx;
290     internalParams.pmaBufferVA = pParams->pmaBufferVA;
291     internalParams.bInputPmaChIdx = NV_FALSE;
292 
293      NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR,
294         pRmApi->Control(pRmApi,
295                         hClient,
296                         hObject,
297                         NVB0CC_CTRL_CMD_INTERNAL_ALLOC_PMA_STREAM,
298                         &internalParams, sizeof(internalParams)), fail);
299 
300     pParams->pmaChannelIdx = internalParams.pmaChannelIdx;
301 
302     if (serverutilGetResourceRef(hClient, pParams->hMemPmaBytesAvailable, &pMemoryRef) == NV_OK &&
303         serverutilGetResourceRef(hClient, pParams->hMemPmaBuffer, &pMemoryRef) == NV_OK)
304     {
305         if (pProfiler->ppBytesAvailable == NULL)
306         {
307             NVB0CC_CTRL_INTERNAL_GET_MAX_PMAS_PARAMS maxPmaParams;
308             portMemSet(&maxPmaParams, 0, sizeof(NVB0CC_CTRL_INTERNAL_GET_MAX_PMAS_PARAMS));
309             NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR,
310                 pRmApi->Control(pRmApi, hClient, hObject,
311                     NVB0CC_CTRL_CMD_INTERNAL_GET_MAX_PMAS,
312                     &maxPmaParams, sizeof(maxPmaParams)), fail);
313 
314             pProfiler->maxPmaChannels = maxPmaParams.maxPmaChannels;
315             pProfiler->ppBytesAvailable = (RsResourceRef**)portMemAllocNonPaged(maxPmaParams.maxPmaChannels * sizeof(RsResourceRef*));
316             pProfiler->ppStreamBuffers = (RsResourceRef**)portMemAllocNonPaged(maxPmaParams.maxPmaChannels * sizeof(RsResourceRef*));
317         }
318 
319         NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR,
320             serverutilGetResourceRef(hClient, pParams->hMemPmaBytesAvailable, &pMemoryRef), fail);
321         pProfiler->ppBytesAvailable[pParams->pmaChannelIdx] = pMemoryRef;
322         refAddDependant(pMemoryRef, RES_GET_REF(pProfiler));
323 
324         NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR,
325             serverutilGetResourceRef(hClient, pParams->hMemPmaBuffer, &pMemoryRef), fail);
326         pProfiler->ppStreamBuffers[pParams->pmaChannelIdx] = pMemoryRef;
327         refAddDependant(pMemoryRef, RES_GET_REF(pProfiler));
328 
329         // Copy output params to external struct.
330         pProfiler->pmaVchIdx = pParams->pmaChannelIdx;
331         pProfiler->bLegacyHwpm = NV_FALSE;
332     }
333 
334     // Copy output params to external struct.
335     pParams->pmaBufferVA = internalParams.pmaBufferVA;
336 
337     return status;
338 
339 fail:
340     if (bMemPmaBufferRegistered)
341     {
342         // These are no-op with BareMetal/No GSP
343         NV_ASSERT_OK(memdescDeregisterFromGSP(pGpu, hClient, hParent, pParams->hMemPmaBuffer));
344     }
345 
346     if (bMemPmaBytesAvailableRegistered)
347     {
348         // These are no-op with BareMetal/No GSP
349         NV_ASSERT_OK(memdescDeregisterFromGSP(pGpu, hClient, hParent, pParams->hMemPmaBytesAvailable));
350     }
351 
352     return status;
353 }
354 
355