1 /*
2  * SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3  * SPDX-License-Identifier: MIT
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21  * DEALINGS IN THE SOFTWARE.
22  */
23 
24 // FIXME XXX
25 #define NVOC_KERNEL_GRAPHICS_MANAGER_H_PRIVATE_ACCESS_ALLOWED
26 
27 #include "kernel/gpu/fifo/kernel_fifo.h"
28 #include "kernel/gpu/fifo/kernel_channel_group_api.h"
29 #include "kernel/gpu/fifo/kernel_channel_group.h"
30 #include "kernel/gpu/ce/kernel_ce_shared.h"
31 #include "kernel/gpu/mig_mgr/kernel_mig_manager.h"
32 #include "kernel/gpu/bus/kern_bus.h"
33 
34 #include "vgpu/vgpu_events.h"
35 
36 #include "published/ampere/ga100/dev_ram.h"
37 #include "published/ampere/ga100/dev_ctrl.h"
38 
39 
40 NV_STATUS
41 kfifoEngineInfoXlate_GA100
42 (
43     OBJGPU          *pGpu,
44     KernelFifo      *pKernelFifo,
45     ENGINE_INFO_TYPE inType,
46     NvU32            inVal,
47     ENGINE_INFO_TYPE outType,
48     NvU32           *pOutVal
49 )
50 {
51     KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu);
52 
53     // We no longer store ENGINE_INFO_TYPE_INTR on Ampere+ (bug 24110055)
54     if (inType == ENGINE_INFO_TYPE_INTR || outType == ENGINE_INFO_TYPE_INTR)
55     {
56         return NV_ERR_NOT_SUPPORTED;
57     }
58 
59     //
60     // We need extra logic for translation when SMC is enabled and input or output is
61     // MMU_FAULT_ID because device Info cannot translate MMU_FAULT_ID to/from any type for GR > 0
62     //
63     if (IS_MIG_IN_USE(pGpu))
64     {
65         NvU32 baseGrFaultId;
66         NV_ASSERT_OK_OR_RETURN(kfifoEngineInfoXlate_GV100(pGpu, pKernelFifo,
67                                                           ENGINE_INFO_TYPE_ENG_DESC, ENG_GR(0),
68                                                           ENGINE_INFO_TYPE_MMU_FAULT_ID, &baseGrFaultId));
69 
70         if (inType == ENGINE_INFO_TYPE_MMU_FAULT_ID)
71         {
72             NvU32 subctxId, grIdx;
73             NvU32 maxSubctx = kfifoGetMaxSubcontext_HAL(pGpu, pKernelFifo, NV_FALSE);
74 
75             // check if input fault ID corresponds to GR
76             if ((inVal >= baseGrFaultId) && (inVal < (baseGrFaultId + maxSubctx)))
77             {
78                 subctxId = inVal - baseGrFaultId;
79                 NV_ASSERT_OK_OR_RETURN(kgrmgrGetGrIdxForVeid(pGpu, pKernelGraphicsManager, subctxId, &grIdx));
80                 inVal = RM_ENGINE_TYPE_GR(grIdx);
81                 inType = ENGINE_INFO_TYPE_RM_ENGINE_TYPE;
82             }
83         }
84 
85         if (outType == ENGINE_INFO_TYPE_MMU_FAULT_ID)
86         {
87             NvU32 grIdx, startSubctxId;
88             NV_STATUS status;
89             RM_ENGINE_TYPE rmEngineType;
90 
91             status = kfifoEngineInfoXlate_GV100(pGpu, pKernelFifo, inType, inVal,
92                                                 ENGINE_INFO_TYPE_RM_ENGINE_TYPE, (NvU32 *)&rmEngineType);
93             if (status != NV_OK)
94                 return status;
95 
96             // check if rmEngineType corresponding to input is GR
97             if (RM_ENGINE_TYPE_IS_GR(rmEngineType))
98             {
99                 grIdx = RM_ENGINE_TYPE_GR_IDX(rmEngineType);
100                 NV_ASSERT_OK_OR_RETURN(kgrmgrGetVeidBaseForGrIdx(pGpu, pKernelGraphicsManager, grIdx, &startSubctxId));
101                 *pOutVal = baseGrFaultId + startSubctxId;
102                 return NV_OK;
103             }
104         }
105     }
106 
107     return kfifoEngineInfoXlate_GV100(pGpu, pKernelFifo, inType, inVal, outType, pOutVal);
108 }
109 
110 
111 /*!
112  * @brief Get the local maximum number of subctx allowed in this TSG
113  *
114  * @param      pGpu
115  * @param      pKernelFifo
116  * @param[in]  pKernelChannelGroup
117  * @param[in]  bLegacyMode          Is TSG in legacy mode.
118  */
119 NvU32
120 kfifoChannelGroupGetLocalMaxSubcontext_GA100
121 (
122     OBJGPU             *pGpu,
123     KernelFifo         *pKernelFifo,
124     KernelChannelGroup *pKernelChannelGroup,
125     NvBool              bLegacyMode
126 )
127 {
128     KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu);
129 
130     NV_ASSERT_OR_RETURN(pKernelChannelGroup != NULL, NV_ERR_INVALID_ARGUMENT);
131 
132     if (IS_MIG_IN_USE(pGpu) && !bLegacyMode &&
133         RM_ENGINE_TYPE_IS_GR(pKernelChannelGroup->engineType))
134     {
135         NvU32 grIdx = RM_ENGINE_TYPE_GR_IDX(pKernelChannelGroup->engineType);
136         return nvPopCount64(pKernelGraphicsManager->grIdxVeidMask[grIdx]);
137     }
138 
139     // In SMC-Legacy mode, revert to pre-Ampere behavior
140     return kfifoChannelGroupGetLocalMaxSubcontext_GM107(pGpu, pKernelFifo,
141                                                         pKernelChannelGroup,
142                                                         bLegacyMode);
143 }
144 
145 /*!
146  * @brief Update the usermode doorbell register with work submit token to notify
147  *        host that work is available on this channel.
148  *
149  * @param[in] pGpu
150  * @param[in] pKernelFifo
151  * @param[in] workSubmitToken Token to update the doorbell with
152  * @param[in] runlistId       Runlist ID
153  */
154 NV_STATUS
155 kfifoUpdateUsermodeDoorbell_GA100
156 (
157     OBJGPU     *pGpu,
158     KernelFifo *pKernelFifo,
159     NvU32       workSubmitToken,
160     NvU32       runlistId
161 )
162 {
163     //
164     // Updating the usermode doorbell is different for CPU vs. GSP.
165     //
166     if (!RMCFG_FEATURE_PLATFORM_GSP)
167     {
168         return kfifoUpdateUsermodeDoorbell_TU102(pGpu, pKernelFifo, workSubmitToken, runlistId);
169     }
170     else
171     {
172         return kfifoUpdateInternalDoorbellForUsermode_HAL(pGpu, pKernelFifo, workSubmitToken, runlistId);
173     }
174 
175     return NV_OK;
176 }
177 
178 /*!
179  * @brief Construct the worksubmit token. Caller cannot make assumption about this handle.
180  *
181  * @param[in]  pGpu
182  * @param[in]  pKernelFifo
183  * @param[in]  pKernelChannel
184  * @param[out] pGeneratedToken Store the generated token
185  * @param[in]  bUsedForHost    Used on Host RM
186  *
187  */
188 NV_STATUS
189 kfifoGenerateWorkSubmitToken_GA100
190 (
191     OBJGPU        *pGpu,
192     KernelFifo    *pKernelFifo,
193     KernelChannel *pKernelChannel,
194     NvU32         *pGeneratedToken,
195     NvBool         bUsedForHost
196 )
197 {
198     NvU32          chId;
199     NvU32          gfId;
200     NvU32          runlistId;
201     NvU32          val = 0;
202 
203     NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_CHANNEL);
204 
205     NV_ASSERT_OR_RETURN(pGeneratedToken != NULL, NV_ERR_INVALID_OBJECT);
206     NV_ASSERT_OR_RETURN((pKernelChannel->pKernelChannelGroupApi != NULL) &&
207                         (pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup != NULL),
208                         NV_ERR_INVALID_STATE);
209 
210     chId = pKernelChannel->ChID;
211 
212     NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &gfId));
213 
214     if (!RMCFG_FEATURE_PLATFORM_GSP || (IS_VGPU_GSP_PLUGIN_OFFLOAD_ENABLED(pGpu) && IS_GFID_VF(gfId)))
215     {
216 
217         // TODO: Remove check on Ampere. Bug 200606706.
218         if (!bUsedForHost && IS_GFID_VF(gfId))
219         {
220             NvU32 vChId;
221 
222             NV_ASSERT_OK_OR_RETURN(kfifoGetVChIdForSChId_HAL(pGpu, pKernelFifo,
223                                                              chId, gfId,
224                                                              kchannelGetEngineType(pKernelChannel),
225                                                              &vChId));
226             chId = vChId;
227         }
228 
229         // TODO: Remove, on Ampere channels should be set to a valid runlist before allocation. Bug 200606706.
230         if (!kchannelIsRunlistSet(pGpu, pKernelChannel))
231         {
232             NV_PRINTF(LEVEL_NOTICE,
233                       "FAILED Channel 0x%x is not assigned to runlist yet\n",
234                       chId);
235             return NV_ERR_INVALID_STATE;
236         }
237 
238         runlistId = kchannelGetRunlistId(pKernelChannel);
239 
240         // Here we construct token to be a concatenation of runlist id and channel id
241         val = FLD_SET_DRF_NUM(_CTRL, _VF_DOORBELL, _RUNLIST_ID, runlistId, val);
242         val = FLD_SET_DRF_NUM(_CTRL, _VF_DOORBELL, _VECTOR,     chId,      val);
243 
244         NV_PRINTF(LEVEL_INFO,
245                   "Generated workSubmitToken 0x%x for channel 0x%x runlist 0x%x\n",
246                   val, chId, runlistId);
247     }
248     else // RMCFG_FEATURE_PLATFORM_GSP
249     {
250         NV_ASSERT_OK_OR_RETURN(kfifoGenerateInternalWorkSubmitToken_HAL(pGpu, pKernelFifo, pKernelChannel));
251     }
252 
253     *pGeneratedToken = val;
254 
255     return NV_OK;
256 }
257 
258 /**
259  * @brief Get the runlist base shift amount
260  *
261  * @param pKernelFifo
262  *
263  * @return shift amount
264  */
265 NvU32
266 kfifoRunlistGetBaseShift_GA100
267 (
268     KernelFifo *pKernelFifo
269 )
270 {
271     return NV_RAMRL_ENTRY_BASE_SHIFT;
272 }
273 
274 /*!
275  * Special function to be used early when the CHID_MGRs aren't and cannot be
276  * constructed in all cases. Do not use otherwise
277  */
278 NvU32
279 kfifoGetMaxCeChannelGroups_GA100
280 (
281     OBJGPU     *pGpu,
282     KernelFifo *pKernelFifo
283 )
284 {
285     ENGDESCRIPTOR eng = 0;
286     NvU32 deviceIndex;
287     const ENGINE_INFO *pEngineInfo = kfifoGetEngineInfo(pKernelFifo);
288     NvU32 maxCeChannels = 0;
289 
290     // If called before kfifoConstructEngineList has executed
291     if (pEngineInfo == NULL)
292         return 0;
293 
294     //
295     // We can't use chidMgr here as this gets called before chidMgr is completely initialized
296     // Use device info table instead
297     //
298     for (deviceIndex = 0; deviceIndex < pEngineInfo->engineInfoListSize; deviceIndex++)
299     {
300         eng = pEngineInfo->engineInfoList[deviceIndex].engineData[ENGINE_INFO_TYPE_ENG_DESC];
301 
302         // All GR CE use the same pool as GR
303         if ((eng == ENG_GR(0)) ||
304             (IS_CE(eng) &&
305              (!ceIsCeGrce(pGpu, pEngineInfo->engineInfoList[deviceIndex].engineData[ENGINE_INFO_TYPE_RM_ENGINE_TYPE]))))
306         {
307             maxCeChannels += kfifoRunlistQueryNumChannels_HAL(pGpu, pKernelFifo, 0);
308         }
309     }
310 
311     // override max channels if we can run out of BAR2 page tables
312     if (kbusIsBug2751296LimitBar2PtSize(GPU_GET_KERNEL_BUS(pGpu)))
313     {
314         // 2k for GR CE and 2k for the rest
315         maxCeChannels = 4096;
316     }
317 
318     return maxCeChannels;
319 }
320