1 /*
2  * SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3  * SPDX-License-Identifier: MIT
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21  * DEALINGS IN THE SOFTWARE.
22  */
23 
24 #include "kernel/gpu/fifo/kernel_fifo.h"
25 #include "kernel/gpu/fifo/kernel_channel.h"
26 
27 #include "gpu/gpu_access.h"
28 #include "gpu/gpu.h"
29 #include "vgpu/vgpu_events.h"
30 
31 #include "published/turing/tu102/dev_fault.h"
32 #include "published/turing/tu102/dev_vm.h"
33 #include "published/turing/tu102/dev_ctrl.h"
34 #include "published/turing/tu102/hwproject.h"
35 
36 /*!
37  * @brief Update the usermode doorbell register with work submit token to notify
38  *        host that work is available on this channel.
39  *
40  * @param[in] pGpu
41  * @param[in] pKernelFifo
42  * @param[in] workSubmitToken Token to update the doorbell with
43  * @param[in] runlistId       Runlist ID
44  */
45 NV_STATUS
kfifoUpdateUsermodeDoorbell_TU102(OBJGPU * pGpu,KernelFifo * pKernelFifo,NvU32 workSubmitToken,NvU32 runlistId)46 kfifoUpdateUsermodeDoorbell_TU102
47 (
48     OBJGPU     *pGpu,
49     KernelFifo *pKernelFifo,
50     NvU32       workSubmitToken,
51     NvU32       runlistId
52 )
53 {
54     NV_PRINTF(LEVEL_INFO, "Poking workSubmitToken 0x%x\n", workSubmitToken);
55 
56     GPU_VREG_WR32(pGpu, NV_VIRTUAL_FUNCTION_DOORBELL, workSubmitToken);
57 
58     return NV_OK;
59 }
60 
61 /*!
62  * @brief Construct the worksubmit token. Caller cannot make assumption about this handle.
63  *
64  * @param[in]  pGpu
65  * @param[in]  pKernelFifo
66  * @param[in]  pKernelChannel
67  * @param[out] pGeneratedToken Store the generated token
68  * @param[in]  bUsedForHost    Used on Host RM
69  *
70  */
71 NV_STATUS
kfifoGenerateWorkSubmitToken_TU102(OBJGPU * pGpu,KernelFifo * pKernelFifo,KernelChannel * pKernelChannel,NvU32 * pGeneratedToken,NvBool bUsedForHost)72 kfifoGenerateWorkSubmitToken_TU102
73 (
74     OBJGPU        *pGpu,
75     KernelFifo    *pKernelFifo,
76     KernelChannel *pKernelChannel,
77     NvU32         *pGeneratedToken,
78     NvBool         bUsedForHost
79 )
80 {
81     NvU32          chId;
82     NvU32          gfid;
83     NvU32          val = 0;
84 
85     NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_CHANNEL);
86 
87     NV_ASSERT_OR_RETURN(pGeneratedToken != NULL, NV_ERR_INVALID_ARGUMENT);
88 
89     chId = pKernelChannel->ChID;
90 
91     NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &gfid));
92 
93     //
94     // In case of vGPU with SR-IOV, host RM is currently generating token using
95     // virtual chid that was allocated inside the guest. This needs to change
96     // once the guest starts managing its own channels. The guest would then
97     // generate its own tokens.
98     //
99     if (!bUsedForHost && IS_GFID_VF(gfid))
100     {
101         NvU32 vChId;
102 
103         NV_ASSERT_OK_OR_RETURN(kfifoGetVChIdForSChId_HAL(pGpu, pKernelFifo,
104                                                          chId, gfid,
105                                                          kchannelGetEngineType(pKernelChannel),
106                                                          &vChId));
107         chId = vChId;
108     }
109 
110     if (!kchannelIsRunlistSet(pGpu, pKernelChannel))
111     {
112         NV_PRINTF(LEVEL_ERROR,
113                   "FAILED Channel 0x%x is not assigned to runlist yet\n",
114                   kchannelGetDebugTag(pKernelChannel));
115         return NV_ERR_INVALID_STATE;
116     }
117 
118     // Here we construct token to be a concatenation of runlist id and channel id
119     val = FLD_SET_DRF_NUM(_CTRL, _VF_DOORBELL, _RUNLIST_ID, kchannelGetRunlistId(pKernelChannel), val);
120     val = FLD_SET_DRF_NUM(_CTRL, _VF_DOORBELL, _VECTOR, chId, val);
121     *pGeneratedToken = val;
122 
123     NV_PRINTF(LEVEL_INFO,
124               "Generated workSubmitToken 0x%x for channel 0x%x runlist 0x%x\n",
125               *pGeneratedToken, chId, kchannelGetRunlistId(pKernelChannel));
126 
127     return NV_OK;
128 }
129 
130 /**
131 * @brief Convert PBDMA ID to string
132 * @param[in] pGpu
133 * @param[in] pKernelFifo
134 * @param[in] pbdmaId
135 *
136 * @return cont string
137 */
138 const char*
kfifoPrintPbdmaId_TU102(OBJGPU * pGpu,KernelFifo * pKernelFifo,NvU32 pbdmaId)139 kfifoPrintPbdmaId_TU102
140 (
141     OBJGPU     *pGpu,
142     KernelFifo *pKernelFifo,
143     NvU32       pbdmaId
144 )
145 {
146     NV_ASSERT_OR_RETURN(pbdmaId < NV_HOST_NUM_PBDMA, "UNKNOWN");
147     static const char* pbdmaIdString[NV_HOST_NUM_PBDMA] = { "HOST0",
148                                                             "HOST1",
149                                                             "HOST2",
150                                                             "HOST3",
151                                                             "HOST4",
152                                                             "HOST5",
153                                                             "HOST6",
154                                                             "HOST7",
155                                                             "HOST8",
156                                                             "HOST9",
157                                                             "HOST10",
158                                                             "HOST11",
159 #if NV_HOST_NUM_PBDMA > 12
160 #error Uninitialized elements of static array.
161 #endif
162                                                           };
163 
164     return pbdmaIdString[pbdmaId];
165 }
166 
167 /*!
168  * @brief Converts a mmu engine id (NV_PFAULT_MMU_ENG_ID_*) into a string.
169  *
170  * @param[in] pGpu
171  * @param[in] pKernelFifo
172  * @param[in] engineID NV_PFAULT_MMU_ENG_ID_*
173  *
174  * @returns a string (always non-null)
175  */
176 const char*
kfifoPrintInternalEngine_TU102(OBJGPU * pGpu,KernelFifo * pKernelFifo,NvU32 engineID)177 kfifoPrintInternalEngine_TU102
178 (
179     OBJGPU     *pGpu,
180     KernelFifo *pKernelFifo,
181     NvU32       engineID
182 )
183 {
184     KernelGmmu *pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu);
185     NvU32 pbdmaId;
186 
187     if (kfifoIsMmuFaultEngineIdPbdma(pGpu, pKernelFifo, engineID))
188     {
189         NV_ASSERT_OR_RETURN(kfifoGetPbdmaIdFromMmuFaultId(pGpu, pKernelFifo, engineID, &pbdmaId) == NV_OK, "UNKNOWN");
190         return kfifoPrintPbdmaId_HAL(pGpu, pKernelFifo, pbdmaId);
191     }
192 
193     if (kgmmuIsFaultEngineBar1_HAL(pKernelGmmu, engineID))
194     {
195         return "BAR1";
196     }
197     else if (kgmmuIsFaultEngineBar2_HAL(pKernelGmmu, engineID))
198     {
199         return "BAR2";
200     }
201     else
202     {
203         switch (engineID)
204         {
205             case NV_PFAULT_MMU_ENG_ID_DISPLAY:
206                 return "DISPLAY";
207             case NV_PFAULT_MMU_ENG_ID_IFB:
208                 return "IFB";
209             case NV_PFAULT_MMU_ENG_ID_SEC:
210                 return "SEC";
211             case NV_PFAULT_MMU_ENG_ID_PERF:
212                 return "PERF";
213             case NV_PFAULT_MMU_ENG_ID_NVDEC0:
214                 return "NVDEC0";
215             case NV_PFAULT_MMU_ENG_ID_NVDEC1:
216                 return "NVDEC1";
217             case NV_PFAULT_MMU_ENG_ID_NVDEC2:
218                 return "NVDEC2";
219             case NV_PFAULT_MMU_ENG_ID_CE0:
220                 return "CE0";
221             case NV_PFAULT_MMU_ENG_ID_CE1:
222                 return "CE1";
223             case NV_PFAULT_MMU_ENG_ID_CE2:
224                 return "CE2";
225             case NV_PFAULT_MMU_ENG_ID_CE3:
226                 return "CE3";
227             case NV_PFAULT_MMU_ENG_ID_CE4:
228                 return "CE4";
229             case NV_PFAULT_MMU_ENG_ID_CE5:
230                 return "CE5";
231             case NV_PFAULT_MMU_ENG_ID_CE6:
232                 return "CE6";
233             case NV_PFAULT_MMU_ENG_ID_CE7:
234                 return "CE7";
235             case NV_PFAULT_MMU_ENG_ID_CE8:
236                 return "CE8";
237             case NV_PFAULT_MMU_ENG_ID_PWR_PMU:
238                 return "PMU";
239             case NV_PFAULT_MMU_ENG_ID_PTP:
240                 return "PTP";
241             case NV_PFAULT_MMU_ENG_ID_NVENC0:
242                 return "NVENC0";
243             case NV_PFAULT_MMU_ENG_ID_NVENC1:
244                 return "NVENC1";
245             case NV_PFAULT_MMU_ENG_ID_PHYSICAL:
246                 return "PHYSICAL";
247             case NV_PFAULT_MMU_ENG_ID_NVJPG0:
248                 return "NVJPG";
249             default:
250             {
251                 NV_STATUS status = NV_OK;
252                 NvU32     engTag;
253 
254                 status = kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo, ENGINE_INFO_TYPE_MMU_FAULT_ID,
255                     engineID, ENGINE_INFO_TYPE_ENG_DESC, &engTag);
256                 if ((status == NV_OK) && (IS_GR(engTag)))
257                 {
258                     return "GRAPHICS";
259                 }
260 
261             }
262         }
263     }
264 
265     return "UNKNOWN";
266 }
267 
268 /*!
269  * @brief Converts a subid/clientid into a client string
270  *
271  * @param[in] pGpu
272  * @param[in] pKernelFifo
273  * @param[in] pMmuExceptData
274 
275  * @returns a string (always non-null)
276  */
277 const char*
kfifoGetClientIdString_TU102(OBJGPU * pGpu,KernelFifo * pKernelFifo,FIFO_MMU_EXCEPTION_DATA * pMmuExceptInfo)278 kfifoGetClientIdString_TU102
279 (
280     OBJGPU                  *pGpu,
281     KernelFifo              *pKernelFifo,
282     FIFO_MMU_EXCEPTION_DATA *pMmuExceptInfo
283 )
284 {
285     if (pMmuExceptInfo->bGpc)
286     {
287         switch (pMmuExceptInfo->clientId)
288         {
289             case NV_PFAULT_CLIENT_GPC_T1_0:
290                 return "GPCCLIENT_T1_0";
291             case NV_PFAULT_CLIENT_GPC_PE_0:
292                 return "GPCCLIENT_PE_0";
293             case NV_PFAULT_CLIENT_GPC_T1_1:
294                 return "GPCCLIENT_T1_1";
295             case NV_PFAULT_CLIENT_GPC_PE_1:
296                 return "GPCCLIENT_PE_1";
297             case NV_PFAULT_CLIENT_GPC_T1_2:
298                 return "GPCCLIENT_T1_2";
299             case NV_PFAULT_CLIENT_GPC_PE_2:
300                 return "GPCCLIENT_PE_2";
301             case NV_PFAULT_CLIENT_GPC_T1_3:
302                 return "GPCCLIENT_T1_3";
303             case NV_PFAULT_CLIENT_GPC_PE_3:
304                 return "GPCCLIENT_PE_3";
305             case NV_PFAULT_CLIENT_GPC_T1_4:
306                 return "GPCCLIENT_T1_4";
307             case NV_PFAULT_CLIENT_GPC_PE_4:
308                 return "GPCCLIENT_PE_4";
309             case NV_PFAULT_CLIENT_GPC_T1_5:
310                 return "GPCCLIENT_T1_5";
311             case NV_PFAULT_CLIENT_GPC_PE_5:
312                 return "GPCCLIENT_PE_5";
313             case NV_PFAULT_CLIENT_GPC_T1_6:
314                 return "GPCCLIENT_T1_6";
315             case NV_PFAULT_CLIENT_GPC_PE_6:
316                 return "GPCCLIENT_PE_6";
317             case NV_PFAULT_CLIENT_GPC_T1_7:
318                 return "GPCCLIENT_T1_7";
319             case NV_PFAULT_CLIENT_GPC_PE_7:
320                 return "GPCCLIENT_PE_7";
321             case NV_PFAULT_CLIENT_GPC_T1_8:
322                 return "GPCCLIENT_T1_8";
323             case NV_PFAULT_CLIENT_GPC_PE_8:
324                 return "GPCCLIENT_PE_8";
325             case NV_PFAULT_CLIENT_GPC_T1_9:
326                 return "GPCCLIENT_T1_9";
327             case NV_PFAULT_CLIENT_GPC_T1_10:
328                  return "GPCCLIENT_T1_10";
329             case NV_PFAULT_CLIENT_GPC_T1_11:
330                  return "GPCCLIENT_T1_11";
331             case NV_PFAULT_CLIENT_GPC_T1_12:
332                  return "GPCCLIENT_T1_12";
333             case NV_PFAULT_CLIENT_GPC_T1_13:
334                  return "GPCCLIENT_T1_13";
335             case NV_PFAULT_CLIENT_GPC_T1_14:
336                  return "GPCCLIENT_T1_14";
337             case NV_PFAULT_CLIENT_GPC_T1_15:
338                  return "GPCCLIENT_T1_15";
339             case NV_PFAULT_CLIENT_GPC_TPCCS_0:
340                 return "GPCCLIENT_TPCCS_0";
341             case NV_PFAULT_CLIENT_GPC_TPCCS_1:
342                 return "GPCCLIENT_TPCCS_1";
343             case NV_PFAULT_CLIENT_GPC_TPCCS_2:
344                 return "GPCCLIENT_TPCCS_2";
345             case NV_PFAULT_CLIENT_GPC_TPCCS_3:
346                 return "GPCCLIENT_TPCCS_3";
347             case NV_PFAULT_CLIENT_GPC_TPCCS_4:
348                 return "GPCCLIENT_TPCCS_4";
349             case NV_PFAULT_CLIENT_GPC_TPCCS_5:
350                 return "GPCCLIENT_TPCCS_5";
351             case NV_PFAULT_CLIENT_GPC_TPCCS_6:
352                 return "GPCCLIENT_TPCCS_6";
353             case NV_PFAULT_CLIENT_GPC_TPCCS_7:
354                 return "GPCCLIENT_TPCCS_7";
355             case NV_PFAULT_CLIENT_GPC_TPCCS_8:
356                 return "GPCCLIENT_TPCCS_8";
357             case NV_PFAULT_CLIENT_GPC_RAST:
358                 return "GPCCLIENT_RAST";
359             case NV_PFAULT_CLIENT_GPC_GCC:
360                 return "GPCCLIENT_GCC";
361             case NV_PFAULT_CLIENT_GPC_GPCCS:
362                 return "GPCCLIENT_GPCCS";
363             case NV_PFAULT_CLIENT_GPC_PROP_0:
364                 return "GPCCLIENT_PROP_0";
365             case NV_PFAULT_CLIENT_GPC_PROP_1:
366                 return "GPCCLIENT_PROP_1";
367             default:
368                 return "UNRECOGNIZED_CLIENT";
369         }
370     }
371     else
372     {
373         switch (pMmuExceptInfo->clientId)
374         {
375             case NV_PFAULT_CLIENT_HUB_CE0:
376                 return "HUBCLIENT_CE0";
377             case NV_PFAULT_CLIENT_HUB_HSCE0:
378                 return "HUBCLIENT_HSCE0";
379             case NV_PFAULT_CLIENT_HUB_CE1:
380                 return "HUBCLIENT_CE1";
381             case NV_PFAULT_CLIENT_HUB_HSCE1:
382                 return "HUBCLIENT_HSCE1";
383             case NV_PFAULT_CLIENT_HUB_CE2:
384                 return "HUBCLIENT_CE2";
385             case NV_PFAULT_CLIENT_HUB_HSCE2:
386                 return "HUBCLIENT_HSCE2";
387             case NV_PFAULT_CLIENT_HUB_HSCE3:
388                 return "HUBCLIENT_HSCE3";
389             case NV_PFAULT_CLIENT_HUB_HSCE4:
390                 return "HUBCLIENT_HSCE4";
391             case NV_PFAULT_CLIENT_HUB_HSCE5:
392                 return "HUBCLIENT_HSCE5";
393             case NV_PFAULT_CLIENT_HUB_HSCE6:
394                 return "HUBCLIENT_HSCE6";
395             case NV_PFAULT_CLIENT_HUB_HSCE7:
396                 return "HUBCLIENT_HSCE7";
397             case NV_PFAULT_CLIENT_HUB_HSCE8:
398                 return "HUBCLIENT_HSCE8";
399             case NV_PFAULT_CLIENT_HUB_HSCE9:
400                 return "HUBCLIENT_HSCE9";
401             case NV_PFAULT_CLIENT_HUB_DNISO:
402                 return "HUBCLIENT_DNISO";
403             case NV_PFAULT_CLIENT_HUB_FE:
404                 return "HUBCLIENT_FE";
405             case NV_PFAULT_CLIENT_HUB_FECS:
406                 return "HUBCLIENT_FECS";
407             case NV_PFAULT_CLIENT_HUB_HOST:
408                 return "HUBCLIENT_HOST";
409             case NV_PFAULT_CLIENT_HUB_HOST_CPU:
410                 return "HUBCLIENT_HOST_CPU";
411             case NV_PFAULT_CLIENT_HUB_HOST_CPU_NB:
412                 return "HUBCLIENT_HOST_CPU_NB";
413             case NV_PFAULT_CLIENT_HUB_ISO:
414                 return "HUBCLIENT_ISO";
415             case NV_PFAULT_CLIENT_HUB_MMU:
416                 return "HUBCLIENT_MMU";
417             case NV_PFAULT_CLIENT_HUB_NVDEC0:
418                 return "HUBCLIENT_NVDEC0";
419             case NV_PFAULT_CLIENT_HUB_NVDEC1:
420                 return "HUBCLIENT_NVDEC1";
421             case NV_PFAULT_CLIENT_HUB_NVDEC2:
422                 return "HUBCLIENT_NVDEC2";
423             case NV_PFAULT_CLIENT_HUB_NVENC0:
424                 return "HUBCLIENT_NVENC0";
425             case NV_PFAULT_CLIENT_HUB_NVENC1:
426                 return "HUBCLIENT_NVENC1";
427             case NV_PFAULT_CLIENT_HUB_NISO:
428                 return "HUBCLIENT_NISO";
429             case NV_PFAULT_CLIENT_HUB_P2P:
430                 return "HUBCLIENT_P2P";
431             case NV_PFAULT_CLIENT_HUB_PD:
432                 return "HUBCLIENT_PD";
433             case NV_PFAULT_CLIENT_HUB_PERF:
434                 return "HUBCLIENT_PERF";
435             case NV_PFAULT_CLIENT_HUB_PMU:
436                 return "HUBCLIENT_PMU";
437             case NV_PFAULT_CLIENT_HUB_RASTERTWOD:
438                 return "HUBCLIENT_RASTERTWOD";
439             case NV_PFAULT_CLIENT_HUB_SCC:
440                 return "HUBCLIENT_SCC";
441             case NV_PFAULT_CLIENT_HUB_SCC_NB:
442                 return "HUBCLIENT_SCC_NB";
443             case NV_PFAULT_CLIENT_HUB_SEC:
444                 return "HUBCLIENT_SEC2";
445             case NV_PFAULT_CLIENT_HUB_SSYNC:
446                 return "HUBCLIENT_SSYNC";
447             case NV_PFAULT_CLIENT_HUB_VIP:
448                 return "HUBCLIENT_VIP";
449             case NV_PFAULT_CLIENT_HUB_XV:
450                 return "HUBCLIENT_XV";
451             case NV_PFAULT_CLIENT_HUB_MMU_NB:
452                 return "HUBCLIENT_MMU_NB";
453             case NV_PFAULT_CLIENT_HUB_DFALCON:
454                 return "HUBCLIENT_DFALCON";
455             case NV_PFAULT_CLIENT_HUB_SKED:
456                 return "HUBCLIENT_SKED";
457             case NV_PFAULT_CLIENT_HUB_AFALCON:
458                 return "HUBCLIENT_AFALCON";
459             case NV_PFAULT_CLIENT_HUB_DONT_CARE:
460                 return "HUBCLIENT_DONT_CARE";
461             case NV_PFAULT_CLIENT_HUB_DWBIF:
462                 return "HUBCLIENT_DWBIF";
463             case NV_PFAULT_CLIENT_HUB_GSP:
464                 return "HUBCLIENT_GSP";
465             case NV_PFAULT_CLIENT_HUB_FBFALCON:
466                 return "HUBCLIENT_FBFLCN";
467             case NV_PFAULT_CLIENT_HUB_NVJPG0:
468                 return "HUBCLIENT_NVJPG0";
469             default:
470                 return "UNRECOGNIZED_CLIENT";
471         }
472     }
473 }
474