1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "kernel/gpu/fifo/kernel_fifo.h"
25 #include "kernel/gpu/fifo/kernel_channel.h"
26
27 #include "gpu/gpu_access.h"
28 #include "gpu/gpu.h"
29 #include "vgpu/vgpu_events.h"
30
31 #include "published/turing/tu102/dev_fault.h"
32 #include "published/turing/tu102/dev_vm.h"
33 #include "published/turing/tu102/dev_ctrl.h"
34 #include "published/turing/tu102/hwproject.h"
35
36 /*!
37 * @brief Update the usermode doorbell register with work submit token to notify
38 * host that work is available on this channel.
39 *
40 * @param[in] pGpu
41 * @param[in] pKernelFifo
42 * @param[in] workSubmitToken Token to update the doorbell with
43 * @param[in] runlistId Runlist ID
44 */
45 NV_STATUS
kfifoUpdateUsermodeDoorbell_TU102(OBJGPU * pGpu,KernelFifo * pKernelFifo,NvU32 workSubmitToken,NvU32 runlistId)46 kfifoUpdateUsermodeDoorbell_TU102
47 (
48 OBJGPU *pGpu,
49 KernelFifo *pKernelFifo,
50 NvU32 workSubmitToken,
51 NvU32 runlistId
52 )
53 {
54 NV_PRINTF(LEVEL_INFO, "Poking workSubmitToken 0x%x\n", workSubmitToken);
55
56 GPU_VREG_WR32(pGpu, NV_VIRTUAL_FUNCTION_DOORBELL, workSubmitToken);
57
58 return NV_OK;
59 }
60
61 /*!
62 * @brief Construct the worksubmit token. Caller cannot make assumption about this handle.
63 *
64 * @param[in] pGpu
65 * @param[in] pKernelFifo
66 * @param[in] pKernelChannel
67 * @param[out] pGeneratedToken Store the generated token
68 * @param[in] bUsedForHost Used on Host RM
69 *
70 */
71 NV_STATUS
kfifoGenerateWorkSubmitToken_TU102(OBJGPU * pGpu,KernelFifo * pKernelFifo,KernelChannel * pKernelChannel,NvU32 * pGeneratedToken,NvBool bUsedForHost)72 kfifoGenerateWorkSubmitToken_TU102
73 (
74 OBJGPU *pGpu,
75 KernelFifo *pKernelFifo,
76 KernelChannel *pKernelChannel,
77 NvU32 *pGeneratedToken,
78 NvBool bUsedForHost
79 )
80 {
81 NvU32 chId;
82 NvU32 gfid;
83 NvU32 val = 0;
84
85 NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_CHANNEL);
86
87 NV_ASSERT_OR_RETURN(pGeneratedToken != NULL, NV_ERR_INVALID_ARGUMENT);
88
89 chId = pKernelChannel->ChID;
90
91 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &gfid));
92
93 //
94 // In case of vGPU with SR-IOV, host RM is currently generating token using
95 // virtual chid that was allocated inside the guest. This needs to change
96 // once the guest starts managing its own channels. The guest would then
97 // generate its own tokens.
98 //
99 if (!bUsedForHost && IS_GFID_VF(gfid))
100 {
101 NvU32 vChId;
102
103 NV_ASSERT_OK_OR_RETURN(kfifoGetVChIdForSChId_HAL(pGpu, pKernelFifo,
104 chId, gfid,
105 kchannelGetEngineType(pKernelChannel),
106 &vChId));
107 chId = vChId;
108 }
109
110 if (!kchannelIsRunlistSet(pGpu, pKernelChannel))
111 {
112 NV_PRINTF(LEVEL_ERROR,
113 "FAILED Channel 0x%x is not assigned to runlist yet\n",
114 kchannelGetDebugTag(pKernelChannel));
115 return NV_ERR_INVALID_STATE;
116 }
117
118 // Here we construct token to be a concatenation of runlist id and channel id
119 val = FLD_SET_DRF_NUM(_CTRL, _VF_DOORBELL, _RUNLIST_ID, kchannelGetRunlistId(pKernelChannel), val);
120 val = FLD_SET_DRF_NUM(_CTRL, _VF_DOORBELL, _VECTOR, chId, val);
121 *pGeneratedToken = val;
122
123 NV_PRINTF(LEVEL_INFO,
124 "Generated workSubmitToken 0x%x for channel 0x%x runlist 0x%x\n",
125 *pGeneratedToken, chId, kchannelGetRunlistId(pKernelChannel));
126
127 return NV_OK;
128 }
129
130 /**
131 * @brief Convert PBDMA ID to string
132 * @param[in] pGpu
133 * @param[in] pKernelFifo
134 * @param[in] pbdmaId
135 *
136 * @return cont string
137 */
138 const char*
kfifoPrintPbdmaId_TU102(OBJGPU * pGpu,KernelFifo * pKernelFifo,NvU32 pbdmaId)139 kfifoPrintPbdmaId_TU102
140 (
141 OBJGPU *pGpu,
142 KernelFifo *pKernelFifo,
143 NvU32 pbdmaId
144 )
145 {
146 NV_ASSERT_OR_RETURN(pbdmaId < NV_HOST_NUM_PBDMA, "UNKNOWN");
147 static const char* pbdmaIdString[NV_HOST_NUM_PBDMA] = { "HOST0",
148 "HOST1",
149 "HOST2",
150 "HOST3",
151 "HOST4",
152 "HOST5",
153 "HOST6",
154 "HOST7",
155 "HOST8",
156 "HOST9",
157 "HOST10",
158 "HOST11",
159 #if NV_HOST_NUM_PBDMA > 12
160 #error Uninitialized elements of static array.
161 #endif
162 };
163
164 return pbdmaIdString[pbdmaId];
165 }
166
167 /*!
168 * @brief Converts a mmu engine id (NV_PFAULT_MMU_ENG_ID_*) into a string.
169 *
170 * @param[in] pGpu
171 * @param[in] pKernelFifo
172 * @param[in] engineID NV_PFAULT_MMU_ENG_ID_*
173 *
174 * @returns a string (always non-null)
175 */
176 const char*
kfifoPrintInternalEngine_TU102(OBJGPU * pGpu,KernelFifo * pKernelFifo,NvU32 engineID)177 kfifoPrintInternalEngine_TU102
178 (
179 OBJGPU *pGpu,
180 KernelFifo *pKernelFifo,
181 NvU32 engineID
182 )
183 {
184 KernelGmmu *pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu);
185 NvU32 pbdmaId;
186
187 if (kfifoIsMmuFaultEngineIdPbdma(pGpu, pKernelFifo, engineID))
188 {
189 NV_ASSERT_OR_RETURN(kfifoGetPbdmaIdFromMmuFaultId(pGpu, pKernelFifo, engineID, &pbdmaId) == NV_OK, "UNKNOWN");
190 return kfifoPrintPbdmaId_HAL(pGpu, pKernelFifo, pbdmaId);
191 }
192
193 if (kgmmuIsFaultEngineBar1_HAL(pKernelGmmu, engineID))
194 {
195 return "BAR1";
196 }
197 else if (kgmmuIsFaultEngineBar2_HAL(pKernelGmmu, engineID))
198 {
199 return "BAR2";
200 }
201 else
202 {
203 switch (engineID)
204 {
205 case NV_PFAULT_MMU_ENG_ID_DISPLAY:
206 return "DISPLAY";
207 case NV_PFAULT_MMU_ENG_ID_IFB:
208 return "IFB";
209 case NV_PFAULT_MMU_ENG_ID_SEC:
210 return "SEC";
211 case NV_PFAULT_MMU_ENG_ID_PERF:
212 return "PERF";
213 case NV_PFAULT_MMU_ENG_ID_NVDEC0:
214 return "NVDEC0";
215 case NV_PFAULT_MMU_ENG_ID_NVDEC1:
216 return "NVDEC1";
217 case NV_PFAULT_MMU_ENG_ID_NVDEC2:
218 return "NVDEC2";
219 case NV_PFAULT_MMU_ENG_ID_CE0:
220 return "CE0";
221 case NV_PFAULT_MMU_ENG_ID_CE1:
222 return "CE1";
223 case NV_PFAULT_MMU_ENG_ID_CE2:
224 return "CE2";
225 case NV_PFAULT_MMU_ENG_ID_CE3:
226 return "CE3";
227 case NV_PFAULT_MMU_ENG_ID_CE4:
228 return "CE4";
229 case NV_PFAULT_MMU_ENG_ID_CE5:
230 return "CE5";
231 case NV_PFAULT_MMU_ENG_ID_CE6:
232 return "CE6";
233 case NV_PFAULT_MMU_ENG_ID_CE7:
234 return "CE7";
235 case NV_PFAULT_MMU_ENG_ID_CE8:
236 return "CE8";
237 case NV_PFAULT_MMU_ENG_ID_PWR_PMU:
238 return "PMU";
239 case NV_PFAULT_MMU_ENG_ID_PTP:
240 return "PTP";
241 case NV_PFAULT_MMU_ENG_ID_NVENC0:
242 return "NVENC0";
243 case NV_PFAULT_MMU_ENG_ID_NVENC1:
244 return "NVENC1";
245 case NV_PFAULT_MMU_ENG_ID_PHYSICAL:
246 return "PHYSICAL";
247 case NV_PFAULT_MMU_ENG_ID_NVJPG0:
248 return "NVJPG";
249 default:
250 {
251 NV_STATUS status = NV_OK;
252 NvU32 engTag;
253
254 status = kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo, ENGINE_INFO_TYPE_MMU_FAULT_ID,
255 engineID, ENGINE_INFO_TYPE_ENG_DESC, &engTag);
256 if ((status == NV_OK) && (IS_GR(engTag)))
257 {
258 return "GRAPHICS";
259 }
260
261 }
262 }
263 }
264
265 return "UNKNOWN";
266 }
267
268 /*!
269 * @brief Converts a subid/clientid into a client string
270 *
271 * @param[in] pGpu
272 * @param[in] pKernelFifo
273 * @param[in] pMmuExceptData
274
275 * @returns a string (always non-null)
276 */
277 const char*
kfifoGetClientIdString_TU102(OBJGPU * pGpu,KernelFifo * pKernelFifo,FIFO_MMU_EXCEPTION_DATA * pMmuExceptInfo)278 kfifoGetClientIdString_TU102
279 (
280 OBJGPU *pGpu,
281 KernelFifo *pKernelFifo,
282 FIFO_MMU_EXCEPTION_DATA *pMmuExceptInfo
283 )
284 {
285 if (pMmuExceptInfo->bGpc)
286 {
287 switch (pMmuExceptInfo->clientId)
288 {
289 case NV_PFAULT_CLIENT_GPC_T1_0:
290 return "GPCCLIENT_T1_0";
291 case NV_PFAULT_CLIENT_GPC_PE_0:
292 return "GPCCLIENT_PE_0";
293 case NV_PFAULT_CLIENT_GPC_T1_1:
294 return "GPCCLIENT_T1_1";
295 case NV_PFAULT_CLIENT_GPC_PE_1:
296 return "GPCCLIENT_PE_1";
297 case NV_PFAULT_CLIENT_GPC_T1_2:
298 return "GPCCLIENT_T1_2";
299 case NV_PFAULT_CLIENT_GPC_PE_2:
300 return "GPCCLIENT_PE_2";
301 case NV_PFAULT_CLIENT_GPC_T1_3:
302 return "GPCCLIENT_T1_3";
303 case NV_PFAULT_CLIENT_GPC_PE_3:
304 return "GPCCLIENT_PE_3";
305 case NV_PFAULT_CLIENT_GPC_T1_4:
306 return "GPCCLIENT_T1_4";
307 case NV_PFAULT_CLIENT_GPC_PE_4:
308 return "GPCCLIENT_PE_4";
309 case NV_PFAULT_CLIENT_GPC_T1_5:
310 return "GPCCLIENT_T1_5";
311 case NV_PFAULT_CLIENT_GPC_PE_5:
312 return "GPCCLIENT_PE_5";
313 case NV_PFAULT_CLIENT_GPC_T1_6:
314 return "GPCCLIENT_T1_6";
315 case NV_PFAULT_CLIENT_GPC_PE_6:
316 return "GPCCLIENT_PE_6";
317 case NV_PFAULT_CLIENT_GPC_T1_7:
318 return "GPCCLIENT_T1_7";
319 case NV_PFAULT_CLIENT_GPC_PE_7:
320 return "GPCCLIENT_PE_7";
321 case NV_PFAULT_CLIENT_GPC_T1_8:
322 return "GPCCLIENT_T1_8";
323 case NV_PFAULT_CLIENT_GPC_PE_8:
324 return "GPCCLIENT_PE_8";
325 case NV_PFAULT_CLIENT_GPC_T1_9:
326 return "GPCCLIENT_T1_9";
327 case NV_PFAULT_CLIENT_GPC_T1_10:
328 return "GPCCLIENT_T1_10";
329 case NV_PFAULT_CLIENT_GPC_T1_11:
330 return "GPCCLIENT_T1_11";
331 case NV_PFAULT_CLIENT_GPC_T1_12:
332 return "GPCCLIENT_T1_12";
333 case NV_PFAULT_CLIENT_GPC_T1_13:
334 return "GPCCLIENT_T1_13";
335 case NV_PFAULT_CLIENT_GPC_T1_14:
336 return "GPCCLIENT_T1_14";
337 case NV_PFAULT_CLIENT_GPC_T1_15:
338 return "GPCCLIENT_T1_15";
339 case NV_PFAULT_CLIENT_GPC_TPCCS_0:
340 return "GPCCLIENT_TPCCS_0";
341 case NV_PFAULT_CLIENT_GPC_TPCCS_1:
342 return "GPCCLIENT_TPCCS_1";
343 case NV_PFAULT_CLIENT_GPC_TPCCS_2:
344 return "GPCCLIENT_TPCCS_2";
345 case NV_PFAULT_CLIENT_GPC_TPCCS_3:
346 return "GPCCLIENT_TPCCS_3";
347 case NV_PFAULT_CLIENT_GPC_TPCCS_4:
348 return "GPCCLIENT_TPCCS_4";
349 case NV_PFAULT_CLIENT_GPC_TPCCS_5:
350 return "GPCCLIENT_TPCCS_5";
351 case NV_PFAULT_CLIENT_GPC_TPCCS_6:
352 return "GPCCLIENT_TPCCS_6";
353 case NV_PFAULT_CLIENT_GPC_TPCCS_7:
354 return "GPCCLIENT_TPCCS_7";
355 case NV_PFAULT_CLIENT_GPC_TPCCS_8:
356 return "GPCCLIENT_TPCCS_8";
357 case NV_PFAULT_CLIENT_GPC_RAST:
358 return "GPCCLIENT_RAST";
359 case NV_PFAULT_CLIENT_GPC_GCC:
360 return "GPCCLIENT_GCC";
361 case NV_PFAULT_CLIENT_GPC_GPCCS:
362 return "GPCCLIENT_GPCCS";
363 case NV_PFAULT_CLIENT_GPC_PROP_0:
364 return "GPCCLIENT_PROP_0";
365 case NV_PFAULT_CLIENT_GPC_PROP_1:
366 return "GPCCLIENT_PROP_1";
367 default:
368 return "UNRECOGNIZED_CLIENT";
369 }
370 }
371 else
372 {
373 switch (pMmuExceptInfo->clientId)
374 {
375 case NV_PFAULT_CLIENT_HUB_CE0:
376 return "HUBCLIENT_CE0";
377 case NV_PFAULT_CLIENT_HUB_HSCE0:
378 return "HUBCLIENT_HSCE0";
379 case NV_PFAULT_CLIENT_HUB_CE1:
380 return "HUBCLIENT_CE1";
381 case NV_PFAULT_CLIENT_HUB_HSCE1:
382 return "HUBCLIENT_HSCE1";
383 case NV_PFAULT_CLIENT_HUB_CE2:
384 return "HUBCLIENT_CE2";
385 case NV_PFAULT_CLIENT_HUB_HSCE2:
386 return "HUBCLIENT_HSCE2";
387 case NV_PFAULT_CLIENT_HUB_HSCE3:
388 return "HUBCLIENT_HSCE3";
389 case NV_PFAULT_CLIENT_HUB_HSCE4:
390 return "HUBCLIENT_HSCE4";
391 case NV_PFAULT_CLIENT_HUB_HSCE5:
392 return "HUBCLIENT_HSCE5";
393 case NV_PFAULT_CLIENT_HUB_HSCE6:
394 return "HUBCLIENT_HSCE6";
395 case NV_PFAULT_CLIENT_HUB_HSCE7:
396 return "HUBCLIENT_HSCE7";
397 case NV_PFAULT_CLIENT_HUB_HSCE8:
398 return "HUBCLIENT_HSCE8";
399 case NV_PFAULT_CLIENT_HUB_HSCE9:
400 return "HUBCLIENT_HSCE9";
401 case NV_PFAULT_CLIENT_HUB_DNISO:
402 return "HUBCLIENT_DNISO";
403 case NV_PFAULT_CLIENT_HUB_FE:
404 return "HUBCLIENT_FE";
405 case NV_PFAULT_CLIENT_HUB_FECS:
406 return "HUBCLIENT_FECS";
407 case NV_PFAULT_CLIENT_HUB_HOST:
408 return "HUBCLIENT_HOST";
409 case NV_PFAULT_CLIENT_HUB_HOST_CPU:
410 return "HUBCLIENT_HOST_CPU";
411 case NV_PFAULT_CLIENT_HUB_HOST_CPU_NB:
412 return "HUBCLIENT_HOST_CPU_NB";
413 case NV_PFAULT_CLIENT_HUB_ISO:
414 return "HUBCLIENT_ISO";
415 case NV_PFAULT_CLIENT_HUB_MMU:
416 return "HUBCLIENT_MMU";
417 case NV_PFAULT_CLIENT_HUB_NVDEC0:
418 return "HUBCLIENT_NVDEC0";
419 case NV_PFAULT_CLIENT_HUB_NVDEC1:
420 return "HUBCLIENT_NVDEC1";
421 case NV_PFAULT_CLIENT_HUB_NVDEC2:
422 return "HUBCLIENT_NVDEC2";
423 case NV_PFAULT_CLIENT_HUB_NVENC0:
424 return "HUBCLIENT_NVENC0";
425 case NV_PFAULT_CLIENT_HUB_NVENC1:
426 return "HUBCLIENT_NVENC1";
427 case NV_PFAULT_CLIENT_HUB_NISO:
428 return "HUBCLIENT_NISO";
429 case NV_PFAULT_CLIENT_HUB_P2P:
430 return "HUBCLIENT_P2P";
431 case NV_PFAULT_CLIENT_HUB_PD:
432 return "HUBCLIENT_PD";
433 case NV_PFAULT_CLIENT_HUB_PERF:
434 return "HUBCLIENT_PERF";
435 case NV_PFAULT_CLIENT_HUB_PMU:
436 return "HUBCLIENT_PMU";
437 case NV_PFAULT_CLIENT_HUB_RASTERTWOD:
438 return "HUBCLIENT_RASTERTWOD";
439 case NV_PFAULT_CLIENT_HUB_SCC:
440 return "HUBCLIENT_SCC";
441 case NV_PFAULT_CLIENT_HUB_SCC_NB:
442 return "HUBCLIENT_SCC_NB";
443 case NV_PFAULT_CLIENT_HUB_SEC:
444 return "HUBCLIENT_SEC2";
445 case NV_PFAULT_CLIENT_HUB_SSYNC:
446 return "HUBCLIENT_SSYNC";
447 case NV_PFAULT_CLIENT_HUB_VIP:
448 return "HUBCLIENT_VIP";
449 case NV_PFAULT_CLIENT_HUB_XV:
450 return "HUBCLIENT_XV";
451 case NV_PFAULT_CLIENT_HUB_MMU_NB:
452 return "HUBCLIENT_MMU_NB";
453 case NV_PFAULT_CLIENT_HUB_DFALCON:
454 return "HUBCLIENT_DFALCON";
455 case NV_PFAULT_CLIENT_HUB_SKED:
456 return "HUBCLIENT_SKED";
457 case NV_PFAULT_CLIENT_HUB_AFALCON:
458 return "HUBCLIENT_AFALCON";
459 case NV_PFAULT_CLIENT_HUB_DONT_CARE:
460 return "HUBCLIENT_DONT_CARE";
461 case NV_PFAULT_CLIENT_HUB_DWBIF:
462 return "HUBCLIENT_DWBIF";
463 case NV_PFAULT_CLIENT_HUB_GSP:
464 return "HUBCLIENT_GSP";
465 case NV_PFAULT_CLIENT_HUB_FBFALCON:
466 return "HUBCLIENT_FBFLCN";
467 case NV_PFAULT_CLIENT_HUB_NVJPG0:
468 return "HUBCLIENT_NVJPG0";
469 default:
470 return "UNRECOGNIZED_CLIENT";
471 }
472 }
473 }
474