1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2017-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24
25 /* ------------------------- System Includes -------------------------------- */
26 #include "gpu/bif/kernel_bif.h"
27 #include "gpu/bus/kern_bus.h"
28 #include "gpu/gpu.h"
29 #include "platform/chipset/chipset.h"
30
31 #include "virtualization/kernel_vgpu_mgr.h"
32 #include "virtualization/hypervisor/hypervisor.h"
33
34 #include "published/turing/tu102/dev_nv_xve.h"
35 #include "published/turing/tu102/dev_vm.h"
36
37 #include "published/turing/tu102/dev_nv_pcfg_xve_regmap.h"
38 #include "published/turing/tu102/dev_boot.h"
39
40 #define RTLSIM_DELAY_SCALE_US 8
41
42 // XVE register map for PCIe config space
43 static const NvU32 xveRegMapValid[] = NV_PCFG_XVE_REGISTER_VALID_MAP;
44 static const NvU32 xveRegMapWrite[] = NV_PCFG_XVE_REGISTER_WR_MAP;
45
46 /* ------------------------ Public Functions -------------------------------- */
47
48 /*!
49 * @brief Check if MSIX is enabled in HW
50 *
51 * @param[in] pGpu GPU object pointer
52 * @param[in] pKernelBif BIF object pointer
53 *
54 * @return True if MSIX enabled else False
55 */
56 NvBool
kbifIsMSIXEnabledInHW_TU102(OBJGPU * pGpu,KernelBif * pKernelBif)57 kbifIsMSIXEnabledInHW_TU102
58 (
59 OBJGPU *pGpu,
60 KernelBif *pKernelBif
61 )
62 {
63 NvU32 data32;
64
65 if (IS_VIRTUAL(pGpu))
66 {
67 // SR-IOV guests only support MSI-X
68 return IS_VIRTUAL_WITH_SRIOV(pGpu);
69 }
70 else
71 {
72 if (GPU_BUS_CFG_RD32(pGpu, NV_XVE_MSIX_CAP_HDR, &data32) != NV_OK)
73 {
74 NV_ASSERT_FAILED("Unable to read NV_XVE_MSIX_CAP_HDR\n");
75 return NV_FALSE;
76 }
77 return FLD_TEST_DRF(_XVE, _MSIX_CAP_HDR, _ENABLE, _ENABLED, data32);
78 }
79 }
80
81 /*!
82 * @brief Disables P2P reads/writes on VF
83 *
84 * @param[in] pGpu GPU object pointer
85 * @param[in] pKernelBif BIF object pointer
86 */
87 void
kbifDisableP2PTransactions_TU102(OBJGPU * pGpu,KernelBif * pKernelBif)88 kbifDisableP2PTransactions_TU102
89 (
90 OBJGPU *pGpu,
91 KernelBif *pKernelBif
92 )
93 {
94 if (IS_VIRTUAL_WITH_SRIOV(pGpu))
95 {
96 pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_P2P_READS_DISABLED, NV_TRUE);
97 pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_P2P_WRITES_DISABLED, NV_TRUE);
98 }
99 }
100
101 NV_STATUS
kbifGetVFSparseMmapRegions_TU102(OBJGPU * pGpu,KernelBif * pKernelBif,KERNEL_HOST_VGPU_DEVICE * pKernelHostVgpuDevice,NvU64 osPageSize,NvU32 * pNumAreas,NvU64 * pOffsets,NvU64 * pSizes)102 kbifGetVFSparseMmapRegions_TU102
103 (
104 OBJGPU *pGpu,
105 KernelBif *pKernelBif,
106 KERNEL_HOST_VGPU_DEVICE *pKernelHostVgpuDevice,
107 NvU64 osPageSize,
108 NvU32 *pNumAreas,
109 NvU64 *pOffsets,
110 NvU64 *pSizes
111 )
112 {
113 NvU64 offsetStart = 0;
114 NvU64 offsetEnd = 0;
115 NvU32 idx = 0;
116 NvU32 maxInstance;
117 NvU32 i;
118 NvBool bDryRun;
119
120 NV_CHECK_OR_RETURN(LEVEL_ERROR, pKernelHostVgpuDevice != NULL, NV_ERR_INVALID_ARGUMENT);
121 NV_CHECK_OR_RETURN(LEVEL_ERROR, pNumAreas != NULL, NV_ERR_INVALID_ARGUMENT);
122
123 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
124 kvgpumgrGetMaxInstanceOfVgpu(pKernelHostVgpuDevice->vgpuType,
125 &maxInstance));
126
127 // Dry run to calculate the total number of areas
128 bDryRun = ((pOffsets == NULL) || (pSizes == NULL));
129 if (bDryRun)
130 {
131 pOffsets = portMemAllocStackOrHeap(NVA084_CTRL_KERNEL_HOST_VGPU_DEVICE_MAX_BAR_MAPPING_RANGES * sizeof(pOffsets[0]));
132 pSizes = portMemAllocStackOrHeap(NVA084_CTRL_KERNEL_HOST_VGPU_DEVICE_MAX_BAR_MAPPING_RANGES * sizeof(pSizes[0]));
133 }
134
135 // For VF TLB emulation, trap MMU FAULT BUFFER page
136 if ((maxInstance > 1) && pGpu->getProperty(pGpu, PDB_PROP_GPU_BUG_3007008_EMULATE_VF_MMU_TLB_INVALIDATE))
137 {
138 offsetEnd = NV_VIRTUAL_FUNCTION_PRIV_MMU_FAULT_BUFFER_LO(0);
139 pOffsets[idx] = offsetStart;
140 pSizes[idx] = offsetEnd - offsetStart;
141 idx++;
142
143 offsetStart = NV_VIRTUAL_FUNCTION_PRIV_MMU_FAULT_BUFFER_LO(0) + osPageSize;
144 }
145
146 // Trap MSI-X table page
147 if (!hypervisorIsType(OS_HYPERVISOR_HYPERV))
148 {
149
150 offsetEnd = NV_VIRTUAL_FUNCTION_PRIV_MSIX_TABLE_ADDR_LO(0);
151
152 // Since MSI-X page is immediately after VGPU_EMU, if both are
153 // trapped, skip creating a 0 size region in between
154 if (offsetEnd > offsetStart)
155 {
156 pOffsets[idx] = offsetStart;
157 pSizes[idx] = offsetEnd - offsetStart;
158 idx++;
159 }
160
161 offsetStart = NV_VIRTUAL_FUNCTION_PRIV_MSIX_TABLE_ADDR_LO(0) + osPageSize;
162 }
163
164 offsetEnd = pGpu->sriovState.vfBarSize[0];
165 pOffsets[idx] = offsetStart;
166 pSizes[idx] = offsetEnd - offsetStart;
167 idx++;
168
169 if (bDryRun)
170 {
171 portMemFreeStackOrHeap(pOffsets);
172 portMemFreeStackOrHeap(pSizes);
173 }
174 else
175 {
176 // It might be too late to check if the passed arrays are big enough,
177 // but better late than never
178 NV_ASSERT_OR_RETURN(idx <= *pNumAreas, NV_ERR_FATAL_ERROR);
179
180 for (i = 0; i < idx; i++)
181 {
182 NV_PRINTF(LEVEL_INFO, "VF Sparse Mmap Region[%u] range 0x%llx - 0x%llx, size 0x%llx\n",
183 i, pOffsets[i], pOffsets[i] + pSizes[i], pSizes[i]);
184 }
185 }
186
187 *pNumAreas = idx;
188 return NV_OK;
189 }
190
191 /*!
192 * @brief Check and cache Function level reset support
193 *
194 * @param[in] pGpu GPU object pointer
195 * @param[in] pKernelBif Kernel BIF object pointer
196 *
197 */
198 void
kbifCacheFlrSupport_TU102(OBJGPU * pGpu,KernelBif * pKernelBif)199 kbifCacheFlrSupport_TU102
200 (
201 OBJGPU *pGpu,
202 KernelBif *pKernelBif
203 )
204 {
205 NvU32 regVal = 0;
206
207 // Read config register
208 if (GPU_BUS_CFG_RD32(pGpu, NV_XVE_DEVICE_CAPABILITY,
209 ®Val) != NV_OK)
210 {
211 NV_PRINTF(LEVEL_ERROR, "Unable to read NV_XVE_DEVICE_CAPABILITY\n");
212 return;
213 }
214
215 // Check if FLR is supported
216 if (FLD_TEST_DRF(_XVE, _DEVICE_CAPABILITY, _FUNCTION_LEVEL_RESET,
217 _SUPPORTED, regVal))
218 {
219 pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_FLR_SUPPORTED, NV_TRUE);
220 }
221 }
222
223 /*!
224 * @brief Fetch VF details such as no. of VFs, First VF offset etc
225 *
226 * @param[in] pGpu GPU object pointer
227 * @param[in] pKernelBif Kernel BIF object pointer
228 */
229 void
kbifCacheVFInfo_TU102(OBJGPU * pGpu,KernelBif * pKernelBif)230 kbifCacheVFInfo_TU102
231 (
232 OBJGPU *pGpu,
233 KernelBif *pKernelBif
234 )
235 {
236 NV_STATUS status = NV_OK;
237 NvU32 regVal = 0;
238 NvU32 saveLo = 0;
239 NvU32 saveHi = 0;
240
241 // Get total VF count
242 GPU_BUS_CFG_RD32(pGpu, NV_XVE_SRIOV_CAP_HDR3, ®Val);
243 pGpu->sriovState.totalVFs = GPU_DRF_VAL(_XVE, _SRIOV_CAP_HDR3,
244 _TOTAL_VFS, regVal);
245
246 // Get first VF offset
247 GPU_BUS_CFG_RD32(pGpu, NV_XVE_SRIOV_CAP_HDR5, ®Val);
248 pGpu->sriovState.firstVFOffset = GPU_DRF_VAL(_XVE, _SRIOV_CAP_HDR5,
249 _FIRST_VF_OFFSET, regVal);
250
251 // Get VF BAR0 first address
252 status = GPU_BUS_CFG_CYCLE_RD32(pGpu, NV_XVE_SRIOV_CAP_HDR9, &saveLo);
253 NV_ASSERT(status == NV_OK);
254 pGpu->sriovState.firstVFBarAddress[0] = saveLo & 0xFFFFFFF0;
255
256 // Get VF BAR1 first address
257 status = GPU_BUS_CFG_CYCLE_RD32(pGpu, NV_XVE_SRIOV_CAP_HDR10, &saveLo);
258 NV_ASSERT(status == NV_OK);
259 status = GPU_BUS_CFG_CYCLE_RD32(pGpu, NV_XVE_SRIOV_CAP_HDR11_VF_BAR1_HI, &saveHi);
260 NV_ASSERT(status == NV_OK);
261 pGpu->sriovState.firstVFBarAddress[1] = (((NvU64)saveHi) << 32) + (saveLo & 0xFFFFFFF0);
262
263 // Get VF BAR2 first address
264 status = GPU_BUS_CFG_CYCLE_RD32(pGpu, NV_XVE_SRIOV_CAP_HDR12, &saveLo);
265 NV_ASSERT(status == NV_OK);
266 status = GPU_BUS_CFG_CYCLE_RD32(pGpu, NV_XVE_SRIOV_CAP_HDR13_VF_BAR2_HI, &saveHi);
267 NV_ASSERT(status == NV_OK);
268 pGpu->sriovState.firstVFBarAddress[2] = (((NvU64)saveHi) << 32) + (saveLo & 0xFFFFFFF0);
269
270 // Get if VF BARs are 64 bit addressable
271 regVal = GPU_REG_RD32(pGpu, DEVICE_BASE(NV_PCFG) + NV_XVE_SRIOV_CAP_HDR10);
272 pGpu->sriovState.b64bitVFBar1 = IS_BAR_64(regVal);
273 regVal = GPU_REG_RD32(pGpu, DEVICE_BASE(NV_PCFG) + NV_XVE_SRIOV_CAP_HDR12);
274 pGpu->sriovState.b64bitVFBar2 = IS_BAR_64(regVal);
275 }
276
277 /*!
278 * @brief Waits for function issued transaction completions(sysmem to GPU) to arrive
279 *
280 * @param[in] pGpu GPU object pointer
281 * @param[in] pKernelBif Kernel BIF object pointer
282 */
283 NV_STATUS
kbifWaitForTransactionsComplete_TU102(OBJGPU * pGpu,KernelBif * pKernelBif)284 kbifWaitForTransactionsComplete_TU102
285 (
286 OBJGPU *pGpu,
287 KernelBif *pKernelBif
288 )
289 {
290 NvU32 domain = gpuGetDomain(pGpu);
291 NvU8 bus = gpuGetBus(pGpu);
292 NvU8 device = gpuGetDevice(pGpu);
293 void *handle = NULL;
294 NvU16 vendorId;
295 NvU16 deviceId;
296 RMTIMEOUT timeout;
297
298 handle = osPciInitHandle(domain, bus, device, 0, &vendorId, &deviceId);
299
300 gpuSetTimeout(pGpu, GPU_TIMEOUT_DEFAULT, &timeout, 0);
301
302 // Wait for number of pending transactions to go to 0
303 while (DRF_VAL(_XVE, _DEVICE_CONTROL_STATUS, _TRANSACTIONS_PENDING,
304 osPciReadDword(handle, NV_XVE_DEVICE_CONTROL_STATUS)) != 0)
305 {
306 if (gpuCheckTimeout(pGpu, &timeout) == NV_ERR_TIMEOUT)
307 {
308 NV_PRINTF(LEVEL_ERROR,
309 "Timeout waiting for transactions pending to go to 0\n");
310 DBG_BREAKPOINT();
311 return NV_ERR_TIMEOUT;
312 }
313 }
314
315 return NV_OK;
316 }
317
318 /*!
319 * @brief Trigger FLR
320 *
321 * @param[in] pGpu GPU object pointer
322 * @param[in] pKernelBif Kernel BIF object pointer
323 *
324 * @return NV_OK if successful.
325 */
326 NV_STATUS
kbifTriggerFlr_TU102(OBJGPU * pGpu,KernelBif * pKernelBif)327 kbifTriggerFlr_TU102
328 (
329 OBJGPU *pGpu,
330 KernelBif *pKernelBif
331 )
332 {
333 NvU32 domain = gpuGetDomain(pGpu);
334 NvU8 bus = gpuGetBus(pGpu);
335 NvU8 device = gpuGetDevice(pGpu);
336 void *handle = NULL;
337 OBJSYS *pSys = SYS_GET_INSTANCE();
338 OBJCL *pCl = SYS_GET_CL(pSys);
339 NvU32 regVal = 0;
340 NvU16 vendorId = 0;
341 NvU16 deviceId = 0;
342
343 handle = osPciInitHandle(domain, bus, device, 0, &vendorId, &deviceId);
344
345 regVal = osPciReadDword(handle, NV_XVE_DEVICE_CONTROL_STATUS);
346
347 regVal = FLD_SET_DRF_NUM(_XVE, _DEVICE_CONTROL_STATUS ,
348 _INITIATE_FN_LVL_RST, 0x1, regVal);
349
350 clPcieWriteDword(pCl, gpuGetDomain(pGpu), gpuGetBus(pGpu),
351 gpuGetDevice(pGpu), 0, NV_XVE_DEVICE_CONTROL_STATUS,
352 regVal);
353
354 return NV_OK;
355 }
356
357 /*!
358 * This function setups the xve register map pointers
359 *
360 * @param[in] pGpu GPU object pointer
361 * @param[in] pKernelBif Pointer to KernelBif object
362 * @param[in] func PCIe function number
363 *
364 * @return 'NV_OK' if successful, an RM error code otherwise.
365 */
366 NV_STATUS
kbifInitXveRegMap_TU102(OBJGPU * pGpu,KernelBif * pKernelBif,NvU8 func)367 kbifInitXveRegMap_TU102
368 (
369 OBJGPU *pGpu,
370 KernelBif *pKernelBif,
371 NvU8 func
372 )
373 {
374 extern NvU32 kbifInitXveRegMap_GM107(OBJGPU *pGpu, KernelBif *pKernelBif, NvU8 func);
375 NV_STATUS status = NV_OK;
376
377 if (func == 0)
378 {
379 pKernelBif->xveRegmapRef[0].nFunc = 0;
380 pKernelBif->xveRegmapRef[0].xveRegMapValid = xveRegMapValid;
381 pKernelBif->xveRegmapRef[0].xveRegMapWrite = xveRegMapWrite;
382 pKernelBif->xveRegmapRef[0].numXveRegMapValid = sizeof(xveRegMapValid)/sizeof(xveRegMapValid[0]);
383 pKernelBif->xveRegmapRef[0].numXveRegMapWrite = sizeof(xveRegMapWrite)/sizeof(xveRegMapWrite[0]);
384 pKernelBif->xveRegmapRef[0].bufBootConfigSpace = pKernelBif->cacheData.gpuBootConfigSpace;
385 // MSIX table buf not used in Turing, but it could be
386 pKernelBif->xveRegmapRef[0].bufMsixTable = NULL;
387 }
388 else if (func == 1)
389 {
390 // Init regmap for Fn1 using older HAL
391 status = kbifInitXveRegMap_GM107(pGpu, pKernelBif, 1);
392 }
393 else
394 {
395 NV_PRINTF(LEVEL_ERROR, "Invalid argument, func: %d.\n", func);
396 NV_ASSERT(0);
397 status = NV_ERR_INVALID_ARGUMENT;
398 }
399
400 return status;
401 }
402
403 /*!
404 * @brief Returns size of MSIX vector control table
405 *
406 * @param[in] pGpu GPU object pointer
407 * @param[in] pKernelBif Pointer to KernelBif object
408 */
409 NvU32
kbifGetMSIXTableVectorControlSize_TU102(OBJGPU * pGpu,KernelBif * pKernelBif)410 kbifGetMSIXTableVectorControlSize_TU102
411 (
412 OBJGPU *pGpu,
413 KernelBif *pKernelBif
414 )
415 {
416 return NV_VIRTUAL_FUNCTION_PRIV_MSIX_TABLE_VECTOR_CONTROL__SIZE_1;
417 }
418
419 /*!
420 * This function saves MSIX vector control masks which can later be restored
421 * using bifRestoreMSIXVectorControlMasks_HAL.
422 *
423 * @param[in] pGpu GPU object pointer
424 * @param[in] pKernelBif Pointer to KernelBif object
425 * @param[out] *msixVectorMask MSIX vector control mask state for all
426 * MSIX table entries
427 * @return 'NV_OK' if successful, an RM error code otherwise.
428 */
429 NV_STATUS
kbifSaveMSIXVectorControlMasks_TU102(OBJGPU * pGpu,KernelBif * pKernelBif,NvU32 * msixVectorMask)430 kbifSaveMSIXVectorControlMasks_TU102
431 (
432 OBJGPU *pGpu,
433 KernelBif *pKernelBif,
434 NvU32 *msixVectorMask
435 )
436 {
437 NvU32 i;
438 NvU32 regVal;
439 NV_STATUS status = NV_OK;
440 NvU32 controlSize = kbifGetMSIXTableVectorControlSize_HAL(pGpu, pKernelBif);
441
442 //Sanity check for size
443 if (controlSize > 32U)
444 {
445 NV_PRINTF(LEVEL_ERROR,
446 "Size of MSIX vector control exceeds maximum assumed size!!\n");
447 DBG_BREAKPOINT();
448
449 status = NV_ERR_BUFFER_TOO_SMALL;
450 goto kbifSaveMSIXVectorControlMasks_TU102_exit;
451 }
452
453 // Set the bits in msixVectorMask if NV_MSIX_TABLE_VECTOR_CONTROL(i) is masked
454 *msixVectorMask = 0U;
455 for (i = 0U; i < controlSize; i++)
456 {
457 regVal = GPU_VREG_RD32(pGpu, NV_VIRTUAL_FUNCTION_PRIV_MSIX_TABLE_VECTOR_CONTROL(i));
458
459 if (FLD_TEST_DRF(_VIRTUAL_FUNCTION_PRIV, _MSIX_TABLE_VECTOR_CONTROL,
460 _MASK_BIT, _MASKED, regVal))
461 {
462 (*msixVectorMask) |= NVBIT(i);
463 }
464 }
465
466 kbifSaveMSIXVectorControlMasks_TU102_exit:
467 return status;
468 }
469
470 /*!
471 * This function restores MSIX vector control masks to the saved state. Vector
472 * control mask needs to be saved using bifSaveMSIXVectorControlMasks_HAL.
473 *
474 * @param[in] pGpu GPU object pointer
475 * @param[in] pKernelBif Pointer to KernelBif object
476 * @param[in] msixVectorMask State of MSIX vector control masks
477 *
478 * @return 'NV_OK' if successful, an RM error code otherwise.
479 */
480 NV_STATUS
kbifRestoreMSIXVectorControlMasks_TU102(OBJGPU * pGpu,KernelBif * pKernelBif,NvU32 msixVectorMask)481 kbifRestoreMSIXVectorControlMasks_TU102
482 (
483 OBJGPU *pGpu,
484 KernelBif *pKernelBif,
485 NvU32 msixVectorMask
486 )
487 {
488 NvU32 i;
489 NV_STATUS status = NV_OK;
490 NvU32 controlSize = kbifGetMSIXTableVectorControlSize_HAL(pGpu, pKernelBif);
491
492 // Initialize the base offset for the virtual registers for physical function
493 NvU32 vRegOffset = pGpu->sriovState.virtualRegPhysOffset;
494
495 //Sanity check for size
496 if (controlSize > 32U)
497 {
498 NV_PRINTF(LEVEL_ERROR,
499 "Size of MSIX vector control exceeds maximum assumed size!!\n");
500 DBG_BREAKPOINT();
501 status = NV_ERR_BUFFER_TOO_SMALL;
502 goto kbifRestoreMSIXVectorControlMasks_TU102_exit;
503 }
504
505 //
506 // Restore NV_MSIX_TABLE_VECTOR_CONTROL(i) based on msixVectorMask
507 // In FLR path, we don't want to use usual register r/w macros
508 //
509 for (i = 0U; i < controlSize; i++)
510 {
511 if ((NVBIT(i) & msixVectorMask) != 0U)
512 {
513 osGpuWriteReg032(pGpu,
514 vRegOffset + NV_VIRTUAL_FUNCTION_PRIV_MSIX_TABLE_VECTOR_CONTROL(i),
515 NV_VIRTUAL_FUNCTION_PRIV_MSIX_TABLE_VECTOR_CONTROL_MASK_BIT_MASKED);
516 }
517 else
518 {
519 osGpuWriteReg032(pGpu,
520 vRegOffset + NV_VIRTUAL_FUNCTION_PRIV_MSIX_TABLE_VECTOR_CONTROL(i),
521 NV_VIRTUAL_FUNCTION_PRIV_MSIX_TABLE_VECTOR_CONTROL_MASK_BIT_UNMASKED);
522 }
523 }
524
525 kbifRestoreMSIXVectorControlMasks_TU102_exit:
526 return status;
527 }
528
529
530 /**
531 *
532 * Do function level reset for Fn0.
533 *
534 */
535 NV_STATUS
kbifDoFunctionLevelReset_TU102(OBJGPU * pGpu,KernelBif * pKernelBif)536 kbifDoFunctionLevelReset_TU102
537 (
538 OBJGPU *pGpu,
539 KernelBif *pKernelBif
540 )
541 {
542 NvU32 domain = gpuGetDomain(pGpu);
543 NvU8 bus = gpuGetBus(pGpu);
544 NvU8 device = gpuGetDevice(pGpu);
545 void *handle = NULL;
546 NV_STATUS status = NV_OK;
547 NvU32 i;
548 NvU32 tempRegVal;
549 NvU16 vendorId;
550 NvU16 deviceId;
551 // 'i'th bit set to 1 indicates NV_MSIX_TABLE_VECTOR_CONTROL(i) is masked
552 NvU32 msixVectorMask;
553 NvBool bMSIXEnabled;
554 // volatile is required to prevent any compiler optimizations
555 volatile NvU32 tempRegValSwReset;
556
557 //
558 // From the experimental data: We need to get the handle before asserting FLR
559 // GPU is always at function 0
560 //
561 handle = osPciInitHandle(domain, bus, device, 0, &vendorId, &deviceId);
562
563 pKernelBif->bPreparingFunctionLevelReset = NV_TRUE;
564
565 status = kbifSavePcieConfigRegisters_HAL(pGpu, pKernelBif);
566 if (status != NV_OK)
567 {
568 DBG_BREAKPOINT();
569 NV_PRINTF(LEVEL_ERROR, "Config registers save failed!\n");
570 goto bifDoFunctionLevelReset_TU102_exit;
571 }
572
573 bMSIXEnabled = kbifIsMSIXEnabledInHW_HAL(pGpu, pKernelBif);
574 if (bMSIXEnabled)
575 {
576 status = kbifSaveMSIXVectorControlMasks_HAL(pGpu, pKernelBif, &msixVectorMask);
577 if (status != NV_OK)
578 {
579 DBG_BREAKPOINT();
580 NV_PRINTF(LEVEL_ERROR, "MSIX vector control registers save failed!\n");
581 goto bifDoFunctionLevelReset_TU102_exit;
582 }
583 }
584
585 pKernelBif->bPreparingFunctionLevelReset = NV_FALSE;
586 if (pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_FLR_PRE_CONDITIONING_REQUIRED))
587 {
588 //
589 // SW WARs required for Function Level Reset -
590 // Clear Bus Master Enable bit in command register so that no more requests to sysmem are made by Fn0
591 // Executing these WARs after save config space so that restore config space does not restore-
592 // incorrect command register
593 // For other WARs which are executed in bifPrepareForFullChipReset_HAL, gpu re-init sequence after FLR makes
594 // sure to revert these WARs
595 //
596 if (kbifStopSysMemRequests_HAL(pGpu, pKernelBif, NV_TRUE) != NV_OK)
597 {
598 NV_PRINTF(LEVEL_ERROR, "BIF Stop Sys Mem requests failed.\n");
599 DBG_BREAKPOINT();
600 }
601 // Wait for number of sysmem transactions pending to go to 0
602 if (kbifWaitForTransactionsComplete_HAL(pGpu, pKernelBif) != NV_OK)
603 {
604 NV_PRINTF(LEVEL_ERROR,
605 "BIF Wait for Transactions complete failed.\n");
606 DBG_BREAKPOINT();
607 }
608 }
609
610 // Trigger FLR now
611 kbifTriggerFlr_HAL(pGpu, pKernelBif);
612
613 pKernelBif->bInFunctionLevelReset = NV_TRUE;
614
615 // wait a bit to make sure reset is propagated properly
616 if (IS_RTLSIM(pGpu))
617 {
618 //
619 // On RTL sims the OS delay functions don't scale well.
620 // Instead we use reg reads as approximate delays
621 //
622 NV_PRINTF(LEVEL_ERROR,
623 "Do config reads of NV_XVE_SW_RESET to add delay\n");
624 // Set 1ms/1000us delay - this would be acceptable delay for RTL sim
625 for( i = 0; i < (1000 * RTLSIM_DELAY_SCALE_US) ; i++)
626 {
627 // Only config reads and that too of sticky registers are supposed to work while Fn0 is under reset
628 tempRegValSwReset = osPciReadDword(handle, NV_XVE_SW_RESET);
629 }
630 // Printing tempRegValSwReset in order to use it and suppress compiler warnings(of variable set but not used)
631 NV_PRINTF(LEVEL_ERROR, "NV_XVE_SW_RESET read returned %x\n",
632 tempRegValSwReset);
633 }
634 else
635 {
636 // Wait for 100 ms before attempting to read PCI config space
637 osDelayUs(100000);
638 }
639
640 status = kbifRestorePcieConfigRegisters_HAL(pGpu, pKernelBif);
641 if (status != NV_OK)
642 {
643 DBG_BREAKPOINT();
644 NV_PRINTF(LEVEL_ERROR, "Config registers restore failed!\n");
645 goto bifDoFunctionLevelReset_TU102_exit;
646 }
647
648 // As of GA10X, miata rom cannot be run on simulation.
649 if (IS_SILICON(pGpu) || IS_EMULATION(pGpu))
650 {
651 // On emulation VBIOS boot can take long so add prints for better visibility
652 if (IS_EMULATION(pGpu))
653 {
654 NV_PRINTF(LEVEL_ERROR, "Entering secure boot completion wait.\n");
655 }
656 status = NV_ERR_NOT_SUPPORTED;
657 if (status != NV_OK)
658 {
659 DBG_BREAKPOINT();
660 NV_PRINTF(LEVEL_ERROR, "VBIOS boot failed!!\n");
661 goto bifDoFunctionLevelReset_TU102_exit;
662 }
663 if (IS_EMULATION(pGpu))
664 {
665 NV_PRINTF(LEVEL_ERROR,
666 "Exited secure boot completion wait with status = NV_OK.\n");
667 }
668 }
669
670 if (bMSIXEnabled)
671 {
672 // TODO-NM: Check if this needed for other NVPM flows like GC6
673 status = kbifRestoreMSIXVectorControlMasks_HAL(pGpu, pKernelBif, msixVectorMask);
674 if (status != NV_OK)
675 {
676 DBG_BREAKPOINT();
677 NV_PRINTF(LEVEL_ERROR, "MSIX vector control restore failed!\n");
678 goto bifDoFunctionLevelReset_TU102_exit;
679 }
680 }
681
682 // Re-init handle as well since gpu is reset
683 handle = osPciInitHandle(domain, bus, device, 0, &vendorId, &deviceId);
684 tempRegVal = osPciReadDword(handle, NV_XVE_ID);
685 if (!FLD_TEST_DRF(_XVE, _ID, _VENDOR, _NVIDIA, tempRegVal))
686 {
687 status = NV_ERR_GENERIC;
688 goto bifDoFunctionLevelReset_TU102_exit;
689 }
690
691 bifDoFunctionLevelReset_TU102_exit:
692 pKernelBif->bPreparingFunctionLevelReset = NV_FALSE;
693 pKernelBif->bInFunctionLevelReset = NV_FALSE;
694 return status;
695 }
696
697 /*!
698 * @brief Get the PMC bit of the valid Engines to reset.
699 *
700 * @return All valid engines
701 */
702 NvU32
kbifGetValidEnginesToReset_TU102(OBJGPU * pGpu,KernelBif * pKernelBif)703 kbifGetValidEnginesToReset_TU102
704 (
705 OBJGPU *pGpu,
706 KernelBif *pKernelBif
707 )
708 {
709 return (DRF_DEF(_PMC, _ENABLE, _PGRAPH, _ENABLED) |
710 DRF_DEF(_PMC, _ENABLE, _PMEDIA, _ENABLED) |
711 DRF_DEF(_PMC, _ENABLE, _CE0, _ENABLED) |
712 DRF_DEF(_PMC, _ENABLE, _CE1, _ENABLED) |
713 DRF_DEF(_PMC, _ENABLE, _CE2, _ENABLED) |
714 DRF_DEF(_PMC, _ENABLE, _CE3, _ENABLED) |
715 DRF_DEF(_PMC, _ENABLE, _CE4, _ENABLED) |
716 DRF_DEF(_PMC, _ENABLE, _CE5, _ENABLED) |
717 DRF_DEF(_PMC, _ENABLE, _CE6, _ENABLED) |
718 DRF_DEF(_PMC, _ENABLE, _CE7, _ENABLED) |
719 DRF_DEF(_PMC, _ENABLE, _CE8, _ENABLED) |
720 DRF_DEF(_PMC, _ENABLE, _PFIFO, _ENABLED) |
721 DRF_DEF(_PMC, _ENABLE, _PWR, _ENABLED) |
722 DRF_DEF(_PMC, _ENABLE, _PDISP, _ENABLED) |
723 DRF_DEF(_PMC, _ENABLE, _NVDEC0, _ENABLED) |
724 DRF_DEF(_PMC, _ENABLE, _NVDEC1, _ENABLED) |
725 DRF_DEF(_PMC, _ENABLE, _NVDEC2, _ENABLED) |
726 DRF_DEF(_PMC, _ENABLE, _NVENC0, _ENABLED) |
727 DRF_DEF(_PMC, _ENABLE, _NVENC1, _ENABLED) |
728 DRF_DEF(_PMC, _ENABLE, _SEC, _ENABLED) |
729 DRF_DEF(_PMC, _ENABLE, _PERFMON, _ENABLED) |
730 DRF_DEF(_PMC, _ENABLE, _NVJPG0, _ENABLED));
731 }