1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2013-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24
25 /*!
26 * @file
27 * @brief Provides KERNEL only functions for OBJBIF
28 */
29
30 /* ------------------------ Includes ---------------------------------------- */
31 #include "gpu/bif/kernel_bif.h"
32 #include "gpu/bus/kern_bus.h"
33 #include "gpu/gpu.h"
34 #include "gpu/intr/intr.h"
35 #include "os/os.h"
36 #include "platform/chipset/chipset.h"
37 #include "core/locks.h"
38 #include "nvrm_registry.h"
39 #include "diagnostics/tracer.h"
40 #include "nvpcie.h"
41 #include "vgpu/vgpu_events.h"
42
43 /* ------------------------ Macros ------------------------------------------ */
44 /* ------------------------ Compile Time Checks ----------------------------- */
45 /* ------------------------ Static Function Prototypes ---------------------- */
46 static void _kbifInitRegistryOverrides(OBJGPU *, KernelBif *);
47 static void _kbifCheckIfGpuExists(OBJGPU *, void*);
48 static NV_STATUS _kbifSetPcieRelaxedOrdering(OBJGPU *, KernelBif *, NvBool);
49
50 /* ------------------------ Public Functions -------------------------------- */
51
52 /*!
53 * @brief KernelBif Constructor
54 *
55 * @param[in] pGpu GPU object pointer
56 * @param[in] pKernelBif BIF object pointer
57 * @param[in] engDesc Engine descriptor
58 */
59 NV_STATUS
kbifConstructEngine_IMPL(OBJGPU * pGpu,KernelBif * pKernelBif,ENGDESCRIPTOR engDesc)60 kbifConstructEngine_IMPL
61 (
62 OBJGPU *pGpu,
63 KernelBif *pKernelBif,
64 ENGDESCRIPTOR engDesc
65 )
66 {
67 // Initialize registry overrides
68 _kbifInitRegistryOverrides(pGpu, pKernelBif);
69
70 // WAR for Bug 3208922 - disables P2P on Ampere NB
71 kbifApplyWARBug3208922_HAL(pGpu, pKernelBif);
72
73 // Disables P2P on VF
74 kbifDisableP2PTransactions_HAL(pGpu, pKernelBif);
75
76 // Cache MNOC interface support
77 kbifCacheMnocSupport_HAL(pGpu, pKernelBif);
78
79 // Cache FLR support
80 kbifCacheFlrSupport_HAL(pGpu, pKernelBif);
81
82 // Cache 64B BAR0 support
83 kbifCache64bBar0Support_HAL(pGpu, pKernelBif);
84
85 // Cache VF info
86 kbifCacheVFInfo_HAL(pGpu, pKernelBif);
87
88 // Used to track when the link has gone into Recovery, which can cause CEs.
89 pKernelBif->EnteredRecoverySinceErrorsLastChecked = NV_FALSE;
90
91 // Default scale is 1 and could be overriden by registry
92 pKernelBif->flrDevInitTimeoutScale = 1;
93
94 return NV_OK;
95 }
96
97 /*!
98 * @brief KernelBif Constructor
99 *
100 * @param[in] pGpu GPU object pointer
101 * @param[in] pKernelBif BIF object pointer
102 */
103 NV_STATUS
kbifStateInitLocked_IMPL(OBJGPU * pGpu,KernelBif * pKernelBif)104 kbifStateInitLocked_IMPL
105 (
106 OBJGPU *pGpu,
107 KernelBif *pKernelBif
108 )
109 {
110 OBJSYS *pSys = SYS_GET_INSTANCE();
111 OBJOS *pOS = SYS_GET_OS(pSys);
112 OBJCL *pCl = SYS_GET_CL(pSys);
113
114 kbifInitXveRegMap_HAL(pGpu, pKernelBif, 1);
115 kbifInitXveRegMap_HAL(pGpu, pKernelBif, 0);
116
117 // Do the HAL dependent init
118 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, kbifInit_HAL(pGpu, pKernelBif));
119
120 // Return early if GPU is connected to an unsupported chipset
121 if (pCl->getProperty(pCl, PDB_PROP_CL_UNSUPPORTED_CHIPSET))
122 {
123 return NV_ERR_NOT_COMPATIBLE;
124 }
125
126 // Initialize OS mapping and core logic
127 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, osInitMapping(pGpu));
128
129 // Initialize BIF static info
130 kbifStaticInfoInit(pGpu, pKernelBif);
131
132 // Initialize DMA caps
133 kbifInitDmaCaps(pGpu, pKernelBif);
134
135 // Check for OS w/o usable PAT support
136 if ((kbifGetBusIntfType_HAL(pKernelBif) ==
137 NV2080_CTRL_BUS_INFO_TYPE_PCI_EXPRESS) &&
138 pOS->getProperty(pOS, PDB_PROP_OS_PAT_UNSUPPORTED))
139 {
140 NV_PRINTF(LEVEL_INFO,
141 "BIF disabling noncoherent on OS w/o usable PAT support\n");
142
143 pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_SUPPORT_NONCOHERENT, NV_FALSE);
144 }
145
146 return NV_OK;
147 }
148
149 /*!
150 * @brief KernelBif state load
151 *
152 * @param[in] pGpu GPU object pointer
153 * @param[in] pKernelBif BIF object pointer
154 * @param[in] flags GPU state flag
155 */
156 NV_STATUS
kbifStateLoad_IMPL(OBJGPU * pGpu,KernelBif * pKernelBif,NvU32 flags)157 kbifStateLoad_IMPL
158 (
159 OBJGPU *pGpu,
160 KernelBif *pKernelBif,
161 NvU32 flags
162 )
163 {
164 NV_PRINTF(LEVEL_INFO, "BIF DMA Caps: %08x\n", kbifGetDmaCaps(pGpu, pKernelBif));
165
166 // Apply C73 chipset WAR
167 kbifExecC73War_HAL(pGpu, pKernelBif);
168
169 // Check for stale PCI-E dev ctrl/status errors and AER errors
170 kbifClearConfigErrors(pGpu, pKernelBif, NV_TRUE, KBIF_CLEAR_XVE_AER_ALL_MASK);
171
172 // Cache PCI config registers to be restored during resume
173 if (!pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_PM_RESUME_CODEPATH))
174 {
175 kbifSavePcieConfigRegisters_HAL(pGpu, pKernelBif);
176 }
177
178 //
179 // A vGPU cannot disappear and these accesses are
180 // particularly expensive on vGPUs
181 //
182 if (pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_CHECK_IF_GPU_EXISTS_DEF) &&
183 !IS_VIRTUAL(pGpu))
184 {
185 osSchedule1HzCallback(pGpu, _kbifCheckIfGpuExists, NULL, NV_OS_1HZ_REPEAT);
186 }
187
188 return NV_OK;
189 }
190
191 /*!
192 * @brief Configure PCIe Relaxed Ordering in BIF
193 *
194 * @param[in] pGpu GPU object pointer
195 * @param[in] pKernelBif KBIF object pointer
196 * @param[in] enableRo Enable/disable RO
197 */
198 static NV_STATUS
_kbifSetPcieRelaxedOrdering(OBJGPU * pGpu,KernelBif * pKernelBif,NvBool enableRo)199 _kbifSetPcieRelaxedOrdering
200 (
201 OBJGPU *pGpu,
202 KernelBif *pKernelBif,
203 NvBool enableRo
204 )
205 {
206 NV2080_CTRL_INTERNAL_BIF_SET_PCIE_RO_PARAMS pcieRo;
207 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
208 NV_STATUS status;
209
210 pcieRo.enableRo = enableRo;
211
212 status = pRmApi->Control(pRmApi, pGpu->hInternalClient, pGpu->hInternalSubdevice,
213 NV2080_CTRL_CMD_INTERNAL_BIF_SET_PCIE_RO,
214 &pcieRo, sizeof(pcieRo));
215 if (status != NV_OK) {
216 NV_PRINTF(LEVEL_ERROR, "NV2080_CTRL_CMD_INTERNAL_BIF_SET_PCIE_RO failed %s (0x%x)\n",
217 nvstatusToString(status), status);
218 return status;
219 }
220
221 return NV_OK;
222 }
223
224 /*!
225 * @brief KernelBif state post-load
226 *
227 * @param[in] pGpu GPU object pointer
228 * @param[in] pKernelBif KBIF object pointer
229 * @param[in] flags GPU state flag
230 */
231 NV_STATUS
kbifStatePostLoad_IMPL(OBJGPU * pGpu,KernelBif * pKernelBif,NvU32 flags)232 kbifStatePostLoad_IMPL
233 (
234 OBJGPU *pGpu,
235 KernelBif *pKernelBif,
236 NvU32 flags
237 )
238 {
239 NV_STATUS status;
240
241 kbifInitRelaxedOrderingFromEmulatedConfigSpace(pGpu, pKernelBif);
242 if (pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_PCIE_RELAXED_ORDERING_SET_IN_EMULATED_CONFIG_SPACE)) {
243 //
244 // This is done from StatePostLoad() to guarantee that BIF's StateLoad()
245 // is already completed for both monolithic RM and GSP RM.
246 //
247 status = _kbifSetPcieRelaxedOrdering(pGpu, pKernelBif, NV_TRUE);
248 if (status != NV_OK)
249 return NV_OK;
250 }
251
252 return NV_OK;
253 }
254
255 /*!
256 * @brief KernelBif state unload
257 *
258 * @param[in] pGpu GPU object pointer
259 * @param[in] pKernelBif BIF object pointer
260 * @param[in] flags GPU state flag
261 */
262 NV_STATUS
kbifStateUnload_IMPL(OBJGPU * pGpu,KernelBif * pKernelBif,NvU32 flags)263 kbifStateUnload_IMPL
264 (
265 OBJGPU *pGpu,
266 KernelBif *pKernelBif,
267 NvU32 flags
268 )
269 {
270
271 return NV_OK;
272 }
273
274 /*!
275 * @brief Initialize DMA caps
276 *
277 * @param[in] pGpu GPU object pointer
278 * @param[in] pKernelBif BIF object pointer
279 */
280 void
kbifInitDmaCaps_IMPL(OBJGPU * pGpu,KernelBif * pKernelBif)281 kbifInitDmaCaps_IMPL
282 (
283 OBJGPU *pGpu,
284 KernelBif *pKernelBif
285 )
286 {
287 OBJSYS *pSys = SYS_GET_INSTANCE();
288 OBJCL *pCl = SYS_GET_CL(pSys);
289
290 pKernelBif->dmaCaps = REF_DEF(BIF_DMA_CAPS_NOSNOOP, _CTXDMA);
291
292 // Set the coherency cap on host RM based on the chipset
293 if (IsAMODEL(pGpu) ||
294 pCl->getProperty(pCl, PDB_PROP_CL_IS_CHIPSET_IO_COHERENT))
295 {
296 pKernelBif->dmaCaps |= REF_DEF(BIF_DMA_CAPS_SNOOP, _CTXDMA);
297 }
298 }
299
300 NvU32
kbifGetDmaCaps_IMPL(OBJGPU * pGpu,KernelBif * pKernelBif)301 kbifGetDmaCaps_IMPL
302 (
303 OBJGPU *pGpu,
304 KernelBif *pKernelBif
305 )
306 {
307 NvU32 retval;
308
309 // Start out with system specific DMA caps
310 retval = pKernelBif->dmaCaps;
311
312 // If noncoherent support is disabled, mask out SNOOP caps
313 if (!pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_SUPPORT_NONCOHERENT))
314 {
315 retval &= ~DRF_SHIFTMASK(BIF_DMA_CAPS_NOSNOOP);
316 }
317
318 return retval;
319 }
320
321 /*!
322 * @brief Initialize BIF static info in Kernel object through RPC
323 *
324 * @param[in] pGpu GPU object pointer
325 * @param[in] pKernelBif BIF object pointer
326 */
327 NV_STATUS
kbifStaticInfoInit_IMPL(OBJGPU * pGpu,KernelBif * pKernelBif)328 kbifStaticInfoInit_IMPL
329 (
330 OBJGPU *pGpu,
331 KernelBif *pKernelBif
332 )
333 {
334 NV2080_CTRL_INTERNAL_BIF_GET_STATIC_INFO_PARAMS *pStaticInfo;
335 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
336 NV_STATUS status = NV_OK;
337
338 // Allocate memory for the command parameter
339 pStaticInfo = portMemAllocNonPaged(sizeof(*pStaticInfo));
340 if (pStaticInfo == NULL)
341 {
342 NV_PRINTF(LEVEL_ERROR, "Could not allocate pStaticInfo for KernelBif");
343 status = NV_ERR_NO_MEMORY;
344 goto kBifStaticInfoInit_IMPL_exit;
345 }
346 portMemSet(pStaticInfo, 0, sizeof(*pStaticInfo));
347
348 // Send the command
349 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR,
350 pRmApi->Control(pRmApi, pGpu->hInternalClient, pGpu->hInternalSubdevice,
351 NV2080_CTRL_CMD_INTERNAL_BIF_GET_STATIC_INFO,
352 pStaticInfo, sizeof(*pStaticInfo)),
353 kBifStaticInfoInit_IMPL_exit);
354
355 // Initialize Kernel object fields with RPC response
356 pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_PCIE_GEN4_CAPABLE,
357 pStaticInfo->bPcieGen4Capable);
358 pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_IS_C2C_LINK_UP,
359 pStaticInfo->bIsC2CLinkUp);
360 pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_DEVICE_IS_MULTIFUNCTION,
361 pStaticInfo->bIsDeviceMultiFunction);
362 pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_GCX_PMU_CFG_SPACE_RESTORE,
363 pStaticInfo->bGcxPmuCfgSpaceRestore);
364 pKernelBif->dmaWindowStartAddress = pStaticInfo->dmaWindowStartAddress;
365
366 kBifStaticInfoInit_IMPL_exit:
367 portMemFree(pStaticInfo);
368
369 return status;
370 }
371
372 /*!
373 * @brief Initialize PCI-E config space bits based on chipset and GPU support.
374 */
375 void
kbifInitPcieDeviceControlStatus_IMPL(OBJGPU * pGpu,KernelBif * pKernelBif)376 kbifInitPcieDeviceControlStatus_IMPL
377 (
378 OBJGPU *pGpu,
379 KernelBif *pKernelBif
380 )
381 {
382 OBJSYS *pSys = SYS_GET_INSTANCE();
383 OBJCL *pCl = SYS_GET_CL(pSys);
384
385 kbifEnableExtendedTagSupport_HAL(pGpu, pKernelBif);
386
387 //
388 // Bug 382675 and 482867: Many SBIOSes default to disabling relaxed
389 // ordering on GPUs, we want to always force it back on unless
390 // the upstream root port is known to be broken with respect to this
391 // feature.
392 //
393 if (!pCl->getProperty(pCl, PDB_PROP_CL_RELAXED_ORDERING_NOT_CAPABLE))
394 {
395 kbifPcieConfigEnableRelaxedOrdering_HAL(pGpu, pKernelBif);
396 }
397 else
398 {
399 kbifPcieConfigDisableRelaxedOrdering_HAL(pGpu, pKernelBif);
400 }
401
402 //
403 // WAR for bug 3661529. All GH100 SKUs will need the NoSnoop WAR.
404 // But currently GSP-RM does not detect this correctly,
405 //
406 if (IsGH100(pGpu))
407 {
408 pCl->setProperty(pCl, PDB_PROP_CL_ROOTPORT_NEEDS_NOSNOOP_WAR, NV_TRUE);
409 }
410
411 if (!pCl->getProperty(pCl, PDB_PROP_CL_NOSNOOP_NOT_CAPABLE) &&
412 !pCl->getProperty(pCl, PDB_PROP_CL_ROOTPORT_NEEDS_NOSNOOP_WAR))
413 {
414 // Bug 393398 - Re-enable DEVICE_CONTROL_STATUS_ENABLE_NO_SNOOP
415 kbifEnableNoSnoop_HAL(pGpu, pKernelBif, NV_TRUE);
416 }
417 else
418 {
419 //
420 // Check for NO_SNOOP P2P bug on specific chipset. More info in bug 332764.
421 // Check for NO_SNOOP enabled by default on specific CPU. Refer bug 1511622.
422 //
423 kbifEnableNoSnoop_HAL(pGpu, pKernelBif, NV_FALSE);
424 }
425 }
426
427 /*!
428 * @brief Check and rearm MSI
429 *
430 * @param[in] pGpu GPU object pointer
431 * @param[in] pKernelBif BIF object pointer
432 *
433 * @return NV_TRUE if MSI is enabled
434 * NV_FALSE if MSI is disabled
435 */
436 void
kbifCheckAndRearmMSI_IMPL(OBJGPU * pGpu,KernelBif * pKernelBif)437 kbifCheckAndRearmMSI_IMPL
438 (
439 OBJGPU *pGpu,
440 KernelBif *pKernelBif
441 )
442 {
443 Intr *pIntr = GPU_GET_INTR(pGpu);
444
445 if (kbifIsMSIEnabled(pGpu, pKernelBif))
446 {
447 if (!IS_VIRTUAL(pGpu) ||
448 pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_ALLOW_REARM_MSI_FOR_VF))
449 {
450 // Send EOI to rearm
451 if (pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_USE_CONFIG_SPACE_TO_REARM_MSI))
452 {
453 kbifRearmMSI_HAL(pGpu, pKernelBif);
454 }
455 else
456 {
457 intrRetriggerTopLevel_HAL(pGpu, pIntr);
458 }
459 }
460 }
461 else if (kbifIsMSIXEnabled(pGpu, pKernelBif))
462 {
463 intrRetriggerTopLevel_HAL(pGpu, pIntr);
464 }
465 }
466
467 /*!
468 * @brief Checks if MSI is enabled. Prefers to check the SW cache, but if
469 * uncached, checks HW state and updates the SW cache for future use
470 *
471 * @param[in] pGpu GPU object pointer
472 * @param[in] pKernelBif BIF object pointer
473 *
474 * @return NV_TRUE if MSI is enabled
475 * NV_FALSE if MSI is disabled
476 */
477 NvBool
kbifIsMSIEnabled_IMPL(OBJGPU * pGpu,KernelBif * pKernelBif)478 kbifIsMSIEnabled_IMPL
479 (
480 OBJGPU *pGpu,
481 KernelBif *pKernelBif
482 )
483 {
484 //
485 // Bug 418883: We shall rely upon value cached at boot, for the value
486 // should not change during execution. If however, we must ever change
487 // this back to be read at every ISR, we *must* read the value through
488 // PCI CFG cycles.
489 //
490 if (!pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_IS_MSI_CACHED))
491 {
492 if (kbifIsMSIEnabledInHW_HAL(pGpu, pKernelBif))
493 {
494 pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_IS_MSI_ENABLED, NV_TRUE);
495
496 if (IS_VIRTUAL(pGpu))
497 {
498 // vGPU guests want an FYI print that re-arming is not required
499 NV_PRINTF(LEVEL_WARNING,
500 "MSI is enabled for vGPU, but no need to re-ARM\n");
501 }
502 }
503 else
504 {
505 pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_IS_MSI_ENABLED, NV_FALSE);
506 }
507 pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_IS_MSI_CACHED, NV_TRUE);
508 }
509
510 return pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_IS_MSI_ENABLED);
511 }
512
513 /*!
514 * @brief Checks if MSI-X is enabled. Prefers to check the SW cache, but if
515 * uncached, checks HW state and updates the SW cache for future use
516 *
517 * @param[in] pGpu GPU object pointer
518 * @param[in] pKernelBif BIF object pointer
519 *
520 * @return NV_TRUE if MSI is enabled
521 * NV_FALSE if MSI is disabled
522 */
523 NvBool
kbifIsMSIXEnabled_IMPL(OBJGPU * pGpu,KernelBif * pKernelBif)524 kbifIsMSIXEnabled_IMPL
525 (
526 OBJGPU *pGpu,
527 KernelBif *pKernelBif
528 )
529 {
530 if (!pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_IS_MSIX_CACHED))
531 {
532 if (kbifIsMSIXEnabledInHW_HAL(pGpu, pKernelBif))
533 {
534 pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_IS_MSIX_ENABLED, NV_TRUE);
535 }
536 else
537 {
538 pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_IS_MSIX_ENABLED, NV_FALSE);
539 }
540 pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_IS_MSIX_CACHED, NV_TRUE);
541 }
542 return pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_IS_MSIX_ENABLED);
543 }
544
545 /*!
546 * @brief Clear PCIe HW PCIe config space error counters.
547 * All of these should be cleared using config cycles.
548 *
549 * @param[in] pGpu GPU object pointer
550 * @param[in] pKernelBif BIF object pointer
551 */
552 void
kbifClearConfigErrors_IMPL(OBJGPU * pGpu,KernelBif * pKernelBif,NvBool bClearStatus,NvU32 xveAerFlagsMask)553 kbifClearConfigErrors_IMPL
554 (
555 OBJGPU *pGpu,
556 KernelBif *pKernelBif,
557 NvBool bClearStatus,
558 NvU32 xveAerFlagsMask
559 )
560 {
561 NvU32 xveStatusFlags = 0;
562 NvU32 xveStatus = 0;
563 NvU32 xveAerFlags = 0;
564
565 if ((bClearStatus) &&
566 (kbifGetXveStatusBits_HAL(pGpu, pKernelBif, &xveStatusFlags, &xveStatus) == NV_OK) &&
567 (xveStatusFlags != 0))
568 {
569 NV_PRINTF(LEVEL_WARNING, "PCI-E device status errors pending (%08X):\n",
570 xveStatusFlags);
571 #ifdef DEBUG
572 if ( xveStatusFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_ERRORS_CORR_ERROR )
573 {
574 NV_PRINTF(LEVEL_WARNING, " _CORR_ERROR_DETECTED\n");
575 }
576 if ( xveStatusFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_ERRORS_NON_FATAL_ERROR )
577 {
578 NV_PRINTF(LEVEL_WARNING, " _NON_FATAL_ERROR_DETECTED\n");
579 }
580 if ( xveStatusFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_ERRORS_FATAL_ERROR )
581 {
582 NV_PRINTF(LEVEL_WARNING, " _FATAL_ERROR_DETECTED\n");
583 }
584 if ( xveStatusFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_ERRORS_UNSUPP_REQUEST )
585 {
586 NV_PRINTF(LEVEL_WARNING, " _UNSUPP_REQUEST_DETECTED\n");
587 }
588 #endif
589 NV_PRINTF(LEVEL_WARNING, "Clearing these errors..\n");
590 kbifClearXveStatus_HAL(pGpu, pKernelBif, &xveStatus);
591 }
592
593 if ((xveAerFlagsMask) &&
594 (kbifGetXveAerBits_HAL(pGpu, pKernelBif, &xveAerFlags) == NV_OK))
595 {
596 xveAerFlags &= xveAerFlagsMask;
597
598 if (xveAerFlags != 0)
599 {
600 NV_PRINTF(LEVEL_WARNING,
601 "PCI-E device AER errors pending (%08X):\n",
602 xveAerFlags);
603 #ifdef DEBUG
604 if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_TRAINING_ERR)
605 {
606 NV_PRINTF(LEVEL_WARNING, " _AER_UNCORR_TRAINING_ERR\n");
607 }
608 if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_DLINK_PROTO_ERR)
609 {
610 NV_PRINTF(LEVEL_WARNING, " _AER_UNCORR_DLINK_PROTO_ERR\n");
611 }
612 if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_POISONED_TLP)
613 {
614 NV_PRINTF(LEVEL_WARNING, " _AER_UNCORR_POISONED_TLP\n");
615 }
616 if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_FC_PROTO_ERR)
617 {
618 NV_PRINTF(LEVEL_WARNING, " _AER_UNCORR_FC_PROTO_ERR\n");
619 }
620 if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_CPL_TIMEOUT)
621 {
622 NV_PRINTF(LEVEL_WARNING, " _AER_UNCORR_CPL_TIMEOUT\n");
623 }
624 if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_CPL_ABORT)
625 {
626 NV_PRINTF(LEVEL_WARNING, " _AER_UNCORR_CPL_ABORT\n");
627 }
628 if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_UNEXP_CPL)
629 {
630 NV_PRINTF(LEVEL_WARNING, " _AER_UNCORR_UNEXP_CPL\n");
631 }
632 if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_RCVR_OVERFLOW)
633 {
634 NV_PRINTF(LEVEL_WARNING, " _AER_UNCORR_RCVR_OVERFLOW\n");
635 }
636 if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_MALFORMED_TLP)
637 {
638 NV_PRINTF(LEVEL_WARNING, " _AER_UNCORR_MALFORMED_TLP\n");
639 }
640 if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_ECRC_ERROR)
641 {
642 NV_PRINTF(LEVEL_WARNING, " _AER_UNCORR_ECRC_ERROR\n");
643 }
644 if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_UNSUPPORTED_REQ)
645 {
646 NV_PRINTF(LEVEL_WARNING, " _AER_UNCORR_UNSUPPORTED_REQ\n");
647 }
648 if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_RCV_ERR)
649 {
650 NV_PRINTF(LEVEL_WARNING, " _AER_CORR_RCV_ERR\n");
651 }
652 if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_BAD_TLP)
653 {
654 NV_PRINTF(LEVEL_WARNING, " _AER_CORR_BAD_TLP\n");
655 }
656 if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_BAD_DLLP)
657 {
658 NV_PRINTF(LEVEL_WARNING, " _AER_CORR_BAD_DLLP\n");
659 }
660 if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_RPLY_ROLLOVER)
661 {
662 NV_PRINTF(LEVEL_WARNING, " _AER_CORR_RPLY_ROLLOVER\n");
663 }
664 if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_RPLY_TIMEOUT)
665 {
666 NV_PRINTF(LEVEL_WARNING, " _AER_CORR_RPLY_TIMEOUT\n");
667 }
668 if (xveAerFlags & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_ADVISORY_NONFATAL)
669 {
670 NV_PRINTF(LEVEL_WARNING, " _AER_CORR_ADVISORY_NONFATAL\n");
671 }
672 #endif
673 NV_PRINTF(LEVEL_WARNING, "Clearing these errors..\n");
674 kbifClearXveAer_HAL(pGpu, pKernelBif, xveAerFlags);
675 }
676 }
677 }
678
679 /*!
680 * @brief The PCI bus family means it has the concept of bus/dev/func
681 * and compatible PCI config space.
682 */
683 NvBool
kbifIsPciBusFamily_IMPL(KernelBif * pKernelBif)684 kbifIsPciBusFamily_IMPL
685 (
686 KernelBif *pKernelBif
687 )
688 {
689 NvU32 busType = kbifGetBusIntfType_HAL(pKernelBif);
690
691 return ((busType == NV2080_CTRL_BUS_INFO_TYPE_PCI) ||
692 (busType == NV2080_CTRL_BUS_INFO_TYPE_PCI_EXPRESS) ||
693 (busType == NV2080_CTRL_BUS_INFO_TYPE_FPCI));
694 }
695
696 /*!
697 * @brief Regkey Overrides for Bif
698 *
699 * @param[in] pGpu GPU object pointer
700 * @param[in] pKernelBif BIF object pointer
701 */
702 static void
_kbifInitRegistryOverrides(OBJGPU * pGpu,KernelBif * pKernelBif)703 _kbifInitRegistryOverrides
704 (
705 OBJGPU *pGpu,
706 KernelBif *pKernelBif
707 )
708 {
709 NvU32 data32;
710
711 // P2P Override
712 pKernelBif->p2pOverride = BIF_P2P_NOT_OVERRIDEN;
713 if (osReadRegistryDword(pGpu, NV_REG_STR_CL_FORCE_P2P, &data32) == NV_OK)
714 {
715 pKernelBif->p2pOverride = data32;
716 pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_P2P_READS_DISABLED, FLD_TEST_DRF(_REG_STR, _CL_FORCE_P2P, _READ, _DISABLE, data32));
717 pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_P2P_WRITES_DISABLED, FLD_TEST_DRF(_REG_STR, _CL_FORCE_P2P, _WRITE, _DISABLE, data32));
718 }
719
720 // P2P force type override
721 pKernelBif->forceP2PType = NV_REG_STR_RM_FORCE_P2P_TYPE_DEFAULT;
722 if (osReadRegistryDword(pGpu, NV_REG_STR_RM_FORCE_P2P_TYPE, &data32) == NV_OK &&
723 (data32 <= NV_REG_STR_RM_FORCE_P2P_TYPE_MAX))
724 {
725 pKernelBif->forceP2PType = data32;
726 }
727
728 // Peer Mapping override
729 pKernelBif->peerMappingOverride = NV_REG_STR_PEERMAPPING_OVERRIDE_DEFAULT;
730 if (osReadRegistryDword(pGpu, NV_REG_STR_PEERMAPPING_OVERRIDE, &data32) == NV_OK)
731 {
732 NV_PRINTF(LEVEL_INFO, "allow peermapping reg key = %d\n", data32);
733 pKernelBif->peerMappingOverride = !!data32;
734 }
735
736 // Check if the PCIe SBR recovery feature is enabled.
737 if (osReadRegistryDword(pGpu, NV_REG_STR_SECONDARY_BUS_RESET_ENABLED, &data32) == NV_OK)
738 {
739 if (data32)
740 {
741 // The regkey was set to a non-zero value, enable the feature.
742 pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_SECONDARY_BUS_RESET_ENABLED, NV_TRUE);
743 }
744 else
745 {
746 // The regkey was set to 0, disable the feature.
747 pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_SECONDARY_BUS_RESET_ENABLED, NV_FALSE);
748 }
749 }
750
751 // Check if saving of the PCIe config space during SBR is enabled.
752 if (osReadRegistryDword(pGpu, NV_REG_STR_FORCE_PCIE_CONFIG_SAVE, &data32) == NV_OK)
753 {
754 if (data32)
755 {
756 // The regkey was set to a non-zero value, enable config save
757 pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_FORCE_PCIE_CONFIG_SAVE, NV_TRUE);
758 }
759 else
760 {
761 // The regkey was set to a 0, disable config save
762 pKernelBif->setProperty(pBif, PDB_PROP_KBIF_FORCE_PCIE_CONFIG_SAVE, NV_FALSE);
763 }
764 }
765
766 // Check if Function Level Reset(FLR) is disabled
767 pKernelBif->bForceDisableFLR = NV_REG_STR_RM_PCIE_FLR_POLICY_DEFAULT;
768 if (osReadRegistryDword(pGpu,
769 NV_REG_STR_RM_PCIE_FLR_POLICY, &data32) == NV_OK)
770 {
771 NV_PRINTF(LEVEL_INFO, "Pcie FLR Policy reg key = %d\n", data32);
772
773 if (data32 == NV_REG_STR_RM_PCIE_FLR_POLICY_FORCE_DISABLE)
774 {
775 pKernelBif->bForceDisableFLR = NV_TRUE;
776 }
777 // There might be a requirement to handle (data32 == NV_REG_STR_RM_PCIE_FLR_POLICY_DEFAULT) case as per platform's policy
778 else
779 {
780 pKernelBif->bForceDisableFLR = NV_FALSE;
781 }
782 }
783
784 // Check for FLR timeout scale override
785 if (osReadRegistryDword(pGpu,
786 NV_REG_STR_RM_PCIE_FLR_DEVINIT_TIMEOUT_SCALE, &data32) == NV_OK)
787 {
788 if ((data32 >= NV_REG_STR_RM_PCIE_FLR_DEVINIT_TIMEOUT_SCALE_MIN_ALLOWED) &&
789 (data32 <= NV_REG_STR_RM_PCIE_FLR_DEVINIT_TIMEOUT_SCALE_MAX_ALLOWED))
790 {
791 pKernelBif->flrDevInitTimeoutScale = data32;
792 }
793 }
794
795 }
796
797 /*!
798 * Callback function to check if GPU exists
799 *
800 * @param[in] pGpu GPU object pointer
801 * @param[in] rsvd Reserved field
802 */
803 static void
_kbifCheckIfGpuExists(OBJGPU * pGpu,void * rsvd)804 _kbifCheckIfGpuExists
805 (
806 OBJGPU *pGpu,
807 void *rsvd
808 )
809 {
810 if (FULL_GPU_SANITY_CHECK(pGpu))
811 {
812 if (gpuVerifyExistence_HAL(pGpu) != NV_OK)
813 {
814 osRemove1HzCallback(pGpu, _kbifCheckIfGpuExists, NULL);
815 }
816 }
817 }
818
819 /*!
820 * @brief Get link capabilities register value
821 *
822 * @param[in] pGpu GPU object pointer
823 * @param[in] pKernelBif KernelBif object pointer
824 *
825 * @return Register value
826 */
827 NvU32
kbifGetGpuLinkCapabilities_IMPL(OBJGPU * pGpu,KernelBif * pKernelBif)828 kbifGetGpuLinkCapabilities_IMPL
829 (
830 OBJGPU *pGpu,
831 KernelBif *pKernelBif
832 )
833 {
834 NvU32 addrLinkCap = 0;
835 NvU32 data = 0;
836
837 if (NV_OK != kbifGetBusOptionsAddr_HAL(pGpu, pKernelBif, BUS_OPTIONS_LINK_CAPABILITIES, &addrLinkCap))
838 {
839 return 0;
840 }
841
842 if (NV_OK != GPU_BUS_CFG_RD32(pGpu, addrLinkCap, &data))
843 {
844 NV_PRINTF(LEVEL_ERROR, "Unable to read %x\n", addrLinkCap);
845 return 0;
846 }
847
848 return data;
849 }
850
851 /*!
852 * @brief Get link control status register value
853 *
854 * @param[in] pGpu GPU object pointer
855 * @param[in] pKernelBif KernelBif object pointer
856 *
857 * @return Register value
858 */
859 NvU32
kbifGetGpuLinkControlStatus_IMPL(OBJGPU * pGpu,KernelBif * pKernelBif)860 kbifGetGpuLinkControlStatus_IMPL
861 (
862 OBJGPU *pGpu,
863 KernelBif *pKernelBif
864 )
865 {
866 NvU32 addrLinkControlStatus = 0;
867 NvU32 data = 0;
868
869 if (NV_OK != kbifGetBusOptionsAddr_HAL(pGpu, pKernelBif, BUS_OPTIONS_LINK_CONTROL_STATUS, &addrLinkControlStatus))
870 {
871 return 0;
872 }
873
874 if (NV_OK != GPU_BUS_CFG_RD32(pGpu, addrLinkControlStatus, &data ))
875 {
876 NV_PRINTF(LEVEL_ERROR, "Unable to read %x\n", addrLinkControlStatus);
877 return 0;
878 }
879
880 return data;
881 }
882
883 /*!
884 * @brief Get device control status register value
885 *
886 * @param[in] pGpu GPU object pointer
887 * @param[in] pKernelBif KernelBif object pointer
888 *
889 * @return Register value
890 */
891 NvU32
kbifGetGpuDevControlStatus_IMPL(OBJGPU * pGpu,KernelBif * pKernelBif)892 kbifGetGpuDevControlStatus_IMPL
893 (
894 OBJGPU *pGpu,
895 KernelBif *pKernelBif
896 )
897 {
898 NvU32 addrDevControlStatus = 0;
899 NvU32 data = 0;
900
901 if (NV_OK != kbifGetBusOptionsAddr_HAL(pGpu, pKernelBif, BUS_OPTIONS_DEV_CONTROL_STATUS, &addrDevControlStatus))
902 {
903 return 0;
904 }
905
906 if (NV_OK != GPU_BUS_CFG_RD32(pGpu, addrDevControlStatus, &data ))
907 {
908 NV_PRINTF(LEVEL_ERROR, "Unable to read %x\n", addrDevControlStatus);
909 return 0;
910 }
911
912 return data;
913 }
914
915 /*!
916 * @brief Get device control status 2 register value
917 *
918 * @param[in] pGpu GPU object pointer
919 * @param[in] pKernelBif KernelBif object pointer
920 *
921 * @return Register value
922 */
923 NvU32
kbifGetGpuDevControlStatus2_IMPL(OBJGPU * pGpu,KernelBif * pKernelBif)924 kbifGetGpuDevControlStatus2_IMPL
925 (
926 OBJGPU *pGpu,
927 KernelBif *pKernelBif
928 )
929 {
930 NvU32 addrDevControlStatus2 = 0;
931 NvU32 data = 0;
932
933 if (NV_OK != kbifGetBusOptionsAddr_HAL(pGpu, pKernelBif, BUS_OPTIONS_DEV_CONTROL_STATUS_2, &addrDevControlStatus2))
934 {
935 return 0;
936 }
937
938 if (NV_OK != GPU_BUS_CFG_RD32(pGpu, addrDevControlStatus2, &data ))
939 {
940 NV_PRINTF(LEVEL_ERROR, "Unable to read %x\n", addrDevControlStatus2);
941 return 0;
942 }
943
944 return data;
945 }
946
947 /*!
948 * @brief Get L1 Substates control register value
949 *
950 * @param[in] pGpu GPU object pointer
951 * @param[in] pKernelBif KernelBif object pointer
952 *
953 * @return Register value
954 */
955 NvU32
kbifGetGpuL1PmSubstatesCtrl1_IMPL(OBJGPU * pGpu,KernelBif * pKernelBif)956 kbifGetGpuL1PmSubstatesCtrl1_IMPL
957 (
958 OBJGPU *pGpu,
959 KernelBif *pKernelBif
960 )
961 {
962 NvU32 addrL1PmSubstatesCtrl1 = 0;
963 NvU32 data = 0;
964
965 if (NV_OK != kbifGetBusOptionsAddr_HAL(pGpu, pKernelBif, BUS_OPTIONS_L1_PM_SUBSTATES_CTRL_1, &addrL1PmSubstatesCtrl1))
966 {
967 return 0;
968 }
969
970 if (NV_OK != GPU_BUS_CFG_RD32(pGpu, addrL1PmSubstatesCtrl1, &data ))
971 {
972 NV_PRINTF(LEVEL_ERROR, "Unable to read %x\n", addrL1PmSubstatesCtrl1);
973 return 0;
974 }
975
976 return data;
977 }
978
979 static NvBool
_doesBoardHaveMultipleGpusAndSwitch(OBJGPU * pGpu)980 _doesBoardHaveMultipleGpusAndSwitch(OBJGPU *pGpu)
981 {
982 if (((gpuIsMultiGpuBoard(pGpu)) ||
983 (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_GEMINI)))&&
984 ((pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_PLX_PRESENT)) ||
985 (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_BR03_PRESENT)) ||
986 (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_BR04_PRESENT))))
987 {
988 return NV_TRUE;
989 }
990 else
991 {
992 return NV_FALSE;
993 }
994 }
995
996 NV_STATUS
kbifControlGetPCIEInfo_IMPL(OBJGPU * pGpu,KernelBif * pKernelBif,NV2080_CTRL_BUS_INFO * pBusInfo)997 kbifControlGetPCIEInfo_IMPL
998 (
999 OBJGPU *pGpu,
1000 KernelBif *pKernelBif,
1001 NV2080_CTRL_BUS_INFO *pBusInfo
1002 )
1003 {
1004 OBJSYS *pSys = SYS_GET_INSTANCE();
1005 OBJCL *pCl = SYS_GET_CL(pSys);
1006 NvU32 index = pBusInfo->index;
1007 NvU32 data = 0;
1008
1009 if ((pKernelBif != NULL) &&
1010 (kbifGetBusIntfType_HAL(pKernelBif) != NV2080_CTRL_BUS_INFO_TYPE_PCI_EXPRESS))
1011 {
1012 // KMD cannot handle error codes for this ctrl call, hence returning
1013 // NV_OK, once KMD fixes the bug:3545197, RM can return NV_ERR_NOT_SUPPORTED
1014 return NV_OK;
1015 }
1016
1017 switch (index)
1018 {
1019 case NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_LINK_CAPS:
1020 {
1021 data = kbifGetGpuLinkCapabilities(pGpu, pKernelBif);
1022 break;
1023 }
1024 case NV2080_CTRL_BUS_INFO_INDEX_PCIE_UPSTREAM_LINK_CAPS:
1025 case NV2080_CTRL_BUS_INFO_INDEX_PCIE_BOARD_LINK_CAPS:
1026 {
1027 if (_doesBoardHaveMultipleGpusAndSwitch(pGpu))
1028 {
1029 if (clPcieReadPortConfigReg(pGpu, pCl,
1030 &pGpu->gpuClData.boardUpstreamPort,
1031 CL_PCIE_LINK_CAP, &data) != NV_OK)
1032 {
1033 data = 0;
1034 }
1035 }
1036 else
1037 {
1038 data = kbifGetGpuLinkCapabilities(pGpu, pKernelBif);
1039 }
1040 break;
1041 }
1042 case NV2080_CTRL_BUS_INFO_INDEX_PCIE_UPSTREAM_GEN_INFO:
1043 case NV2080_CTRL_BUS_INFO_INDEX_PCIE_BOARD_GEN_INFO:
1044 {
1045 NvU32 temp;
1046
1047 if (_doesBoardHaveMultipleGpusAndSwitch(pGpu))
1048 {
1049 if (clPcieReadPortConfigReg(pGpu, pCl,
1050 &pGpu->gpuClData.boardUpstreamPort,
1051 CL_PCIE_LINK_CTRL_STATUS, &temp) != NV_OK)
1052 {
1053 data = 0;
1054 break;
1055 }
1056 else
1057 {
1058 temp = REF_VAL(NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_LINK_SPEED, temp);
1059 if (temp == NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_LINK_SPEED_64000MBPS)
1060 {
1061 data = FLD_SET_DRF(2080, _CTRL_BUS_INFO_PCIE_LINK_CAP,
1062 _CURR_LEVEL, _GEN6, data);
1063 }
1064 else if (temp == NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_LINK_SPEED_32000MBPS)
1065 {
1066 data = FLD_SET_DRF(2080, _CTRL_BUS_INFO_PCIE_LINK_CAP,
1067 _CURR_LEVEL, _GEN5, data);
1068 }
1069 else if (temp == NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_LINK_SPEED_16000MBPS)
1070 {
1071 data = FLD_SET_DRF(2080, _CTRL_BUS_INFO_PCIE_LINK_CAP,
1072 _CURR_LEVEL, _GEN4, data);
1073 }
1074 else if (temp == NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_LINK_SPEED_8000MBPS)
1075 {
1076 data = FLD_SET_DRF(2080, _CTRL_BUS_INFO_PCIE_LINK_CAP,
1077 _CURR_LEVEL, _GEN3, data);
1078 }
1079 else if (temp == NV2080_CTRL_BUS_INFO_PCIE_LINK_CTRL_STATUS_LINK_SPEED_5000MBPS)
1080 {
1081 data = FLD_SET_DRF(2080, _CTRL_BUS_INFO_PCIE_LINK_CAP,
1082 _CURR_LEVEL, _GEN2, data);
1083 }
1084 else
1085 {
1086 data = FLD_SET_DRF(2080, _CTRL_BUS_INFO_PCIE_LINK_CAP,
1087 _CURR_LEVEL, _GEN1, data);
1088 }
1089 }
1090
1091 if (clPcieReadPortConfigReg(pGpu, pCl,
1092 &pGpu->gpuClData.boardUpstreamPort,
1093 CL_PCIE_LINK_CAP, &temp) != NV_OK)
1094 {
1095 data = 0;
1096 break;
1097 }
1098 else
1099 {
1100 temp = REF_VAL(NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED, temp);
1101 if (temp == NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED_64000MBPS)
1102 {
1103 data = FLD_SET_DRF(2080, _CTRL_BUS_INFO_PCIE_LINK_CAP,
1104 _GEN, _GEN6, data);
1105 }
1106 else if (temp == NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED_32000MBPS)
1107 {
1108 data = FLD_SET_DRF(2080, _CTRL_BUS_INFO_PCIE_LINK_CAP,
1109 _GEN, _GEN5, data);
1110 }
1111 else if (temp == NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED_16000MBPS)
1112 {
1113 data = FLD_SET_DRF(2080, _CTRL_BUS_INFO_PCIE_LINK_CAP,
1114 _GEN, _GEN4, data);
1115 }
1116 else if (temp == NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED_8000MBPS)
1117 {
1118 data = FLD_SET_DRF(2080, _CTRL_BUS_INFO_PCIE_LINK_CAP,
1119 _GEN, _GEN3, data);
1120 }
1121 else if (temp == NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED_5000MBPS)
1122 {
1123 data = FLD_SET_DRF(2080, _CTRL_BUS_INFO_PCIE_LINK_CAP,
1124 _GEN, _GEN2, data);
1125 }
1126 else
1127 {
1128 data = FLD_SET_DRF(2080, _CTRL_BUS_INFO_PCIE_LINK_CAP,
1129 _GEN, _GEN1, data);
1130 }
1131 }
1132 }
1133 else
1134 {
1135 if (IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu))
1136 {
1137 NV2080_CTRL_BUS_INFO busInfo = {0};
1138 NV_STATUS rmStatus = NV_OK;
1139
1140 busInfo.index = NV2080_CTRL_BUS_INFO_INDEX_PCIE_GEN_INFO;
1141
1142 if ((rmStatus = kbusSendBusInfo(pGpu, GPU_GET_KERNEL_BUS(pGpu), &busInfo)) != NV_OK)
1143 {
1144 NV_PRINTF(LEVEL_INFO, "Squashing rmStatus: %x \n", rmStatus);
1145 rmStatus = NV_OK;
1146 busInfo.data = 0;
1147 }
1148 data = busInfo.data;
1149 }
1150 }
1151 break;
1152 }
1153 case NV2080_CTRL_BUS_INFO_INDEX_PCIE_ROOT_LINK_CAPS:
1154 {
1155 if (clPcieReadPortConfigReg(pGpu, pCl,
1156 &pGpu->gpuClData.rootPort,
1157 CL_PCIE_LINK_CAP, &data) != NV_OK)
1158 {
1159 data = 0;
1160 }
1161 break;
1162 }
1163 case NV2080_CTRL_BUS_INFO_INDEX_PCIE_DOWNSTREAM_LINK_CAPS:
1164 {
1165 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_BR03_PRESENT) ||
1166 pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_BR04_PRESENT))
1167 {
1168 if (clPcieReadPortConfigReg(pGpu, pCl,
1169 &pGpu->gpuClData.boardDownstreamPort,
1170 CL_PCIE_LINK_CAP, &data) != NV_OK)
1171 {
1172 data = 0;
1173 }
1174 }
1175 else
1176 {
1177 // no br03/br04, same as link from RC
1178 if (clPcieReadPortConfigReg(pGpu, pCl,
1179 &pGpu->gpuClData.rootPort,
1180 CL_PCIE_LINK_CAP, &data) != NV_OK)
1181 {
1182 data = 0;
1183 }
1184 }
1185 break;
1186 }
1187 case NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_LINK_CTRL_STATUS:
1188 {
1189 data = kbifGetGpuLinkControlStatus(pGpu, pKernelBif);
1190 break;
1191 }
1192 case NV2080_CTRL_BUS_INFO_INDEX_PCIE_UPSTREAM_LINK_CTRL_STATUS:
1193 case NV2080_CTRL_BUS_INFO_INDEX_PCIE_BOARD_LINK_CTRL_STATUS:
1194 {
1195 if (_doesBoardHaveMultipleGpusAndSwitch(pGpu))
1196 {
1197 if (clPcieReadPortConfigReg(pGpu, pCl,
1198 &pGpu->gpuClData.boardUpstreamPort,
1199 CL_PCIE_LINK_CTRL_STATUS, &data) != NV_OK)
1200 {
1201 data = 0;
1202 }
1203 }
1204 else
1205 {
1206 data = kbifGetGpuLinkControlStatus(pGpu, pKernelBif);
1207 }
1208 break;
1209 }
1210 case NV2080_CTRL_BUS_INFO_INDEX_PCIE_ROOT_LINK_CTRL_STATUS:
1211 {
1212 if (clPcieReadPortConfigReg(pGpu, pCl,
1213 &pGpu->gpuClData.rootPort,
1214 CL_PCIE_LINK_CTRL_STATUS,
1215 &data) != NV_OK)
1216 {
1217 data = 0;
1218 }
1219 break;
1220 }
1221 case NV2080_CTRL_BUS_INFO_INDEX_PCIE_DOWNSTREAM_LINK_CTRL_STATUS:
1222 {
1223 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_BR03_PRESENT) ||
1224 pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_BR04_PRESENT))
1225 {
1226 if (clPcieReadPortConfigReg(pGpu, pCl,
1227 &pGpu->gpuClData.boardDownstreamPort,
1228 CL_PCIE_LINK_CTRL_STATUS, &data) != NV_OK)
1229 {
1230 data = 0;
1231 }
1232 }
1233 else
1234 {
1235 // no br03/br04, same as link from RC
1236 if (clPcieReadPortConfigReg(pGpu, pCl,
1237 &pGpu->gpuClData.rootPort,
1238 CL_PCIE_LINK_CTRL_STATUS,
1239 &data) != NV_OK)
1240 {
1241 data = 0;
1242 }
1243 }
1244 break;
1245 }
1246 case NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_LINK_ERRORS:
1247 {
1248 NvU32 xveStatus = 0;
1249
1250 if (pKernelBif != NULL)
1251 {
1252 if (kbifGetXveStatusBits_HAL(pGpu, pKernelBif, &data, &xveStatus) != NV_OK)
1253 {
1254 data = 0;
1255 break;
1256 }
1257 if (kbifClearXveStatus_HAL(pGpu, pKernelBif, &xveStatus) != NV_OK)
1258 {
1259 data = 0;
1260 }
1261 }
1262 break;
1263 }
1264 case NV2080_CTRL_BUS_INFO_INDEX_PCIE_ROOT_LINK_ERRORS:
1265 {
1266 NvU32 clStatus = 0;
1267
1268 if (clPcieReadDevCtrlStatus(pGpu, pCl, &data, &clStatus) != NV_OK)
1269 {
1270 data = 0;
1271 break;
1272 }
1273 if (clPcieClearDevCtrlStatus(pGpu, pCl, &clStatus) != NV_OK)
1274 {
1275 data = 0;
1276 }
1277 break;
1278 }
1279 case NV2080_CTRL_BUS_INFO_INDEX_PCIE_GPU_LINK_AER:
1280 {
1281 if (pKernelBif != NULL)
1282 {
1283 if (kbifGetXveAerBits_HAL(pGpu, pKernelBif, &data) != NV_OK)
1284 {
1285 data = 0;
1286 break;
1287 }
1288 if (kbifClearXveAer_HAL(pGpu, pKernelBif, data) != NV_OK)
1289 {
1290 data = 0;
1291 }
1292 }
1293 break;
1294 }
1295 case NV2080_CTRL_BUS_INFO_INDEX_MSI_INFO:
1296 {
1297 if (kbifIsMSIEnabledInHW_HAL(pGpu, pKernelBif))
1298 {
1299 data = FLD_SET_DRF(2080, _CTRL_BUS_INFO_MSI,
1300 _STATUS, _ENABLED, data);
1301 }
1302 else
1303 {
1304 data = FLD_SET_DRF(2080, _CTRL_BUS_INFO_MSI,
1305 _STATUS, _DISABLED, data);
1306 }
1307 break;
1308 }
1309
1310 default:
1311 break;
1312 }
1313
1314 pBusInfo->data = data;
1315 return NV_OK;
1316 }
1317
1318 /*!
1319 * @brief To ensure GPU is back on bus and accessible by polling device ID
1320 *
1321 * @param[in] pGpu GPU object pointer
1322 * @param[in] pKernelBif Kernel BIF object pointer
1323 *
1324 * @returns NV_OK
1325 * @returns NV_ERR_TIMEOUT
1326 */
1327 NV_STATUS
kbifPollDeviceOnBus_IMPL(OBJGPU * pGpu,KernelBif * pKernelBif)1328 kbifPollDeviceOnBus_IMPL
1329 (
1330 OBJGPU *pGpu,
1331 KernelBif *pKernelBif
1332 )
1333 {
1334 RMTIMEOUT timeout;
1335
1336 gpuSetTimeout(pGpu, GPU_TIMEOUT_DEFAULT, &timeout, 0);
1337
1338 while (osPciInitHandle(gpuGetDomain(pGpu),
1339 gpuGetBus(pGpu),
1340 gpuGetDevice(pGpu), 0, NULL, NULL) == NULL)
1341 {
1342 if (gpuCheckTimeout(pGpu, &timeout) == NV_ERR_TIMEOUT)
1343 {
1344 NV_PRINTF(LEVEL_ERROR, "Timeout polling GPU back on bus\n");
1345 DBG_BREAKPOINT();
1346 return NV_ERR_TIMEOUT;
1347 }
1348 osDelayUs(100);
1349 }
1350
1351 return NV_OK;
1352 }
1353
1354 /*!
1355 * @brief To get PCI link max spead from PCI link Gen info
1356 *
1357 * @param[in] pGpu GPU object pointer
1358 * @param[in] pKernelBif Kernel BIF object pointer
1359 * @param[in] pciLinkGenInfo PCI link Gen info
1360 * @param[in] pciLinkMaxSpeed pointer to PCI link max spead
1361
1362 * @returns NV_OK
1363 * @returns NV_ERR_INVALID_STATE
1364 */
1365 NV_STATUS
kbifGetPciLinkMaxSpeedByPciGenInfo_IMPL(OBJGPU * pGpu,KernelBif * pKernelBif,NvU32 pciLinkGenInfo,NvU32 * pciLinkMaxSpeed)1366 kbifGetPciLinkMaxSpeedByPciGenInfo_IMPL
1367 (
1368 OBJGPU *pGpu,
1369 KernelBif *pKernelBif,
1370 NvU32 pciLinkGenInfo,
1371 NvU32 *pciLinkMaxSpeed
1372 )
1373 {
1374 NV_STATUS rmStatus = NV_OK;
1375
1376 switch (pciLinkGenInfo)
1377 {
1378 case NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_GEN_GEN1:
1379 *pciLinkMaxSpeed = NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED_2500MBPS;
1380 break;
1381
1382 case NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_GEN_GEN2:
1383 *pciLinkMaxSpeed = NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED_5000MBPS;
1384 break;
1385
1386 case NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_GEN_GEN3:
1387 *pciLinkMaxSpeed = NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED_8000MBPS;
1388 break;
1389
1390 case NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_GEN_GEN4:
1391 *pciLinkMaxSpeed = NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED_16000MBPS;
1392 break;
1393
1394 case NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_GEN_GEN5:
1395 *pciLinkMaxSpeed = NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED_32000MBPS;
1396 break;
1397
1398 case NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_GEN_GEN6:
1399 *pciLinkMaxSpeed = NV2080_CTRL_BUS_INFO_PCIE_LINK_CAP_MAX_SPEED_64000MBPS;
1400 break;
1401
1402 default:
1403 rmStatus = NV_ERR_INVALID_STATE;
1404 NV_PRINTF(LEVEL_ERROR, "Unknown PCIe Gen Info\n");
1405 }
1406
1407 return rmStatus;
1408 }
1409